diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index dcc8595932..0f2ef680df 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -51,6 +51,9 @@ outputs: bootstrap: description: Run the bootstrap installer lane (install.sh sandbox + stamp verification). value: ${{ steps.classify.outputs.bootstrap }} + desktop_updater: + description: Run the Windows desktop-update hand-off (windows.ps1) integration tests. + value: ${{ steps.classify.outputs.desktop_updater }} rust: description: Run `cargo test` for the Tauri bootstrap installer. value: ${{ steps.classify.outputs.rust }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e148debf07..1b1a3dd300 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,6 +50,7 @@ jobs: npm_lock: ${{ steps.classify.outputs.npm_lock }} installer: ${{ steps.classify.outputs.installer }} bootstrap: ${{ steps.classify.outputs.bootstrap }} + desktop_updater: ${{ steps.classify.outputs.desktop_updater }} rust: ${{ steps.classify.outputs.rust }} docker_meta: ${{ steps.classify.outputs.docker_meta }} mcp_catalog: ${{ steps.classify.outputs.mcp_catalog }} @@ -76,6 +77,7 @@ jobs: if: needs.detect.outputs.python == 'true' uses: ./.github/workflows/tests.yml + lint: name: Python lints needs: detect @@ -123,7 +125,13 @@ jobs: # single job in the workflow — while still running the full pytest lanes. # # Re-disabled (Sep 2026): the Sep 1 re-enable is still incredibly flaky. - if: ${{ false && (needs.detect.outputs.python_prod == 'true' || needs.detect.outputs.frontend == 'true' })} + # Keep this a bare `if: false`. The earlier + # `${{ false && (... || ...) }}` form on this reusable-workflow job made + # GitHub's workflow parser fail at startup ("An unexpected error has + # occurred") — every ci.yaml run repo-wide dispatched 0 jobs from + # 24f5a60ed1 until this line changed. To re-enable, restore: + # if: ${{ needs.detect.outputs.python_prod == 'true' || needs.detect.outputs.frontend == 'true' }} + if: false uses: ./.github/workflows/e2e-desktop.yml docs-site: diff --git a/.gitignore b/.gitignore index aa9b7f8f63..86d1fa906f 100644 --- a/.gitignore +++ b/.gitignore @@ -238,6 +238,9 @@ apps/shared/src/**/*.js apps/shared/src/**/*.js.map apps/shared/src/**/*.d.ts apps/desktop/release/ +# stage-and-swap Desktop rebuild output (#86443); removed after the swap, but +# a killed build must not leave the checkout dirty +apps/desktop/.staging-*/ *.tsbuildinfo # Web UI assets — synced from @nous-research/ui at build time via diff --git a/agent/agent_init.py b/agent/agent_init.py index 8bba984e38..0e7cdc27e0 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1831,6 +1831,9 @@ def init_agent( except Exception: agent.show_commentary = True + # Window (seconds) for the bounded /fast auto|cold modes (agent.fast_mode). + agent.fast_auto_seconds = (_agent_cfg.get("agent") or {}).get("fast_auto_seconds", 60) + # LM Studio can either be explicitly preloaded through LM Studio's # management API (the historical Hermes behavior) or left to LM Studio's # just-in-time / Auto-Evict chat-completions path. Keep the default @@ -1851,10 +1854,39 @@ def init_agent( except Exception: agent.lmstudio_load_mode = "explicit" + # API-transport streaming (``model.streaming``, default true). The + # conversation loop prefers ``stream=True`` for every turn — including + # subagent turns — to get fine-grained liveness health-checking (#3120), + # but self-hosted OpenAI-compatible backends with broken streaming + # tool-call paths (e.g. vLLM ``--tool-call-parser qwen3_xml`` + a + # reasoning parser can leak tool-call markup into plain text and return + # zero ``tool_calls``, #72901) silently no-op instead of executing. + # ``model.streaming: false`` seeds ``_disable_streaming`` so the session + # uses the non-streaming path, which the loop already falls back to at + # runtime when a provider rejects streaming. The setting is + # session-scoped: it persists across mid-session model switches, mirroring + # the runtime fallback's semantics. Orthogonal to ``display.streaming`` + # (token rendering) — display-only settings are untouched. + agent._disable_streaming = False + try: + _model_section = _agent_cfg.get("model", {}) + if isinstance(_model_section, dict): + _streaming = str(_model_section.get("streaming", "true")).strip().lower() + if _streaming in {"false", "0", "no", "off"}: + agent._disable_streaming = True + elif _streaming not in {"true", "1", "yes", "on"}: + logger.warning( + "Invalid model.streaming=%r; expected a boolean. Using streaming (default).", + _model_section.get("streaming"), + ) + except Exception: + agent._disable_streaming = False + try: agent._tool_guardrails = ToolCallGuardrailController( ToolCallGuardrailConfig.from_mapping( - _agent_cfg.get("tool_loop_guardrails", {}) + _agent_cfg.get("tool_loop_guardrails", {}), + platform=platform, ) ) except Exception as _tlg_err: @@ -2999,8 +3031,10 @@ def init_agent( except Exception as _ce_err: _ra().logger.debug("Context engine on_session_start: %s", _ce_err) + from agent.runtime_cwd import scope_terminal_cwd as _scope_terminal_cwd + agent._subdirectory_hints = SubdirectoryHintTracker( - working_dir=os.getenv("TERMINAL_CWD") or None, + working_dir=_scope_terminal_cwd() or None, ) agent._user_turn_count = 0 # Copilot x-initiator flag: first API call of a user turn sends "user" (#3040). diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index ffd9ba61fe..1880698e81 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -100,6 +100,17 @@ _STRAY_TOOL_CALL_CLOSER_PATTERN = re.compile( re.IGNORECASE, ) +# A tool-call opener with no closer, or GLM-style argument markup +# (/) outside any closed block, means the stream was +# cut mid-serialization of a text-channel tool call (#101899). The call +# can't be recovered; strip from the block-boundary opener (or the line +# holding the first stray argument tag) to the end of the text. +_UNTERMINATED_TOOL_CALL_PATTERN = re.compile( + rf'(?:^|\n)[ \t]*<(?:{"|".join(_TOOL_CALL_TAG_NAMES)})\b[^>]*>.*$' + r'|(?:^|\n)[^\n<]* str: # during streaming may still be valuable to the user; matches # OpenClaw's intentional asymmetry.) content = _STRAY_TOOL_CALL_CLOSER_PATTERN.sub('', content) + # 3c. Tool-call openers or argument markup surviving 1b belong to a + # block that never closed — a mid-serialization stream cut (#101899). + content = _UNTERMINATED_TOOL_CALL_PATTERN.sub('', content) return content @@ -2709,6 +2723,71 @@ def anthropic_prompt_cache_policy( +def _provider_supplied_client(agent, client_kwargs: dict) -> Any | None: + """Ask the registered ProviderProfile for a custom client, if any. + + Resolves by provider name first, then by the ``base_url`` scheme prefix so a + runtime configured only by URL (``acp://…``) still reaches its profile. + A profile that raises is logged and skipped: a third-party plugin must not + be able to take the turn down, it can only fail to provide a client. + """ + try: + from providers import get_provider_profile + except Exception: + return None + + profile = None + provider_name = (getattr(agent, "provider", "") or "").strip() + if provider_name: + try: + profile = get_provider_profile(provider_name) + except Exception: + profile = None + if profile is None: + base_url = str(client_kwargs.get("base_url", "") or "").strip() + if base_url: + profile = _profile_for_base_url(base_url) + if profile is None: + return None + + try: + return profile.create_client(**client_kwargs) + except Exception: + _ra().logger.warning( + "Provider profile %r failed to create a client; falling back to the " + "standard client path", + getattr(profile, "name", provider_name) or "?", + exc_info=True, + ) + return None + + +def _profile_for_base_url(base_url: str) -> Any | None: + """Find a registered profile whose own base_url matches ``base_url``. + + Only used when the provider name did not resolve. Matches on exact base_url + so a non-HTTP scheme (``acp://copilot``) routes to its profile even when the + caller passed no provider name. + """ + try: + from providers import list_providers + except Exception: + return None + target = base_url.rstrip("/").lower() + try: + candidates = list_providers() + except Exception: + return None + for candidate in candidates or []: + own = str(getattr(candidate, "base_url", "") or "").rstrip("/").lower() + # Prefix match, not equality: the replaced copilot-acp branch keyed on + # ``startswith("acp://copilot")``, so a base_url carrying a path or a + # user override under the same root must still resolve. + if own and (target == own or target.startswith(own + "/")): + return candidate + return None + + def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: bool) -> Any: from agent.auxiliary_client import _validate_base_url, _validate_proxy_env_urls from agent.ssl_verify import resolve_httpx_verify @@ -2737,17 +2816,24 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo httpx_verify = resolve_httpx_verify(ca_bundle=ssl_ca_cert, ssl_verify=ssl_verify_cfg) _validate_proxy_env_urls() _validate_base_url(client_kwargs.get("base_url")) - if agent.provider == "copilot-acp" or str(client_kwargs.get("base_url", "")).startswith("acp://copilot"): - from agent.copilot_acp_client import CopilotACPClient - - client = CopilotACPClient(**client_kwargs) + # ── Provider-supplied client (registration seam) ────────────────────── + # A provider whose wire protocol is not OpenAI-over-HTTP supplies its own + # client from its ProviderProfile.create_client(). Consulted before the + # built-in ladder so a profile registered from ~/.hermes/plugins/ or a pip + # entry point can ship a transport without editing this function — that is + # what makes an out-of-tree ACP provider possible at all. Returning None + # (the default) falls through to the paths below, so every existing + # provider is unaffected. + provider_client = _provider_supplied_client(agent, client_kwargs) + if provider_client is not None: _ra().logger.info( - "Copilot ACP client created (%s, shared=%s) %s", + "%s client created from provider profile (%s, shared=%s) %s", + agent.provider, reason, shared, agent._client_log_context(), ) - return client + return provider_client if agent.provider == "gemini": from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url @@ -2788,6 +2874,12 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo # constructs a fresh one — no stale closed transport can be reused. # Tests in ``tests/run_agent/test_create_openai_client_reuse.py`` and # ``tests/run_agent/test_sequential_chats_live.py`` pin this invariant. + # What IS shared across those per-client wrappers is the underlying + # connection pool: ``build_keepalive_http_client`` mounts a + # process-shared ``HTTPTransport`` behind a per-client view whose + # ``close()`` is a no-op for the pool, so a closed wrapper never takes + # a sibling's (or the successor's) connections with it + # (tests/agent/test_shared_http_transport.py). if "http_client" not in client_kwargs: keepalive_http = agent._build_keepalive_http_client( client_kwargs.get("base_url", ""), verify=httpx_verify, @@ -3529,7 +3621,7 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i pass return result - if function_name == "todo": + if function_name == "todo_list": def _execute(next_args: dict) -> Any: from tools.todo_tool import todo_tool as _todo_tool return _finish_agent_tool( @@ -3671,7 +3763,7 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i ), next_args, ) - elif function_name == "tour": + elif function_name == "gui_tour": def _execute(next_args: dict) -> Any: from tools.tour_tool import tour_tool as _tour_tool return _finish_agent_tool( @@ -4836,8 +4928,8 @@ def reapply_reasoning_echo_for_provider(agent, api_messages: list) -> int: ) -def _iter_httpx_pool_objects(http_client: Any): - """Yield httpcore pool objects reachable from an httpx client. +def _iter_httpx_pools_with_owner(http_client: Any): + """Yield ``(pool, owner)`` pairs reachable from an httpx client. Hermes' keepalive client (#10324 / ``_build_keepalive_http_client``) and any ``HTTP(S)_PROXY`` configuration put live connections on *mounted* @@ -4846,31 +4938,37 @@ def _iter_httpx_pool_objects(http_client: Any): ``force_close_tcp_sockets`` return 0 while a stream is still mid-recv — the interrupt logs success and the provider keeps burning the slot (#72975). + + ``owner`` is ``None`` for a pool this client owns outright, or the + ``_SharedTransport`` view id when the pool is process-shared with other + clients (``process_bootstrap.build_keepalive_http_client``). Callers must + then touch only the in-flight requests stamped with that owner. """ seen_pools: set[int] = set() - def _emit(pool: Any): + def _emit(pool: Any, owner: Any): if pool is None: return marker = id(pool) if marker in seen_pools: return seen_pools.add(marker) - yield pool + yield pool, owner def _pools_for_transport(transport: Any): if transport is None: return + owner = id(transport) if type(transport).__name__ == "_SharedTransport" else None # Normal httpx.HTTPTransport / HTTPProxy-as-transport: connections # live under ``_pool``. HTTPProxy itself *is* a ConnectionPool and # may be mounted directly — then ``_connections`` is on the # transport. pool = getattr(transport, "_pool", None) if pool is not None: - yield from _emit(pool) + yield from _emit(pool, owner) return if getattr(transport, "_connections", None) is not None: - yield from _emit(transport) + yield from _emit(transport, owner) try: yield from _pools_for_transport(getattr(http_client, "_transport", None)) @@ -4881,6 +4979,12 @@ def _iter_httpx_pool_objects(http_client: Any): return +def _iter_httpx_pool_objects(http_client: Any): + """Yield httpcore pool objects reachable from an httpx client.""" + for pool, _owner in _iter_httpx_pools_with_owner(http_client): + yield pool + + def _connection_candidates(conn: Any): """Walk nested ``_connection`` wrappers (proxy tunnel → HTTP11/2).""" seen: set[int] = set() @@ -4919,23 +5023,32 @@ def _iter_pool_sockets(client: Any): # Some SDK wrappers *are* the httpx client (or expose the pool # directly). Fall through so mount-aware discovery still runs. http_client = client - pools = list(_iter_httpx_pool_objects(http_client)) + pools = list(_iter_httpx_pools_with_owner(http_client)) except Exception: return if not pools: return + from agent.process_bootstrap import HERMES_TRANSPORT_OWNER_EXT + seen: set[int] = set() - for pool in pools: + for pool, owner in pools: # Empty-list is falsy: use ``is None`` so an empty ``_connections`` # still lets us walk in-flight ``_requests`` rather than skipping # the pool entirely. raw_conns = getattr(pool, "_connections", None) if raw_conns is None: raw_conns = getattr(pool, "_pool", None) - connections = list(raw_conns or []) + # A process-shared pool carries other clients' idle + in-flight + # connections: only this client's own in-flight requests (stamped by + # ``_SharedTransport.handle_request``) may be shut down. + connections = [] if owner is not None else list(raw_conns or []) for pool_req in list(getattr(pool, "_requests", None) or []): + if owner is not None: + exts = getattr(getattr(pool_req, "request", None), "extensions", None) or {} + if exts.get(HERMES_TRANSPORT_OWNER_EXT) != owner: + continue conn = getattr(pool_req, "connection", None) if conn is not None: connections.append(conn) diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 7712f57198..0c6a885a55 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -226,7 +226,7 @@ def _is_claude_model(model: str | None) -> bool: return "claude" in (model or "").lower() -_FAST_MODE_SUPPORTED_SUBSTRINGS = ("opus-4-6", "opus-4.6") +_FAST_MODE_SUPPORTED_SUBSTRINGS = ("opus-4-8", "opus-4.8", "opus-5") # ── Max output token limits per Anthropic model ─────────────────────── # Source: Anthropic docs + Cline model catalog. Anthropic's API requires @@ -433,13 +433,28 @@ def _forbids_sampling_params(model: str) -> bool: def _supports_fast_mode(model: str) -> bool: - """Return True for models that support Anthropic Fast Mode (speed=fast). + """Return True for models that accept the ``speed: "fast"`` request param. - Per Anthropic docs, fast mode is currently supported on Opus 4.6 only. - Sending ``speed: "fast"`` to any other Claude model (including Opus 4.7) - returns HTTP 400. This guard prevents silently 400'ing when stale config - or older callers leave fast mode enabled across a model upgrade. + Per the Anthropic fast-mode docs (research preview), the ``speed`` param + is supported on Opus 4.8 and Opus 5 — Claude API only. The matrix has + changed with nearly every Opus release, in both directions: + + - Opus 4.6 HAD fast mode at launch and LOST it (2026-06-29): requests + with ``speed: "fast"`` do not error — they silently run at standard + speed and bill standard rates (``usage.speed: "standard"``). Keeping + 4.6 in this allowlist would show users a fast toggle that does + nothing. + - Opus 4.7 never had it and hard-400s on the parameter. + - Dedicated ``…-fast`` model ids (e.g. OpenRouter's + ``claude-opus-4.8-fast``) select fast inference via the model field + itself and must NOT also receive the speed parameter. + + Keep this an explicit allowlist rather than a version-floor check so a + model that drops fast mode again fails closed (standard speed) instead + of silently 400'ing. """ + if "-fast" in model: + return False return any(v in model for v in _FAST_MODE_SUPPORTED_SUBSTRINGS) @@ -935,9 +950,9 @@ def build_anthropic_kwargs( thinking block signatures are stripped (they are Anthropic-proprietary). When *fast_mode* is True, adds ``extra_body["speed"] = "fast"`` and the - fast-mode beta header for ~2.5x faster output throughput on Opus 4.6. - Currently only supported on native Anthropic endpoints (not third-party - compatible ones). + fast-mode beta header for ~2.5x faster output throughput on Opus 4.8 / + Opus 5. Currently only supported on native Anthropic endpoints (not + third-party compatible ones). """ system, anthropic_messages = convert_messages_to_anthropic( messages, base_url=base_url, model=model @@ -1148,12 +1163,15 @@ def build_anthropic_kwargs( for _sampling_key in ("temperature", "top_p", "top_k"): kwargs.pop(_sampling_key, None) - # ── Fast mode (Opus 4.6 only) ──────────────────────────────────── + # ── Fast mode (Opus 4.8 / Opus 5) ──────────────────────────────── # Adds extra_body.speed="fast" + the fast-mode beta header for ~2.5x - # output speed. Per Anthropic docs, fast mode is only supported on - # Opus 4.6 — Opus 4.7 and other models 400 on the speed parameter. + # output speed. Per Anthropic docs the speed param is supported on + # Opus 4.8 and Opus 5 (research preview); Opus 4.7 400s on it and + # Opus 4.6 silently ignores it (standard speed, standard billing). # Only for native Anthropic endpoints — third-party providers would - # reject the unknown beta header and speed parameter. + # reject the unknown beta header and speed parameter, and Anthropic + # itself scopes fast mode to the Claude API (not Bedrock/Vertex/ + # Foundry). if ( fast_mode and not _is_third_party_anthropic_endpoint(base_url) @@ -1282,12 +1300,21 @@ def create_anthropic_message( for _event in stream: try: on_stream_event(_event) + except TimeoutError: + # The callback is the caller's deadline seam + # (#99692: the host waiting on this summary has + # already given up). Abandon the stream — the + # ``with`` closes it — instead of streaming an + # answer nobody will read. + raise except Exception: logger.debug( "%son_stream_event callback failed", log_prefix, exc_info=True, ) return stream.get_final_message() + except TimeoutError: + raise except Exception as exc: if not _is_stream_unavailable_error(exc): raise diff --git a/agent/anthropic_credentials.py b/agent/anthropic_credentials.py index e778e2a306..57fc0663b1 100644 --- a/agent/anthropic_credentials.py +++ b/agent/anthropic_credentials.py @@ -917,6 +917,22 @@ def _get_hermes_oauth_file() -> Path: return get_hermes_home() / ".anthropic_oauth.json" +def _root_hermes_oauth_file() -> Optional[Path]: + """Global-root ``.anthropic_oauth.json`` when running inside a named profile. + + ``None`` in classic mode (profile == root). Used to commit a rotation of a + grant the profile borrowed through the credential-pool root fallback. + """ + try: + from hermes_constants import get_default_hermes_root + root = get_default_hermes_root() + if root.resolve(strict=False) == get_hermes_home().resolve(strict=False): + return None + return root / ".anthropic_oauth.json" + except Exception: + return None + + def _generate_pkce() -> tuple: """Generate PKCE code_verifier and code_challenge (S256).""" import base64 @@ -1077,9 +1093,16 @@ def _write_hermes_oauth_credentials( access_token: str, refresh_token: Optional[str], expires_at_ms: Optional[int], + *, + target: Optional[Path] = None, ) -> None: """Write refreshed hermes_pkce tokens back to ~/.hermes/.anthropic_oauth.json. + ``target`` overrides the destination: a named profile that rotated a grant + it BORROWED from the global root (credential-pool root fallback) must + commit the new pair to the ROOT singleton, not create a forked copy under + its own HERMES_HOME (#100339). + Without this, a successful pool-level refresh of a ``hermes_pkce``-sourced entry is invisible to this singleton file. The next ``load_pool()`` call runs ``_seed_from_singletons()``, which reads the stale file and @@ -1090,7 +1113,7 @@ def _write_hermes_oauth_credentials( file, for the same reason ``_write_claude_code_credentials`` does: this is the commit step of the refresh transaction. """ - oauth_file = _get_hermes_oauth_file() + oauth_file = target if target is not None else _get_hermes_oauth_file() try: oauth_data = { "accessToken": access_token, diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 2aabeac76e..9a859bfc95 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -452,6 +452,16 @@ _aux_progress = threading.local() _aux_dispatch = threading.local() _aux_provider_response = threading.local() +# Absolute wall-clock deadline (time.monotonic) of the HOST waiting for this +# auxiliary call, when it has one (#99692). Liveness alone is not enough: a +# host also stops waiting at its own total ceiling, and the streamed consumer +# below bounds itself only by _aux_stream_total_ceiling() — a budget derived +# from the aux request timeout, which is >= the host ceiling for every +# configured value AND starts counting later. So the stream that outlives its +# abandoned host is not an edge case; it is the guaranteed outcome of every +# total-ceiling timeout. +_aux_stream_deadline = threading.local() + def _notify_aux_progress() -> None: """Tick the installed forward-progress hook, if any. Never raises.""" @@ -524,6 +534,37 @@ def _anthropic_event_has_content(event: Any) -> bool: return False +def _anthropic_aux_stream_event_hook() -> Callable[[Any], None]: + """Per-event callback for the Anthropic auxiliary wire. + + Records provider-response timing for every frame, ticks the forward-progress + hook only for substantive payloads (keepalive pings must not keep a stalled + summary alive), and — #99692 — stops the stream at the waiting host's + absolute deadline (``aux_stream_deadline``) or on an explicit hard cancel, + the same two stop conditions the chat.completions and Codex wires honour. + The ``TimeoutError`` is phrased with "timed out" so ``_is_timeout_error`` + classifies it like any other request timeout. + """ + host_deadline = _current_aux_stream_deadline() + started = time.monotonic() + + def _on_event(event: Any) -> None: + if _anthropic_event_has_content(event): + _notify_aux_provider_response() + else: + _notify_aux_timing_response() + if _aux_interrupt_cancel_requested(): + raise AuxiliaryExplicitCancellation() + if host_deadline is not None and time.monotonic() >= host_deadline: + raise TimeoutError( + "Anthropic auxiliary stream timed out at the host compression " + f"deadline after {time.monotonic() - started:.0f}s " + "(the caller already stopped waiting)" + ) + + return _on_event + + _CODEX_PROGRESS_DELTA_TYPES = frozenset( { "response.output_text.delta", @@ -586,6 +627,38 @@ def aux_progress_hook(hook): yield +def _current_aux_stream_deadline() -> Optional[float]: + """The waiting host's absolute monotonic deadline, if one is installed.""" + return getattr(_aux_stream_deadline, "value", None) + + +@contextlib.contextmanager +def aux_stream_deadline(deadline: Optional[float]): + """Publish the waiting host's absolute deadline to the stream consumer. + + *deadline* is a ``time.monotonic()`` timestamp — the same instant the host + itself stops waiting — or ``None`` for callers with no host deadline (a + no-op passthrough, so callers can wire it unconditionally). Re-entrant-safe. + + #99692: the progress hook is a one-way channel (worker -> host). This is the + return leg. ``8207862212`` releases the compression OWNER when the fence is + cancelled, but the isolated provider daemon + (:func:`_run_protected_sync_provider_call`) that holds the socket keeps + streaming to its own ``_aux_stream_total_ceiling`` budget — >= the host's + ceiling by construction — billing an abandoned summary the commit fence is + already guaranteed to refuse, and stacking one fresh orphan per turn on a + session that compression never managed to shrink. + """ + previous = getattr(_aux_stream_deadline, "value", None) + _aux_stream_deadline.value = ( + deadline if isinstance(deadline, (int, float)) else previous + ) + try: + yield + finally: + _aux_stream_deadline.value = previous + + # Back-compat alias — the timing hooks were introduced with this name. _aux_timing_hook = _aux_thread_local_hook @@ -628,6 +701,11 @@ def _run_protected_sync_provider_call( # the protected daemon path is taken. dispatch_hook = getattr(_aux_dispatch, "hook", None) provider_response_hook = getattr(_aux_provider_response, "hook", None) + # #99692: the stream is consumed on the daemon below, and thread-locals do + # not cross that boundary — an owner-thread-only deadline would leave the + # fix inert on exactly the path large-session compression takes (protected + # call + hard-cancel source installed). + host_deadline = _current_aux_stream_deadline() provider_context = contextvars.copy_context() done = threading.Event() outcome: dict[str, Any] = {} @@ -638,6 +716,7 @@ def _run_protected_sync_provider_call( aux_progress_hook(progress_hook), _aux_thread_local_hook(_aux_dispatch, dispatch_hook), _aux_thread_local_hook(_aux_provider_response, provider_response_hook), + aux_stream_deadline(host_deadline), aux_interrupt_protection(cancel_check=cancel_check), ): outcome["result"] = callback(kwargs) @@ -669,6 +748,23 @@ def _run_protected_sync_provider_call( return outcome.get("result") +def _client_declares(client_obj: Any, flag: str) -> bool: + """Whether ``client_obj`` (or its class) sets ``flag`` truthy. + + Capability declaration instead of isinstance: a client shipped by an + out-of-tree provider profile can opt out of the transport/async wrappers + without this module importing it. Mirrors ``SUPPORTS_HERMES_TOOL_CALLS`` in + ``agent/background_review.py``. Absent attribute → False, so every ordinary + client keeps its existing behaviour. + """ + if client_obj is None: + return False + try: + return bool(getattr(client_obj, flag, False)) + except Exception: + return False + + def _safe_isinstance(obj: Any, maybe_type: Any) -> bool: """Return False instead of raising when a patched symbol is not a type.""" try: @@ -1703,6 +1799,10 @@ class _CodexCompletionsAdapter: timeout = kwargs.get("timeout") if timeout is not None: resp_kwargs["timeout"] = timeout + # Per-request HTTP headers (OpenCode session affinity, Copilot + # x-initiator) map to real headers via the SDK kwarg — forward them. + if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]: + resp_kwargs["extra_headers"] = dict(kwargs["extra_headers"]) # Note: the Codex endpoint (chatgpt.com/backend-api/codex) does NOT # support max_output_tokens or temperature — omit to avoid 400 errors. @@ -1888,6 +1988,15 @@ class _CodexCompletionsAdapter: if total_timeout is not None: no_progress_timeout = min(no_progress_timeout, float(total_timeout)) hard_deadline = _start_monotonic + _aux_stream_total_ceiling(total_timeout) + # #99692: the waiting host's absolute deadline (compress_context + # publishes its commit-fence ceiling via aux_stream_deadline) clamps + # the hard ceiling so the re-armable watchdog Timer wakes and severs + # the socket at the instant the host stops waiting — a live Codex + # stream cannot otherwise be stopped by a per-event cancel check + # while it is blocked between events. + _host_deadline = _current_aux_stream_deadline() + if isinstance(_host_deadline, (int, float)) and _host_deadline < hard_deadline: + hard_deadline = float(_host_deadline) deadline_lock = threading.Lock() progress_deadline = [_start_monotonic + no_progress_timeout] saw_content = threading.Event() @@ -2447,6 +2556,13 @@ class _AnthropicCompletionsAdapter: from agent.anthropic_adapter import _forbids_sampling_params if not _forbids_sampling_params(model): anthropic_kwargs["temperature"] = temperature + # Per-request HTTP headers (OpenCode session affinity) — the Anthropic + # SDK accepts ``extra_headers`` on messages.create/stream too. + if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]: + anthropic_kwargs["extra_headers"] = { + **(anthropic_kwargs.get("extra_headers") or {}), + **kwargs["extra_headers"], + } # Pass through caller-supplied extra_body so providers behind # Anthropic-compatible gateways receive their per-vendor request @@ -2503,13 +2619,7 @@ class _AnthropicCompletionsAdapter: # stalled summary open. No-op when no hook is installed (None # keeps the fast get_final_message path). on_stream_event=( - ( - lambda event: ( - _notify_aux_provider_response() - if _anthropic_event_has_content(event) - else _notify_aux_timing_response() - ) - ) + _anthropic_aux_stream_event_hook() if _aux_progress_active() else None ), @@ -2750,7 +2860,9 @@ def _maybe_wrap_anthropic( Returns ``client_obj`` unchanged when: - - It's already an Anthropic/Codex/Gemini/CopilotACP wrapper. + - It's already a complete client — an Anthropic/Codex wrapper, or any + client declaring ``HERMES_SKIP_TRANSPORT_WRAP`` (the native and ACP + shims, in-tree or from a provider plugin). - The endpoint is an OpenAI-wire endpoint. - ``api_mode`` is explicitly set to a non-Anthropic transport. - The ``anthropic`` SDK is not installed (falls back to OpenAI wire). @@ -2768,18 +2880,12 @@ def _maybe_wrap_anthropic( # Other specialized adapters we should never re-dispatch. if _safe_isinstance(client_obj, CodexAuxiliaryClient): return client_obj - try: - from agent.gemini_native_adapter import GeminiNativeClient - if _safe_isinstance(client_obj, GeminiNativeClient): - return client_obj - except ImportError: - pass - try: - from agent.copilot_acp_client import CopilotACPClient - if _safe_isinstance(client_obj, CopilotACPClient): - return client_obj - except ImportError: - pass + # A client that declares itself complete is never re-dispatched through a + # wire adapter. Declared as a class attribute rather than isinstance-checked + # so an out-of-tree provider's client is covered too — and so this hot path + # no longer imports the native/ACP client modules just to type-test. + if _client_declares(client_obj, "HERMES_SKIP_TRANSPORT_WRAP"): + return client_obj # Explicit non-anthropic api_mode wins over URL heuristics. if api_mode and api_mode != "anthropic_messages": @@ -2935,13 +3041,19 @@ def _resolve_nous_pool_runtime_api(*, force_refresh: bool = False) -> Optional[t return api_key, base_url -def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]: +def _resolve_nous_runtime_api( + *, force_refresh: bool = False, stale_access_token: Optional[str] = None +) -> Optional[tuple[str, str]]: """Return fresh Nous runtime credentials when available. This mirrors the main agent's 401 recovery path and keeps auxiliary clients aligned with the singleton auth store + JWT refresh flow instead of relying only on whatever raw tokens happen to be sitting in auth.json or the credential pool. + + ``stale_access_token`` is the bearer that just 401'd; with ``force_refresh`` + it lets the auth store adopt a sibling process's rotation instead of + re-POSTing the shared grant. """ pooled = _resolve_nous_pool_runtime_api(force_refresh=force_refresh) if pooled is not None: @@ -2953,6 +3065,7 @@ def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[ creds = resolve_nous_runtime_credentials( timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=force_refresh, + stale_access_token=stale_access_token or None, ) except Exception as exc: logger.debug("Auxiliary Nous runtime credential resolution failed: %s", exc) @@ -4939,6 +5052,38 @@ def _is_invalid_aux_response_error(exc: Exception) -> bool: ) +# Auxiliary tasks that sit on a user-visible critical path. A same-provider +# retry after a full-budget timeout costs another whole ``timeout`` window +# before the fallback chain is reached, so these skip it and fall through +# immediately. Fast blips (a streaming-close or a 5xx) still retry, since +# those are cheap. See issue #54465 for the compression case. +_TIMEOUT_NO_RETRY_TASKS = frozenset({"compression", "vision"}) + + +def _should_skip_same_provider_retry(task: Optional[str], exc: Exception) -> bool: + """True when a transient error should go straight to fallback. + + Compression is on the critical preflight path: a user cannot continue or + resume an oversized session until it compacts. Vision is on the + interactive path: the turn holding the image cannot answer, and because + turns are serialised the following user messages stall behind it. For + those tasks a same-provider retry on a full-budget timeout means another + whole ``timeout`` of wall-clock before the fallback chain runs, doubling + the user-visible stall (#54465). + + Carve-out: a fast first-token fail (dead stream detected within the 60s + no-progress window, zero output seen — see ``_timeout_message``) is cheap, + so it keeps the normal same-provider retry; the provider is often fine + and only that one stream was stillborn. Mid-stream stalls and hard-ceiling + timeouts skip to fallback. + """ + return ( + task in _TIMEOUT_NO_RETRY_TASKS + and _is_timeout_error(exc) + and "no-progress timeout" not in str(exc) + ) + + def _evict_cached_clients(provider: str) -> None: """Drop cached auxiliary clients for a provider so fresh creds are used.""" normalized = _normalize_aux_provider(provider) @@ -6577,12 +6722,10 @@ def _to_async_client(sync_client, model: str, is_vision: bool = False): return AsyncGeminiNativeClient(sync_client), model except ImportError: pass - try: - from agent.copilot_acp_client import CopilotACPClient - if isinstance(sync_client, CopilotACPClient): - return sync_client, model - except ImportError: - pass + # Clients that are already usable from async code (the ACP shims drive a + # subprocess, not an HTTP connection pool) opt out of the async wrapper. + if _client_declares(sync_client, "HERMES_SKIP_ASYNC_WRAP"): + return sync_client, model async_kwargs = { "api_key": sync_client.api_key, @@ -7396,34 +7539,55 @@ def resolve_provider_client( or _read_main_model_for_aux(), provider, ) - if provider == "copilot-acp": + # Any external-process provider whose registered profile supplies a + # client is served here — keyed on the profile, not on a provider name, + # so an out-of-tree ACP provider reaches the auxiliary path (compression, + # vision, background review) exactly like the in-tree one. + _extproc_profile = None + try: + from providers import get_provider_profile as _get_provider_profile + + _extproc_profile = _get_provider_profile(provider) + except Exception: + _extproc_profile = None + if _extproc_profile is not None: api_key = str(creds.get("api_key", "")).strip() base_url = str(creds.get("base_url", "")).strip() command = str(creds.get("command", "")).strip() or None args = list(creds.get("args") or []) if not final_model: logger.warning( - "resolve_provider_client: copilot-acp requested but no model " - "was provided or configured" + "resolve_provider_client: %s requested but no model " + "was provided or configured", + provider, ) return None, None if not api_key or not base_url: logger.warning( - "resolve_provider_client: copilot-acp requested but external " - "process credentials are incomplete" + "resolve_provider_client: %s requested but external " + "process credentials are incomplete", + provider, ) return None, None - from agent.copilot_acp_client import CopilotACPClient - - client = CopilotACPClient( - api_key=api_key, - base_url=base_url, - command=command, - args=args, - ) - logger.debug("resolve_provider_client: %s (%s)", provider, final_model) - return (_to_async_client(client, final_model, is_vision=is_vision) if async_mode - else (client, final_model)) + try: + client = _extproc_profile.create_client( + api_key=api_key, + base_url=base_url, + command=command, + args=args, + ) + except Exception: + logger.warning( + "resolve_provider_client: profile %r failed to create an " + "external-process client", + provider, + exc_info=True, + ) + client = None + if client is not None: + logger.debug("resolve_provider_client: %s (%s)", provider, final_model) + return (_to_async_client(client, final_model, is_vision=is_vision) if async_mode + else (client, final_model)) if provider not in _LOGGED_UNSUPPORTED_EXTPROC_KEYS: _LOGGED_UNSUPPORTED_EXTPROC_KEYS.add(provider) logger.debug("resolve_provider_client: external-process provider %s not " @@ -8128,9 +8292,30 @@ def _refresh_nous_auxiliary_client( api_mode: Optional[str] = None, main_runtime: Optional[Dict[str, Any]] = None, is_vision: bool = False, + lookup_model: Optional[str] = None, + lookup_task: Optional[str] = None, ) -> Tuple[Optional[Any], Optional[str]]: - """Refresh Nous runtime creds, rebuild the client, and replace the cache entry.""" - runtime = _resolve_nous_runtime_api(force_refresh=True) + """Refresh Nous runtime creds, rebuild the client, and replace the cache entry. + + ``model`` is the resolved model actually sent on the wire (e.g. the provider + default ``"Hermes-4-405B"``); it is stored as the entry's usable model and + returned to the caller. ``lookup_model`` is the model as it was passed to + ``_get_cached_client`` when the (now stale) client was acquired -- ``None`` + on the default Nous config, where ``call_llm`` looks up with + ``resolved_model=None``. The cache KEY MUST be built from ``lookup_model`` so + the fresh client overwrites the exact entry the stale client is served from. + Keying on the resolved ``model`` instead stored under a different key (model + element ``"Hermes-4-405B"`` vs the lookup's ``""``), leaving the expired + client immortal so every auxiliary call 401s forever (#56889). + + ``lookup_task`` is the task the stale client was acquired under. For + ``provider == "auto"`` the task participates in the cache key (task-specific + fallback policy), so it MUST be carried into the key here for the same + reason as ``lookup_model``; otherwise an auto-provider client refreshed on a + 401 lands under the ``task=""`` key while the stale entry survives under the + task-scoped key (#58894). + """ + runtime = _resolve_nous_runtime_api(force_refresh=True, stale_access_token=api_key) if runtime is None: return None, model @@ -8157,7 +8342,8 @@ def _refresh_nous_auxiliary_client( api_mode=api_mode, main_runtime=main_runtime, is_vision=is_vision, - model=final_model, + task=lookup_task, + model=lookup_model, ) _store_cached_client(cache_key, client, final_model, bound_loop=current_loop) return client, final_model @@ -9387,7 +9573,13 @@ def _build_call_kwargs( ): kwargs["_reasoning_config"] = dict(reasoning_config) - return kwargs + # OpenCode relay session affinity — same key as the main turn so + # compression/title/vision calls stay on the conversation's warm backend. + from agent.opencode_affinity import merge_opencode_session_headers + + return merge_opencode_session_headers( + kwargs, provider, base_url, _runtime_main_value("session_id") or None + ) def _validate_llm_response( @@ -9871,7 +10063,11 @@ def _aggregate_chat_stream( Accumulation is shared with the async mirror via :class:`_ChatStreamAccumulator`. """ - acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + acc = _ChatStreamAccumulator( + model=model, + total_ceiling=total_ceiling, + host_deadline=_current_aux_stream_deadline(), + ) try: for chunk in chunks: acc.feed(chunk) @@ -9893,9 +10089,20 @@ class _ChatStreamAccumulator: tool-call delta reassembly, same "timed out" ceiling phrasing). """ - def __init__(self, model: str = "", total_ceiling: Optional[float] = None): + def __init__( + self, + model: str = "", + total_ceiling: Optional[float] = None, + host_deadline: Optional[float] = None, + ): self._started = time.monotonic() self._total_ceiling = total_ceiling + # #99692: absolute instant the WAITING HOST gives up. Checked as well + # as (not instead of) the ceiling above: the ceiling still bounds + # callers with no host deadline, and the host deadline is absolute, so + # it is unaffected by however long dispatch and TTFT took before this + # accumulator was constructed. + self._host_deadline = host_deadline self.content_parts: List[str] = [] self.reasoning_parts: List[str] = [] self.reasoning_details: List[Any] = [] @@ -9919,6 +10126,16 @@ class _ChatStreamAccumulator: f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s " "total ceiling (stream still open but over budget)" ) + if ( + self._host_deadline is not None + and time.monotonic() >= self._host_deadline + ): + raise TimeoutError( + "Auxiliary streamed call timed out at the host compression " + f"deadline after {time.monotonic() - self._started:.0f}s " + "(the caller already stopped waiting; streaming on would only " + "pin its session lease)" + ) self.resp_id = getattr(chunk, "id", None) or self.resp_id self.resp_model = getattr(chunk, "model", None) or self.resp_model chunk_usage = getattr(chunk, "usage", None) @@ -10030,7 +10247,11 @@ async def _aggregate_chat_stream_async( the sync helper raises. Same accumulation and ceiling semantics via :class:`_ChatStreamAccumulator`. """ - acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + acc = _ChatStreamAccumulator( + model=model, + total_ceiling=total_ceiling, + host_deadline=_current_aux_stream_deadline(), + ) try: async for chunk in chunks: acc.feed(chunk) @@ -10455,29 +10676,15 @@ def _call_llm_impl( except Exception as transient_err: if not _is_transient_transport_error(transient_err): raise - # Compression is on the critical preflight path: a user cannot - # continue or resume an oversized session until it compacts. A - # same-provider retry on a timeout means another full ``timeout``- - # long wall-clock block before the except-chain below can fall - # back — doubling the user-visible stall (issue #54465). Skip the - # same-provider retry for compression on a full-budget timeout and - # fall straight through to provider/model fallback; fast blips (a - # streaming-close or a 5xx) still retry, since those are cheap. - if task == "compression" and _is_timeout_error(transient_err): - # A fast first-token fail (dead stream detected within the - # 60s no-progress window, zero output seen) is cheap — take - # the normal same-provider retry chain first; the provider - # is often fine and only that one stream was stillborn. A - # mid-stream stall or hard-ceiling timeout skips straight to - # fallback, because re-running a multi-minute summary on the - # same provider doubles the user-visible stall (#54465). - if "no-progress timeout" not in str(transient_err): - logger.info( - "Auxiliary compression: timeout on the critical path; " - "skipping same-provider retry and falling back: %s", - transient_err, - ) - raise + # Critical-path tasks skip the same-provider retry on a + # full-budget timeout; see _should_skip_same_provider_retry. + if _should_skip_same_provider_retry(task, transient_err): + logger.info( + "Auxiliary %s: timeout on the critical path; " + "skipping same-provider retry and falling back: %s", + task, transient_err, + ) + raise _max_transient_retries = _transient_retry_count() _last_transient = transient_err for _attempt in range(1, _max_transient_retries + 1): @@ -10656,6 +10863,8 @@ def _call_llm_impl( refreshed_client, refreshed_model = _refresh_nous_auxiliary_client( cache_provider=resolved_provider or "nous", model=final_model, + lookup_model=resolved_model, + lookup_task=task, async_mode=False, base_url=resolved_base_url, api_key=resolved_api_key, @@ -10692,6 +10901,8 @@ def _call_llm_impl( refreshed_client, refreshed_model = _refresh_nous_auxiliary_client( cache_provider=resolved_provider or "nous", model=final_model, + lookup_model=resolved_model, + lookup_task=task, async_mode=False, base_url=resolved_base_url, api_key=resolved_api_key, @@ -11289,14 +11500,13 @@ async def _async_call_llm_impl( except Exception as transient_err: if not _is_transient_transport_error(transient_err): raise - # See call_llm(): compression is on the critical preflight path, - # so skip the same-provider retry on a full-budget timeout and - # fall straight through to fallback (issue #54465). - if task == "compression" and _is_timeout_error(transient_err): + # Same rule as call_llm(); the async Codex adapter wraps the sync + # stream via to_thread, so the same TimeoutError reaches here. + if _should_skip_same_provider_retry(task, transient_err): logger.info( - "Auxiliary compression (async): timeout on the critical " + "Auxiliary %s (async): timeout on the critical " "path; skipping same-provider retry and falling back: %s", - transient_err, + task, transient_err, ) raise logger.info( @@ -11452,10 +11662,13 @@ async def _async_call_llm_impl( refreshed_client, refreshed_model = _refresh_nous_auxiliary_client( cache_provider=resolved_provider or "nous", model=final_model, + lookup_model=resolved_model, + lookup_task=task, async_mode=True, base_url=resolved_base_url, api_key=resolved_api_key, api_mode=resolved_api_mode, + main_runtime=main_runtime, is_vision=(task == "vision"), ) if refreshed_client is not None: @@ -11487,10 +11700,13 @@ async def _async_call_llm_impl( refreshed_client, refreshed_model = _refresh_nous_auxiliary_client( cache_provider=resolved_provider or "nous", model=final_model, + lookup_model=resolved_model, + lookup_task=task, async_mode=True, base_url=resolved_base_url, api_key=resolved_api_key, api_mode=resolved_api_mode, + main_runtime=main_runtime, is_vision=(task == "vision"), ) if refreshed_client is not None: diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index cbd21b46e8..ab6a8aa47a 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -34,6 +34,7 @@ from agent.error_classifier import ( PROVIDER_STREAM_NON_JSON_ERROR_CODE, ) from agent.errors import EmptyStreamError +from agent.fast_mode import effective_request_overrides from agent.turn_context import substitute_api_content from agent.gemini_native_adapter import is_native_gemini_base_url from agent.model_metadata import is_local_endpoint @@ -1432,6 +1433,35 @@ def interruptible_api_call(agent, api_kwargs: dict): # a network bug and surfaced to the caller. (PR #6600 — cascading interrupt # hang.) _request_cancelled = {"value": False} + # Codex Responses retirement token (codex_responses only). The worker + # thread reads it through ``agent._active_codex_stream_request_token`` to + # tell whether it still owns the turn. When a watchdog below force-closes + # the connection it clears the agent-level token, so a worker still + # draining SSE frames raises instead of returning its partial output as a + # "completed" response (see run_codex_stream's _request_is_current). + # ``_codex_request_retired`` is the request-local mirror, used to swallow + # the transport error our own force-close causes — same split as + # ``_request_cancelled`` above. + _codex_request_token = object() if agent.api_mode == "codex_responses" else None + _codex_request_retired = {"value": False} + + def _install_codex_request_token() -> None: + if _codex_request_token is None: + return + if _codex_request_retired["value"]: + # Already retired before the worker got going — do not re-publish. + return + agent._active_codex_stream_request_token = _codex_request_token + + def _retire_codex_request_token() -> None: + if _codex_request_token is None: + return + _codex_request_retired["value"] = True + if ( + getattr(agent, "_active_codex_stream_request_token", None) + is _codex_request_token + ): + agent._active_codex_stream_request_token = None def _set_request_client(client, *, kind: str = "openai"): with request_client_lock: @@ -1491,6 +1521,7 @@ def interruptible_api_call(agent, api_kwargs: dict): def _call(): try: + _install_codex_request_token() # _set_request_client registers each per-request client with the # stranger-thread abort machinery above; the shared dispatch helper # builds it via this callback (openai- or anthropic-kind) so the @@ -1513,15 +1544,34 @@ def interruptible_api_call(agent, api_kwargs: dict): # handler, the transport error is the expected consequence of our # own force-close, NOT a network bug. Swallow it instead of # surfacing — the main thread raises InterruptedError. (#6600) - if _request_cancelled["value"]: - logger.debug( - "Non-streaming worker caught %s after request cancellation — " - "exiting without surfacing a network error.", - type(e).__name__, - ) + if _request_cancelled["value"] or _codex_request_retired["value"]: + # Retirement is logged at info: it means a watchdog discarded + # output the provider had already sent, which is exactly the + # event an operator debugging a truncated reply needs to see. + # Cancellation stays at debug — a user interrupt is a normal, + # high-frequency outcome and the caller already surfaces it. + if _codex_request_retired["value"]: + logger.info( + "Codex worker caught %s after request retirement — " + "discarding the stale partial instead of surfacing it " + "as a completed response. %s", + type(e).__name__, + agent._client_log_context(), + ) + else: + logger.debug( + "Non-streaming worker caught %s after request " + "cancellation — exiting without surfacing a network " + "error.", + type(e).__name__, + ) return result["error"] = e finally: + # Retire first: _close_request_client_once can raise (every other + # call site wraps it in try/except), and a leaked token would let a + # later worker mistake itself for the owning attempt. + _retire_codex_request_token() # Reuse reason only on a clean response; any other outcome — # error, or the cancel-swallow return above (which leaves both # result slots None) — really closes so the next attempt builds @@ -1734,6 +1784,7 @@ def interruptible_api_call(agent, api_kwargs: dict): _close_request_client_once("codex_ttfb_kill") except Exception: pass + _retire_codex_request_token() agent._emit_wait_notice( f"⚠ no response from provider in {int(_elapsed)}s — " f"reconnecting..." @@ -1784,6 +1835,7 @@ def interruptible_api_call(agent, api_kwargs: dict): _close_request_client_once("codex_stream_idle_kill") except Exception: pass + _retire_codex_request_token() agent._touch_activity( f"codex stream killed after {int(_event_stale_elapsed)}s with no SSE events" ) @@ -1815,6 +1867,7 @@ def interruptible_api_call(agent, api_kwargs: dict): _close_request_client_once("stale_call_kill") except Exception: pass + _retire_codex_request_token() # Circuit breaker (#58962): count the stale kill. See the # canonical comment block above ``_stale_streak()``. _bump_stale_streak(agent) @@ -1862,6 +1915,7 @@ def interruptible_api_call(agent, api_kwargs: dict): _close_request_client_once("interrupt_abort") except Exception: pass + _retire_codex_request_token() # #81521 (sibling of the streaming-path fix): wait for the worker # to unwind Relay-managed scopes before surfacing # InterruptedError, so turn teardown cannot race a still-open @@ -1879,10 +1933,97 @@ def interruptible_api_call(agent, api_kwargs: dict): +def _consume_ephemeral_reasoning_off(agent) -> bool: + """Consume the one-shot "answer without thinking" continuation flag. + + Set by the length-continuation path when a request returned reasoning + but NO visible content — the thinking phase consumed the entire output + cap (GLM-5.3 on ollama-cloud with reasoning_effort=high: reported live as + finish_reason="length", content="", completion_tokens == max_tokens). + + Continuation turns never replay the prior reasoning, so re-running with + thinking ON re-derives — and re-burns — the whole thinking budget from + scratch instead of writing the answer (observed: 4 futile continuations + then "Response remained truncated after 4 continuation attempts"). + When True is returned the caller must override the wire reasoning_config + with ``{"enabled": False, "effort": "none"}`` for exactly the next call. + + Prompt-cache cost (deliberate, bounded): the reasoning parameter is part + of the provider's cache key on config-sensitive providers — Anthropic + renders thinking/effort into the prompt, OpenAI lists reasoning.effort + among prefix-affecting settings — so THAT one request misses the prefix + cache and pays a cold write of the full prefix (1.25x input instead of + the 0.1x read). The next request goes out with the configured reasoning + again and hits the thinking-on entry written by the truncated request + (still within TTL), so the damage is exactly one write. Template-tail + providers (GLM/Qwen/Kimi-style, where thinking on/off is a chat-template + switch at the tail) see no prefix change at all. The system prompt bytes + are never touched. This is far cheaper than what the flag prevents: four + full-output-budget requests that produce nothing and end the turn with an + error. + """ + if getattr(agent, "_ephemeral_reasoning_off", False): + agent._ephemeral_reasoning_off = False + return True + return False + + +def _reasoning_config_for_wire(agent): + """``agent.reasoning_config`` with the one-shot reasoning-off override applied. + + Once the route has answered a disable with "reasoning is mandatory" + (``agent._reasoning_disable_rejected``), every disable — configured or + the one-shot continuation override — is dropped for the rest of the + session: the request goes out without a reasoning config and the route + applies its own default. + """ + cfg = agent.reasoning_config + ephemeral_off = _consume_ephemeral_reasoning_off(agent) + if getattr(agent, "_reasoning_disable_rejected", False): + # The route rejects disables. Resend exactly what the session has + # been sending — the user's own config — so the retry lands on the + # same provider cache key as every prior request. Only a config that + # is itself a disable is dropped (omitted → route default), and that + # session has never sent anything else, so nothing warm is lost. + if isinstance(cfg, dict) and ( + cfg.get("enabled") is False or cfg.get("effort") == "none" + ): + return None + return cfg + if ephemeral_off: + cfg = {**(cfg or {}), "enabled": False, "effort": "none"} + return cfg + + def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = None) -> dict: - """Build the keyword arguments dict for the active API mode.""" + """Build the keyword arguments dict for the active API mode. + + Wraps the per-api_mode builder so the OpenCode ``x-opencode-session`` + affinity header rides on every OpenCode request regardless of transport + (chat_completions / codex_responses / anthropic_messages all route + OpenCode models). No-op for every other provider. + """ + from agent.opencode_affinity import merge_opencode_session_headers + + kwargs = _build_api_kwargs_for_mode(agent, api_messages, tools_for_api) + return merge_opencode_session_headers( + kwargs, + getattr(agent, "provider", None), + getattr(agent, "base_url", None), + getattr(agent, "session_id", None), + ) + + +def _build_api_kwargs_for_mode(agent, api_messages: list, tools_for_api: list | None = None) -> dict: + # One-shot continuation override — consumed exactly once, on the FIRST + # request this call builds (only one api_mode branch runs per invocation). + _wire_reasoning_config = _reasoning_config_for_wire(agent) if tools_for_api is None: tools_for_api = agent.tools + # The one place request_overrides are consumed: static /fast values are + # already pinned in agent.request_overrides; auto/cold windows layer the + # fast override here, per request, only while the window is open. + _request_overrides = effective_request_overrides(agent) if agent.api_mode == "anthropic_messages": _transport = agent._get_transport() @@ -1897,12 +2038,12 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non messages=anthropic_messages, tools=tools_for_api, max_tokens=ephemeral_out if ephemeral_out is not None else agent.max_tokens, - reasoning_config=agent.reasoning_config, + reasoning_config=_wire_reasoning_config, is_oauth=agent._is_anthropic_oauth, preserve_dots=agent._anthropic_preserve_dots(), context_length=ctx_len, base_url=getattr(agent, "_anthropic_base_url", None), - fast_mode=(agent.request_overrides or {}).get("speed") == "fast", + fast_mode=_request_overrides.get("speed") == "fast", drop_context_1m_beta=bool(getattr(agent, "_oauth_1m_beta_disabled", False)), ) # Nous Portal reads ``tags`` and ``session_id`` as top-level body fields @@ -1989,13 +2130,13 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non model=agent.model, messages=_msgs_for_codex, tools=tools_for_api, - reasoning_config=agent.reasoning_config, + reasoning_config=_wire_reasoning_config, session_id=getattr(agent, "session_id", None), cache_scope_id=_cache_scope_id, base_url=agent.base_url, max_tokens=agent.max_tokens, timeout=agent._resolved_api_call_timeout(), - request_overrides=agent.request_overrides, + request_overrides=_request_overrides, provider=getattr(agent, "provider", None), is_github_responses=is_github_responses, is_codex_backend=is_codex_backend, @@ -2146,8 +2287,8 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non max_tokens=agent.max_tokens, ephemeral_max_output_tokens=_ephemeral_out, max_tokens_param_fn=agent._max_tokens_param, - reasoning_config=agent.reasoning_config, - request_overrides=agent.request_overrides, + reasoning_config=_wire_reasoning_config, + request_overrides=_request_overrides, session_id=getattr(agent, "session_id", None), cache_scope_id=_cache_scope_id, provider_profile=_profile, @@ -2179,8 +2320,8 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non max_tokens=agent.max_tokens, ephemeral_max_output_tokens=_ephemeral_out, max_tokens_param_fn=agent._max_tokens_param, - reasoning_config=agent.reasoning_config, - request_overrides=agent.request_overrides, + reasoning_config=_wire_reasoning_config, + request_overrides=_request_overrides, session_id=getattr(agent, "session_id", None), cache_scope_id=_cache_scope_id, model_lower=(agent.model or "").lower(), @@ -3163,6 +3304,10 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: # tool_call was summarized away; Responses API rejects that as # "No tool call found for function call output". api_messages = agent._sanitize_api_messages(api_messages) + # Same send-path vision eviction as the main loop (#89296). + from agent.context_compressor import evict_stale_outbound_tool_images + + evict_stale_outbound_tool_images(api_messages) # Same safety net as the main loop: drop thinking-only assistant # turns so Anthropic-family providers don't 400 the summary call. @@ -3546,14 +3691,11 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if emit is not None: emit(final_text=final_text, finished=finished, error=error) - # Cron and other non-interactive, nested-pool contexts deadlock on the - # spawned worker thread (#62151). They also have no stream consumer, so the - # deltas this path produces go nowhere. Delegate to the non-streaming entry - # (which runs inline via should_use_direct_api_call) exactly like the codex - # branch below — routing through the _interruptible_api_call method keeps the - # outer loop's per-request retry/refresh seam intact. - if should_use_direct_api_call(agent): - return agent._interruptible_api_call(api_kwargs) + # Cron turns and delegated children (should_use_direct_api_call) used to be + # short-circuited here onto the NON-streaming wire. They now stay on this + # streaming path and run the request inline — see the ``_inline`` block + # before the poll loop below. Only the codex branch still detours through + # _interruptible_api_call (it streams internally). if agent.api_mode == "codex_responses": # Codex streams internally via _run_codex_stream. The main dispatch @@ -4118,6 +4260,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= _conn_cap = min(_base_timeout, 60.0) if _provider_timeout_cfg is not None else 30.0 content_parts: list = [] tool_calls_acc: dict = {} + tool_argument_parts: dict[int, list[str]] = {} tool_gen_notified: set = set() # Ollama-compatible endpoints reuse index 0 for every tool call # in a parallel batch, distinguishing them only by id. Track @@ -4224,7 +4367,12 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= last_chunk_time["t"] = time.time() return True + def _materialize_tool_arguments() -> None: + for index, parts in tool_argument_parts.items(): + tool_calls_acc[index]["function"]["arguments"] = "".join(parts) + def _relay_final_response() -> dict[str, Any]: + _materialize_tool_arguments() tool_calls = [tool_calls_acc[index] for index in sorted(tool_calls_acc)] return { "model": model_name, @@ -4493,6 +4641,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= "function": {"name": "", "arguments": ""}, "extra_content": None, } + tool_argument_parts[idx] = [] entry = tool_calls_acc[idx] tc_id = getattr(tc_delta, "id", None) if tc_id is not None: @@ -4516,7 +4665,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= entry["function"]["name"] = function_name function_arguments = getattr(tc_function, "arguments", None) if function_arguments: - entry["function"]["arguments"] += function_arguments + tool_argument_parts[idx].append(function_arguments) extra = getattr(tc_delta, "extra_content", None) if extra is None and hasattr(tc_delta, "model_extra"): extra = (tc_delta.model_extra if isinstance(tc_delta.model_extra, dict) else {}).get("extra_content") @@ -4592,6 +4741,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= mock_tool_calls = None has_truncated_tool_args = False if tool_calls_acc: + _materialize_tool_arguments() mock_tool_calls = [] for idx in sorted(tool_calls_acc): tc = tool_calls_acc[idx] @@ -5371,8 +5521,42 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if _reasoning_floor is not None: _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor) - t = threading.Thread(target=_context_thread_target(_call), daemon=True) - t.start() + # Delegated children and gateway cron turns run the streaming request + # INLINE on the conversation thread: spawning the interrupt worker inside + # their nested thread pools wedges before the socket opens (#62151, + # #60203). They used to be routed to the non-streaming wire for that + # reason — but streaming is also the transport keepalive and the + # liveness signal: a non-streaming POST that stays silent through a + # reasoning model's thinking phase is killed by edge proxies (z.ai 524, + # #90202) and by our own stale watchdog, which cannot tell thinking from + # a hang when no bytes ever arrive (#100260). Inline mode keeps the + # stream (per-token liveness) and moves ONLY the lightweight poll loop + # below — heartbeat, stale detector, interrupt abort — onto a monitor + # thread. The monitor never issues a request, so the no-worker property + # that fixes the deadlock class is preserved (same shape as the + # direct_api_call watchdog timer). + _inline = should_use_direct_api_call(agent) + _call_done = threading.Event() + _monitor_interrupted = {"yes": False} + + def _run_call(): + try: + _call() + finally: + _call_done.set() + + if _inline: + t = None + else: + t = threading.Thread(target=_context_thread_target(_run_call), daemon=True) + t.start() + + def _call_alive() -> bool: + return not _call_done.is_set() + + def _wait_call(timeout: float) -> None: + _call_done.wait(timeout=timeout) + _last_heartbeat = time.time() _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches # Managed local server: a cold model streams weights off disk for tens @@ -5385,173 +5569,198 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= _load_notice_shown = False _load_notice_misses = 0 _is_local_base = bool(agent.base_url) and is_local_endpoint(agent.base_url) - while t.is_alive(): - t.join(timeout=0.3) - _hb_now = time.time() - # Cold-load window: last_chunk_time is touched at request-client - # creation and then only by REAL chunks, so "no chunk for 2s+" is - # true through a model load (nothing can stream while the child is - # still mapping weights) and false during healthy token flow — - # which is what keeps this poll off the streaming hot path. The - # probe itself is an in-memory snapshot read. - if ( - _is_local_base - and _hb_now - last_chunk_time["t"] >= 2.0 - and _hb_now - _last_load_poll >= 1.0 - ): - _last_load_poll = _hb_now - _load_notice = _managed_local_load_notice(agent, api_kwargs) - if _load_notice is not None: - agent._emit_wait_notice(_load_notice) - agent._touch_activity("local model loading") - _load_notice_shown = True - _load_notice_misses = 0 - # Loading IS liveness for the heartbeat; the stale detector - # needs no help — the local floor (900s) dwarfs any load. - _last_heartbeat = _hb_now - continue - if _load_notice_shown: - # One missed sample is routine (a /slots read straddling a - # batch boundary, a 2s probe timeout under load) — clearing - # on it made the status line strobe blank once every few - # seconds mid-prefill. Only a SUSTAINED absence means the - # phase really ended. - _load_notice_misses += 1 - if _load_notice_misses >= 3: - _load_notice_shown = False + def _monitor_loop() -> None: + nonlocal _last_heartbeat, _last_load_poll, _load_notice_shown, _load_notice_misses + while _call_alive(): + _wait_call(0.3) + + _hb_now = time.time() + # Cold-load window: last_chunk_time is touched at request-client + # creation and then only by REAL chunks, so "no chunk for 2s+" is + # true through a model load (nothing can stream while the child is + # still mapping weights) and false during healthy token flow — + # which is what keeps this poll off the streaming hot path. The + # probe itself is an in-memory snapshot read. + if ( + _is_local_base + and _hb_now - last_chunk_time["t"] >= 2.0 + and _hb_now - _last_load_poll >= 1.0 + ): + _last_load_poll = _hb_now + _load_notice = _managed_local_load_notice(agent, api_kwargs) + if _load_notice is not None: + agent._emit_wait_notice(_load_notice) + agent._touch_activity("local model loading") + _load_notice_shown = True _load_notice_misses = 0 - agent._emit_wait_notice("") + # Loading IS liveness for the heartbeat; the stale detector + # needs no help — the local floor (900s) dwarfs any load. + _last_heartbeat = _hb_now + continue + if _load_notice_shown: + # One missed sample is routine (a /slots read straddling a + # batch boundary, a 2s probe timeout under load) — clearing + # on it made the status line strobe blank once every few + # seconds mid-prefill. Only a SUSTAINED absence means the + # phase really ended. + _load_notice_misses += 1 + if _load_notice_misses >= 3: + _load_notice_shown = False + _load_notice_misses = 0 + agent._emit_wait_notice("") - # Periodic heartbeat: touch the agent's activity tracker so the - # gateway's inactivity monitor knows we're alive while waiting - # for stream chunks. Without this, long thinking pauses (e.g. - # reasoning models) or slow prefill on local providers (Ollama) - # trigger false inactivity timeouts. The _call thread touches - # activity on each chunk, but the gap between API call start - # and first chunk can exceed the gateway timeout — especially - # when the stale-stream timeout is disabled (local providers). - if _hb_now - _last_heartbeat >= _HEARTBEAT_INTERVAL: - _last_heartbeat = _hb_now - _waiting_secs = int(_hb_now - last_chunk_time["t"]) - if _waiting_secs >= _HEARTBEAT_INTERVAL: - # No chunks for 30s+ — rewrite the live spinner/status line - # so CLI/TUI/Desktop users see WHAT the wait is (slow or - # overloaded provider / long thinking pause) instead of an - # unexplained generic spinner, and WHEN recovery kicks in. - if ( - _stream_stale_timeout is not None - and _stream_stale_timeout != float("inf") - ): - _recovery = f"; auto-reconnect at {int(_stream_stale_timeout)}s" + # Periodic heartbeat: touch the agent's activity tracker so the + # gateway's inactivity monitor knows we're alive while waiting + # for stream chunks. Without this, long thinking pauses (e.g. + # reasoning models) or slow prefill on local providers (Ollama) + # trigger false inactivity timeouts. The _call thread touches + # activity on each chunk, but the gap between API call start + # and first chunk can exceed the gateway timeout — especially + # when the stale-stream timeout is disabled (local providers). + if _hb_now - _last_heartbeat >= _HEARTBEAT_INTERVAL: + _last_heartbeat = _hb_now + _waiting_secs = int(_hb_now - last_chunk_time["t"]) + if _waiting_secs >= _HEARTBEAT_INTERVAL: + # No chunks for 30s+ — rewrite the live spinner/status line + # so CLI/TUI/Desktop users see WHAT the wait is (slow or + # overloaded provider / long thinking pause) instead of an + # unexplained generic spinner, and WHEN recovery kicks in. + if ( + _stream_stale_timeout is not None + and _stream_stale_timeout != float("inf") + ): + _recovery = f"; auto-reconnect at {int(_stream_stale_timeout)}s" + else: + _recovery = "" + agent._emit_wait_notice( + f"⏳ waiting on {api_kwargs.get('model', 'the provider')} — " + f"{_waiting_secs}s with no output yet (provider may be " + f"slow or overloaded, or the model is thinking{_recovery})" + ) else: - _recovery = "" + # Chunks are flowing — keep the activity tracker fresh but + # leave the live display alone. + agent._touch_activity( + f"waiting for stream response ({_waiting_secs}s, no chunks yet)" + ) + + # Detect stale streams: connections kept alive by SSE pings + # but delivering no real chunks. Kill the client so the + # inner retry loop can start a fresh connection. + _stale_elapsed = time.time() - last_chunk_time["t"] + if _stale_elapsed > _stream_stale_timeout: + _est_ctx = estimate_request_context_tokens(api_kwargs) + logger.warning( + "Stream stale for %.0fs (threshold %.0fs) — no chunks received. " + "model=%s context=~%s tokens. Killing connection.", + _stale_elapsed, _stream_stale_timeout, + api_kwargs.get("model", "unknown"), f"{_est_ctx:,}", + ) + agent._buffer_status( + f"⚠️ No response from provider for {int(_stale_elapsed)}s " + f"(model: {api_kwargs.get('model', 'unknown')}, " + f"context: ~{_est_ctx:,} tokens). " + f"Reconnecting..." + ) + try: + _cancel_current_stream_attempt("stale_stream_kill") + _close_request_client_once("stale_stream_kill") + except Exception: + pass + # Circuit breaker (#58962): count the stale kill. See the + # canonical comment block above ``_stale_streak()``. + _bump_stale_streak(agent) + # Rebuild the primary client too — its connection pool + # may hold dead sockets from the same provider outage. + if agent.api_mode == "anthropic_messages": + # #67142: the stale stream ran on a request-local anthropic + # client, already socket-aborted above via + # _close_request_client_once (which unblocks the worker and + # preserves the #28161 no-hang guarantee). The shared + # _anthropic_client is NOT the in-flight transport, so we must + # not close it from this poll (stranger) thread — that was the + # FD-recycle corruption vector. Nothing further is needed. + pass + else: + # #70773: same FD-recycle corruption vector as #67142. + # The shared OpenAI client's connection pool must NOT be + # closed from this watchdog/poll thread — worker threads + # from previous stale-killed attempts may still be + # unwinding their SSL BIOs. The request-local client is + # already closed above via _close_request_client_once. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next request. + pass + # Reset the timer so we don't kill repeatedly while + # the inner thread processes the closure. + last_chunk_time["t"] = time.time() agent._emit_wait_notice( - f"⏳ waiting on {api_kwargs.get('model', 'the provider')} — " - f"{_waiting_secs}s with no output yet (provider may be " - f"slow or overloaded, or the model is thinking{_recovery})" + f"⚠ no output from provider for {int(_stale_elapsed)}s — " + f"reconnecting..." ) - else: - # Chunks are flowing — keep the activity tracker fresh but - # leave the live display alone. agent._touch_activity( - f"waiting for stream response ({_waiting_secs}s, no chunks yet)" + f"stale stream detected after {int(_stale_elapsed)}s, reconnecting" ) - # Detect stale streams: connections kept alive by SSE pings - # but delivering no real chunks. Kill the client so the - # inner retry loop can start a fresh connection. - _stale_elapsed = time.time() - last_chunk_time["t"] - if _stale_elapsed > _stream_stale_timeout: - _est_ctx = estimate_request_context_tokens(api_kwargs) - logger.warning( - "Stream stale for %.0fs (threshold %.0fs) — no chunks received. " - "model=%s context=~%s tokens. Killing connection.", - _stale_elapsed, _stream_stale_timeout, - api_kwargs.get("model", "unknown"), f"{_est_ctx:,}", - ) - agent._buffer_status( - f"⚠️ No response from provider for {int(_stale_elapsed)}s " - f"(model: {api_kwargs.get('model', 'unknown')}, " - f"context: ~{_est_ctx:,} tokens). " - f"Reconnecting..." - ) - try: - _cancel_current_stream_attempt("stale_stream_kill") - _close_request_client_once("stale_stream_kill") - except Exception: - pass - # Circuit breaker (#58962): count the stale kill. See the - # canonical comment block above ``_stale_streak()``. - _bump_stale_streak(agent) - # Rebuild the primary client too — its connection pool - # may hold dead sockets from the same provider outage. - if agent.api_mode == "anthropic_messages": - # #67142: the stale stream ran on a request-local anthropic - # client, already socket-aborted above via - # _close_request_client_once (which unblocks the worker and - # preserves the #28161 no-hang guarantee). The shared - # _anthropic_client is NOT the in-flight transport, so we must - # not close it from this poll (stranger) thread — that was the - # FD-recycle corruption vector. Nothing further is needed. - pass - else: - # #70773: same FD-recycle corruption vector as #67142. - # The shared OpenAI client's connection pool must NOT be - # closed from this watchdog/poll thread — worker threads - # from previous stale-killed attempts may still be - # unwinding their SSL BIOs. The request-local client is - # already closed above via _close_request_client_once. - # The shared client will be replaced lazily by - # _ensure_primary_openai_client on the next request. - pass - # Reset the timer so we don't kill repeatedly while - # the inner thread processes the closure. - last_chunk_time["t"] = time.time() - agent._emit_wait_notice( - f"⚠ no output from provider for {int(_stale_elapsed)}s — " - f"reconnecting..." - ) - agent._touch_activity( - f"stale stream detected after {int(_stale_elapsed)}s, reconnecting" - ) - - if agent._interrupt_requested: - # The stale branch above already counted this iteration when its - # deadline won the race; do not double-count a simultaneous stop. - if _stale_elapsed <= _stream_stale_timeout: - _record_interrupted_provider_wait( - agent, - _stale_elapsed, - response_started=deltas_were_sent["yes"], + if agent._interrupt_requested: + # The stale branch above already counted this iteration when its + # deadline won the race; do not double-count a simultaneous stop. + if _stale_elapsed <= _stream_stale_timeout: + _record_interrupted_provider_wait( + agent, + _stale_elapsed, + response_started=deltas_were_sent["yes"], + ) + # Mark THIS request cancelled before force-closing so the worker's + # exception handler recognizes the forced transport error as a + # cancel and exits without retrying or surfacing a network error. + # (#6600) + _request_cancelled["value"] = True + logger.debug( + "Force-closing streaming httpx client due to interrupt " + "(not a network error)." ) - # Mark THIS request cancelled before force-closing so the worker's - # exception handler recognizes the forced transport error as a - # cancel and exits without retrying or surfacing a network error. - # (#6600) - _request_cancelled["value"] = True - logger.debug( - "Force-closing streaming httpx client due to interrupt " - "(not a network error)." - ) - try: - _cancel_current_stream_attempt("stream_interrupt_abort") - # #67142: kind-aware — anthropic aborts the request-local - # client's socket from this poll thread; the shared - # _anthropic_client is never closed here. - _close_request_client_once("stream_interrupt_abort") - except Exception: - pass - # Wait for the worker to unwind Relay-managed stream scopes - # (physical LLM + deferred logical) before surfacing - # InterruptedError. Raising immediately lets turn teardown - # (finish_logical_calls / end_turn / close_session) race a - # still-open physical scope and corrupt the LIFO stack — - # "scope handle is not at the top of the stack" → CLI EIO / - # redraw storm (#81521). No-op when Relay managed execution - # is not live. - _join_worker_for_relay_teardown(t, label="Streaming") - raise InterruptedError("Agent interrupted during streaming API call") + try: + _cancel_current_stream_attempt("stream_interrupt_abort") + # #67142: kind-aware — anthropic aborts the request-local + # client's socket from this poll thread; the shared + # _anthropic_client is never closed here. + _close_request_client_once("stream_interrupt_abort") + except Exception: + pass + # Wait for the worker to unwind Relay-managed stream scopes + # (physical LLM + deferred logical) before surfacing + # InterruptedError. Raising immediately lets turn teardown + # (finish_logical_calls / end_turn / close_session) race a + # still-open physical scope and corrupt the LIFO stack — + # "scope handle is not at the top of the stack" → CLI EIO / + # redraw storm (#81521). No-op when Relay managed execution + # is not live. (Inline mode has no worker: the request runs + # on the caller's thread and has already unwound by the time + # the InterruptedError below is raised.) + if t is not None: + _join_worker_for_relay_teardown(t, label="Streaming") + _monitor_interrupted["yes"] = True + return + + if _inline: + # Request on THIS thread; heartbeat / stale / interrupt monitor on a + # side thread that only ever aborts sockets (never dispatches). + monitor = threading.Thread( + target=_context_thread_target(_monitor_loop), + name="stream-inline-monitor", + daemon=True, + ) + monitor.start() + try: + _run_call() + finally: + monitor.join(timeout=2.0) + else: + _monitor_loop() + if _monitor_interrupted["yes"]: + raise InterruptedError("Agent interrupted during streaming API call") # Worker thread exited before the main thread's poll loop could check # the interrupt flag. If the worker returned early due to an interrupt # (e.g. _call_anthropic() detected _interrupt_requested and returned diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index 1fdd2d153e..30b86c6171 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -1101,7 +1101,9 @@ def _consume_codex_event_stream( * ``on_first_delta()`` — one-shot, fires on the first text delta only. * ``on_event(event)`` — fires for every event before any other processing. Used for watchdog activity, debug logging, anything wire-shape-agnostic. - * ``interrupt_check()`` — returns True to break the loop early. + * ``interrupt_check()`` — returns True to break the loop early, or raises + ``TimeoutError`` / ``InterruptedError`` for request-retirement control + flow that must not be converted into a partial final response. """ collected_output_items: List[Any] = [] # output_index of each collected_output_items entry, appended in lockstep @@ -1606,18 +1608,39 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta max_stream_retries = 1 # Accumulate streamed text so callers / compat shims can read it. agent._codex_streamed_text_parts: list = [] + # Retirement token for THIS request, installed by + # ``interruptible_api_call`` before it hands off to the worker thread. When + # a watchdog (TTFB / stream-idle / stale-call) kills the connection it + # clears the agent-level token, so a worker that is still draining frames + # can tell it has been retired. ``None`` means no watchdog owns this call + # (auxiliary callers drive this function directly) — then every check + # passes and behavior is unchanged. + request_token = getattr(agent, "_active_codex_stream_request_token", None) + + def _request_is_current() -> bool: + if request_token is None: + return True + return getattr(agent, "_active_codex_stream_request_token", None) is request_token def _on_text_delta(text: str) -> None: + if not _request_is_current(): + return agent._codex_streamed_text_parts.append(text) agent._fire_stream_delta(text) def _on_reasoning_delta(text: str) -> None: + if not _request_is_current(): + return agent._fire_reasoning_delta(text) def _on_commentary_message(text: str) -> None: + if not _request_is_current(): + return agent._fire_streamed_codex_commentary(text) def _on_event(event: Any) -> None: + if not _request_is_current(): + return # TTFB watchdog and activity touch — runs once per SSE event. agent._codex_stream_last_event_ts = time.time() agent._touch_activity("receiving stream response") @@ -1721,6 +1744,14 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta raise def _interrupt_or_superseded() -> bool: + # A retired request must NOT break out of the consume loop: breaking + # returns the partial `final` (status defaults to "completed"), which + # the caller persists as a finished assistant turn. Raise so the + # watchdog's own TimeoutError is what the retry path sees. + if not _request_is_current(): + raise TimeoutError( + "Codex Responses stream request retired before terminal response" + ) return bool(agent._interrupt_requested) try: diff --git a/agent/coding_context.py b/agent/coding_context.py index 200d576cb2..21050c4126 100644 --- a/agent/coding_context.py +++ b/agent/coding_context.py @@ -253,7 +253,7 @@ CODING_AGENT_GUIDANCE = ( "paths for the same flaw and fix the class, not just the reported site.\n" "- When fixing linter/type errors on a file, stop after about three " "attempts on the same file and ask the user rather than looping.\n" - "- Track multi-step work with `todo`. Reference code as `path:line` instead " + "- Track multi-step work with `todo_list`. Reference code as `path:line` instead " "of pasting whole files.\n" "\n" "Respect the user's repo: don't commit, push, or rewrite history unless " @@ -547,9 +547,9 @@ class RuntimeMode: trailing: list[str] = [] if self.profile.guidance: brief = self.profile.guidance - if valid_tool_names is not None and "todo" not in valid_tool_names: + if valid_tool_names is not None and "todo_list" not in valid_tool_names: brief = brief.replace( - "- Track multi-step work with `todo`. Reference code as " + "- Track multi-step work with `todo_list`. Reference code as " "`path:line` instead of pasting whole files.", "- Reference code as `path:line` instead of pasting " "whole files.", diff --git a/agent/context_compressor.py b/agent/context_compressor.py index be38dae5f9..c166f4757c 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -210,6 +210,20 @@ _TRUNCATED_SUMMARY_MARKER = "finish_reason=length" def _is_summary_access_or_quota_error(exc: Exception) -> bool: """Return True for non-retryable summary auth, permission, or quota errors.""" + # A credential read that failed closed because no profile secret scope + # was active (multiplexed gateway, worker thread without the caller's + # ContextVars) is a missing-credential failure of our own making: the + # summary model cannot be reached until the spawn site is fixed, and a + # placeholder summary would only destroy the middle window for nothing. + # Classify it with the credential class so compress() preserves the + # session unchanged (#100849 bundle: every hygiene pass truncated). + try: + from agent.secret_scope import UnscopedSecretError + except Exception: # pragma: no cover - import guard + UnscopedSecretError = () # type: ignore[assignment] + if UnscopedSecretError and isinstance(exc, UnscopedSecretError): + return True + classified = classify_api_error(exc) if classified.reason is FailoverReason.rate_limit: return False @@ -372,6 +386,21 @@ def _template_visible_role(message: Any) -> Optional[str]: return role +def _last_template_visible_role(messages: List[Dict[str, Any]]) -> Optional[str]: + """Last role a strict alternation template would count in *messages*. + + ``None`` when every row is template-exempt (tool flow only). + """ + return next( + ( + role + for role in (_template_visible_role(m) for m in reversed(messages)) + if role is not None + ), + None, + ) + + def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: """Enforce the compaction invariant: no assembled message carries a session-store persistence marker. @@ -507,6 +536,22 @@ _SUMMARY_END_MARKER = ( _MERGED_PRIOR_CONTEXT_HEADER = "[PRIOR CONTEXT — for reference only; not a new message]" _MERGED_SUMMARY_DELIMITER = "[END OF PRIOR CONTEXT — COMPACTION SUMMARY BELOW]" +# Prefixes the copy of a still-running user task that compaction re-states after +# the handoff boundary (#100818). A cron run's only user turn is the job prompt +# in the protected head, so compaction leaves it BEFORE the summary — and +# SUMMARY_PREFIX tells the model to do nothing when no user message follows. +# Set on a compaction carrier when the in-flight task was merged onto it (the +# carrier ends the list, so a standalone user row would break alternation). +# conversation_compression._ensure_compressed_has_user_turn treats it as +# "intent present" so it does not insert a second copy of the same request. +_INFLIGHT_REPLAY_MERGED_KEY = "_inflight_replay_merged" + +_INFLIGHT_TASK_REPLAY_HEADER = ( + "[STILL IN PROGRESS — this is the active request, restated after the " + "compaction boundary because it was not finished yet. Continue it; do not " + "start over.]" +) + _SALVAGE_SUMMARY_MAX_CHARS = 8_000 _SALVAGE_KEEP_RECENT_TOOLS = 2 @@ -1758,6 +1803,23 @@ def _retire_stale_tool_result_images( return pruned +def evict_stale_outbound_tool_images( + api_messages: List[Dict[str, Any]], + keep_newest: int = _MAX_KEEP_TOOL_IMAGES, +) -> int: + """Drop stale screenshot/vision payloads from the per-call API copy. + + Compression's keep-newest pass only runs when prune/compress fires, and + the Anthropic adapter's screenshot eviction only sees nested + ``tool_result`` blocks. OpenAI-style ``image_url`` tool results + otherwise ride every subsequent request until a 413 forces the reactive + strip (#89286). Call this on the cloned ``api_messages`` list after + sanitization so older frames never leave the box (#89296). Do not pass + persisted history — the rewrite is send-path only. + """ + return _retire_stale_tool_result_images(api_messages, keep_newest=keep_newest) + + def _truncate_tool_call_args_json(args: str, head_chars: int = 200) -> str: """Shrink long string values inside a tool-call arguments JSON blob while preserving JSON validity. @@ -2172,7 +2234,7 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten target = args.get("target", "?") return f"[memory] {action} on {target}" - if tool_name == "todo": + if tool_name == "todo_list": return "[todo] updated task list" if tool_name == "clarify": @@ -2225,11 +2287,11 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten if tool_name == "text_to_speech": return f"[text_to_speech] generated audio ({content_len:,} chars)" - if tool_name == "cronjob": + if tool_name == "cronjob_manage": action = args.get("action", "?") return f"[cronjob] {action}" - if tool_name == "process": + if tool_name == "process_manage": action = args.get("action", "?") sid = args.get("session_id", "?") return f"[process] {action} session={sid}" @@ -2318,7 +2380,7 @@ class ContextCompressor(ContextEngine): self._last_compression_telemetry = None self._active_compression_telemetry = None self._compression_telemetry_seed = None - self._proactive_prune_rearm_tokens = 0 + self._reset_proactive_prune_rearm() # Micro-compaction state reset self._micro_compact_cursor = 0 @@ -2623,7 +2685,7 @@ class ContextCompressor(ContextEngine): self._last_compression_telemetry = None self._active_compression_telemetry = None self._compression_telemetry_seed = None - self._proactive_prune_rearm_tokens = 0 + self._reset_proactive_prune_rearm() def bind_session_state(self, session_db: Any = None, session_id: str = "") -> None: """Bind the current session row so durable cooldowns can round-trip.""" @@ -2638,10 +2700,11 @@ class ContextCompressor(ContextEngine): self._prellm_skip_count = 0 self._anti_thrash_recovery_deadline = 0.0 self._structural_no_op_backoff_until = 0.0 - self._proactive_prune_rearm_tokens = 0 + self._reset_proactive_prune_rearm() self.get_active_compression_failure_cooldown() self._load_fallback_compression_streak() self._load_ineffective_compression_count() + self._load_anti_thrash_recovery_deadline() self._load_proactive_prune_rearm_tokens() def on_session_start(self, session_id: str, **kwargs) -> None: @@ -2807,6 +2870,45 @@ class ContextCompressor(ContextEngine): except Exception as exc: logger.debug("compression ineffective count persist failed (non-sqlite): %s", exc) + def _load_anti_thrash_recovery_deadline(self) -> None: + """Restore the durable recovery deadline (wall-clock epoch, #100185). + + Missing/absent storage leaves the in-memory clock disarmed, so the + next blocked evaluation arms a full fresh window (#54923). + """ + session_db = getattr(self, "_session_db", None) + session_id = getattr(self, "_session_id", "") + getter = getattr(session_db, "get_compression_recovery_deadline", None) + if not session_id or not callable(getter): + return + try: + stored = getter(session_id) + self._anti_thrash_recovery_deadline = max( + 0.0, + float(stored) if isinstance(stored, (int, float, str)) else 0.0, + ) + except (TypeError, ValueError, sqlite3.Error) as exc: + logger.debug("compression recovery deadline lookup failed: %s", exc) + except Exception as exc: + logger.debug("compression recovery deadline lookup failed (non-sqlite): %s", exc) + + def _set_anti_thrash_recovery_deadline(self, deadline: float) -> None: + """Set the recovery deadline, persisting on change only (0 = disarmed).""" + if deadline == self._anti_thrash_recovery_deadline: + return + self._anti_thrash_recovery_deadline = deadline + session_db = getattr(self, "_session_db", None) + session_id = getattr(self, "_session_id", "") + setter = getattr(session_db, "set_compression_recovery_deadline", None) + if not session_id or not callable(setter): + return + try: + setter(session_id, deadline) + except sqlite3.Error as exc: + logger.debug("compression recovery deadline persist failed: %s", exc) + except Exception as exc: + logger.debug("compression recovery deadline persist failed (non-sqlite): %s", exc) + def _record_ineffective_compression_verdict(self, count: int) -> None: """Set the anti-thrash strike counter, keeping the durable copy in sync. @@ -3213,7 +3315,7 @@ class ContextCompressor(ContextEngine): # sizes. Same durable-sync discipline as the strike reset above: clear # the model_config copy too, so a restart doesn't resurrect a runway # this recalibration just voided. - self._proactive_prune_rearm_tokens = 0 + self._reset_proactive_prune_rearm() self._clear_durable_proactive_prune_rearm() # When the MINIMUM_CONTEXT_LENGTH floor meets/exceeds a small context @@ -3446,6 +3548,10 @@ class ContextCompressor(ContextEngine): # A committed prune is a prompt-cache boundary. Do not permit the next # one until the prompt has regrown the tokens just reclaimed. self._proactive_prune_rearm_tokens: int = 0 + # Dedup key for the over-threshold "reclamation no-oped" warning + # (#101889) so a tool loop riding above the threshold warns once per + # distinct reason + rearm snapshot instead of every iteration. + self._last_reclaim_block_warn: "tuple[str, int] | None" = None self.min_tail_user_messages = min_tail_user_messages self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80)) self.quiet_mode = quiet_mode @@ -3900,9 +4006,17 @@ class ContextCompressor(ContextEngine): except Exception as exc: logger.debug("compression ineffective-count refresh failed: %s", exc) - def _automatic_compression_blocked(self) -> bool: - """Return whether automatic compaction is in cooldown or tripped.""" - if not self._automatic_compression_blocked_locally(): + def _automatic_compression_blocked(self, *, ignore_cooldown: bool = False) -> bool: + """Return whether automatic compaction is in cooldown or tripped. + + ``ignore_cooldown=True`` evaluates only the breakers that are NOT the + summary-failure cooldown. Used by provider-proven overflow recovery + (#100661): the provider already rejected the request, so waiting out + the cooldown just wedges the session — every turn defers and the next + failure extends the ladder. The overflow path gets one real attempt; + the ineffective/structural breakers still apply. + """ + if not self._automatic_compression_blocked_locally(ignore_cooldown=ignore_cooldown): return False # Blocked on the in-memory snapshot. Durable guard rows may have # been cleared by another agent since bind_session_state() — a @@ -3912,9 +4026,9 @@ class ContextCompressor(ContextEngine): # local block outlive the durable state that justified it. The # unblocked hot path above never pays for the DB reads. self._refresh_durable_guards() - return self._automatic_compression_blocked_locally() + return self._automatic_compression_blocked_locally(ignore_cooldown=ignore_cooldown) - def _automatic_compression_blocked_locally(self) -> bool: + def _automatic_compression_blocked_locally(self, *, ignore_cooldown: bool = False) -> bool: """Evaluate the automatic-compaction gate on in-memory state only.""" # Do not trigger compression while the summary LLM is in cooldown. # On a 429/transient failure _generate_summary() sets a cooldown and @@ -3926,7 +4040,7 @@ class ContextCompressor(ContextEngine): # force=True, which clears this cooldown in compress() before running, # so it still retries immediately. _cooldown_remaining = self._summary_failure_cooldown_until - time.monotonic() - if _cooldown_remaining > 0: + if _cooldown_remaining > 0 and not ignore_cooldown: if not self.quiet_mode: logger.debug( "Compression deferred — summary LLM in cooldown for %.0fs more", @@ -3964,21 +4078,34 @@ class ContextCompressor(ContextEngine): # the worst case in the truly-incompressible state is one compaction # attempt per recovery window — bounded, not thrash. # - # The clock is armed lazily on the first BLOCKED evaluation rather - # than persisted at trip time: a fresh process that loads a durable - # tripped counter (#69872) therefore starts a full window blocked, - # preserving the restart-must-not-disarm contract (#54923). + # The clock is armed lazily on the first BLOCKED evaluation and + # persisted on the session row (#100185): a fresh process/compressor + # that loads a durable tripped counter (#69872) with no stored + # deadline starts a full window blocked, preserving the + # restart-must-not-disarm contract (#54923) — but one that loads an + # already-armed deadline resumes that window instead of restarting it. if ( self._ineffective_compression_count >= 2 or self._fallback_compression_streak >= 2 ): - _now = time.monotonic() - if self._anti_thrash_recovery_deadline <= 0.0: - self._anti_thrash_recovery_deadline = ( + # Wall clock, not monotonic: the deadline is persisted on the + # session row (#100185) so a fresh compressor bound to the same + # session — the gateway rebuilds the AIAgent on every cache + # eviction — resumes the SAME window instead of restarting it. + # Without that, a blocked messaging session never earned its + # probe and stayed blocked forever. + _now = time.time() + if self._anti_thrash_recovery_deadline <= 0.0 or ( + # Clock jumped backwards past a full window: never wait + # longer than one window from now. + self._anti_thrash_recovery_deadline - _now + > self._ANTI_THRASH_RECOVERY_SECONDS + ): + self._set_anti_thrash_recovery_deadline( _now + self._ANTI_THRASH_RECOVERY_SECONDS ) elif _now >= self._anti_thrash_recovery_deadline: - self._anti_thrash_recovery_deadline = 0.0 + self._set_anti_thrash_recovery_deadline(0.0) if self._ineffective_compression_count >= 2: self._record_ineffective_compression_verdict(1) if self._fallback_compression_streak >= 2: @@ -4009,7 +4136,7 @@ class ContextCompressor(ContextEngine): # Guard not tripped (counters were cleared by an effective compaction # or a fitting real-usage reading) — disarm any pending recovery clock # so a LATER trip starts its own full window. - self._anti_thrash_recovery_deadline = 0.0 + self._set_anti_thrash_recovery_deadline(0.0) return False # ------------------------------------------------------------------ @@ -4319,6 +4446,76 @@ class ContextCompressor(ContextEngine): return result, pruned + def _reset_proactive_prune_rearm(self) -> None: + """Fully rearm the proactive prune and let a future lockout warn again. + + Every path that zeroes the rearm mark (compaction, session + reset/end/rebind, model recalibration) is a reclamation or a fresh + start, so the over-threshold no-op dedup key must not survive it — + otherwise an identical lockout after a full compaction (rearm back + at 0) would be silent (#101889). + """ + self._proactive_prune_rearm_tokens = 0 + self._last_reclaim_block_warn = None + + def _billed_basis_over_threshold(self, current_tokens: "int | None") -> bool: + """Whether a provider-billed reading says the session is over threshold. + + ``current_tokens`` is the provider's ``prompt_tokens`` (or the + overhead-aware fallback estimate): it counts the system prompt and tool + schemas, which the message-only estimate behind + ``_proactive_prune_rearm_tokens`` does not. Used to stop schema + overhead from parking the prune rearm gate above a real request that is + already over ``threshold_tokens`` (#101889). + """ + return ( + current_tokens is not None + and self.threshold_tokens > 0 + and current_tokens >= self.threshold_tokens + ) + + def _warn_reclamation_no_op( + self, + reason: str, + current_tokens: "int | None", + before: "int | None" = None, + ) -> None: + """Warn when an over-threshold session's reclamation path no-ops. + + A session sitting above ``threshold_tokens`` with every reclamation + path declining is the failure mode from #101889: context keeps growing + until the provider's hard limit rejects the request, with nothing in + the log to explain it. Silent below the threshold (a declined prune + there is ordinary hysteresis, not a lockout). Deduped on + ``reason`` + the rearm snapshot so a busy tool loop logs once per + distinct state, not once per iteration; the key is cleared whenever + the session drops back under threshold or any reclamation resets the + rearm mark (prune commit, compaction, session reset/rebind, model + recalibration) so a later lockout warns again. + """ + # The explicit None check is redundant with the predicate; it narrows + # ``current_tokens`` for the type checker on the format below. + if current_tokens is None or not self._billed_basis_over_threshold( + current_tokens + ): + self._last_reclaim_block_warn = None + return + key = (reason, int(self._proactive_prune_rearm_tokens)) + if self._last_reclaim_block_warn == key: + return + self._last_reclaim_block_warn = key + logger.warning( + "Context is over the compression threshold (~%s of %s tokens) but " + "reclamation did not run: %s (message-token estimate %s, prune " + "rearm mark %s). The session may keep growing until the provider " + "rejects the request — /compact to compress history now.", + f"{int(current_tokens):,}", + f"{int(self.threshold_tokens):,}", + reason, + "n/a" if before is None else f"{int(before):,}", + f"{int(self._proactive_prune_rearm_tokens):,}", + ) + def prune_tool_results_only( self, messages: List[Dict[str, Any]], current_tokens: int | None = None, ) -> tuple[List[Dict[str, Any]], int]: @@ -4360,6 +4557,13 @@ class ContextCompressor(ContextEngine): object is returned unchanged — the standard no-op caller contract (callers gate bookkeeping on ``result is not input``). + The rearm gate is measured on message bodies only, so it is bypassed + (never the reclaim gate) when a provider-billed ``current_tokens`` + reading already puts the request over ``threshold_tokens``: schema + overhead must not park an over-threshold session below the rearm mark + forever with no reclamation and no log (#101889). Every no-op taken + while over threshold is logged once per distinct reason. + Returns ``(messages, 0)`` — the input object — when disabled, below the trigger, or when the reclaim gate rejects the commit. """ @@ -4369,10 +4573,17 @@ class ContextCompressor(ContextEngine): return messages, 0 # Nothing to reclaim until there are messages outside the protected tail. if len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1: + self._warn_reclamation_no_op("prune:tail_only", current_tokens) return messages, 0 before = sum(_estimate_msg_budget_tokens(m) for m in messages) if before < self._proactive_prune_rearm_tokens: - return messages, 0 + # Message-only estimate is short of the runway. Honour it as + # prompt-cache hysteresis only while the real (billed) request is + # still under threshold — above it, the lockout is the bug. The + # under-threshold skip stays silent on purpose: ordinary + # hysteresis, not a stuck session. + if not self._billed_basis_over_threshold(current_tokens): + return messages, 0 # Capability gate BEFORE the expensive multi-pass scan: a bound store that # can't persist the prune atomically (duck-typed/plugin session store # without archive_and_compact) makes every prune a permanent no-op, so @@ -4384,6 +4595,7 @@ class ContextCompressor(ContextEngine): and session_id and not callable(getattr(session_db, "archive_and_compact", None)) ): + self._warn_reclamation_no_op("prune:store_cannot_persist", current_tokens) return messages, 0 pruned_msgs, pruned_count = self._prune_old_tool_results( messages, @@ -4394,6 +4606,7 @@ class ContextCompressor(ContextEngine): if not pruned_count: # Standard no-op contract: hand back the INPUT object so callers # can gate bookkeeping on `result is not input`. + self._warn_reclamation_no_op("prune:nothing_eligible", current_tokens) return messages, 0 # Measured-savings gate (prompt-cache hysteresis): only commit when # the prune reclaims a meaningful batch of tokens. Estimated on the @@ -4401,6 +4614,9 @@ class ContextCompressor(ContextEngine): after = sum(_estimate_msg_budget_tokens(m) for m in pruned_msgs) reclaimed = max(0, before - after) if reclaimed < self.proactive_prune_min_reclaim_tokens: + self._warn_reclamation_no_op( + "prune:reclaim_below_minimum", current_tokens, before=before + ) return messages, 0 # ``after`` includes the tool batch appended since the provider's last # usage reading, so both the low-water mark and future gate use the @@ -4433,6 +4649,8 @@ class ContextCompressor(ContextEngine): # the micro-compaction sync (#98450) — one stamp site for the class. stamp_db_persisted_markers(pruned_msgs) self._proactive_prune_rearm_tokens = next_rearm_tokens + # Reclamation just ran: let a future lockout warn again. + self._last_reclaim_block_warn = None return pruned_msgs, pruned_count # ------------------------------------------------------------------ @@ -4946,6 +5164,7 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb turns_to_summarize: List[Dict[str, Any]], focus_topic: Optional[str] = None, memory_context: str = "", + bypass_cooldown: bool = False, ) -> Optional[str]: """Generate a structured summary of conversation turns. @@ -4968,7 +5187,10 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb if self._compression_cancelled(): raise AuxiliaryExplicitCancellation() now = prompt_started_at - if now < self._summary_failure_cooldown_until: + # bypass_cooldown (#100661): provider-proven overflow gets ONE real + # summary attempt while the cooldown is armed; a failure below still + # records/extends the cooldown normally. + if now < self._summary_failure_cooldown_until and not bypass_cooldown: logger.debug( "Skipping context summary during cooldown (%.0fs remaining)", self._summary_failure_cooldown_until - now, @@ -6592,6 +6814,164 @@ This compaction should PRIORITISE preserving all information related to the focu return max(pair_end, head_end + 1) return adjusted + @classmethod + def _find_inflight_user_task( + cls, messages: List[Dict[str, Any]] + ) -> Optional[Dict[str, Any]]: + """Return the user turn that is still awaiting completion, or ``None``. + + Scans the WHOLE transcript, not just the compressible region: a cron + run's only user turn is the job prompt sitting in the protected head + (``protect_first_n`` keeps system + first user), which is exactly the + turn ``_find_last_user_message_idx`` cannot see (#100818). + + A turn is in-flight when the transcript does not already end with a + completed assistant reply — i.e. a text-bearing assistant message with + no pending ``tool_calls``. A trailing ``tool`` result or an assistant + message that still has ``tool_calls`` outstanding means the run was + interrupted mid-task and the instruction is still owed an answer. + + Handoff carriers and synthetic scaffolding rows are excluded via the + same filter pair as ``_find_last_user_message_idx``, so an idle session + whose only user-role row is an inherited summary yields ``None`` and is + never re-animated (#80622). + """ + from agent.conversation_compression import _is_real_user_message + + last_user_idx = -1 + for i in range(len(messages) - 1, -1, -1): + msg = messages[i] + # _is_real_user_message also rejects metadata-flagged scaffolding + # (_todo_snapshot_synthetic, recovery nudges, ...) that + # _is_actionable_user_turn cannot see. + if cls._is_actionable_user_turn(msg) and _is_real_user_message(msg): + last_user_idx = i + break + if isinstance(msg, dict) and msg.get(_INFLIGHT_REPLAY_MERGED_KEY): + # A previous cycle merged the live request onto this summary + # carrier; it is the only copy left, so it is still the task. + last_user_idx = i + break + if last_user_idx < 0: + return None + + for msg in reversed(messages[last_user_idx + 1:]): + if not isinstance(msg, dict) or msg.get("role") != "assistant": + # Trailing tool result (or anything else): still mid-task. + break + if msg.get("tool_calls"): + break + if _content_text_for_contains(msg.get("content")).strip(): + # Final answer already delivered — replaying the ask would + # hand the model finished work as a fresh instruction. + return None + # Empty assistant row (a bare reasoning/stub turn): keep looking. + return messages[last_user_idx] + + def _reappend_inflight_user_task( + self, + compressed: List[Dict[str, Any]], + inflight: Optional[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """Restate an unfinished user task after the compaction handoff. + + ``SUMMARY_PREFIX`` instructs the model to act only on a user message + that appears AFTER the summary, and to do nothing when none does. When + the single in-flight instruction lived in the protected head, the + assembled transcript orders it before the handoff and the run ends in a + ``[SILENT]`` no-op that the scheduler records as success (#100818). + + Re-append a copy of that turn after the surviving tail so the prefix's + "latest user message" pointer resolves to it again. If the transcript + already ends on a template-visible user row, appending a second one + would break user/assistant alternation, so the restatement is merged + onto the handoff carrier instead — after ``_SUMMARY_END_MARKER``, which + is the boundary the prefix's rule is written against. + """ + if inflight is None or not compressed: + return compressed + + carrier_idx = -1 + for idx in range(len(compressed) - 1, -1, -1): + if self._is_context_summary_message(compressed[idx]): + carrier_idx = idx + break + if carrier_idx < 0: + # No handoff was emitted — nothing reordered the instruction. + return compressed + + for msg in compressed[carrier_idx + 1:]: + if self._is_actionable_user_turn( + msg + ) and not self._is_synthetic_compression_user_turn(msg): + # A real request already follows the summary. + return compressed + + carrier = compressed[carrier_idx] + carrier_text = _content_text_for_contains(carrier.get("content")) + if _SUMMARY_END_MARKER not in carrier_text: + return compressed + if carrier_text.split(_SUMMARY_END_MARKER, 1)[1].strip(): + # The _force_user_leading layout keeps the live request on the + # carrier itself, after the marker. Already actionable. + return compressed + + task_text = _content_text_for_contains(inflight.get("content")).strip() + if _INFLIGHT_TASK_REPLAY_HEADER in task_text: + # Already a restatement from an earlier compaction (standalone row + # or merged onto a carrier): take the text after the header so a + # task that survives >1 cycle never stacks headers or drags the + # old summary along. + task_text = task_text.rsplit(_INFLIGHT_TASK_REPLAY_HEADER, 1)[1].strip() + if not task_text: + return compressed + + if not self.quiet_mode: + logger.info( + "Re-appending the in-flight user task after the compaction " + "handoff so it stays actionable (#100818)" + ) + + last_visible_role = _last_template_visible_role(compressed) + if inflight.get(_INFLIGHT_REPLAY_MERGED_KEY): + # Never copy a summary carrier (metadata would mark the replay + # synthetic): restate as a plain user row. + replay = {"role": "user", "content": task_text} + else: + replay = _fresh_compaction_message_copy(inflight) + replay.pop(_COMPACTION_TAIL_MARKER, None) + if isinstance(replay.get("content"), str): + # Plain text: rebuild from the header-stripped task text so a + # task surviving several compactions never stacks headers. + replay["content"] = _INFLIGHT_TASK_REPLAY_HEADER + "\n" + task_text + else: + # Multimodal parts: keep them, prepend the header text part. + replay["content"] = _append_text_to_content( + replay.get("content"), + _INFLIGHT_TASK_REPLAY_HEADER + "\n", + prepend=True, + ) + drop_stale_api_content(replay) + + if last_visible_role == "user": + # Alternation is judged on template-visible rows only (tool_calls / + # tool rows are exempt), so a user-pinned summary followed by a + # tool tail still "ends on user": a standalone user row would break + # the Mistral-style pre-flight check (#58753). Merge onto the + # carrier instead and flag it — the carrier's own metadata marks it + # synthetic, and without the flag _ensure_compressed_has_user_turn + # would insert a second copy of the same request. + carrier["content"] = _append_text_to_content( + carrier.get("content"), + "\n\n" + _INFLIGHT_TASK_REPLAY_HEADER + "\n" + task_text, + ) + carrier[_INFLIGHT_REPLAY_MERGED_KEY] = True + drop_stale_api_content(carrier) + return compressed + + compressed.append(replay) + return compressed + def _ensure_last_n_user_messages_in_tail( self, messages: List[Dict[str, Any]], @@ -7662,6 +8042,7 @@ This compaction should PRIORITISE preserving all information related to the focu focus_topic: Optional[str] = None, force: bool = False, memory_context: str = "", + bypass_cooldown: bool = False, ) -> List[Dict[str, Any]]: """Compress conversation messages by summarizing middle turns. @@ -7698,6 +8079,10 @@ This compaction should PRIORITISE preserving all information related to the focu summary path. Auto-compress callers pass False. memory_context: Optional provider-supplied context to preserve in the summary prompt. Whitespace-only values are ignored. + bypass_cooldown: If True, run the summary LLM even while the + summary-failure cooldown is armed, WITHOUT clearing it + (#100661). Set by provider-proven overflow recovery, which + is already bounded by the caller's attempt budget. """ # Reset per-call summary failure state — callers inspect these fields # after compress() returns to decide whether to surface a warning. @@ -8035,6 +8420,7 @@ This compaction should PRIORITISE preserving all information related to the focu turns_to_summarize, focus_topic=summary_focus_topic, memory_context=memory_context, + bypass_cooldown=bypass_cooldown, ) except AuxiliaryExplicitCancellation: # Explicit cancellation is a true no-op. Restore state mutated by @@ -8222,20 +8608,10 @@ This compaction should PRIORITISE preserving all information related to the focu # Jinja alternation 500, permanently poisoning the session. last_head_role: Optional[str] = "user" if compressed: - last_head_role = next( - ( - role - for role in ( - _template_visible_role(m) for m in reversed(compressed) - ) - if role is not None - ), - # Head holds only template-exempt messages: the summary will - # be the first message the template counts, and the sequence - # must open with "user" (handled below alongside the forced - # cases). - None, - ) + # None: head holds only template-exempt messages, so the summary + # will be the first message the template counts and the sequence + # must open with "user" (handled below alongside the forced cases). + last_head_role = _last_template_visible_role(compressed) first_tail_role = None first_tail_visible_idx: Optional[int] = None if tail_messages: @@ -8422,9 +8798,20 @@ This compaction should PRIORITISE preserving all information related to the focu _merge_summary_into_tail = False compressed.append(msg) - self.compression_count += 1 - + # The assembled list can order the only live instruction BEFORE the + # handoff (single-prompt cron shape: the job prompt is pinned in the + # protected head). SUMMARY_PREFIX reads that as "no user message after + # the summary → do nothing", so restate it past the boundary (#100818). + # Run BEFORE the in-flight re-append: the sanitizer's trailing-in-flight + # exemption (#79278) walks back from the list end, and a replay user row + # sitting there would make a genuinely pending assistant(tool_calls) look + # orphaned and get its calls stripped. compressed = self._sanitize_tool_pairs(compressed) + compressed = self._reappend_inflight_user_task( + compressed, self._find_inflight_user_task(messages) + ) + + self.compression_count += 1 # Replace image parts in all compressed messages before the newest # image-bearing user turn with a short text placeholder. Without @@ -8520,7 +8907,7 @@ This compaction should PRIORITISE preserving all information related to the focu self._micro_compact_cursor = 0 self._micro_compact_consecutive_failures = 0 self._micro_compact_last_failure_cursor = -1 - self._proactive_prune_rearm_tokens = 0 + self._reset_proactive_prune_rearm() return compressed diff --git a/agent/context_references.py b/agent/context_references.py index 4ce6fd5809..e4561ac743 100644 --- a/agent/context_references.py +++ b/agent/context_references.py @@ -12,7 +12,12 @@ from pathlib import Path from typing import Awaitable, Callable from agent.model_metadata import estimate_tokens_rough -from hermes_cli._subprocess_compat import IS_WINDOWS, windows_hide_flags +from hermes_cli._subprocess_compat import ( + IS_WINDOWS, + harden_git_argv, + noninteractive_git_env, + windows_hide_flags, +) from hermes_cli.sizefmt import format_bytes from abc import ABC, abstractmethod @@ -425,12 +430,13 @@ def _expand_git_reference( _popen_kwargs = {"creationflags": windows_hide_flags()} if IS_WINDOWS else {} try: result = subprocess.run( - ["git", *args], + ["git", *harden_git_argv(args)], cwd=cwd, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=30, stdin=subprocess.DEVNULL, + env=noninteractive_git_env(), **_popen_kwargs, ) except subprocess.TimeoutExpired: diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index fd15799838..e6ef77e996 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -109,6 +109,12 @@ COMPACTION_STATUS_MARKER = "Compacting context" COMPACTION_STATUS = ( f"🗜️ {COMPACTION_STATUS_MARKER} — summarizing earlier conversation so I can continue..." ) +# Periodic heartbeat re-emitted while a long compression is still running so +# remote transports with idle-turn watchdogs (#98371) see progress. Same +# marker as COMPACTION_STATUS so every consumer classifies it identically. +COMPACTION_HEARTBEAT_STATUS = ( + f"🗜️ {COMPACTION_STATUS_MARKER} — still summarizing earlier conversation so I can continue..." +) COMPACTION_DONE_STATUS = "✓ Context compaction complete — continuing turn..." @@ -201,6 +207,7 @@ CONTEXT_OVERFLOW_BLOCKED_WARNING_TEMPLATE = ( # same constants the emission sites use) through the gateway noise filter. ROUTINE_COMPRESSION_STATUS_SAMPLES = ( COMPACTION_STATUS, + COMPACTION_HEARTBEAT_STATUS, COMPACTION_DONE_STATUS, PRE_API_COMPRESSION_STATUS_TEMPLATE.format(tokens=123456), PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(tokens=120000, threshold=100000), @@ -718,6 +725,17 @@ class CompressionCommitFence: self._progress_observed = False self._deadline: float | None = None self._retain_cancelled_lock_until_worker_done = False + # #97963: set by the worker (mark_commit_watermark_fenced) once its + # commit path is watermark-fenced — i.e. it captured the session's + # active-row watermark at compression start, so any row appended + # AFTER that point survives a late commit verbatim as concurrent + # tail (archive_and_compact / publish_compression_child clone rows + # above the watermark instead of archiving them). Hosts read this + # at the turn-hold boundary to decide whether a detached worker may + # KEEP its commit admission (safe: newer turns cannot be clobbered) + # or must be cancelled as before (unfenced commit; discard is the + # only safe outcome). Plain bool store — atomic in CPython. + self._commit_watermark_fenced = False if total_ceiling_seconds is not None: self.set_total_ceiling_seconds(total_ceiling_seconds) @@ -748,6 +766,20 @@ class CompressionCommitFence: deadline = self._deadline return deadline is not None and time.monotonic() >= deadline + @property + def deadline_monotonic(self) -> float | None: + """The armed deadline as an absolute ``time.monotonic()`` instant. + + :meth:`set_total_ceiling_seconds` documents this deadline as "shared by + the host and worker", but until #99692 only the host could read it — + ``deadline_exceeded`` answers "is it past?" for a caller that is already + polling, which is useless to a worker blocked inside a provider stream. + Publishing the instant itself lets the worker's stream consumer stop at + exactly the moment the host stops waiting (see + ``auxiliary_client.aux_stream_deadline``). + """ + return self._deadline + def seconds_since_progress(self) -> float: """Seconds since the worker last reported forward progress.""" return max(0.0, time.monotonic() - self._last_progress) @@ -843,6 +875,24 @@ class CompressionCommitFence: """Prevent a timed-out live worker from overlapping a retry.""" self._retain_cancelled_lock_until_worker_done = True + def mark_commit_watermark_fenced(self) -> None: + """Record that this attempt's commit is bounded by a start watermark. + + Called by the compression worker right after it captures + ``get_active_message_watermark()`` under the durable compression + lock (#75316/#87484). A watermark-fenced commit archives ONLY rows + at or below the watermark; rows appended later — e.g. the user turn + the host released at the turn-hold boundary (#97963) — are cloned + as live concurrent tail. That is exactly the property a host needs + before letting a detached worker keep its commit admission. + """ + self._commit_watermark_fenced = True + + @property + def commit_watermark_fenced(self) -> bool: + """Lock-free read: the worker's commit is watermark-bounded.""" + return self._commit_watermark_fenced + def allow_cancelled_lock_release(self) -> None: """Undo :meth:`retain_compression_lock_until_worker_done`. @@ -1969,6 +2019,25 @@ def context_compression_timed_out(agent: Any) -> bool: return getattr(agent, "_last_compression_timed_out", None) is True +def _automatic_gate_blocked( + blocked: Any, compressor: Any, bypass_cooldown: bool +) -> bool: + """Evaluate the automatic breaker gate, optionally ignoring the cooldown. + + Provider-proven overflow recovery (#100661) passes ``bypass_cooldown``; + engines whose gate predates the kwarg (plugins, test doubles) are called + with the legacy no-argument shape. + """ + if bypass_cooldown: + try: + accepts = "ignore_cooldown" in inspect.signature(blocked).parameters + except (TypeError, ValueError): + accepts = False + if accepts: + return bool(blocked(compressor, ignore_cooldown=True)) + return bool(blocked(compressor)) + + def compression_blocked_transiently(agent: Any) -> bool: """Type-pinned read of the transient-block signal (#97488). @@ -2205,6 +2274,7 @@ def _supported_compression_kwargs( focus_topic: Optional[str], force: bool, memory_context: str, + bypass_cooldown: bool = False, ) -> dict: """Return only compression kwargs accepted by an engine callable. @@ -2218,6 +2288,8 @@ def _supported_compression_kwargs( "focus_topic": focus_topic, "force": force, } + if bypass_cooldown: + candidates["bypass_cooldown"] = True if memory_context: candidates["memory_context"] = memory_context try: @@ -2244,6 +2316,8 @@ class _CompressionActivityHeartbeat: self, agent: Any, interval_seconds: float | None = None, + *, + emit_client_status: bool = False, commit_fence: Optional[CompressionCommitFence] = None, ) -> None: self._agent = agent @@ -2260,6 +2334,10 @@ class _CompressionActivityHeartbeat: if not math.isfinite(interval_seconds): interval_seconds = 60.0 self._interval_seconds = max(0.1, interval_seconds) + # Only a compression that opened a VISIBLE compaction phase (the + # routine start status was emitted) keeps it alive with heartbeats; + # quiet context engines emit neither (#98371 follow-up). + self._emit_client_status = emit_client_status self._stop = threading.Event() self._thread = threading.Thread( target=self._run, @@ -2332,11 +2410,40 @@ class _CompressionActivityHeartbeat: except Exception: logger.debug("compression activity heartbeat touch failed", exc_info=True) + def _emit_progress_status(self) -> None: + """Re-publish the compacting status so remote transports see progress. + + Compression can stream for minutes with no deltas, tool events, or + status lines reaching remote transports. Idle-progress watchdogs on + those clients (e.g. the Android relay app's 180s turn watchdog) + treat the silence as a dead turn and fire ``session.interrupt`` — + killing a healthy compression mid-flight and rolling back its work, + which retriggers on the next prompt and loops forever on sessions + near the context ceiling (#98371). + + Routed through ``agent._emit_status`` like every other compaction + status: same "lifecycle" key (the TUI gateway re-tags it to + ``compacting``; Telegram edits one bubble per key), same chat-platform + filter, same CLI print path. + """ + if not self._emit_client_status: + return + emit = getattr(self._agent, "_emit_status", None) + if not callable(emit): + return + try: + emit(COMPACTION_HEARTBEAT_STATUS) + except Exception: + logger.debug( + "status emit error in compression heartbeat", exc_info=True + ) + def _run(self) -> None: while not self._stop.wait(self._interval_seconds): if self._should_suppress(): return self._touch("context compression in progress") + self._emit_progress_status() def _direct_messages_for_pre_compress_memory(messages: Any) -> list[dict[str, Any]]: """Return direct user/assistant evidence safe for memory checkpointing. @@ -2826,6 +2933,79 @@ def _is_real_user_message(message: Any) -> bool: return not ContextCompressor._is_synthetic_compression_user_turn(message) +def _message_contains_busy_steer(message: Any) -> bool: + """Return whether *message* carries a busy-steer marker. + + With ``display.busy_input_mode: steer`` the follow-up is embedded as an + out-of-band marker inside a ``role=tool`` result (see + ``agent_runtime_helpers.apply_pending_steer_to_tool_results``). That marker + carries real user intent but lives outside ``role=user``, so the + ``_is_real_user_message`` / ``_transcript_has_real_user_turn`` checks + alone would miss it. + """ + text = _message_text(message) + if not text: + return False + try: + from agent.prompt_builder import STEER_MARKER_CLOSE, STEER_MARKER_OPEN + + return STEER_MARKER_OPEN in text and STEER_MARKER_CLOSE in text + except Exception: + return "[OUT-OF-BAND USER MESSAGE" in text and "[/OUT-OF-BAND USER MESSAGE]" in text + + +def _extract_steer_text_from_message(message: Any) -> Optional[str]: + """Extract the inner user text from a steer marker, or None.""" + text = _message_text(message) + if not text: + return None + try: + from agent.prompt_builder import STEER_MARKER_CLOSE, STEER_MARKER_OPEN + + open_marker = STEER_MARKER_OPEN + close_marker = STEER_MARKER_CLOSE + except Exception: + open_marker = "[OUT-OF-BAND USER MESSAGE" + close_marker = "[/OUT-OF-BAND USER MESSAGE]" + start = text.find(open_marker) + if start == -1: + # Fallback: marker wording may evolve; look for the stable prefix. + fallback_open = "[OUT-OF-BAND USER MESSAGE" + start = text.find(fallback_open) + if start == -1: + return None + # Skip to end of the opening line. + nl = text.find("\n", start) + if nl != -1: + start = nl + 1 + else: + start += len(fallback_open) + else: + start += len(open_marker) + end = text.find(close_marker, start) + if end == -1: + end = text.find("[/OUT-OF-BAND USER MESSAGE]", start) + if end == -1: + return None + extracted = text[start:end].strip() + return extracted if extracted else None + + +def _compressed_has_busy_steer(messages: list) -> bool: + """Whether *messages* already carries a steer marker (intent present). + + Only ``role=tool`` rows count: that is the sole place the runtime ever + delivers a steer, so a compaction summary that merely quotes the marker + text must not be mistaken for live intent. + """ + for msg in messages: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + if _message_contains_busy_steer(msg): + return True + return False + + def _strip_stale_todo_snapshot(content: Any) -> Any: """Remove a previously merged todo-snapshot block from message content. @@ -3028,17 +3208,43 @@ def _ensure_compressed_has_user_turn( """Preserve human intent, not merely a synthetic user-role placeholder.""" if any(_is_real_user_message(message) for message in compressed): return "already_present" + if _compressed_has_busy_steer(compressed): + return "already_present" + from agent.context_compressor import _INFLIGHT_REPLAY_MERGED_KEY + + if any( + isinstance(message, dict) and message.get(_INFLIGHT_REPLAY_MERGED_KEY) + for message in compressed + ): + # The in-flight request was restated onto the summary carrier + # (#100818); inserting an anchor would duplicate it. + return "already_present" from agent.context_compressor import ( COMPRESSION_CONTINUATION_USER_CONTENT, _fresh_compaction_message_copy, ) + # One reversed positional scan: the anchor is whichever intent-bearing + # row is LAST in the original transcript — a real ``role=user`` turn or + # a steer marker riding inside a ``role=tool`` result. Scanning the two + # kinds separately (steer first, then user) would let an older, already + # consumed steer outrank a newer real user request and replay it + # (#100053 follow-up: ``[user A, tool(steer B), ..., user C]`` must + # anchor C, not B). for message in reversed(original_messages): if _is_real_user_message(message): return _insert_real_user_anchor( compressed, _fresh_compaction_message_copy(message), ) + if not isinstance(message, dict) or message.get("role") != "tool": + continue + steer_text = _extract_steer_text_from_message(message) + if steer_text: + return _insert_real_user_anchor( + compressed, + {"role": "user", "content": steer_text}, + ) from agent.message_metadata import append_message append_message( @@ -3156,6 +3362,7 @@ def compress_context( task_id: str = "default", focus_topic: Optional[str] = None, force: bool = False, + bypass_cooldown: bool = False, defer_context_engine_notification: bool = False, commit_fence: Optional[CompressionCommitFence] = None, ) -> Tuple[list, str]: @@ -3175,6 +3382,13 @@ def compress_context( by the manual ``/compress`` slash command so users can retry immediately after an auto-compress abort. Auto-compress callers use the default ``False``. + bypass_cooldown: If True, the automatic breaker gates ignore ONLY the + summary-failure cooldown for this attempt (#100661). Set by the + provider-proven overflow recovery path: the provider already + rejected the request, so deferring until the cooldown lapses + wedges the session. Unlike ``force`` it does not clear the + cooldown, and the ineffective/structural breakers still apply; + a failed attempt records its cooldown normally. defer_context_engine_notification: Delay the existing context-engine hook until a manual host commits its outer history transaction. commit_fence: Optional cooperative fence for executor callers that @@ -3292,7 +3506,9 @@ def compress_context( "_automatic_compression_blocked", None, ) - if callable(blocked) and blocked(agent.context_compressor): + if callable(blocked) and _automatic_gate_blocked( + blocked, agent.context_compressor, bypass_cooldown + ): _mark_compression_blocked_transient(agent, agent.context_compressor) existing_prompt = getattr(agent, "_cached_system_prompt", None) if not existing_prompt: @@ -3516,6 +3732,18 @@ def compress_context( _commit_watermark = _lock_db.get_active_message_watermark( _lock_sid ) + # #97963: a captured watermark makes the eventual + # commit safe against rows appended after this + # point (they survive as cloned concurrent tail on + # BOTH commit paths — archive_and_compact and + # publish_compression_child). Tell the fence so a + # host at the turn-hold boundary can keep this + # attempt's commit admission instead of burning it. + if commit_fence is not None: + try: + commit_fence.mark_commit_watermark_fenced() + except AttributeError: + pass # test doubles without the method except Exception as _wm_err: # Watermark capture is safety-additive: without it the # commit falls back to archive-everything (historical @@ -3751,7 +3979,9 @@ def compress_context( "_automatic_compression_blocked", None, ) - if callable(blocked) and blocked(compressor): + if callable(blocked) and _automatic_gate_blocked( + blocked, compressor, bypass_cooldown + ): _mark_compression_blocked_transient(agent, compressor) _release_lock() existing_prompt = getattr(agent, "_cached_system_prompt", None) @@ -3948,6 +4178,7 @@ def compress_context( focus_topic=focus_topic, force=force, memory_context=memory_context, + bypass_cooldown=bypass_cooldown, ) if memory_context.strip() and "memory_context" not in compress_kwargs: engine_name = getattr( @@ -3968,7 +4199,9 @@ def compress_context( messages_before_compression = copy.deepcopy(messages) _activity_heartbeat = _CompressionActivityHeartbeat( - agent, commit_fence=commit_fence + agent, + commit_fence=commit_fence, + emit_client_status=_compaction_status_emitted, ).start() # Publish forward progress to the commit fence while the summary LLM # call streams. Async hosts (gateway session hygiene) poll @@ -3992,11 +4225,28 @@ def compress_context( from agent.auxiliary_client import ( aux_interrupt_protection, aux_progress_hook, + aux_stream_deadline, ) _progress_hook = ( commit_fence.touch_progress if commit_fence is not None else (lambda: None) ) + # #99692: the progress hook above is the worker -> host leg; this is the + # return leg. _compression_cancel_requested (below) releases the compression + # OWNER when the host gives up, but the isolated provider daemon that + # actually holds the socket keeps streaming to its own budget — + # ``_aux_stream_total_ceiling`` = max(600, 4 * aux_timeout), which is >= + # the host's total ceiling for every configured timeout and starts + # counting later (after admission, serialization, prompt build and TTFT). + # With ``auxiliary.compression.timeout: 600`` that is 2400s of an + # orphaned 500K-token summary the commit fence is already guaranteed to + # refuse: paid tokens, a pinned HTTP connection, and — since every new + # turn re-triggers compression on a session that never shrank — a fresh + # orphan stacked on top of the last one. Sharing the host's absolute + # deadline makes the stream stop when the host it serves stops waiting. + _host_stream_deadline = ( + commit_fence.deadline_monotonic if commit_fence is not None else None + ) # F4 state-ordering (#76354): a LATE successful summary must not undo # the timeout cooldown the host recorded. Install a cancellation # check the compressor consults BEFORE clearing the failure cooldown; @@ -4042,7 +4292,9 @@ def compress_context( ) compressed = messages else: - with aux_progress_hook(_progress_hook), aux_interrupt_protection( + with aux_progress_hook(_progress_hook), aux_stream_deadline( + _host_stream_deadline + ), aux_interrupt_protection( cancel_check=_compression_cancel_requested ): compressed = compress_fn(messages, **compress_kwargs) @@ -5375,9 +5627,10 @@ def compress_context( else: agent.context_compressor._verify_compaction_cleared_threshold = True - # Clear the file-read dedup cache. After compression the original - # read content is summarised away — if the model re-reads the same - # file it needs the full content, not a "file unchanged" stub. + # Advance file-read dedup to a fresh generation while preserving the + # mtime map. The first read of each unchanged key returns full content + # that compaction may have omitted; later reads return lightweight + # stubs. Stub-hit counters restart at the same boundary (#84857). try: from tools.file_tools import reset_file_dedup reset_file_dedup(task_id) @@ -5544,7 +5797,9 @@ def _compress_context_via_codex_app_server( _activity_heartbeat: Optional[_CompressionActivityHeartbeat] = None try: - _activity_heartbeat = _CompressionActivityHeartbeat(agent).start() + _activity_heartbeat = _CompressionActivityHeartbeat( + agent, emit_client_status=True + ).start() result = codex_session.compact_thread() except BaseException: if _activity_heartbeat is not None: @@ -5914,6 +6169,7 @@ def try_shrink_image_parts_in_messages( __all__ = [ "COMPACTION_STATUS", "COMPACTION_DONE_STATUS", + "COMPACTION_HEARTBEAT_STATUS", "COMPACTION_STATUS_MARKER", "is_compaction_progress_status", "check_compression_model_feasibility", diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index ac4f03deb6..74e1a6f3fc 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -41,6 +41,7 @@ from agent.conversation_compression import ( from agent.context_engine import automatic_compaction_status_message from agent.display import KawaiiSpinner from agent.error_classifier import FailoverReason, classify_api_error +from agent.fast_mode import begin_turn as begin_fast_mode_turn from agent.message_metadata import append_message from agent.turn_context import ( PreflightCompressionTimedOut, @@ -608,6 +609,36 @@ def _image_error_max_dimension(error: Exception) -> Optional[int]: return None +def _pressure_with_real_floor(compressor: Any, rough_tokens: int) -> int: + """Floor the ROUGH pre-API pressure estimate at the last REAL prompt size. + + Applied only on the fallback path -- when ``anchored_context_tokens`` has + no valid anchor (first request, transcript rewritten under the anchor, + provider never reported usage). A valid anchor is provider-exact and is + used as-is; in particular on MoA turns the anchor deliberately uses the + pre-fold aggregator usage while ``last_real_prompt_tokens`` holds the + folded figure, so flooring an anchored value would re-add fan-out tokens + the anchor exists to exclude. + + On the rough path, non-ASCII text (Cyrillic, Greek, Polish, ...) + under-counts by up to ~2x, so a session can sit at the provider's real + context ceiling while the rough figure stays under the compaction + threshold -- on silent-clip providers (ollama /v1) that is a truncation + death spiral the reactive overflow handler never sees (observed live: + real prompts 64,842->64,995 against a 55,705 threshold). The provider's + last reported prompt_tokens is authoritative; never let the rough figure + fall below it. Skipped for exactly one turn after a compaction, when + last_real_prompt_tokens still holds the stale pre-compression value + (#36718's awaiting_real_usage_after_compression window). + """ + last_real = int(getattr(compressor, "last_real_prompt_tokens", 0) or 0) + if last_real > rough_tokens and not getattr( + compressor, "awaiting_real_usage_after_compression", False + ): + return last_real + return rough_tokens + + def _ollama_context_limit_error(agent: Any, request_tokens: int) -> Optional[str]: """Return a user-facing error when Ollama is loaded with too little context.""" if not getattr(agent, "tools", None): @@ -988,6 +1019,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) """ stored_prompt = None stored_state = "missing" + session_row = None if conversation_history and agent._session_db: try: session_row = agent._session_db.get_session(agent.session_id) @@ -1090,6 +1122,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # Continuing session — reuse the exact system prompt from the # previous turn so the Anthropic cache prefix matches. agent._cached_system_prompt = stored_prompt + # Same contract for tools[]: a fresh AIAgent for an existing session + # (gateway agent-cache eviction) re-probed every check_fn, so pin the + # array back to the order this session already sent (tools freeze). + try: + saved_tools = session_row.get("tool_names") if session_row else None + if saved_tools: + from tools.mcp_tool import restore_agent_tool_prefix + + restore_agent_tool_prefix(agent, json.loads(saved_tools)) + except Exception: + logger.debug("tool prefix restore skipped", exc_info=True) # Prompt-section callbacks are new-session-only. Recover their frozen # bytes from the persisted full prompt so a later compression rebuild # keeps them without evaluating plugin state in this resumed process. @@ -1172,6 +1215,9 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) if agent._session_db: try: agent._session_db.update_system_prompt(agent.session_id, agent._cached_system_prompt) + from tools.mcp_tool import persist_agent_tool_names + + persist_agent_tool_names(agent) except Exception as exc: logger.warning( "Session DB update_system_prompt failed for session %s: " @@ -1624,6 +1670,41 @@ def _compression_deferred_result( } +def _provider_overflow_exhausted_result( + agent, + messages: List[Dict], + conversation_history, + api_call_count: int, + request_pressure_tokens: int, + max_compression_attempts: int, +) -> Dict[str, Any]: + """Fail closed when a rebuilt request is still too large after recovery.""" + agent._flush_status_buffer() + logger.error( + "%sContext compression failed after %d attempts; rebuilt request " + "remains over threshold at ~%s tokens.", + agent.log_prefix, + max_compression_attempts, + f"{request_pressure_tokens:,}", + ) + agent._persist_session(messages, conversation_history) + final_response = ( + "Context length exceeded: compression could not reduce the rebuilt " + "request below the safe threshold." + ) + return { + "final_response": final_response, + "messages": messages, + "completed": False, + "api_calls": api_call_count, + "error": final_response, + "partial": True, + "failed": True, + "compression_exhausted": True, + "turn_exit_reason": "context_compression_exhausted", + } + + def _rewrite_system_content_blocks(system_message: dict, effective: str) -> bool: """Rewrite a cache-decorated system message in place, keeping its blocks. @@ -2007,6 +2088,7 @@ def run_conversation( agent._last_compaction_in_place = False agent._last_compression_attempt_recorded = False agent._last_compression_attempt_in_place = None + begin_fast_mode_turn(agent, conversation_history) # Adopt any ~/.hermes/.env credential/base-url edits made since the last # turn — a Settings save updates .env but not this worker's client, which @@ -2125,6 +2207,12 @@ def run_conversation( failed = False codex_ack_continuations = 0 length_continue_retries = 0 + # One-shot "continue without thinking" override is turn-scoped: a + # thinking-only truncation arms it right before the continuation restart, + # and build_api_kwargs consumes it on that call. If the turn is + # interrupted/errors between arm and consume, it must not fire on the + # next turn's first request. + agent._ephemeral_reasoning_off = False # Total outer-loop exceptions this turn (#92450) — see _MAX_OUTER_LOOP_ERRORS. _outer_error_count = 0 truncated_tool_call_retries = 0 @@ -2142,6 +2230,13 @@ def run_conversation( max_compression_attempts = getattr(agent, "max_compression_attempts", 3) _last_preflight_pressure: Optional[int] = None _preflight_compression_blocked = _ctx.preflight_compression_blocked + # A provider overflow is stronger evidence than the rough-estimate + # calibration that normally defers preflight immediately after compaction. + # Keep recovery armed until the rebuilt, complete request is below the + # configured compression threshold. Without this handoff, a compaction + # that drops rows but grows the actual prompt can be sent straight back to + # the provider while awaiting_real_usage_after_compression is true. + _provider_overflow_recovery_pending = False # Armed when a compression host-timeout terminates the turn (#98722, # salvaged from #98741); finalize below reuses the gateway's existing # context-recovery contract (error/partial/compression_exhausted). @@ -2648,6 +2743,13 @@ def run_conversation( # gated on context_compressor — so orphans from session loading or # manual message manipulation are always caught. api_messages = agent._sanitize_api_messages(api_messages) + # Send-path vision eviction (#89296): compression only strips stale + # screenshots when prune fires, and the Anthropic adapter's keep-window + # never sees OpenAI-style tool-result image_url parts. The per-call + # clone is rewritten in place; persisted history is untouched. + from agent.context_compressor import evict_stale_outbound_tool_images + + evict_stale_outbound_tool_images(api_messages) # One-time repeated-heal escalation notice (#96870): if the sanitizer # above just crossed the per-session heal threshold, deliver the @@ -2817,6 +2919,10 @@ def run_conversation( ) if _anchored_pressure is not None: request_pressure_tokens = _anchored_pressure + else: + request_pressure_tokens = _pressure_with_real_floor( + agent.context_compressor, request_pressure_tokens + ) total_chars = approx_tokens * 4 # Stash this request's rough estimate so update_from_response() can # pair it with the provider's real prompt count — the (rough, real) @@ -2866,6 +2972,21 @@ def run_conversation( _preflight_threshold = int( getattr(_compressor, "threshold_tokens", 0) or 0 ) + _provider_overflow_preflight = ( + _provider_overflow_recovery_pending + and ( + _preflight_threshold <= 0 + or request_pressure_tokens >= _preflight_threshold + ) + ) + if ( + _provider_overflow_recovery_pending + and not _provider_overflow_preflight + ): + # The outer-loop rebuild includes the active system prompt, + # request-only injections, and tool schemas. Once that complete + # request has real output runway again, the provider may be tried. + _provider_overflow_recovery_pending = False # A previous mid-turn preflight pass deliberately continued the loop so # API-only context and all sanitization could be rebuilt. Compare that # fully assembled request with the fully assembled request that caused @@ -2905,8 +3026,14 @@ def run_conversation( and not _review_fork_first_request_pending(agent) and len(messages) > 1 and compression_attempts < max_compression_attempts - and not _preflight_compression_blocked - and not _defer_preflight(request_pressure_tokens) + and ( + not _preflight_compression_blocked + or _provider_overflow_preflight + ) + and ( + not _defer_preflight(request_pressure_tokens) + or _provider_overflow_preflight + ) and not _compression_cooldown and _compressor.should_compress(request_pressure_tokens) ): @@ -3073,6 +3200,34 @@ def run_conversation( _turn_exit_reason = "compaction_handoff_not_actionable" break continue + elif _provider_overflow_preflight and _compression_cooldown: + # The provider already proved this request cannot fit, while the + # compressor is temporarily unavailable. Do not send the known- + # oversized request again; let the next user turn retry after the + # cooldown instead of turning this into compression exhaustion. + agent._persist_session(messages, conversation_history) + return _compression_deferred_result( + agent, + messages, + api_call_count, + reason="transient_block", + ) + elif ( + _provider_overflow_preflight + and compression_attempts >= max_compression_attempts + ): + # Every bounded recovery pass has been consumed and the rebuilt + # request is still over threshold. Fail closed before another + # provider call; llama.cpp can silently truncate an oversized + # retry instead of returning a second actionable overflow error. + return _provider_overflow_exhausted_result( + agent, + messages, + conversation_history, + api_call_count, + request_pressure_tokens, + max_compression_attempts, + ) elif ( agent.compression_enabled and len(messages) > 1 @@ -3127,6 +3282,20 @@ def run_conversation( if callable(_warn_fn): _warn_fn(request_pressure_tokens, _ctx_len) + if _provider_overflow_preflight: + # Any other gate that prevented the forced preflight (for example, + # an uncompressible one-message request) must also fail closed. + # Falling through would send a request that the provider already + # proved cannot fit. + return _provider_overflow_exhausted_result( + agent, + messages, + conversation_history, + api_call_count, + request_pressure_tokens, + max_compression_attempts, + ) + # Thinking spinner for quiet mode (animated during API call) thinking_spinner = None @@ -4058,7 +4227,7 @@ def run_conversation( "The model used all its output tokens on reasoning " "and had none left for the actual response.\n\n" "To fix this:\n" - "→ Lower reasoning effort: `/thinkon low` or `/thinkon minimal`\n" + "→ Lower reasoning effort: `/reasoning low` or `/reasoning minimal`\n" "→ Or switch to a larger/non-reasoning model with `/model`" ) agent._cleanup_task_resources(effective_task_id) @@ -4185,29 +4354,46 @@ def run_conversation( ) if assistant_message is not None and not _trunc_has_tool_calls: length_continue_retries += 1 - # An EMPTY partial-stream stub (stream dropped - # mid tool-call before any text was delivered) - # must not be appended as an interim assistant - # message: it would serialize as - # {"role": "assistant", "content": ""}, and + # An interim assistant message with NO visible + # content must not be appended — whichever way it + # got that way. An empty partial-stream stub + # (stream dropped before any text was delivered) + # and a response whose whole output budget went to + # reasoning delivered in a separate field (GLM-5.3 + # on ollama-cloud with reasoning_effort=high: + # finish_reason="length", content="", + # completion_tokens == max_tokens) both serialize + # as {"role": "assistant", "content": ""}, and # strict providers (Moonshot/Kimi via OpenRouter) # reject empty assistant content with HTTP 400 # ("message ... with role 'assistant' must not be # empty") on the very next replay — permanently - # poisoning the session history. There is no - # partial text to continue from anyway, so only - # the continuation user-message is appended. + # poisoning the session history until the pre-call + # sanitizer "heals" the hole (observed 3+ healings + # per turn). There is no partial text to continue + # from anyway, so only the continuation + # user-message is appended. + _interim_content = getattr(assistant_message, "content", None) _is_empty_partial_stub = ( getattr(response, "id", "") == PARTIAL_STREAM_STUB_ID - and not getattr(assistant_message, "content", None) + and not _interim_content ) - if not _is_empty_partial_stub: + if not _interim_content and not _is_empty_partial_stub: + # Thinking-only truncation: the model spent the + # entire output cap on reasoning and produced no + # visible text. A continuation with thinking + # ON would re-think the whole context from + # scratch (continuations never replay prior + # reasoning) and re-burn the same budget, so + # the next call drops thinking for one request + # — the answer must be written, not re-derived. + agent._ephemeral_reasoning_off = True + if _interim_content: interim_msg = agent._build_assistant_message(assistant_message, finish_reason) # Marked so the ceiling exit can drop the fragment trail. interim_msg["_length_continuation_fragment"] = True append_message(messages, interim_msg) - if assistant_message.content: - truncated_response_parts.append(assistant_message.content) + truncated_response_parts.append(_interim_content) if length_continue_retries < 4: _is_partial_stream_stub = ( @@ -4251,13 +4437,43 @@ def run_conversation( break partial_response = agent._strip_think_blocks(_join_truncated_parts(truncated_response_parts)).strip() + # The pending one-shot reasoning-off override must + # not leak into the next turn when the 4th + # truncation goes straight to the ceiling exit + # without scheduling a continuation call to + # consume it. + agent._ephemeral_reasoning_off = False if partial_response: agent._vprint( f"{agent.log_prefix}⚠️ Response still truncated " - f"after 4 continuation attempts — keeping the " + f"after {length_continue_retries} continuation attempts — keeping the " f"partial response received so far.", force=True, ) + _ceiling_final = partial_response + else: + # Every fragment was empty — e.g. a thinking + # model that spent each attempt's whole cap on + # reasoning (GLM-5.3 on ollama-cloud). Return + # an actionable message instead of an invisible + # None result, which only surfaces as a bare + # error card. + agent._vprint( + f"{agent.log_prefix}⚠️ Response still truncated " + f"after {length_continue_retries} continuation attempts — no visible " + f"text was produced.", + force=True, + ) + _ceiling_final = ( + "⚠️ **No visible answer was produced.** The " + "model hit its output-token limit on every " + "continuation attempt — its reasoning " + "consumed the entire budget each time.\n\n" + "To fix this:\n" + "→ Lower reasoning effort: `/reasoning low` " + "or `/reasoning none`\n" + "→ Or raise max_tokens for this model" + ) # Unanswered continue nudges made every later turn re-truncate. _turn_start = ( current_turn_user_idx + 1 @@ -4285,7 +4501,7 @@ def run_conversation( agent._cleanup_task_resources(effective_task_id) agent._persist_session(messages, conversation_history) return { - "final_response": partial_response or None, + "final_response": _ceiling_final, "messages": messages, "api_calls": api_call_count, "completed": False, @@ -4393,6 +4609,11 @@ def run_conversation( "error": "First response truncated due to output length limit" } + # Count every completed provider attempt, including providers + # that omit usage. Token/cost accounting below remains gated on + # real usage, but the request itself must stay observable. + agent.session_api_calls += 1 + # Track actual token usage from response for context management if hasattr(response, 'usage') and response.usage: canonical_usage = normalize_usage( @@ -4567,7 +4788,6 @@ def run_conversation( agent.session_prompt_tokens += prompt_tokens agent.session_completion_tokens += completion_tokens agent.session_total_tokens += total_tokens - agent.session_api_calls += 1 agent.session_input_tokens += canonical_usage.input_tokens agent.session_output_tokens += canonical_usage.output_tokens agent.session_cache_read_tokens += canonical_usage.cache_read_tokens @@ -4714,6 +4934,15 @@ def run_conversation( f"{cached:,}/{prompt:,} tokens " f"({hit_pct:.0f}% hit, {written:,} written)" ) + else: + logger.info( + "API call #%d: model=%s provider=%s in=? out=? total=? " + "latency=%.1fs usage=unavailable", + agent.session_api_calls, + agent.model, + agent.provider or "unknown", + api_duration, + ) _retry.has_retried_429 = False # Reset on success # Note: don't clear the retry buffer here — an "API call @@ -5254,7 +5483,7 @@ def run_conversation( ): _retry.nous_auth_retry_attempted = True if agent._try_refresh_nous_client_credentials(force=True): - print(f"{agent.log_prefix}🔐 Nous agent key refreshed after 401. Retrying request...") + agent._buffer_vprint(f"🔐 Nous agent key refreshed after 401. Retrying request...") continue # Credential refresh didn't help — show diagnostic info. # Most common causes: Portal OAuth expired/revoked, @@ -5419,6 +5648,37 @@ def run_conversation( ) continue + # ── Reasoning-mandatory route rejected a disable ────── + # The route (Nous Portal / OpenRouter, e.g. GLM-5.3) answers + # ``reasoning: {enabled: false}`` with HTTP 400. The catalog + # guard in the provider profile normally swallows the + # disable, but a process that warmed its capability cache + # before the route flipped to mandatory keeps sending it. + # One-shot: never send a disable again this session (the + # wire builder omits it → upstream default thinking), queue + # a catalog refresh so the guard is right next time, retry. + if ( + classified.reason == FailoverReason.reasoning_mandatory + and not _retry.reasoning_mandatory_retry_attempted + ): + _retry.reasoning_mandatory_retry_attempted = True + agent._reasoning_disable_rejected = True + try: + from hermes_cli.models import refresh_reasoning_caps_async + refresh_reasoning_caps_async(agent.provider) + except Exception: + pass + agent._vprint( + f"{agent.log_prefix}⚠️ {agent.model} requires reasoning — " + f"thinking stays on for this session, retrying...", + force=True, + ) + logger.warning( + "%sReasoning-mandatory recovery: dropping reasoning disable for %s", + agent.log_prefix, agent.model, + ) + continue + # ── Native compaction rejection recovery ────────────── # Provider explicitly rejected the ``context_management`` # field (structured 400 naming the param). One-shot: turn @@ -5720,6 +5980,12 @@ def run_conversation( ) ) time.sleep(2) + # Same class as the generic overflow handler below: + # the provider proved the request does not fit the + # (now-reduced) window, and row count alone is not + # proof the rebuilt request does. Recheck the + # complete request before the next provider call. + _provider_overflow_recovery_pending = True _retry.restart_with_compressed_messages = True break # Fall through to normal error handling if compression @@ -6005,6 +6271,11 @@ def run_conversation( messages, system_message, approx_tokens=estimate_request_tokens_rough(api_messages, tools=agent.tools or None), task_id=effective_task_id, + # #100661: the provider proved the request does not fit. + # Ignore the summary-failure cooldown for this ONE + # attempt (bounded by max_compression_attempts) instead + # of deferring every turn until the ladder lapses. + bypass_cooldown=True, ) if messages is _overflow_input and compression_skipped_due_to_lock(agent): # #69870 lock-skip: the provider proved the request @@ -6181,6 +6452,7 @@ def run_conversation( messages, system_message, approx_tokens=request_input_estimate, task_id=effective_task_id, + bypass_cooldown=True, # #100661 provider-proven overflow ) if messages is _overflow_input and compression_skipped_due_to_lock(agent): compression_attempts -= 1 @@ -6344,6 +6616,11 @@ def run_conversation( messages, system_message, approx_tokens=estimate_request_tokens_rough(api_messages, tools=agent.tools or None), task_id=effective_task_id, + # #100661: the provider proved the request does not fit. + # Ignore the summary-failure cooldown for this ONE + # attempt (bounded by max_compression_attempts) instead + # of deferring every turn until the ladder lapses. + bypass_cooldown=True, ) if messages is _overflow_input and compression_skipped_due_to_lock(agent): # #69870 lock-skip: the provider proved the request @@ -6407,6 +6684,11 @@ def run_conversation( elif new_tokens > 0 and new_tokens < original_tokens * 0.95: agent._buffer_status(COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE.format(before=original_tokens, after=new_tokens)) time.sleep(2) # Brief pause between compression retries + # Rebuild the complete request before the next provider + # call and force normal preflight to honor it. Message + # count alone is not proof that system/tool-inclusive + # token pressure fell. + _provider_overflow_recovery_pending = True _retry.restart_with_compressed_messages = True break else: @@ -7713,7 +7995,7 @@ def run_conversation( # This classification is needed regardless of whether the turn has visible content, # because a substantive tool-only turn must invalidate any older housekeeping fallback. _HOUSEKEEPING_TOOLS = frozenset({ - "memory", "todo", "skill_manage", "session_search", + "memory", "todo_list", "skill_manage", "session_search", }) _all_housekeeping = all( tc.function.name in _HOUSEKEEPING_TOOLS @@ -8037,9 +8319,19 @@ def run_conversation( _info = getattr(_compressor, "should_compress_info", None) if _info is not None: try: - _block_reason = _info(_real_tokens)[1] + _should_now, _block_reason = _info(_real_tokens) except Exception: - _block_reason = None + _should_now, _block_reason = False, None + if _should_now and not _block_reason: + # The engine says compression SHOULD run, yet this + # branch was taken — the per-turn attempt budget is + # spent. Over threshold with no reclamation left is + # exactly the silent-lockout case, so name it + # instead of dropping the (True, None) on the floor + # (#101889). + _block_reason = ( + f"attempts_exhausted:{compression_attempts}" + ) if _block_reason: agent._warn_context_overflow_blocked( _block_reason, @@ -8304,6 +8596,7 @@ def run_conversation( agent, finish_reason=finish_reason, response=response, + observed_generation=_has_structured, ) _empty_retry_budget = ( _empty_guard.empty_retry_budget(agent, response) @@ -8371,15 +8664,14 @@ def run_conversation( if _truly_empty and _deterministic_empty: logger.warning( - "Deterministic empty response detected " - "(consecutive zero-output completions, " - "model=%s provider=%s finish_reason=%s) — " + "Repeated empty response detected " + "(model=%s provider=%s finish_reason=%s) — " "skipping remaining retries", agent.model, agent.provider, finish_reason, ) agent._buffer_status( - "⚠️ Model is deterministically returning empty " - "(zero output tokens) — skipping further retries " + "⚠️ Model is repeatedly returning empty content — " + "skipping further retries " "to avoid repeat charges" ) diff --git a/agent/copilot_acp_client.py b/agent/copilot_acp_client.py index b569233682..996b58dd96 100644 --- a/agent/copilot_acp_client.py +++ b/agent/copilot_acp_client.py @@ -9,6 +9,7 @@ back into the minimal shape Hermes expects from an OpenAI client. from __future__ import annotations import json +import logging import os import queue import re @@ -31,6 +32,7 @@ from agent.redact import redact_sensitive_text from tools.environments.local import hermes_subprocess_env ACP_MARKER_BASE_URL = "acp://copilot" +logger = logging.getLogger(__name__) _DEFAULT_TIMEOUT_SECONDS = 900.0 # Stderr fingerprint of the deprecated `gh copilot` CLI extension @@ -189,6 +191,71 @@ def _permission_denied(message_id: Any) -> dict[str, Any]: } +def _model_selection_request( + session: dict[str, Any], requested_model: str +) -> tuple[str, dict[str, str]] | None: + """Return the ACP request that selects ``requested_model`` for ``session``. + + Prefer stable v1 ``session/set_config_option``. Fall back to Copilot's + pre-stabilization ``session/set_model`` extension only when no model + config option is advertised. A reported model list is authoritative: + unknown and policy-disabled ids return None instead of being sent. + """ + session_id = str(session.get("sessionId") or "").strip() + requested_model = str(requested_model or "").strip() + if not session_id or not requested_model or requested_model == "copilot-acp": + return None + + config_options = [ + o for o in (session.get("configOptions") or []) if isinstance(o, dict) + ] + model_option = next( + ( + o for o in config_options + if o.get("category") == "model" or o.get("id") == "model" + ), + None, + ) + if model_option is not None: + enabled_values = { + str(o.get("value") or "").strip() + for o in (model_option.get("options") or []) + if isinstance(o, dict) + and str( + ((o.get("_meta") or {}).get("copilotEnablement")) or "" + ).strip().lower() != "disabled" + } + if requested_model not in enabled_values: + return None + return ( + "session/set_config_option", + { + "sessionId": session_id, + "configId": str(model_option.get("id") or "model"), + "value": requested_model, + }, + ) + + advertised = [ + m + for m in ((session.get("models") or {}).get("availableModels") or []) + if isinstance(m, dict) + ] + available = { + str(m.get("modelId") or "").strip() + for m in advertised + if str( + ((m.get("_meta") or {}).get("copilotEnablement")) or "" + ).strip().lower() != "disabled" + } + if available and requested_model not in available: + return None + return ( + "session/set_model", + {"sessionId": session_id, "modelId": requested_model}, + ) + + def _format_messages_as_prompt( messages: list[dict[str, Any]], model: str | None = None, @@ -201,8 +268,11 @@ def _format_messages_as_prompt( "IMPORTANT: If you take an action with a tool, you MUST output tool calls using {...} blocks with JSON exactly in OpenAI function-call shape.", "If no tool is needed, answer normally.", ] - if model: - sections.append(f"Hermes requested model hint: {model}") + # Deliberately no "requested model" line in the prompt: the model is + # applied for real via ACP session/set_model, and when the backend can't + # honor it (org-policy-disabled id) a prompt-text mention makes the + # serving model FALSELY self-identify as the requested one. Identity + # must come from the backend, not from prompt suggestion. # Copilot has no tools of its own that would collide with Hermes', so it # forwards the whole toolset (no allowlist). @@ -292,6 +362,12 @@ class _ACPChatNamespace: class CopilotACPClient: """Minimal OpenAI-client-compatible facade for Copilot ACP.""" + # Declared for agent/auxiliary_client.py: this shim drives an ACP subprocess + # over stdio, so it is already a complete client (never re-dispatch it + # through a wire adapter) and is safe to use from async code as-is. + HERMES_SKIP_TRANSPORT_WRAP = True + HERMES_SKIP_ASYNC_WRAP = True + def __init__( self, *, @@ -369,6 +445,7 @@ class CopilotACPClient: response_text, reasoning_text = self._run_prompt( prompt_text, timeout_seconds=_effective_timeout, + model=model, ) tool_calls, cleaned_text = _extract_tool_calls_from_text(response_text) @@ -397,7 +474,13 @@ class CopilotACPClient: return _completion_to_stream_chunks(completion) return completion - def _run_prompt(self, prompt_text: str, *, timeout_seconds: float) -> tuple[str, str]: + def _run_prompt( + self, + prompt_text: str, + *, + timeout_seconds: float, + model: str | None = None, + ) -> tuple[str, str]: # Fast-fail when the CLI doesn't support the ACP args we'd pass. # Without this guard, a CLI like Claude Code v2.x exits with # ``error: unknown option '--acp'`` immediately, then the parent @@ -420,6 +503,13 @@ class CopilotACPClient: f"to a working pair." ) + # Note the model Hermes selected; it is applied after session/new via + # the ACP-native `session/set_model` call. The CLI's `--model` spawn + # flag is deliberately NOT used here: `copilot --acp` validates it + # (an unknown id aborts the spawn) but then ignores it for the actual + # session, so it adds a failure mode without selecting anything. + requested_model = str(model or "").strip() + try: # Hide the console the CLI child would otherwise flash on Windows # (#56747). Hide-only — stdio pipes stay intact for the ACP wire. @@ -564,6 +654,31 @@ class CopilotACPClient: if not session_id: raise RuntimeError("Copilot ACP did not return a sessionId.") + # Select the model Hermes asked for. Prefer the stable ACP v1 + # session-config API: session/new advertises a category="model" + # select option and session/set_config_option updates it. Copilot + # still exposes the older models/session/set_model extension too, + # so retain that only as compatibility fallback for older agents. + if requested_model and requested_model != "copilot-acp": + try: + selection = _model_selection_request(session, requested_model) + if selection is not None: + method, params = selection + _request(method, params) + else: + logger.warning( + "Copilot ACP does not offer model %r; using the " + "session default.", + requested_model, + ) + except Exception as exc: + logger.warning( + "Copilot ACP model selection for %r failed; continuing " + "with the session default: %s", + requested_model, + exc, + ) + text_parts: list[str] = [] reasoning_parts: list[str] = [] _request( diff --git a/agent/credential_pool.py b/agent/credential_pool.py index c28cabd2ba..df2f1fdd80 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -12,7 +12,7 @@ import re from dataclasses import dataclass, fields, replace from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from hermes_constants import OPENROUTER_BASE_URL from hermes_cli.config import load_env @@ -26,6 +26,7 @@ import hermes_cli.auth as auth_mod from hermes_cli.auth import ( CODEX_ACCESS_TOKEN_REFRESH_SKEW_SECONDS, PROVIDER_REGISTRY, + SINGLE_USE_REFRESH_POOL_PROVIDERS, _auth_store_lock, _codex_access_token_is_expiring, _decode_jwt_claims, @@ -807,11 +808,134 @@ def _write_through_provider_state_to_global_root( ) +def _singleton_target_for_entry(pool: "CredentialPool", entry: "PooledCredential") -> Optional[Path]: + """Root ``.anthropic_oauth.json`` when *entry* is a borrowed hermes_pkce row, else None.""" + if entry.source != "hermes_pkce" or entry.id not in getattr(pool, "_borrowed_root_ids", ()): + return None + try: + from agent.anthropic_credentials import _root_hermes_oauth_file + return _root_hermes_oauth_file() + except Exception: + return None + + +def _profile_owns_pool_provider(provider: str) -> bool: + """True when the ACTIVE auth.json has its own rows for *provider*. + + Named profiles with no local rows read the provider through the + ``read_credential_pool`` global-root fallback ("borrowing"). + """ + try: + pool = _load_auth_store().get("credential_pool") + except Exception: + return True # unreadable store: assume ownership, keep legacy path + entries = pool.get(provider) if isinstance(pool, dict) else None + return isinstance(entries, list) and bool(entries) + + +def _borrowed_single_use_pool_root() -> Optional[Path]: + """Return the global-root auth.json when persisting a BORROWED single-use pool. + + ``None`` means "persist to the active store as usual": classic mode + (profile == root), or the profile owns its own rows for this provider. + Pytest seat belt mirrors ``_write_through_provider_state_to_global_root``. + """ + try: + global_path = _global_auth_file_path() + except Exception: + return None + if global_path is None: + return None + if os.environ.get("PYTEST_CURRENT_TEST"): + real_home_env = os.environ.get("HOME", "") + if real_home_env: + real_root = Path(real_home_env) / ".hermes" / "auth.json" + try: + if global_path.resolve(strict=False) == real_root.resolve(strict=False): + return None + except Exception: + return None + return global_path + + +def persist_pool_entries( + provider: str, + payloads: List[Dict[str, Any]], + *, + removed_ids: Optional[Iterable[str]] = None, +) -> None: + """Persist a provider's pool rows to the store that OWNS them. + + A named profile that sees a single-use-refresh provider (Anthropic, + Codex, xAI OAuth) only through the global-root fallback must not + materialize a local ``credential_pool.`` copy on its first + persist: that copy forks the single-use refresh token, the first profile + to rotate commits the new pair only to its own file, and root plus every + sibling die with ``invalid_grant`` on their next refresh (#100339). Such + rows are written back to the root store (under the root lock) so the + rotation is visible to every profile; everything else goes to the active + store exactly as before. + """ + if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS and not _profile_owns_pool_provider(provider): + global_path = _borrowed_single_use_pool_root() + if global_path is not None: + removed = {rid for rid in (removed_ids or ()) if rid} + try: + with _auth_store_lock(target_path=global_path): + store = _load_auth_store(global_path) + pool = store.get("credential_pool") + if not isinstance(pool, dict): + pool = {} + store["credential_pool"] = pool + existing = pool.get(provider) + existing_list = existing if isinstance(existing, list) else [] + incoming_by_id = { + p.get("id"): p for p in payloads + if isinstance(p, dict) and p.get("id") + } + # UPDATE-ONLY: a borrower may refresh the root's rows + # (rotation, cooldown state) but never add or delete + # them — the root owns their lifecycle. In particular a + # profile's singleton-prune (it has no + # .anthropic_oauth.json of its own) must not delete the + # root grant, and ``removed_ids`` is ignored here. + merged: List[Dict[str, Any]] = [] + changed = False + for disk_entry in existing_list: + did = disk_entry.get("id") if isinstance(disk_entry, dict) else None + incoming = incoming_by_id.get(did) if did else None + if incoming is None: + merged.append(disk_entry) + continue + updated = auth_mod._merge_disk_cooldown_state(incoming, disk_entry, provider) + if updated != disk_entry: + changed = True + merged.append(updated) + if changed: + pool[provider] = merged + _save_auth_store(store, target_path=global_path) + return + except Exception as exc: + # Fail closed on the FORK, not on the save: never fall back to + # writing a local copy (that IS the bug). The in-memory pool + # still holds the rotated pair for this process. + logger.warning( + "%s pool: write-through of borrowed root grant failed (%s); " + "not materializing a profile-local copy", + provider, exc, + ) + return + write_credential_pool(provider, payloads, removed_ids=removed_ids) + + class CredentialPool: def __init__(self, provider: str, entries: List[PooledCredential]): self.provider = provider self._entries = sorted(entries, key=lambda entry: entry.priority) self._current_id: Optional[str] = None + # Ids of rows read via the global-root fallback (single-use OAuth + # providers only); set by load_pool(), consumed by add_entry(). + self._borrowed_root_ids: Set[str] = set() self._strategy = get_pool_strategy(provider) # RLock: the mutation primitives below (_replace_entry/_persist) # self-acquire this lock so the DEFERRED single-use-token refresh @@ -938,7 +1062,7 @@ class CredentialPool: # Self-locking (RLock): snapshotting self._entries must not race a # concurrent rotation when called from the deferred refresh path. with self._lock: - write_credential_pool( + persist_pool_entries( self.provider, [entry.to_dict() for entry in self._entries], removed_ids=removed_ids, @@ -1786,10 +1910,14 @@ class CredentialPool: elif entry.source == "hermes_pkce": try: from agent.anthropic_credentials import _write_hermes_oauth_credentials + # A borrowed row was seeded from the ROOT's singleton + # (this profile has none); commit the rotation there, + # never into a new profile-local copy (#100339). _write_hermes_oauth_credentials( refreshed["access_token"], refreshed["refresh_token"], refreshed["expires_at_ms"], + target=_singleton_target_for_entry(self, entry), ) except Exception as wexc: # Same transaction rule as claude_code above. @@ -1834,11 +1962,19 @@ class CredentialPool: last_refresh=refreshed.get("last_refresh"), ) elif self.provider == "nous": + stale_key = entry.runtime_api_key or entry.agent_key or entry.access_token synced = self._sync_nous_entry_from_auth_store(entry) if synced is not entry: entry = synced + # A peer already rotated and persisted a usable key while + # this process was still holding the failed one: adopt it + # without consuming the (single-use) refresh token again. + if force and entry.runtime_api_key and entry.runtime_api_key != stale_key: + logger.debug("Nous entry %s: adopting peer-rotated token, skipping refresh", entry.id) + return entry auth_mod.resolve_nous_runtime_credentials( force_refresh=force, + stale_access_token=stale_key or None, ) updated = self._sync_nous_entry_from_auth_store(entry) else: @@ -2089,6 +2225,15 @@ class CredentialPool: self._persist() self._sync_device_code_entry_to_auth_store(updated) return updated + if isinstance(exc, TimeoutError): + # Lost the auth-store lock race under heavy fan-out. That + # says nothing about the credential — benching it here is + # what emptied the pool for ~120 sessions on Sep 2 2026 + # ("matched no nous entry ... pool size 0"). Leave the + # entry untouched; the caller's retry re-syncs once the + # winner has persisted. + logger.debug("Nous refresh skipped: auth store lock busy; not benching entry") + return entry if auth_mod._is_terminal_nous_refresh_error(exc): logger.debug("Nous refresh token is terminally invalid; clearing local token state") try: @@ -2806,7 +2951,7 @@ class CredentialPool: replace(entry, priority=new_priority) for new_priority, entry in enumerate(self._entries) ] - write_credential_pool( + persist_pool_entries( self.provider, [entry.to_dict() for entry in self._entries], removed_ids=[removed.id], @@ -2845,7 +2990,22 @@ class CredentialPool: with self._lock: entry = replace(entry, priority=_next_priority(self._entries)) self._entries.append(entry) - self._persist() + borrowed_ids = getattr(self, "_borrowed_root_ids", None) + if borrowed_ids: + # ``hermes -p auth add ``: the + # profile is claiming its OWN credential. Persist only the + # profile-owned rows locally — copying the borrowed root + # grant alongside them would fork its single-use refresh + # token (#100339). Once the profile owns rows, the root + # fallback for this provider is shadowed (existing contract). + write_credential_pool( + self.provider, + [e.to_dict() for e in self._entries if e.id not in borrowed_ids], + ) + self._entries = [e for e in self._entries if e.id not in borrowed_ids] + self._borrowed_root_ids = set() + else: + self._persist() return entry @@ -3631,6 +3791,12 @@ def _seed_custom_pool(pool_key: str, entries: List[PooledCredential]) -> Tuple[b def load_pool(provider: str) -> CredentialPool: provider = (provider or "").strip().lower() + if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS: + # One-time heal for installs that forked this grant across profiles + # BEFORE the clone-strip / root-write-through existed: consolidate the + # profile's copy into root so the read below borrows root's grant + # (#100339). No-op in classic mode or once the profile is clean. + auth_mod.heal_forked_single_use_oauth_grants(provider) raw_entries = read_credential_pool(provider) disk_ids = { entry.get("id") @@ -3678,18 +3844,41 @@ def load_pool(provider: str) -> CredentialPool: # process missing a provider env var must not delete the persisted # pool entry for every other process (#9331). File-backed singletons # still prune when their backing file is gone. - changed |= _prune_stale_seeded_entries( - entries, - singleton_sources | env_sources, - prune_env_sources=False, + borrowing_root_grant = ( + provider in SINGLE_USE_REFRESH_POOL_PROVIDERS + and bool(disk_ids) + and not _profile_owns_pool_provider(provider) ) + if borrowing_root_grant: + # Rows read through the global-root fallback are seeded from the + # ROOT's singleton files, which this profile cannot see; pruning + # them as "backing file gone" would hide (and, via write-through, + # delete) the shared grant. The root's own load_pool() prunes. + borrowed = [e for e in entries if e.id in disk_ids] + others = [e for e in entries if e.id not in disk_ids] + changed |= _prune_stale_seeded_entries( + others, singleton_sources | env_sources, prune_env_sources=False, + ) + entries[:] = borrowed + others + else: + changed |= _prune_stale_seeded_entries( + entries, + singleton_sources | env_sources, + prune_env_sources=False, + ) changed |= _normalize_pool_priorities(provider, entries) if changed: new_ids = {entry.id for entry in entries} - write_credential_pool( + persist_pool_entries( provider, [entry.to_dict() for entry in sorted(entries, key=lambda item: item.priority)], removed_ids=disk_ids - new_ids, ) - return CredentialPool(provider, entries) + pool = CredentialPool(provider, entries) + # Remember which rows are the root's grant (borrowed via fallback) so a + # later ``add_entry`` in this profile can leave them out of the profile's + # own store (#100339). + if provider in SINGLE_USE_REFRESH_POOL_PROVIDERS and not _profile_owns_pool_provider(provider): + pool._borrowed_root_ids = set(disk_ids) + return pool diff --git a/agent/curator_backup.py b/agent/curator_backup.py index 7934ebdbcc..323b76e48a 100644 --- a/agent/curator_backup.py +++ b/agent/curator_backup.py @@ -11,6 +11,7 @@ is undoable, then extracts the chosen snapshot into place. The snapshot does NOT include: - ``.curator_backups/`` (would recurse) - ``.hub/`` (hub-installed skills — managed by the hub, not us) + - ``.git/`` (repository metadata — managed by git, not the curator) It DOES include: - all SKILL.md files + their directories (``scripts/``, ``references/``, @@ -41,6 +42,7 @@ from __future__ import annotations import json import logging +import os import re import shutil import tarfile @@ -60,7 +62,13 @@ DEFAULT_KEEP = 5 # Entries under skills/ that should NEVER be rolled up into a snapshot. # .hub/ is managed by the skills hub; rolling it back would break lockfile # invariants. .curator_backups is the backup dir itself — recursion bomb. -_EXCLUDE_TOP_LEVEL = {".curator_backups", ".hub"} +# .git is repository metadata — rolling it back would break git tracking, +# and snapshots that include it grow with the full history: once backups +# are committed back, the history contains prior backups, so each snapshot +# is bigger than the last (observed: 38MB of skills inflating to 24GB in +# weeks, #91449). ``_tar_filter`` below applies the same set to nested +# paths, so a ``.git`` inside an individual skill dir is skipped too. +_EXCLUDE_TOP_LEVEL = {".curator_backups", ".hub", ".git"} # Snapshot id regex: UTC ISO with colons replaced by dashes so the filename # is portable (Windows-safe). An optional ``-NN`` suffix handles two @@ -260,6 +268,12 @@ def snapshot_skills(reason: str = "manual", *, protect_ids: Optional[Set[str]] = return None archive = dest / "skills.tar.gz" + def _tar_filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: + parts = Path(tarinfo.name).parts + if any(p in _EXCLUDE_TOP_LEVEL for p in parts): + return None + return tarinfo + try: # Stream into the tarball — no tempdir copy needed. with tarfile.open(archive, "w:gz", compresslevel=6) as tf: @@ -268,7 +282,7 @@ def snapshot_skills(reason: str = "manual", *, protect_ids: Optional[Set[str]] = continue # arcname: store paths relative to skills/ so extraction # drops cleanly back into the skills dir. - tf.add(str(entry), arcname=entry.name, recursive=True) + tf.add(str(entry), arcname=entry.name, recursive=True, filter=_tar_filter) # Capture cron/jobs.json alongside the tarball. Never fails the # snapshot — the skills side is the core guarantee; cron is # additive. We still record in the manifest whether it was @@ -542,6 +556,40 @@ def _restore_cron_skill_links(snapshot_dir: Path) -> Dict[str, Any]: +def _restore_excluded_subtrees(staged: Path, skills: Path) -> None: + """Move excluded entries (nested ``.git``/``.hub``/...) from *staged* + back under *skills* after a successful extract. + + Snapshots never contain these, so the extract cannot restore them; the + staged copy of the live tree is the only source. ``.git`` may be a dir + or a file (submodule / worktree ``gitdir:`` pointer) — both are moved. + Best-effort and deliberately conditional: an entry is carried over only + when its parent skill dir was restored and nothing sits at the target. + If the target snapshot predates the skill, the entry is dropped with the + staging dir rather than left as an orphan; note the safety snapshot + excludes these paths too, so that case is not undoable. + """ + def _carry(src: Path) -> None: + dest = skills / src.relative_to(staged) + if dest.parent.is_dir() and not dest.exists(): + try: + shutil.move(str(src), str(dest)) + except OSError as e: + logger.debug("Could not restore excluded entry %s: %s", src, e) + + for dirpath, dirnames, filenames in os.walk(staged): + keep = [] + for name in dirnames: + if name in _EXCLUDE_TOP_LEVEL: + _carry(Path(dirpath) / name) + else: + keep.append(name) + dirnames[:] = keep + for name in filenames: + if name in _EXCLUDE_TOP_LEVEL: + _carry(Path(dirpath) / name) + + def _unstage(moved: List[Tuple[Path, Path]]) -> List[str]: """Move staged entries back to their original paths. @@ -699,8 +747,12 @@ def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path] pass return (False, f"snapshot extract failed (state restored): {e}", None) - # Extract succeeded — the staging dir has served its purpose. The - # user's undo handle is the safety snapshot tarball we took earlier. + # Extract succeeded. Snapshots never contain excluded subtrees (nested + # ``.git``, ``.hub``, ...), so the extract cannot restore them — carry + # them over from the staged copy of the live tree (top-level ``.git`` is + # simpler: it is never staged). Then the staging dir has served its + # purpose; the user's undo handle is the safety snapshot tarball. + _restore_excluded_subtrees(staged, skills) try: shutil.rmtree(staged, ignore_errors=True) except OSError: diff --git a/agent/display.py b/agent/display.py index 8296b42243..f03b87dd2a 100644 --- a/agent/display.py +++ b/agent/display.py @@ -462,7 +462,7 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) - "image_generate": "prompt", "text_to_speech": "text", "vision_analyze": "question", "skill_view": "name", "skills_list": "category", - "cronjob": "action", + "cronjob_manage": "action", "execute_code": "code", "browser_exec": "code", "delegate_task": "goal", "clarify": "question", "skill_manage": "name", } @@ -496,7 +496,7 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) - preview = _oneline(str(goal)) return _truncate_preview(preview, max_len) if preview else None - if tool_name == "process": + if tool_name == "process_manage": action = args.get("action", "") sid = args.get("session_id", "") data = args.get("data", "") @@ -511,7 +511,7 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) - parts = [p for p in parts if p] return " ".join(parts) if parts else None - if tool_name == "todo": + if tool_name == "todo_list": todos_arg = args.get("todos") merge = args.get("merge", False) if todos_arg is None: @@ -657,10 +657,10 @@ _TOOL_VERBS: dict[str, str] = { "skills_list": "Listing skills", "skill_manage": "Updating skill", "delegate_task": "Delegating", - "cronjob": "Scheduling", + "cronjob_manage": "Scheduling", "clarify": "Asking", "memory": "Updating memory", - "todo": "Updating tasks", + "todo_list": "Updating tasks", } # Verbs that read better without the raw argument preview appended. @@ -1433,7 +1433,7 @@ def _get_cute_tool_message( return _wrap(f"┊ 📄 fetch pages {dur}") if tool_name == "terminal": return _wrap(f"┊ 💻 $ {_trunc(build_tool_preview(tool_name, args) or args.get('command', ''), 42)} {dur}") - if tool_name == "process": + if tool_name == "process_manage": action = args.get("action", "?") sid = args.get("session_id", "")[:12] labels = {"list": "ls processes", "poll": f"poll {sid}", "log": f"log {sid}", @@ -1473,7 +1473,7 @@ def _get_cute_tool_message( return _wrap(f"┊ 🖼️ images extracting {dur}") if tool_name == "browser_vision": return _wrap(f"┊ 👁️ vision analyzing page {dur}") - if tool_name == "todo": + if tool_name == "todo_list": todos_arg = args.get("todos") merge = args.get("merge", False) # Parse result for completion progress @@ -1532,7 +1532,7 @@ def _get_cute_tool_message( return _wrap(f"┊ 👁️ vision {_trunc(args.get('question', ''), 30)} {dur}") if tool_name == "send_message": return _wrap(f"┊ 📨 send {args.get('target', '?')}: \"{_trunc(args.get('message', ''), 25)}\" {dur}") - if tool_name == "cronjob": + if tool_name == "cronjob_manage": action = args.get("action", "?") if action == "create": skills = args.get("skills") or ([] if not args.get("skill") else [args.get("skill")]) diff --git a/agent/empty_response_guard.py b/agent/empty_response_guard.py index fbde4b58b0..293d677bfd 100644 --- a/agent/empty_response_guard.py +++ b/agent/empty_response_guard.py @@ -15,15 +15,13 @@ like this). Two independent guards, both failing OPEN to today's behaviour: -1. **Deterministic-empty detection** — two consecutive empty attempts, - both with usage present and ``output_tokens == 0``, from the same - (model, provider, finish_reason), are treated as deterministic: the - same prompt will keep producing the same empty. Remaining retries are - skipped and the loop proceeds straight to the fallback chain (a - different model may behave differently). Attempts with missing usage - or ``output_tokens > 0`` (model generated *something* — think-block - stripping, whitespace, flaky decoding) never classify as deterministic - and keep the full retry budget. +1. **Deterministic-empty detection** — two consecutive empty attempts from + the same (model, provider, finish_reason) are treated as deterministic + when usage proves zero output, or when usage is absent and the assembled + responses contain neither content nor reasoning. Remaining retries are + skipped and the loop proceeds straight to the fallback chain (a different + model may behave differently). Mixed evidence or any generated tokens keep + the full retry budget. 2. **Cost-aware retry budget** — when the estimated input cost of a single empty attempt exceeds the configured threshold (default @@ -78,6 +76,7 @@ class EmptyAttempt: finish_reason: str usage_present: bool zero_output: bool + observed_generation: bool @property def signature(self) -> tuple: @@ -201,7 +200,13 @@ def _zero_output(agent: Any, response: Any) -> tuple: return (True, (output + reasoning) == 0) -def record_empty_attempt(agent: Any, *, finish_reason: str, response: Any) -> None: +def record_empty_attempt( + agent: Any, + *, + finish_reason: str, + response: Any, + observed_generation: bool = True, +) -> None: """Record one empty completion in the current streak. Must be called before ``_empty_content_retries`` is incremented for @@ -222,6 +227,7 @@ def record_empty_attempt(agent: Any, *, finish_reason: str, response: Any) -> No finish_reason=str(finish_reason or ""), usage_present=usage_present, zero_output=zero_output, + observed_generation=bool(observed_generation), ) ) @@ -234,10 +240,10 @@ def record_empty_attempt(agent: Any, *, finish_reason: str, response: Any) -> No def deterministic_empty(agent: Any) -> bool: """True when the current streak looks deterministic. - Requires >= 2 consecutive attempts, ALL with usage present, zero - output tokens, and an identical (model, provider, finish_reason) - signature. Any attempt with missing usage or non-zero output keeps - this False (fail open — transients deserve their retries). + Requires >= 2 consecutive attempts with an identical (model, provider, + finish_reason) signature. Usage-backed attempts must all prove zero output. + Usage-absent attempts must all have no observed content or reasoning. Mixed + evidence fails open so ambiguous transients keep their retries. """ if not guard_enabled(agent): return False @@ -245,10 +251,12 @@ def deterministic_empty(agent: Any) -> bool: if len(attempts) < 2: return False first = attempts[0] - return all( - a.usage_present and a.zero_output and a.signature == first.signature - for a in attempts + same_signature = all(a.signature == first.signature for a in attempts) + usage_proves_empty = all(a.usage_present and a.zero_output for a in attempts) + response_proves_empty = all( + not a.usage_present and not a.observed_generation for a in attempts ) + return same_signature and (usage_proves_empty or response_proves_empty) def empty_retry_budget(agent: Any, response: Any) -> int: diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 7bcfbaf0cb..94ade338b2 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -68,6 +68,7 @@ class FailoverReason(enum.Enum): format_error = "format_error" # 400 bad request — abort or strip + retry invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type content in tool messages (e.g. Xiaomi MiMo) — downgrade to text and retry + reasoning_mandatory = "reasoning_mandatory" # Route rejects reasoning: {enabled: false} — send the disable no more this session and retry # Provider-specific thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid @@ -504,6 +505,10 @@ _REQUEST_VALIDATION_PATTERNS = [ "unsupported_parameter", ] +# A reasoning-mandatory route answering ``reasoning: {enabled: false}`` +# (Nous Portal + OpenRouter wording; ``error_msg`` is lowercased upstream). +_REASONING_MANDATORY_PATTERN = "reasoning is mandatory" + # Request parameters that Hermes sends on SOME routes only, paired with the # providers/hosts where sending them is deliberate. # @@ -1666,6 +1671,20 @@ def _classify_400( should_fallback=False, ) + # Reasoning-mandatory route rejecting a disable (Nous Portal / OpenRouter + # for GLM-5.3 etc.: "Reasoning is mandatory for this endpoint and cannot + # be disabled"). Deterministic for the request shape, but the only bad + # field is ``reasoning: {enabled: false}`` — the conversation_loop drops + # the disable and retries once. Must precede the request-validation + # branch, which would abort the turn as a format_error. + if _REASONING_MANDATORY_PATTERN in error_msg: + return result_fn( + FailoverReason.reasoning_mandatory, + retryable=True, + should_compress=False, + should_fallback=False, + ) + # Server-injected parameter rejection: a 400 blaming a request field the # client never sent. MUST be checked BEFORE the request-validation branch # below, which would otherwise class it as a deterministic format_error and diff --git a/agent/fast_mode.py b/agent/fast_mode.py new file mode 100644 index 0000000000..b8121f6295 --- /dev/null +++ b/agent/fast_mode.py @@ -0,0 +1,63 @@ +"""Bounded fast-mode windows (``/fast auto`` and ``/fast cold``). + +``agent.service_tier`` is ``None`` (normal), ``"priority"`` (static fast), +``"auto"`` or ``"cold"``. The static value is pinned into +``agent.request_overrides`` at agent build time; the two bounded modes +instead open a wall-clock window at each user-turn boundary and layer the +provider's fast override onto the request kwargs only while it is open: + +- ``auto`` — every user turn opens a window of ``agent.fast_auto_seconds``. +- ``cold`` — only the first turn of a session (no prior history) opens it. + +Only per-request params (``service_tier`` / ``speed``) vary between requests; +the system prompt, tools, and messages are untouched, so the prompt cache is +preserved across the window boundary. +""" + +from __future__ import annotations + +import time +from typing import Any + +BOUNDED_MODES = frozenset({"auto", "cold"}) +DEFAULT_WINDOW_SECONDS = 60 + + +def begin_turn(agent: Any, conversation_history: Any) -> None: + """Open (or refuse) the fast window at a user-turn boundary.""" + mode = getattr(agent, "service_tier", None) + agent._fast_until = 0.0 + if mode not in BOUNDED_MODES: + return + if mode == "cold" and any( + isinstance(m, dict) and m.get("role") in ("user", "assistant", "tool") + for m in (conversation_history or ()) + ): + return + try: + window = float(getattr(agent, "fast_auto_seconds", DEFAULT_WINDOW_SECONDS)) + except (TypeError, ValueError): + window = DEFAULT_WINDOW_SECONDS + agent._fast_until = time.monotonic() + max(window, 0.0) + + +def effective_request_overrides(agent: Any) -> dict[str, Any]: + """``agent.request_overrides`` plus the fast override while the window is open.""" + overrides = dict(getattr(agent, "request_overrides", None) or {}) + if getattr(agent, "service_tier", None) not in BOUNDED_MODES: + return overrides + if time.monotonic() >= getattr(agent, "_fast_until", 0.0): + return overrides + from hermes_cli.models import resolve_fast_mode_overrides + + base_url = getattr(agent, "base_url", None) + if getattr(agent, "api_mode", None) == "anthropic_messages": + base_url = getattr(agent, "_anthropic_base_url", None) or base_url + fast = resolve_fast_mode_overrides( + getattr(agent, "model", None), + provider=getattr(agent, "provider", None), + base_url=base_url, + ) + if fast: + overrides.update(fast) + return overrides diff --git a/agent/gemini_native_adapter.py b/agent/gemini_native_adapter.py index 94558f87c8..821ae6b6ce 100644 --- a/agent/gemini_native_adapter.py +++ b/agent/gemini_native_adapter.py @@ -1102,6 +1102,11 @@ class _AsyncGeminiChatNamespace: class GeminiNativeClient: """Minimal OpenAI-SDK-compatible facade over Gemini's native REST API.""" + # Declared for agent/auxiliary_client.py: already a complete client, so it + # is never re-dispatched through a wire adapter. (No HERMES_SKIP_ASYNC_WRAP + # — the async path has a real conversion, AsyncGeminiNativeClient.) + HERMES_SKIP_TRANSPORT_WRAP = True + def __init__( self, *, diff --git a/agent/lsp/client.py b/agent/lsp/client.py index 59de160f18..daa6cd0998 100644 --- a/agent/lsp/client.py +++ b/agent/lsp/client.py @@ -103,6 +103,11 @@ def file_uri(path: str) -> str: return "file://" + quote(abs_path, safe="/:") +def _folder(root: str) -> Dict[str, str]: + """Build an LSP ``WorkspaceFolder`` for ``root``.""" + return {"name": os.path.basename(root.rstrip(os.sep)) or root, "uri": file_uri(root)} + + def uri_to_path(uri: str) -> str: """Inverse of :func:`file_uri`.""" if not uri.startswith("file://"): @@ -197,6 +202,10 @@ class LSPClient: ) -> None: self.server_id = server_id self.workspace_root = workspace_root + # Roots this server is serving. Single-root servers only ever + # hold ``workspace_root``; multi-root servers (pyright) grow this + # via :meth:`add_workspace_folder` instead of a second process. + self.workspace_folders: List[str] = [workspace_root] self._command = list(command) self._env = env self._cwd = cwd or workspace_root @@ -402,9 +411,7 @@ class LSPClient: "rootUri": file_uri(self.workspace_root), "rootPath": self.workspace_root, "processId": os.getpid(), - "workspaceFolders": [ - {"name": "workspace", "uri": file_uri(self.workspace_root)} - ], + "workspaceFolders": [_folder(r) for r in self.workspace_folders], "initializationOptions": self._init_options, "capabilities": { "window": {"workDoneProgress": True}, @@ -701,7 +708,21 @@ class LSPClient: return None async def _handle_workspace_folders(self, params: Any) -> Any: - return [{"name": "workspace", "uri": file_uri(self.workspace_root)}] + return [_folder(r) for r in self.workspace_folders] + + async def add_workspace_folder(self, root: str) -> None: + """Attach another root to a running multi-root server. + + Idempotent; the folder is recorded before the notification is + sent so concurrent callers for the same root only announce once. + """ + if root in self.workspace_folders: + return + self.workspace_folders.append(root) + await self._send_notification( + "workspace/didChangeWorkspaceFolders", + {"event": {"added": [_folder(root)], "removed": []}}, + ) async def _handle_diagnostic_refresh(self, params: Any) -> Any: # We don't honour refresh — we re-pull on every touchFile. diff --git a/agent/lsp/manager.py b/agent/lsp/manager.py index 7ba1b914f7..7dbb5c38bb 100644 --- a/agent/lsp/manager.py +++ b/agent/lsp/manager.py @@ -12,7 +12,10 @@ Design choices: - One client per ``(server_id, workspace_root)`` key. Lazy spawn: the first request for a key spawns the client; subsequent requests - re-use it. + re-use it. Servers flagged ``multi_root`` (pyright) get ONE client + per ``server_id``; further roots — typically sibling git worktrees — + are attached to the running process via + ``workspace/didChangeWorkspaceFolders`` instead of a new spawn. - A **broken-set** records ``(server_id, workspace_root)`` pairs that failed to spawn or initialize. These are never retried for the @@ -448,9 +451,10 @@ class LSPService: # cancelled future never reached the broken-set add inside # ``_get_or_spawn`` so the client may still be hanging in # ``_clients`` with a half-initialized state. + ckey = _client_key(srv, per_server_root) with self._state_lock: - client = self._clients.pop(key, None) - self._last_used.pop(key, None) + client = self._clients.pop(ckey, None) + self._last_used.pop(ckey, None) if client is not None: try: # Fire-and-forget shutdown — give it a second to cleanup, @@ -527,7 +531,7 @@ class LSPService: if not (ws and gated and srv): return [] with self._state_lock: - client = self._clients.get((srv.server_id, ws)) + client = self._clients.get(_client_key(srv, ws)) if client is None: return [] return list(client.diagnostics_for(file_path, fresh_only=True)) @@ -550,21 +554,26 @@ class LSPService: ) return None # exclude marker hit, server gated off - key = (srv.server_id, per_server_root) - if key in self._broken: + if (srv.server_id, per_server_root) in self._broken: return None + key = _client_key(srv, per_server_root) with self._state_lock: client = self._clients.get(key) if client is not None and client.is_running: self._last_used[key] = time.time() eventlog.log_active(srv.server_id, per_server_root) - return client + else: + client = None spawning = self._spawning.get(key) - if spawning is not None: + if client is None and spawning is not None: try: - return await spawning + client = await spawning except Exception: # noqa: BLE001 return None + if client is not None: + if srv.multi_root: + await client.add_workspace_folder(per_server_root) + return client # Begin spawn loop = asyncio.get_running_loop() @@ -586,7 +595,7 @@ class LSPService: # or install attempt failed). Surface this once via # the structured logger so the user can act on it. eventlog.log_server_unavailable(srv.server_id, srv.server_id) - self._broken.add(key) + self._broken.add((srv.server_id, per_server_root)) spawn_future.set_result(None) return None client = LSPClient( @@ -602,7 +611,7 @@ class LSPService: await client.start() except Exception as e: # noqa: BLE001 eventlog.log_spawn_failed(srv.server_id, per_server_root, e) - self._broken.add(key) + self._broken.add((srv.server_id, per_server_root)) spawn_future.set_result(None) return None with self._state_lock: @@ -626,10 +635,10 @@ class LSPService: the key. All writers and the reaper run on the background loop thread; the lock keeps this consistent with the reader anyway. """ - key = (client.server_id, client.workspace_root) with self._state_lock: - if key in self._clients: - self._last_used[key] = time.time() + for key, c in self._clients.items(): + if c is client: + self._last_used[key] = time.time() async def _idle_reaper_loop(self) -> None: interval = min(60.0, self._idle_timeout) @@ -691,12 +700,13 @@ class LSPService: with self._state_lock: clients = [ { - "server_id": k[0], - "workspace_root": k[1], + "server_id": c.server_id, + "workspace_root": c.workspace_root, + "workspace_folders": list(c.workspace_folders), "state": c.state, "running": c.is_running, } - for k, c in self._clients.items() + for c in self._clients.values() ] broken = list(self._broken) return { @@ -710,6 +720,15 @@ class LSPService: } +def _client_key(srv, root: str) -> Tuple[str, str]: + """Cache key for the client serving ``root``. + + Multi-root servers share one process per ``server_id``; everything + else is keyed per resolved project root. + """ + return (srv.server_id, "" if srv.multi_root else root) + + def _diag_key(d: Dict[str, Any]) -> str: """Content equality key used for cross-edit delta filtering. diff --git a/agent/lsp/servers.py b/agent/lsp/servers.py index 4612d18cf9..d95c716cc7 100644 --- a/agent/lsp/servers.py +++ b/agent/lsp/servers.py @@ -146,6 +146,10 @@ class ServerDef: build_spawn: Callable[[str, "ServerContext"], Optional[SpawnSpec]] seed_first_push: bool = False description: str = "" + # Server handles ``workspace/didChangeWorkspaceFolders``: one process + # serves every project root (git worktrees included) as extra + # workspaceFolders instead of one process per root. + multi_root: bool = False def matches(self, file_path: str) -> bool: """Return True iff this server handles ``file_path``.""" @@ -1000,6 +1004,7 @@ SERVERS: List[ServerDef] = [ extensions=(".py", ".pyi"), resolve_root=_root_python, build_spawn=_spawn_pyright, + multi_root=True, description="Python — Microsoft pyright", ), ServerDef( diff --git a/agent/lsp/workspace.py b/agent/lsp/workspace.py index 4f5beacfbb..b25f02f7e0 100644 --- a/agent/lsp/workspace.py +++ b/agent/lsp/workspace.py @@ -153,13 +153,17 @@ def nearest_root( return None except OSError: continue - # Then check markers. - for marker in markers_list: - try: - if (cur / marker).exists(): - return str(cur) - except OSError: - continue + # Then check markers. A directory holding __init__.py is a Python + # package, never a project root: hermes_cli/setup.py matched the + # python marker list and gave every package dir its own pyright, + # doubling servers per worktree (Sep 2026). + if not (cur / "__init__.py").exists(): + for marker in markers_list: + try: + if (cur / marker).exists(): + return str(cur) + except OSError: + continue # Stop conditions. if ceiling_path is not None and cur == ceiling_path: return None diff --git a/agent/model_metadata.py b/agent/model_metadata.py index bdd217f3f0..aa85f2d99d 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -359,6 +359,58 @@ def _save_model_metadata_disk_cache(data: Dict[str, Dict[str, Any]]) -> None: except Exception as e: logger.debug("Failed to save OpenRouter model metadata disk cache: %s", e) +def _get_endpoint_metadata_cache_path() -> Path: + """On-disk memo of remote ``/models`` probes (see ``_endpoint_disk_cache_get``).""" + from hermes_constants import get_hermes_home + return get_hermes_home() / "cache" / "endpoint_model_metadata.json" + + +def _endpoint_disk_cache_get(normalized: str) -> Optional[Dict[str, Dict[str, Any]]]: + """Return a still-fresh (``_ENDPOINT_MODEL_CACHE_TTL``) disk memo for one endpoint. + + The in-memory endpoint cache only helps within a process. One-shot runs + (``hermes -q``, cron, every Bot Mode DM hop) start cold and re-probed the + live ``/models`` endpoint on every launch — 0.3–0.6s of pure network per + process on Nous, whose persistent context cache is bypassed by design so + the portal stays authoritative. This memo keeps that authority (same TTL + as the in-memory cache, so reconciliation still lands within 5 minutes) + while sharing the answer across processes. Local endpoints are never + memoized: their loaded context is transient (LM Studio reloads). + """ + try: + with _get_endpoint_metadata_cache_path().open("r", encoding="utf-8") as f: + data = json.load(f) + entry = data.get(normalized) if isinstance(data, dict) else None + if not isinstance(entry, dict): + return None + if (time.time() - float(entry.get("at", 0))) >= _ENDPOINT_MODEL_CACHE_TTL: + return None + models = entry.get("models") + return models if isinstance(models, dict) else None + except Exception: + return None + + +def _endpoint_disk_cache_put(normalized: str, cache: Dict[str, Dict[str, Any]]) -> None: + """Memoize a successful remote ``/models`` probe; expired siblings are dropped.""" + try: + path = _get_endpoint_metadata_cache_path() + data: Dict[str, Any] = {} + if path.exists(): + with path.open("r", encoding="utf-8") as f: + loaded = json.load(f) + if isinstance(loaded, dict): + now = time.time() + data = { + k: v for k, v in loaded.items() + if isinstance(v, dict) and (now - float(v.get("at", 0))) < _ENDPOINT_MODEL_CACHE_TTL + } + data[normalized] = {"at": time.time(), "models": cache} + atomic_json_write(path, data, indent=0, separators=(",", ":")) + except Exception as e: + logger.debug("Failed to save endpoint model metadata disk cache: %s", e) + + # Descending tiers for context length probing when the model is unknown. # We start at 256K (covers GPT-5.x, many current large-context models) and # step down on context-length errors until one works. Tier[0] is also the @@ -482,6 +534,13 @@ DEFAULT_CONTEXT_LENGTHS = { "deepseek": 128000, # Meta "llama": 131072, + # Muse Spark family (1.1/1.2/1.3 + contributor tiers) ships with a 1M + # context window: 1,048,576 per OpenRouter live metadata (verified + # 2026-09-02). The family key covers every checkpoint; live endpoint / + # models.dev metadata still wins when available. Substring match also + # covers -contributor and provider-prefixed ids (meta/...). + "muse-spark-1.3": 1_048_576, + "muse-spark": 1_048_576, # Thinking Machines — Inkling family ships with a 1M context window # (max output 256K). Verified against OpenRouter live metadata # (context_length 1,048,576 for inkling, inkling-small, and the @@ -607,6 +666,12 @@ DEFAULT_CONTEXT_LENGTHS = { "mimo-v2-omni": 262144, "mimo-v2-flash": 262144, "zai-org/GLM-5": 202752, + # Meta Muse Spark — 1M context (1,048,576; verified via models.dev + # opencode/opencode-go/meta and api.commandcode.ai /models). Covers every + # variant: 1.1, 1.2, 1.3, -contributor, -contributor-free. Kept to the + # "muse-spark" prefix on purpose: a bare "muse" key would also match + # muse-image / muse-voice. + "muse-spark": 1_048_576, } # xAI Grok models that ACCEPT the `reasoning.effort` parameter on @@ -1351,6 +1416,12 @@ def fetch_endpoint_model_metadata( cached_at = _endpoint_model_metadata_cache_time.get(normalized, 0) if cached is not None and (time.time() - cached_at) < _ENDPOINT_MODEL_CACHE_TTL: return cached + if not is_local_endpoint(normalized): + memo = _endpoint_disk_cache_get(normalized) + if memo is not None: + _endpoint_model_metadata_cache[normalized] = memo + _endpoint_model_metadata_cache_time[normalized] = time.time() + return memo # Blackholed endpoint: every candidate below would spend its full 5s # connect budget. Returned empty rather than cached, so the endpoint is @@ -1525,6 +1596,8 @@ def fetch_endpoint_model_metadata( _endpoint_model_metadata_cache[normalized] = cache _endpoint_model_metadata_cache_time[normalized] = time.time() + if cache and not is_local_endpoint(normalized): + _endpoint_disk_cache_put(normalized, cache) return cache except Exception as exc: last_error = exc @@ -2233,6 +2306,8 @@ def _model_name_suggests_minimax_m3(model: str) -> bool: # catch-all can never be listed here. _PRE_CATALOG_STALE_KEYS = frozenset({ "minimax-m3", # 1M; older builds persisted the "minimax" catch-all (204,800) + "muse-spark-1.3", # 1M; builds before this entry fell through to the 256K fallback + "muse-spark", # 1M; 1.1/1.2 builds fell through to the 256K fallback "grok-4.3", # 1M; pre-2026-05-15 builds persisted the "grok-4" catch-all (256,000) "grok-4.6", # 500K; pre-catalog builds persisted the "grok-4" catch-all (256,000) "grok-4-fast", # 2M; pre-2026-04-10 builds fell through to the 256K probe fallback @@ -3389,9 +3464,11 @@ def get_model_context_length( if base_url and codex_source == "live": save_context_length(model, base_url, codex_ctx) return codex_ctx - if effective_provider == "gmi" and base_url: - # GMI exposes authoritative context_length via /models, but it is not - # in models.dev yet. Preserve that higher-fidelity endpoint lookup. + if effective_provider in {"gmi", "commandcode", "commandcode-anthropic"} and base_url: + # GMI and CommandCode (api.commandcode.ai) expose authoritative + # context_length via /models (e.g. muse-spark 1M) but are not in + # models.dev, and as known providers they skip step 2's + # custom-endpoint probe — without this they fell to the 256K fallback. ctx = _resolve_endpoint_context_length(model, base_url, api_key=api_key) if ctx is not None: return ctx @@ -3581,13 +3658,29 @@ def estimate_tokens_rough(text: str) -> int: if text.isascii(): # O(1) fast path — ASCII text cannot contain token-dense CJK chars. return (len(text) + 3) // 4 - dense = len(text) - len(_CJK_DENSE_RE.sub("", text)) + stripped = _CJK_DENSE_RE.sub("", text) + dense = len(text) - len(stripped) if not dense: - # Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): keep the - # classic ~4 chars/token rule. - return (len(text) + 3) // 4 - sparse = len(text) - dense - return dense + ((sparse + 3) // 4) + # Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): count UTF-8 + # BYTES at ~4/token instead of characters. The byte width is the + # corrective: Cyrillic/Greek/Arabic are 2 bytes per char, so they + # count as ~chars/2 — matching their real BPE cost (~2-3 chars per + # token) where chars/4 under-counted them ~2x and let sessions ride + # the provider's context ceiling below the compaction threshold. + # ASCII spans inside mixed text still count at 1 byte each. + # + # Calibrated against cl100k/o200k/Qwen2.5 (estimate / mean real): + # Russian 0.67->1.24, Ukrainian 0.55->1.03, Arabic 0.53->0.96, + # Hindi 0.34->0.90, Greek 0.37->0.68, Polish 0.63->0.69; accented + # Latin barely moves (French 1.02->1.03, German 0.99->1.02, + # Spanish 1.04->1.07) because only the accented chars widen. + # Pure-ASCII prose already over-counts at ~1.4 on the same rule. + # errors="replace": lone surrogates (routine in tool output; see + # message_sanitization) must not turn an estimate into a raise. + return (len(text.encode("utf-8", "replace")) + 3) // 4 + # Mixed CJK + other: dense chars stay ~1 token each; the sparse + # remainder is byte-counted for the same corrective. + return dense + ((len(stripped.encode("utf-8", "replace")) + 3) // 4) def estimate_messages_tokens_rough( diff --git a/agent/models_dev.py b/agent/models_dev.py index 79046d5ae2..85200d688a 100644 --- a/agent/models_dev.py +++ b/agent/models_dev.py @@ -188,6 +188,11 @@ PROVIDER_TO_MODELS_DEV: Dict[str, str] = { "ai-gateway": "vercel", "opencode-zen": "opencode", "opencode-go": "opencode-go", + # opencode-free is Zen-hosted (the free tier lives on the Zen relay, see + # hermes_cli/models.py), and models.dev's "opencode" catalog is the one + # that lists its *-contributor-free SKUs — without this alias every + # opencode-free lookup missed models.dev entirely. + "opencode-free": "opencode", "kilocode": "kilo", "fireworks": "fireworks-ai", "huggingface": "huggingface", diff --git a/agent/opencode_affinity.py b/agent/opencode_affinity.py new file mode 100644 index 0000000000..4f0cc5522e --- /dev/null +++ b/agent/opencode_affinity.py @@ -0,0 +1,84 @@ +"""``x-opencode-session`` — OpenCode relay session-affinity header. + +OpenCode (opencode.ai Zen/Go/free relay) pins requests that share an +``x-opencode-session`` value to the same upstream backend, which is what +keeps its prompt cache warm across the turns of one conversation. The value +only has to be opaque and consistent per conversation, so it is derived the +same way as the other conversation-affinity hints Hermes already sends +(OpenRouter's sticky ``session_id``, xAI's ``x-grok-conv-id``): the +host-declared routing scope first, then the ambient conversation root, then +the physical session id — normalized through ``_cache_scope_from_session_id`` +so cron fires of one job share a scope. + +Every OpenCode request — main turn on any transport, auxiliary calls +(compression, titles, vision, MoA) — goes through :func:`opencode_session_headers` +so the header cannot drift per code path. +""" + +from __future__ import annotations + +from typing import Any, Optional + +OPENCODE_SESSION_HEADER = "x-opencode-session" + + +def is_opencode_target(provider: Optional[str], base_url: Optional[str]) -> bool: + """True when *provider* or *base_url* addresses the OpenCode relay. + + Matches the built-in opencode-zen/go/free providers, custom + ``opencode--*`` providers, and any base_url hosted on opencode.ai. + """ + try: + from hermes_cli.models import opencode_provider_family + + if opencode_provider_family(provider) is not None: + return True + except Exception: + pass + try: + from agent.anthropic_endpoints import _is_opencode_endpoint + + return _is_opencode_endpoint(str(base_url or "")) + except Exception: + return False + + +def opencode_session_headers( + provider: Optional[str], + base_url: Optional[str], + session_id: Optional[str] = None, +) -> dict[str, str]: + """Return ``{"x-opencode-session": }`` for OpenCode targets, else ``{}``.""" + if not is_opencode_target(provider, base_url): + return {} + try: + from agent.portal_tags import get_affinity_scope, get_conversation_context + from agent.transports.codex import _cache_scope_from_session_id + + key = _cache_scope_from_session_id( + get_affinity_scope() or get_conversation_context() or session_id + ) + except Exception: + key = str(session_id or "") + return {OPENCODE_SESSION_HEADER: key} if key else {} + + +def merge_opencode_session_headers( + kwargs: dict[str, Any], + provider: Optional[str], + base_url: Optional[str], + session_id: Optional[str] = None, +) -> dict[str, Any]: + """Merge the affinity header into ``kwargs["extra_headers"]`` (in place). + + Existing per-request headers win, so a caller-pinned value is preserved. + Non-OpenCode targets are left untouched. + """ + headers = opencode_session_headers(provider, base_url, session_id) + if headers: + existing = kwargs.get("extra_headers") + merged = dict(existing) if isinstance(existing, dict) else {} + for key, value in headers.items(): + merged.setdefault(key, value) + kwargs["extra_headers"] = merged + return kwargs diff --git a/agent/outbound_webhooks.py b/agent/outbound_webhooks.py index f437b809f3..cd600832cb 100644 --- a/agent/outbound_webhooks.py +++ b/agent/outbound_webhooks.py @@ -98,8 +98,12 @@ _TOOL_SCOPED_EVENTS = {"pre_tool_call", "post_tool_call"} # kwargs promoted to top-level payload keys (mirrors shell hooks wire). _TOP_LEVEL_PAYLOAD_KEYS = {"tool_name", "args", "session_id", "parent_session_id"} -# (event, url) pairs already wired to the plugin manager in this process. -_registered: Set[Tuple[str, str]] = set() +# (home, event, url) triples already wired to the plugin manager in this +# process. Home is part of the key so a multiplexed gateway's secondary +# profiles — each with their own plugin manager (see +# hermes_cli.plugins.get_plugin_manager) — can register identical webhook +# targets without the first profile's registration shadowing the rest. +_registered: Set[Tuple[str, str, str]] = set() _registered_lock = threading.Lock() _delivery_queue: "queue.Queue[Optional[Dict[str, Any]]]" = queue.Queue( @@ -180,15 +184,17 @@ def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: return [] from hermes_cli.plugins import get_plugin_manager + from hermes_constants import get_hermes_home manager = get_plugin_manager() + home_key = str(get_hermes_home().expanduser().resolve()) registered: List[WebhookTarget] = [] with _registered_lock: for target in targets: wired_any = False for event in target.events: - key = (event, target.url) + key = (home_key, event, target.url) if key in _registered: continue manager._hooks.setdefault(event, []).append( @@ -231,6 +237,29 @@ def flush(timeout: float = 5.0) -> bool: return _delivery_queue.unfinished_tasks == 0 +def re_register_config_hooks() -> None: + """Re-register outbound webhooks from config after a plugin force-reload. + + Mirrors ``agent.shell_hooks.re_register_config_hooks``: config-owned + outbound-webhook callbacks live in the same ``_hooks`` dict that + ``PluginManager.discover_and_load(force=True)`` clears via ``unload()``, + so without this the force-reloaded profile's outbound webhooks go + silently inert (#92682 review). Only the current home's idempotence + keys are cleared so a force-reload in one profile cannot invalidate + another profile's still-live registration. + """ + from hermes_cli.config import load_config + from hermes_constants import get_hermes_home + + home_key = str(get_hermes_home().expanduser().resolve()) + with _registered_lock: + _registered.difference_update( + {key for key in _registered if key[0] == home_key} + ) + + register_from_config(load_config()) + + def reset_for_tests() -> None: """Clear the idempotence set and drain the queue. Test-only helper.""" with _registered_lock: @@ -416,8 +445,13 @@ def _serialize_payload( cwd = str(Path.cwd()) except OSError: cwd = "" + # Resolved at fire time from the bound home so a multiplexed gateway's + # receivers can tell which profile emitted the event (#92674). + from hermes_cli.profiles import get_active_profile_name + payload = { "hook_event_name": event, + "profile": get_active_profile_name(), "tool_name": kwargs.get("tool_name"), "tool_input": kwargs.get("args") if isinstance(kwargs.get("args"), dict) else None, "session_id": kwargs.get("session_id") or kwargs.get("parent_session_id") or "", diff --git a/agent/periodic_scheduler.py b/agent/periodic_scheduler.py new file mode 100644 index 0000000000..a1ba07d666 --- /dev/null +++ b/agent/periodic_scheduler.py @@ -0,0 +1,118 @@ +"""One process-wide timer thread for periodic maintenance callbacks. + +Replaces the per-child ``while not stop.wait(interval): body()`` daemon +threads (delegate heartbeat, durable turn-lease refresher, turn-liveness +watchdog). With ~130 in-process subagents those added 2-3 sleeping OS +threads per child; this module runs every periodic body on ONE daemon +thread ordered by a heap of due times. + +Semantics match the loop they replace: the first call happens ``interval`` +seconds after :func:`schedule`, and each following call ``interval`` seconds +after the previous body *returned* (drift-free wrt. body duration was never +a property of the old loops either). A body that returns ``False`` stops +itself; a body that raises is logged at debug and rescheduled — one bad +callback must never kill the shared thread. +""" + +from __future__ import annotations + +import heapq +import itertools +import logging +import threading +import time +from typing import Callable, Optional + +logger = logging.getLogger(__name__) + +_THREAD_NAME = "hermes-periodic-scheduler" + + +class ScheduledHandle: + """Cancel token for one scheduled periodic callback.""" + + __slots__ = ("_fn", "_interval", "_cancelled", "_scheduler") + + def __init__(self, scheduler: "PeriodicScheduler", fn: Callable[[], object], interval: float): + self._scheduler = scheduler + self._fn = fn + self._interval = interval + self._cancelled = False + + @property + def cancelled(self) -> bool: + return self._cancelled + + def cancel(self, wait: Optional[float] = None) -> None: + """Stop future runs. ``wait`` (seconds) additionally blocks until an + in-flight run of this callback finishes — the analogue of + ``thread.join(timeout=wait)`` on the old per-child thread.""" + self._scheduler._cancel(self, wait) + + +class PeriodicScheduler: + def __init__(self) -> None: + self._cond = threading.Condition() + self._heap: list = [] # (due, seq, handle) + self._seq = itertools.count() + self._thread: Optional[threading.Thread] = None + self._running: Optional[ScheduledHandle] = None + + def schedule(self, fn: Callable[[], object], interval: float) -> ScheduledHandle: + handle = ScheduledHandle(self, fn, float(interval)) + with self._cond: + heapq.heappush(self._heap, (time.monotonic() + handle._interval, next(self._seq), handle)) + if self._thread is None or not self._thread.is_alive(): + self._thread = threading.Thread(target=self._run, name=_THREAD_NAME, daemon=True) + self._thread.start() + self._cond.notify() + return handle + + def _cancel(self, handle: ScheduledHandle, wait: Optional[float]) -> None: + with self._cond: + handle._cancelled = True + self._cond.notify() + if wait and self._running is handle and threading.current_thread() is not self._thread: + self._cond.wait_for(lambda: self._running is not handle, timeout=wait) + + def _run(self) -> None: + while True: + with self._cond: + while True: + if not self._heap: + self._cond.wait() + continue + due, _, handle = self._heap[0] + if handle._cancelled: + heapq.heappop(self._heap) + continue + delay = due - time.monotonic() + if delay > 0: + self._cond.wait(delay) + continue + heapq.heappop(self._heap) + self._running = handle + break + stop = False + try: + stop = handle._fn() is False + except Exception: + logger.debug("periodic callback %r raised", handle._fn, exc_info=True) + with self._cond: + self._running = None + if stop: + handle._cancelled = True + elif not handle._cancelled: + heapq.heappush( + self._heap, + (time.monotonic() + handle._interval, next(self._seq), handle), + ) + self._cond.notify_all() + + +_DEFAULT = PeriodicScheduler() + + +def schedule(fn: Callable[[], object], interval: float) -> ScheduledHandle: + """Run ``fn()`` every ``interval`` seconds on the shared scheduler thread.""" + return _DEFAULT.schedule(fn, interval) diff --git a/agent/process_bootstrap.py b/agent/process_bootstrap.py index 7760d124e7..68b12aae99 100644 --- a/agent/process_bootstrap.py +++ b/agent/process_bootstrap.py @@ -31,6 +31,7 @@ import os import selectors import socket import sys +import threading import time import urllib.request from typing import Any, Optional @@ -43,6 +44,19 @@ from utils import base_url_hostname, normalize_proxy_url _OPENAI_CLS_CACHE = None _HAPPY_EYEBALLS_DELAY_SECONDS = 0.25 +# Process-wide pool of sync ``httpx.HTTPTransport`` objects shared by every +# keepalive client with the same (verify, proxy, happy-eyeballs) identity. +# Each delegated child AIAgent used to get its own transport = its own TLS +# pool, so a fan-out of N children held N separate socket sets to the same +# provider. Bounded: past the cap, callers get a private transport again. +_SHARED_TRANSPORTS: dict[tuple, Any] = {} +_SHARED_TRANSPORTS_LOCK = threading.Lock() +_SHARED_TRANSPORTS_MAX = 32 +# ``request.extensions`` key stamped by ``_SharedTransport.handle_request``; +# the socket-abort walker in agent_runtime_helpers uses it to find only the +# owning client's in-flight connections on a shared pool. +HERMES_TRANSPORT_OWNER_EXT = "hermes_transport_owner" + def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]: """Interleave resolved address families while preserving resolver order.""" @@ -418,6 +432,93 @@ def _get_proxy_for_base_url(base_url: Optional[str]) -> Optional[str]: return proxy +def _shared_transport_cls(): + """Lazily define the per-client transport view (httpx import is deferred).""" + global _SharedTransport + if _SharedTransport is not None: + return _SharedTransport + import httpx + + class _SharedTransportImpl(httpx.BaseTransport): + """Per-client view of a process-shared ``httpx.HTTPTransport``. + + ``httpx.Client.close()`` closes every mounted transport. Each OpenAI + client still owns its own ``httpx.Client`` (the #10933 contract: + closing one client must never poison the next), so the object we + mount must absorb that close while the underlying connection pool + keeps serving every other client. ``handle_request`` stamps the + owning view into ``request.extensions`` so socket-abort sweeps can + target only this client's in-flight connections on the shared pool. + """ + + __slots__ = ("_inner", "_closed") + + def __init__(self, inner: Any) -> None: + self._inner = inner + self._closed = False + + # httpx-private ``_pool`` is what our socket walkers and the + # happy-eyeballs / ssl-verify tests introspect: expose the shared one. + @property + def _pool(self) -> Any: + return getattr(self._inner, "_pool", None) + + def handle_request(self, request: Any) -> Any: + if self._closed: + raise RuntimeError("Cannot send a request, as the client has been closed.") + request.extensions[HERMES_TRANSPORT_OWNER_EXT] = id(self) + return self._inner.handle_request(request) + + def close(self) -> None: + # Deliberately does NOT close ``_inner``: it is shared. Idle + # connections are reaped by ``keepalive_expiry``; the pool lives + # for the process (see ``close_shared_transports``). + self._closed = True + + _SharedTransportImpl.__name__ = _SharedTransportImpl.__qualname__ = "_SharedTransport" + _SharedTransport = _SharedTransportImpl + return _SharedTransport + + +_SharedTransport: Any = None + + +def _shared_transport_key(base_url: str, verify: Any, proxy: Optional[str]) -> tuple: + """Identity under which sync direct transports are pooled process-wide.""" + if verify is True or verify is False: + verify_key: Any = verify + elif isinstance(verify, str): + verify_key = ("path", verify) + else: + # An ssl.SSLContext (or custom object): share only by object identity, + # which is what a caller passing the same context twice expects. + verify_key = ("id", id(verify)) + return (verify_key, proxy, _uses_codex_cloud_transport(base_url)) + + +def _get_shared_transport(key: tuple, build) -> Any: + with _SHARED_TRANSPORTS_LOCK: + transport = _SHARED_TRANSPORTS.get(key) + if transport is None: + transport = build() + if len(_SHARED_TRANSPORTS) < _SHARED_TRANSPORTS_MAX: + _SHARED_TRANSPORTS[key] = transport + return transport + + +def close_shared_transports() -> int: + """Really close every process-shared transport (test teardown / atexit).""" + with _SHARED_TRANSPORTS_LOCK: + transports = list(_SHARED_TRANSPORTS.values()) + _SHARED_TRANSPORTS.clear() + for transport in transports: + try: + transport.close() + except Exception: + pass + return len(transports) + + def build_keepalive_http_client( base_url: str = "", *, @@ -444,6 +545,14 @@ def build_keepalive_http_client( ``ssl_ca_cert`` / ``ssl_verify`` settings the main client uses. It is passed on the client AND on the plain no-proxy mounts (a mounted transport owns the SSL context for its scheme). + + Every call returns a NEW ``httpx.Client`` (per-client close semantics are + what #10933 pins), but sync clients with the same + (verify, proxy, happy-eyeballs) identity mount the SAME underlying + ``HTTPTransport`` through a :class:`_SharedTransport` view, so N delegated + children share one connection pool + SSL context instead of N. Async + clients are never shared: an httpcore async pool is bound to the event + loop that first used it. """ try: import httpx @@ -462,16 +571,47 @@ def build_keepalive_http_client( client_cls = httpx.AsyncClient if async_mode else httpx.Client mounts = {} if proxy is None: - http_transport = transport_cls(verify=verify) - https_transport = transport_cls(verify=verify) - # Async transports need no explicit racing: httpcore's anyio - # backend already implements RFC 8305 natively - # (``anyio.connect_tcp(happy_eyeballs_delay=0.25)``), covered by - # tests/agent/test_codex_happy_eyeballs.py. - if not async_mode and _uses_codex_cloud_transport(base_url): - _enable_happy_eyeballs(http_transport) - _enable_happy_eyeballs(https_transport) - mounts = {"http://": http_transport, "https://": https_transport} + happy_eyeballs = not async_mode and _uses_codex_cloud_transport(base_url) + # One pool now serves every agent in the process, so its ceiling + # must cover a whole fan-out of concurrently streaming children, + # not one client. (Note: previously the mounts silently ran on + # httpx defaults — keepalive_expiry=5s — since Client-level + # ``limits`` only reach the default transport.) + direct_limits = limits if async_mode else httpx.Limits( + max_keepalive_connections=50, + max_connections=1000, + keepalive_expiry=20.0, + ) + + def _build_direct(): + transport = transport_cls(verify=verify, limits=direct_limits) + # Async transports need no explicit racing: httpcore's anyio + # backend already implements RFC 8305 natively + # (``anyio.connect_tcp(happy_eyeballs_delay=0.25)``), covered + # by tests/agent/test_codex_happy_eyeballs.py. + if happy_eyeballs: + _enable_happy_eyeballs(transport) + return transport + + if async_mode: + mounts = {"http://": _build_direct(), "https://": _build_direct()} + else: + key = _shared_transport_key(base_url, verify, proxy) + view_cls = _shared_transport_cls() + mounts = { + f"{scheme}://": view_cls( + _get_shared_transport((scheme, *key), _build_direct) + ) + for scheme in ("http", "https") + } + # Without this httpx builds a third, never-used direct + # transport (and pool + SSL context) per client. + return client_cls( + limits=limits, + timeout=timeout, + transport=mounts["https://"], + mounts=mounts, + ) return client_cls( limits=limits, timeout=timeout, @@ -506,5 +646,6 @@ __all__ = [ "_get_proxy_from_env", "_get_proxy_for_base_url", "build_keepalive_http_client", + "close_shared_transports", "enable_happy_eyeballs_on_client", ] diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index b62c6782cf..781a3caa35 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -7,6 +7,7 @@ assemble pieces, then combines them with memory and ephemeral prompts. import json import logging import os +import queue import sys import threading import contextvars @@ -58,6 +59,58 @@ logger = logging.getLogger(__name__) from tools.threat_patterns import scan_for_threats as _scan_for_threats +# Default read deadline for context files (SOUL.md, AGENTS.md, .cursorrules, +# ...); overridable via ``context_file_read_timeout`` in config.yaml. +# Intentionally short: network-backed filesystems (iCloud Drive, OneDrive, +# NFS) can fault-in an evicted file and block a cold read indefinitely, which +# stalls system-prompt assembly before the first turn. +_CONTEXT_FILE_READ_TIMEOUT_SECS = 5.0 + + +def _get_context_file_read_timeout() -> float: + """``context_file_read_timeout`` from config.yaml, else the 5s default.""" + try: + from hermes_cli.config import load_config_readonly + + val = load_config_readonly().get("context_file_read_timeout") + if isinstance(val, (int, float)) and val > 0: + return float(val) + except Exception as e: + logger.debug("Could not read context_file_read_timeout from config: %s", e) + return _CONTEXT_FILE_READ_TIMEOUT_SECS + + +def _read_text_with_timeout( + path: Path, timeout: Optional[float] = None, encoding: str = "utf-8-sig" +) -> Optional[str]: + """``path.read_text()`` on a daemon thread so a slow file can't stall startup. + + Returns the text, or ``None`` after *timeout* seconds (logged at WARNING; + the orphaned reader thread finishes on its own). Read errors propagate to + the caller exactly as a direct ``read_text`` would, so existing + ``try/except`` handling at each site is unchanged. + """ + if timeout is None: + timeout = _get_context_file_read_timeout() + result: "queue.Queue[tuple[bool, object]]" = queue.Queue(maxsize=1) + + def _reader() -> None: + try: + result.put((True, path.read_text(encoding=encoding))) + except Exception as exc: # re-raised on the caller thread + result.put((False, exc)) + + threading.Thread(target=_reader, daemon=True, name=f"context-read:{path.name}").start() + try: + ok, value = result.get(timeout=timeout) + except queue.Empty: + logger.warning("Context file %s read timed out after %.1fs; skipping", path, timeout) + return None + if ok: + return value # type: ignore[return-value] + raise value # type: ignore[misc] + + def _scan_context_content(content: str, filename: str) -> str: """Scan context file content for injection. Returns sanitized content. @@ -230,14 +283,20 @@ def build_memory_guidance(memory_enabled: bool = True, profile_enabled: bool = T "disabled, so never target='memory'. " ) return frame + ( - "Save proactively — storage has a hard character budget, and when " - "it fills, replace or consolidate stale entries in the same batch " + "Skills come first: when you learn something while doing a task — a " + "procedure, a pitfall, and the user's preferences and corrections " + "for that kind of work — record it in the skill you used or built " + "for the task (skill_manage), where it loads only when relevant. " + "Memory is the narrow exception for facts that apply to EVERY " + "session regardless of task (who the user is, environment facts, " + "standing conventions with no task home); it has a hard character " + "budget, so when it fills, replace or consolidate stale entries " "rather than skipping the save. Write entries as declarative facts, " "not instructions to yourself: 'User prefers concise responses' ✓ — " "'Always respond concisely' ✗ (imperative phrasing gets re-read as " "a directive in later sessions and can override the user's current " - "request). Route by longevity: a fact stale within a week belongs " - "in session history; procedures and workflows belong in skills." + "request). A fact stale within a week belongs in session history; " + "procedures and workflows belong in skills." ) @@ -416,7 +475,9 @@ TOOL_USE_ENFORCEMENT_GUIDANCE = ( # Model name substrings that trigger tool-use enforcement guidance. # Add new patterns here when a model family needs explicit steering. -TOOL_USE_ENFORCEMENT_MODELS = ("gpt", "codex", "gemini", "gemma", "grok", "glm", "qwen", "deepseek") +# "muse" = Meta Muse Spark: on defaults it answers in prose with 0 tool calls +# and the turn closes on finish_reason=stop (#96550). +TOOL_USE_ENFORCEMENT_MODELS = ("gpt", "codex", "gemini", "gemma", "grok", "glm", "qwen", "deepseek", "muse") # Model name substrings whose sessions receive OPENAI_MODEL_EXECUTION_GUIDANCE # (execution discipline: tool persistence, mandatory tool use for arithmetic, @@ -428,13 +489,14 @@ TOOL_USE_ENFORCEMENT_MODELS = ("gpt", "codex", "gemini", "gemma", "grok", "glm", # failure modes on those families (financial math in prose, no read-back after # external writes, identifier "repair", completeness claims despite count # mismatches). GLM's tool-calls-as-plain-text stall (#53847) and MiMo (#41874) -# are covered here too. Gemini/Gemma are excluded — they get the more specific +# are covered here too. Muse Spark (#96550) stops after a chat-only turn on +# defaults. Gemini/Gemma are excluded — they get the more specific # GOOGLE_MODEL_OPERATIONAL_GUIDANCE block instead. Claude is excluded because # it does not exhibit these failure modes; users can opt any model in via # config.yaml `agent.execution_guidance: true` or a substring list. EXECUTION_GUIDANCE_MODELS = ( "gpt", "codex", "grok", - "deepseek", "kimi", "qwen", "glm", "minimax", "mimo", "mistral", + "deepseek", "kimi", "qwen", "glm", "minimax", "mimo", "mistral", "muse", ) # Universal "finish the job" guidance — applied to ALL models, not gated @@ -1173,6 +1235,24 @@ _WINDOWS_BASH_SHELL_HINT = ( ) +def _tenv_read(name: str, default: str = "") -> str: + """Scope-aware TERMINAL_* read (tools.terminal_scope.terminal_env). + + The per-turn terminal scope installed by the multiplexing gateway carries + the active profile's terminal settings; a raw os.getenv would read a value + a previous profile's turn pinned into the process env. + + Only an import failure falls back: an active refusal scope must raise — + swapping it for the ambient process value would defeat the fail-closed + boundary. + """ + try: + from tools.terminal_scope import terminal_env + except ImportError: + return os.getenv(name, default) + return terminal_env(name, default) + + def _probe_remote_backend(env_type: str) -> str | None: """Run a tiny introspection command inside the active terminal backend. @@ -1181,7 +1261,7 @@ def _probe_remote_backend(env_type: str) -> str | None: per process. Used only for non-local backends where the agent's tools operate on a different machine than the host Hermes runs on. """ - cwd_hint = os.getenv("TERMINAL_CWD", "") + cwd_hint = _tenv_read("TERMINAL_CWD", "") cache_key = (env_type, cwd_hint) cached = _BACKEND_PROBE_CACHE.get(cache_key) if cached is not None: @@ -1196,6 +1276,7 @@ def _probe_remote_backend(env_type: str) -> str | None: _BACKEND_PROBE_CACHE[cache_key] = "" return None + env = None try: config = _get_env_config() # Build the environment the same way tools/terminal_tool.py does for a @@ -1276,6 +1357,20 @@ def _probe_remote_backend(env_type: str) -> str | None: logger.debug("Backend probe failed: %s", e) _BACKEND_PROBE_CACHE[cache_key] = "" return None + finally: + # The probe only needs a one-shot `uname`; without teardown the + # backend leaves a second idle sandbox (task_id="prompt-backend-probe") + # running for the whole process lifetime next to the agent's own one. + # ssh is left alone: it has no task-scoped sandbox and its cleanup() + # closes a ControlMaster socket (keyed by user@host:port) shared with + # the agent's real environment; ControlPersist expires it anyway. + if env is not None and env_type != "ssh": + try: + from tools.terminal_tool import _cleanup_env + + _cleanup_env(env, force_remove=True) + except Exception: + logger.debug("Backend probe cleanup failed", exc_info=True) # Parse key=value lines back into a tidy summary. parsed: dict[str, str] = {} @@ -1330,7 +1425,7 @@ def build_environment_hints() -> str: hints: list[str] = [] - backend = (os.getenv("TERMINAL_ENV") or "local").strip().lower() + backend = (_tenv_read("TERMINAL_ENV") or "local").strip().lower() is_remote_backend = backend in _REMOTE_TERMINAL_BACKENDS or _plugin_backend_is_remote(backend) if not is_remote_backend: @@ -2217,7 +2312,7 @@ def load_soul_md( if not soul_path.exists(): return None try: - content = soul_path.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(soul_path) or "").strip() if not content: return None content = _scan_context_content(content, "SOUL.md") @@ -2237,7 +2332,7 @@ def _load_hermes_md(cwd_path: Path, context_length: Optional[int] = None) -> str if not hermes_md_path: return "" try: - content = hermes_md_path.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(hermes_md_path) or "").strip() if not content: return "" content = _strip_yaml_frontmatter(content) @@ -2307,7 +2402,7 @@ def _load_agents_md(cwd_path: Path, context_length: Optional[int] = None) -> str if not candidate.exists(): continue try: - content = candidate.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(candidate) or "").strip() except Exception as e: logger.debug("Could not read %s: %s", candidate, e) continue @@ -2348,7 +2443,7 @@ def _load_claude_md(cwd_path: Path, context_length: Optional[int] = None) -> str candidate = cwd_path / name if candidate.exists(): try: - content = candidate.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(candidate) or "").strip() if content: content = _scan_context_content(content, name) result = f"## {name}\n\n{content}" @@ -2367,7 +2462,7 @@ def _load_cursorrules(cwd_path: Path, context_length: Optional[int] = None) -> s cursorrules_file = cwd_path / ".cursorrules" if cursorrules_file.exists(): try: - content = cursorrules_file.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(cursorrules_file) or "").strip() if content: content = _scan_context_content(content, ".cursorrules") cursorrules_content += f"## .cursorrules\n\n{content}\n\n" @@ -2379,7 +2474,7 @@ def _load_cursorrules(cwd_path: Path, context_length: Optional[int] = None) -> s mdc_files = sorted(cursor_rules_dir.glob("*.mdc")) for mdc_file in mdc_files: try: - content = mdc_file.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(mdc_file) or "").strip() if content: content = _scan_context_content(content, f".cursor/rules/{mdc_file.name}") cursorrules_content += f"## .cursor/rules/{mdc_file.name}\n\n{content}\n\n" diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 2d8c37a18f..d9454500f2 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -1119,6 +1119,19 @@ def _provider_request( for key, value in headers.items() if str(key).lower() not in _RELAY_INTERNAL_PROVIDER_HEADERS } + # Relay's managed-call trace header maps to ``extra_headers`` for known SDK + # adapters and custom requests that already use that container. Other + # native transports receive protocol kwargs directly and may reject a new + # SDK-only argument. Preserve non-trace middleware headers as before. + supports_extra_headers = ( + _relay_protocol(metadata) is not None or "extra_headers" in original + ) + if headers and not supports_extra_headers: + headers = { + key: value + for key, value in headers.items() + if str(key).lower() != "traceparent" + } if headers: final["extra_headers"] = { **dict(final.get("extra_headers") or {}), diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py index 3dbbd397a0..405c30e8af 100644 --- a/agent/relay_runtime.py +++ b/agent/relay_runtime.py @@ -343,8 +343,8 @@ class _ProcessRelayPluginConfiguration: if self._activation is None: # Hermes only enters Relay's initialization path after an - # explicit opt-in. Relay currently owns any subsequent ambient - # layering; a future discovery=False API can make this exact. + # explicit opt-in. Relay 0.8 no longer layers repository-local + # configuration onto this explicitly selected payload. _resolve_plugin_awaitable(relay.plugin.initialize(plugin_config)) except Exception as exc: self._activation = None diff --git a/agent/relay_tools.py b/agent/relay_tools.py index 91c0926b7e..8f4aaff3f2 100644 --- a/agent/relay_tools.py +++ b/agent/relay_tools.py @@ -21,6 +21,7 @@ def execute( callback: Callable[[dict[str, Any]], Any], *, session_id: str, + tool_call_id: str | None = None, metadata: dict[str, Any] | None = None, ) -> tuple[Any, dict[str, Any]]: """Run one tool call through Relay and return its final arguments.""" @@ -52,7 +53,7 @@ def execute( raise raw_result["value"] = result raw_result["json"] = _jsonable(result) - return raw_result["json"] + return runtime.relay.ToolExecutionResult(raw_result["json"]) try: managed = _run_awaitable( @@ -64,6 +65,7 @@ def execute( invoke, handle=parent, metadata=_jsonable(metadata or {}), + tool_call_id=tool_call_id or None, ) ) except BaseException as exc: @@ -85,11 +87,12 @@ def execute( return raw_result["value"], observed_args raise - if "value" in raw_result and _json_equal(managed, raw_result["json"]): + managed_result = managed.result + if "value" in raw_result and _json_equal(managed_result, raw_result["json"]): return raw_result["value"], observed_args - if isinstance(managed, str): - return managed, observed_args - return json.dumps(_jsonable(managed), ensure_ascii=False), observed_args + if isinstance(managed_result, str): + return managed_result, observed_args + return json.dumps(_jsonable(managed_result), ensure_ascii=False), observed_args def _jsonable(value: Any) -> Any: diff --git a/agent/runtime_cwd.py b/agent/runtime_cwd.py index 712e38ed13..bcd776e65b 100644 --- a/agent/runtime_cwd.py +++ b/agent/runtime_cwd.py @@ -57,6 +57,31 @@ def _session_cwd_override() -> str: return str(value).strip() +def _terminal_cwd_env() -> str: + """Scope-aware TERMINAL_CWD read (tools.terminal_scope.terminal_env). + + Under gateway multiplexing the per-turn terminal scope carries the active + profile's cwd; the process-global env var may hold another profile's + value. Only an import failure falls back: an active refusal scope must + raise, not silently resolve the launch profile's cwd. + """ + try: + from tools.terminal_scope import terminal_env + except ImportError: + return os.environ.get("TERMINAL_CWD", "") + return terminal_env("TERMINAL_CWD", "") + + +def scope_terminal_cwd() -> str: + """Public wrapper — the scope-aware TERMINAL_CWD value (may be empty). + + Shared by agent_init / skill_utils / code_execution_tool so every cwd + consumer reads through the per-turn terminal scope under gateway + multiplexing instead of the process-global env var. + """ + return _terminal_cwd_env() + + def resolve_agent_cwd() -> Path: override = _session_cwd_override() if override: @@ -64,7 +89,7 @@ def resolve_agent_cwd() -> Path: if p.is_dir(): return p logger.warning("configured working directory does not exist: %s", override) - raw = os.environ.get("TERMINAL_CWD", "").strip() + raw = _terminal_cwd_env().strip() if raw: p = Path(raw).expanduser() if p.is_dir(): @@ -90,7 +115,7 @@ def resolve_context_cwd() -> Path | None: else: return p return None - raw = os.environ.get("TERMINAL_CWD", "").strip() + raw = _terminal_cwd_env().strip() if raw: p = Path(raw).expanduser() if not p.is_dir(): diff --git a/agent/search_policy.py b/agent/search_policy.py new file mode 100644 index 0000000000..2972d5cae8 --- /dev/null +++ b/agent/search_policy.py @@ -0,0 +1,30 @@ +"""Shared directory pruning policy for broad recursive scans. + +These names identify version-control internals, dependency trees, generated +artifacts, caches, and backup copies that are not useful results for broad +agent-facing discovery. Ordinary search callers may still target an explicit +path; broad diagnostic probes should apply this policy to recursive walks. +""" + +from __future__ import annotations + + +# Keep this policy conservative and name-based so it works for local and remote +# shell backends alike. The same set is used by context discovery and search +# probes; adding a directory here protects every broad recursive consumer. +SEARCH_PRUNE_DIR_NAMES = frozenset({ + # Version-control internals. + ".git", ".hg", ".svn", + # Dependency and vendored trees. + "node_modules", "venv", ".venv", "site-packages", "dist-packages", + "vendor", "third_party", + # Generated/build output. + "build", "dist", "target", "out", "coverage", + ".next", ".turbo", ".parcel-cache", ".nuxt", ".svelte-kit", + # Python and package-manager caches. + "__pycache__", ".cache", ".Trash", ".tox", ".nox", ".mypy_cache", + ".pytest_cache", ".ruff_cache", ".npm", ".yarn", ".pnpm-store", + ".gradle", ".m2", ".nuget", + # Backup copies. + "backups", "backup", ".backups", +}) diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index aed3d252ff..010a49a6c9 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -182,13 +182,17 @@ _BLOCKING_EVENTS = frozenset({"pre_tool_call"}) _STDERR_MESSAGE_LIMIT = 400 -# (event, matcher, command) triples that have been wired to the plugin +# (home, event, matcher, command) tuples that have been wired to the plugin # manager in the current process. Matcher is part of the key because # the same script can legitimately register for different matchers under -# the same event (e.g. one entry per tool the user wants to gate). -# Second registration attempts for the exact same triple become no-ops +# the same event (e.g. one entry per tool the user wants to gate). Home is +# part of the key so a multiplexed gateway's secondary profiles — each with +# their own plugin manager (see hermes_cli.plugins.get_plugin_manager) — can +# register identical hook triples without the first profile's registration +# silently shadowing the rest. +# Second registration attempts for the exact same tuple become no-ops # so the CLI and gateway can both call register_from_config() safely. -_registered: Set[Tuple[str, Optional[str], str]] = set() +_registered: Set[Tuple[str, str, Optional[str], str]] = set() _registered_lock = threading.Lock() # Intra-process lock for allowlist read-modify-write on platforms that @@ -289,13 +293,14 @@ def register_from_config( from hermes_cli.plugins import get_plugin_manager manager = get_plugin_manager() + home_key = str(get_hermes_home().expanduser().resolve()) # Idempotence + allowlist read happen under the lock; the TTY # prompt runs outside so other threads aren't parked on a blocking # input(). Mutation re-takes the lock with a defensive idempotence # re-check in case two callers ever race through the prompt. for spec in specs: - key = (spec.event, spec.matcher, spec.command) + key = (home_key, spec.event, spec.matcher, spec.command) with _registered_lock: if key in _registered: continue @@ -349,11 +354,20 @@ def re_register_config_hooks() -> None: are wired again (#60036 / PR #60267; tracking #64178 — salvaged from PR #64188). + Only the idempotence keys for the *current* Hermes home are cleared — + ``discover_and_load(force=True)`` only unloads the manager scoped to + that one home, so clearing every home's keys would make a force-reload + in profile A drop profile B's still-live registration from the ledger + and duplicate it on B's next registration call (#92682 review). + Commands already allowlisted stay allowlisted, so this never re-prompts at a TTY for hooks the user previously approved. """ + home_key = str(get_hermes_home().expanduser().resolve()) with _registered_lock: - _registered.clear() + _registered.difference_update( + {key for key in _registered if key[0] == home_key} + ) from hermes_cli.config import load_config register_from_config(load_config()) diff --git a/agent/skill_commands.py b/agent/skill_commands.py index ea150dcc00..d86f26e501 100644 --- a/agent/skill_commands.py +++ b/agent/skill_commands.py @@ -236,7 +236,7 @@ def _load_skill_payload(skill_identifier: str, task_id: str | None = None) -> tu return None try: - from tools.skills_tool import SKILLS_DIR, skill_view + from tools.skills_tool import _skills_dir, skill_view from agent.skill_utils import normalize_skill_lookup_name normalized = normalize_skill_lookup_name(raw_identifier) @@ -262,7 +262,7 @@ def _load_skill_payload(skill_identifier: str, task_id: str | None = None) -> tu skill_dir = Path(abs_skill_dir) elif skill_path: try: - skill_dir = SKILLS_DIR / Path(skill_path).parent + skill_dir = _skills_dir() / Path(skill_path).parent except Exception: skill_dir = None @@ -317,7 +317,7 @@ def _build_skill_message( session_id: str | None = None, ) -> str: """Format a loaded skill into a user/system message payload.""" - from tools.skills_tool import SKILLS_DIR + from tools.skills_tool import _skills_dir content = str(loaded_skill.get("content") or "") @@ -388,7 +388,7 @@ def _build_skill_message( if supporting and skill_dir: try: - skill_view_target = str(skill_dir.relative_to(SKILLS_DIR)) + skill_view_target = str(skill_dir.relative_to(_skills_dir())) except ValueError: # Skill is from an external dir — use the skill name instead skill_view_target = skill_dir.name @@ -443,7 +443,7 @@ def scan_skill_commands() -> Dict[str, Dict[str, Any]]: # each naming the same skill as its own incumbent (#74574). commands: Dict[str, Dict[str, Any]] = {} try: - from tools.skills_tool import SKILLS_DIR, _parse_frontmatter, skill_matches_platform, skill_matches_environment, _get_disabled_skill_names + from tools.skills_tool import _skills_dir, _parse_frontmatter, skill_matches_platform, skill_matches_environment, _get_disabled_skill_names from agent.skill_utils import ( get_external_skills_dirs, get_project_skills_dirs, @@ -458,8 +458,12 @@ def scan_skill_commands() -> Dict[str, Dict[str, Any]]: # Project dirs iterate through the quarantine chokepoint. project_dirs = list(get_project_skills_dirs()) dirs_to_scan = list(project_dirs) - if SKILLS_DIR.exists(): - dirs_to_scan.append(SKILLS_DIR) + # Resolve at call time: the import-time SKILLS_DIR is frozen to the + # launch home, so a multiplexed profile scope (set_hermes_home_override) + # would still scan the default profile's skills (#67277). + skills_dir = _skills_dir() + if skills_dir.exists(): + dirs_to_scan.append(skills_dir) dirs_to_scan.extend(get_external_skills_dirs()) for scan_dir in dirs_to_scan: diff --git a/agent/skill_utils.py b/agent/skill_utils.py index cafd8fb6b0..ba39b166fa 100644 --- a/agent/skill_utils.py +++ b/agent/skill_utils.py @@ -31,6 +31,7 @@ EXCLUDED_SKILL_DIRS = frozenset( ".github", ".hub", ".archive", + ".curator_backups", ".venv", "venv", "node_modules", @@ -737,7 +738,9 @@ def find_project_root(start: Optional[Path] = None) -> Optional[Path]: """ try: if start is None: - env_cwd = os.environ.get("TERMINAL_CWD") + from agent.runtime_cwd import scope_terminal_cwd + + env_cwd = scope_terminal_cwd() start = Path(env_cwd) if env_cwd else Path.cwd() cur = Path(start).resolve() except OSError: @@ -976,12 +979,15 @@ def normalize_skill_lookup_name(identifier: str) -> str: # Look the primary skills root up on tools.skills_tool at CALL time # (not via get_skills_dir()): callers and tests patch # ``tools.skills_tool.SKILLS_DIR`` and skill_view() itself resolves - # against that module attribute, so normalization must agree with the - # exact root skill_view() will enforce. Import deferred to avoid a - # module cycle (tools.skills_tool imports agent.skill_utils). + # against ``_skills_dir()`` — which honors that patch and otherwise + # follows the live profile-scoped HERMES_HOME (the import-time + # SKILLS_DIR is frozen to the launch home, #67277) — so normalization + # must agree with the exact root skill_view() will enforce. Import + # deferred to avoid a module cycle (tools.skills_tool imports + # agent.skill_utils). try: from tools import skills_tool as _skills_tool - primary_root = Path(_skills_tool.SKILLS_DIR) + primary_root = _skills_tool._skills_dir() except Exception: primary_root = get_skills_dir() diff --git a/agent/ssl_verify.py b/agent/ssl_verify.py index ee6d06a94a..d14f395be2 100644 --- a/agent/ssl_verify.py +++ b/agent/ssl_verify.py @@ -22,6 +22,7 @@ from __future__ import annotations import logging import ssl +import threading from pathlib import Path from typing import Any, Optional @@ -68,6 +69,27 @@ def _coerce_insecure(ssl_verify: Any) -> bool: return False +_CA_CONTEXTS: dict[str, ssl.SSLContext] = {} +_CA_CONTEXTS_LOCK = threading.Lock() + + +def _context_for_ca_bundle(ca_path: str) -> ssl.SSLContext: + """One ``SSLContext`` per CA bundle path, process-wide. + + ``ssl.create_default_context(cafile=...)`` parses the whole bundle each + call. Every AIAgent (and every delegated child) resolves verify for its + own client, so an env/config CA bundle used to cost one parsed context — + and, because sharing keys on context identity, one private connection + pool — per agent. An ``SSLContext`` is safe to share across connections. + """ + with _CA_CONTEXTS_LOCK: + ctx = _CA_CONTEXTS.get(ca_path) + if ctx is None: + ctx = ssl.create_default_context(cafile=ca_path) + _CA_CONTEXTS[ca_path] = ctx + return ctx + + def resolve_httpx_verify( *, ca_bundle: Optional[str] = None, @@ -95,9 +117,15 @@ def resolve_httpx_verify( ) return False - bundle = (ca_bundle or "").strip() - if bundle: - path = Path(bundle).expanduser() + effective_ca = ( + (ca_bundle or "").strip() + or os.getenv("HERMES_CA_BUNDLE", "").strip() + or os.getenv("SSL_CERT_FILE", "").strip() + or os.getenv("REQUESTS_CA_BUNDLE", "").strip() + or os.getenv("CURL_CA_BUNDLE", "").strip() + ) + if effective_ca: + path = Path(effective_ca).expanduser() if path.is_file(): # An explicit bundle REPLACES the platform store for this client. # inject_into_ssl() rebinds ssl.SSLContext to truststore's class, @@ -118,7 +146,7 @@ def resolve_httpx_verify( return ctx logger.warning( "ssl_ca_cert path does not exist: %s — using the OS trust store instead", - bundle, + effective_ca, ) return True diff --git a/agent/subagent_lifecycle.py b/agent/subagent_lifecycle.py index 319e85a784..f7ba0146c9 100644 --- a/agent/subagent_lifecycle.py +++ b/agent/subagent_lifecycle.py @@ -17,6 +17,7 @@ import math import secrets import threading import time +import weakref from contextlib import contextmanager from concurrent.futures import Future, TimeoutError from typing import Any, Callable, Mapping, Optional @@ -171,8 +172,19 @@ _ACTIVE_PARENT_AGENT: contextvars.ContextVar[Any] = contextvars.ContextVar( @contextmanager def bind_subagent_parent(parent_agent: Any): - """Bind the host-owned parent for the current agent turn.""" - token = _ACTIVE_PARENT_AGENT.set(parent_agent) + """Bind the host-owned parent for the current agent turn. + + Stored as a weakref: every asyncio Handle/Future scheduled from the turn + (LSP reader loops, kernel pipes, ...) snapshots the Context, and those + snapshots outlive the turn. A strong ref there pinned finished delegate + children — each of which binds itself here for its own turn — in the + parent process heap for the life of the background loop. + """ + try: + ref = weakref.ref(parent_agent) + except TypeError: + ref = lambda: parent_agent # noqa: E731 — non-weakrefable test doubles + token = _ACTIVE_PARENT_AGENT.set(ref) try: yield finally: @@ -181,7 +193,8 @@ def bind_subagent_parent(parent_agent: Any): def get_active_subagent_parent() -> Any: """Return the parent bound to this execution context, if any.""" - return _ACTIVE_PARENT_AGENT.get() + ref = _ACTIVE_PARENT_AGENT.get() + return ref() if ref is not None else None class SubagentLifecycleService: diff --git a/agent/subdirectory_hints.py b/agent/subdirectory_hints.py index 24dd6ad4df..fec41517c2 100644 --- a/agent/subdirectory_hints.py +++ b/agent/subdirectory_hints.py @@ -20,7 +20,8 @@ import shlex from pathlib import Path from typing import Dict, Any, Optional, Set -from agent.prompt_builder import _scan_context_content +from agent.prompt_builder import _read_text_with_timeout, _scan_context_content +from agent.search_policy import SEARCH_PRUNE_DIR_NAMES logger = logging.getLogger(__name__) @@ -47,17 +48,9 @@ _COMMAND_TOOLS = {"terminal"} # Prevents scanning all the way to / for deeply nested paths. _MAX_ANCESTOR_WALK = 5 -# Directory names that never contain authoritative project context. -# Backups, vendored deps, VCS internals, and caches routinely hold *copies* of -# AGENTS.md; loading those duplicates real context and inflates the prompt. -_EXCLUDED_DIR_NAMES = frozenset({ - "node_modules", "venv", ".venv", "__pycache__", - ".git", ".hg", ".svn", - ".Trash", ".cache", ".tox", ".mypy_cache", ".pytest_cache", - "site-packages", "dist-packages", - "backups", "backup", ".backups", - "vendor", "third_party", -}) +# Shared with broad recursive search probes so context discovery and search do +# not drift into different dependency/cache/build trees. +_EXCLUDED_DIR_NAMES = SEARCH_PRUNE_DIR_NAMES def _is_ancestor_or_same(a: Path, b: Path) -> bool: @@ -105,7 +98,7 @@ class SubdirectoryHintTracker: try: if not candidate.is_file(): continue - content = candidate.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(candidate) or "").strip() except (OSError, UnicodeDecodeError): continue if content: @@ -285,7 +278,7 @@ class SubdirectoryHintTracker: except OSError: continue try: - content = hint_path.read_text(encoding="utf-8-sig").strip() + content = (_read_text_with_timeout(hint_path) or "").strip() if not content: continue # Skip content we've already injected. The same AGENTS.md is diff --git a/agent/tool_executor.py b/agent/tool_executor.py index 5ee9da8444..fb0265310c 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -771,6 +771,7 @@ def _run_agent_tool_execution_middleware( function_args, _hermes_pipeline, session_id=str(getattr(agent, "session_id", "") or ""), + tool_call_id=tool_call_id or None, metadata={ "task_id": effective_task_id or "", "turn_id": getattr(agent, "_current_turn_id", "") or "", @@ -1151,6 +1152,10 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe parsed_calls = [] for tool_call in tool_calls: function_name = tool_call.function.name + # Legacy tool-name aliases (2026-08 renames) — map BEFORE the + # agent-loop branches (todo_list etc. dispatch above the registry). + from model_tools import _LEGACY_TOOL_ALIASES as _lta + function_name = _lta.get(function_name, function_name) function_args, malformed_args_result = _parse_tool_arguments( tool_call.function.arguments @@ -1192,9 +1197,9 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe _underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args) if not _err and _underlying: if _underlying in _tool_search_scoped_names(agent): - # Probe-validate before unwrapping (ironclaw#5149): - # missing required args return the parameter schema - # instead of dispatching into an opaque failure. + # Validate before unwrapping: the generic bridge hides + # the concrete parameter schema from provider-native + # tool-call validation. _probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args) if _probe_err is not None: _ts_scope_block = _probe_err @@ -2007,6 +2012,10 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe break function_name = tool_call.function.name + # Legacy tool-name aliases (2026-08 renames) — map BEFORE the + # agent-loop branches (todo_list etc. dispatch above the registry). + from model_tools import _LEGACY_TOOL_ALIASES as _lta + function_name = _lta.get(function_name, function_name) function_args, malformed_args_result = _parse_tool_arguments( tool_call.function.arguments @@ -2048,9 +2057,9 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe _underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args) if not _err and _underlying: if _underlying in _tool_search_scoped_names(agent): - # Probe-validate before unwrapping (ironclaw#5149): - # missing required args return the parameter schema - # instead of dispatching into an opaque failure. + # Validate before unwrapping: the generic bridge hides + # the concrete parameter schema from provider-native + # tool-call validation. _probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args) if _probe_err is not None: # This path wraps _block_msg in {"error": ...} — @@ -2081,7 +2090,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe tool_start_time = time.time() - if function_name == "todo": + if function_name == "todo_list": def _execute(next_args: dict) -> Any: from tools.todo_tool import todo_tool as _todo_tool return _todo_tool( @@ -2101,7 +2110,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): - agent._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") + agent._vprint(f" {_get_cute_tool_message_impl('todo_list', function_args, tool_duration, result=function_result)}") elif function_name == "message_agent": # Bot Mode teammate DM (tools/bot_mode_dm.py) — injected, not # registered: only a canonical Bot Chat session carries the @@ -2336,7 +2345,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_window_below', function_args, tool_duration, result=function_result)}") - elif function_name == "tour": + elif function_name == "gui_tour": def _execute(next_args: dict) -> Any: from tools.tour_tool import tour_tool as _tour_tool return _tour_tool( @@ -2362,7 +2371,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): - agent._vprint(f" {_get_cute_tool_message_impl('tour', function_args, tool_duration, result=function_result)}") + agent._vprint(f" {_get_cute_tool_message_impl('gui_tour', function_args, tool_duration, result=function_result)}") elif function_name == "setup_mcp": def _execute(next_args: dict) -> Any: from tools.setup_mcp_tool import setup_mcp_tool as _setup_mcp_tool diff --git a/agent/tool_guardrails.py b/agent/tool_guardrails.py index ff30ee516c..b432c21d9f 100644 --- a/agent/tool_guardrails.py +++ b/agent/tool_guardrails.py @@ -24,6 +24,8 @@ IDEMPOTENT_TOOL_NAMES = frozenset( "web_search", "web_extract", "session_search", + "skill_view", + "skills_list", "browser_snapshot", "browser_console", "browser_get_images", @@ -44,7 +46,7 @@ MUTATING_TOOL_NAMES = frozenset( "execute_code", "write_file", "patch", - "todo", + "todo_list", "memory", "skill_manage", "browser_click", @@ -53,9 +55,9 @@ MUTATING_TOOL_NAMES = frozenset( "browser_scroll", "browser_navigate", "send_message", - "cronjob", + "cronjob_manage", "delegate_task", - "process", + "process_manage", } ) @@ -67,7 +69,7 @@ MUTATING_TOOL_NAMES = frozenset( # unannotated. STALL_GUARD_REPEATABLE_TOOLS = frozenset( { - "process", + "process_manage", } ) @@ -98,6 +100,49 @@ IDENTICAL_RESULT_STUB_MIN_CHARS = 512 _RESULT_STUB_ARGS_PREVIEW_CHARS = 120 +# Tools whose "failure" is a normal, informative outcome of legitimate work: +# a red test run, a grep with no matches, a failing build during a fix loop, a +# page that times out. Hard stops never fire on these from failure counts of +# DIFFERENT commands (same_tool_failure) — only an exact-args replay with NO +# intervening change, or an identical-result streak, can halt them. +FAILURE_TOLERANT_TOOL_NAMES = frozenset( + { + "terminal", + "execute_code", + "process_manage", + "process", + "browser_navigate", + "web_extract", + } +) + +# A landed mutation between two attempts means the retry is a NEW experiment +# (edit -> re-run) rather than a replay. A successful call to one of these +# marks progress for every failing signature still being counted this turn. +PROGRESS_RESET_TOOL_NAMES = frozenset( + { + "write_file", + "patch", + "terminal", + "execute_code", + "browser_click", + "browser_type", + "browser_press", + "browser_navigate", + "process_manage", + "process", + "delegate_task", + "send_message", + "cronjob", + "cronjob_manage", + "todo", + "todo_list", + "memory", + "skill_manage", + } +) + + def is_stall_guard_repeatable(tool_name: str) -> bool: """Whether a tool is exempt from the identical-call loop notice.""" if tool_name in STALL_GUARD_REPEATABLE_TOOLS: @@ -110,12 +155,14 @@ class ToolCallGuardrailConfig: """Thresholds for per-turn tool-call loop detection. Warnings are enabled by default and never prevent tool execution. Hard stops - are explicit opt-in so interactive CLI/TUI sessions get a gentle nudge unless - the user enables circuit-breaker behavior in config.yaml. + stay opt-in for interactive CLI/TUI/Desktop/ACP sessions, but default on for + non-interactive gateway/cron platforms where nobody is present to interrupt + a model that ignores loop warnings. """ warnings_enabled: bool = True hard_stop_enabled: bool = False + non_interactive_hard_stop_enabled: bool = True exact_failure_warn_after: int = 2 exact_failure_block_after: int = 5 same_tool_failure_warn_after: int = 3 @@ -127,10 +174,15 @@ class ToolCallGuardrailConfig: loop_caps: "LoopCapConfig" = field(default_factory=lambda: LoopCapConfig()) @classmethod - def from_mapping(cls, data: Mapping[str, Any] | None) -> "ToolCallGuardrailConfig": + def from_mapping( + cls, + data: Mapping[str, Any] | None, + *, + platform: str | None = None, + ) -> "ToolCallGuardrailConfig": """Build config from the `tool_loop_guardrails` config.yaml section.""" if not isinstance(data, Mapping): - return cls() + data = {} warn_after = data.get("warn_after") if not isinstance(warn_after, Mapping): @@ -140,9 +192,18 @@ class ToolCallGuardrailConfig: hard_stop_after = {} defaults = cls() + hard_stop_enabled = _as_bool(data.get("hard_stop_enabled"), defaults.hard_stop_enabled) + non_interactive_hard_stop_enabled = _as_bool( + data.get("non_interactive_hard_stop_enabled"), + defaults.non_interactive_hard_stop_enabled, + ) + if _is_non_interactive_platform(platform) and non_interactive_hard_stop_enabled: + hard_stop_enabled = True + return cls( warnings_enabled=_as_bool(data.get("warnings_enabled"), defaults.warnings_enabled), - hard_stop_enabled=_as_bool(data.get("hard_stop_enabled"), defaults.hard_stop_enabled), + hard_stop_enabled=hard_stop_enabled, + non_interactive_hard_stop_enabled=non_interactive_hard_stop_enabled, exact_failure_warn_after=_positive_int( warn_after.get("exact_failure", data.get("exact_failure_warn_after")), defaults.exact_failure_warn_after, @@ -218,6 +279,25 @@ class LoopCapConfig: ) +_INTERACTIVE_PLATFORMS = frozenset({"cli", "tui", "desktop", "acp"}) + +# Platforms that are not chat gateways but whose work is a bounded, supervised +# task loop: a subagent inherits its parent's budget and is stopped by the +# parent; api_server runs have a live client holding the request. Both do +# real edit -> re-run work, so they keep the interactive (warn-only) default. +_SUPERVISED_TASK_PLATFORMS = frozenset({"subagent", "api_server"}) + + +def _is_non_interactive_platform(platform: str | None) -> bool: + """Return true for gateway/cron sessions where tool loops are unattended.""" + if not isinstance(platform, str) or not platform.strip(): + return False + key = platform.strip().lower() + if key in _INTERACTIVE_PLATFORMS or key in _SUPERVISED_TASK_PLATFORMS: + return False + return True + + @dataclass(frozen=True) class IdenticalCallObservation: """Outcome of observing one completed tool call for the stall guards. @@ -340,6 +420,8 @@ class ToolCallGuardrailController: def reset_for_turn(self) -> None: self._exact_failure_counts: dict[ToolCallSignature, int] = {} self._same_tool_failure_counts: dict[str, int] = {} + # signature -> a mutating call succeeded since its last failure + self._progress_since_failure: dict[ToolCallSignature, bool] = {} self._no_progress: dict[ToolCallSignature, tuple[str, int]] = {} self._halt_decision: ToolGuardrailDecision | None = None # Identical-call loop-breaker state (agent.stall_guards): tracks the @@ -389,6 +471,10 @@ class ToolCallGuardrailController: return ToolGuardrailDecision(tool_name=tool_name, signature=signature) exact_count = self._exact_failure_counts.get(signature, 0) + if self._progress_since_failure.get(signature): + # Something landed since this call last failed — let it run; the + # streak restarts in after_call if it fails again. + exact_count = 0 if exact_count >= self.config.exact_failure_block_after: decision = ToolGuardrailDecision( action="block", @@ -441,6 +527,12 @@ class ToolCallGuardrailController: failed, _ = classify_tool_failure(tool_name, result) if failed: + # An identical failing call is only a REPLAY if nothing landed in + # between. If any mutating call succeeded since the previous + # identical failure (edit -> re-run pytest, click -> re-snapshot), + # the retry is a new experiment: restart the exact-args streak. + if self._progress_since_failure.pop(signature, False): + self._exact_failure_counts.pop(signature, None) exact_count = self._exact_failure_counts.get(signature, 0) + 1 self._exact_failure_counts[signature] = exact_count self._no_progress.pop(signature, None) @@ -448,7 +540,17 @@ class ToolCallGuardrailController: same_count = self._same_tool_failure_counts.get(tool_name, 0) + 1 self._same_tool_failure_counts[tool_name] = same_count - if self.config.hard_stop_enabled and same_count >= self.config.same_tool_failure_halt_after: + # same_tool_failure counts DIFFERENT args on one tool. For tools + # whose non-zero exit is ordinary work output (terminal, + # execute_code, pollers) a run of distinct red commands is + # diagnosis, not a loop — warn, never halt. The exact-args replay + # path still applies to them. + same_tool_halt_eligible = tool_name not in FAILURE_TOLERANT_TOOL_NAMES + if ( + self.config.hard_stop_enabled + and same_tool_halt_eligible + and same_count >= self.config.same_tool_failure_halt_after + ): decision = ToolGuardrailDecision( action="halt", code="same_tool_failure_halt", @@ -492,6 +594,16 @@ class ToolCallGuardrailController: self._exact_failure_counts.pop(signature, None) self._same_tool_failure_counts.pop(tool_name, None) + # A successful mutation is progress for every failing signature still + # being counted this turn: the next identical retry runs against + # changed state, so it is a fresh attempt rather than a replay. Pure + # loops never mutate anything between attempts, so the replay detector + # keeps its teeth. + if tool_name in PROGRESS_RESET_TOOL_NAMES or file_mutation_result_landed(tool_name, result): + for sig in list(self._exact_failure_counts): + self._progress_since_failure[sig] = True + self._same_tool_failure_counts.clear() + if not self._is_idempotent(tool_name): self._no_progress.pop(signature, None) return ToolGuardrailDecision(tool_name=tool_name, signature=signature) @@ -604,6 +716,31 @@ class ToolCallGuardrailController: "Do not repeat it — change arguments, use a different tool, or " "proceed with what you have.]" ) + # Hard-stop widening (#89069 / #100849 bundle): the per-turn + # no-progress BLOCK above only covers tools in idempotent_tools, so + # a model replaying the same successful `terminal`/`skill_view` + # call with a byte-identical result ran until the iteration budget. + # The consecutive-identical streak is tool-agnostic; when hard + # stops are enabled, halt at the same idempotent_no_progress + # threshold. Pollers stay exempt (an unchanged poll is progress). + if ( + self.config.hard_stop_enabled + and count >= self.config.no_progress_block_after + and self._halt_decision is None + ): + self._halt_decision = ToolGuardrailDecision( + action="halt", + code="identical_call_streak_halt", + message=( + f"Stopped {tool_name}: the same call with identical arguments " + f"returned the same result {count} times in a row. Stop " + "repeating it unchanged; use the result already provided or " + "change strategy." + ), + tool_name=tool_name, + count=count, + signature=signature, + ) stub = None if ( diff --git a/agent/transports/chat_completions.py b/agent/transports/chat_completions.py index d2284b64d2..25b41be44d 100644 --- a/agent/transports/chat_completions.py +++ b/agent/transports/chat_completions.py @@ -767,9 +767,18 @@ class ChatCompletionsTransport(ProviderTransport): extra_body["reasoning"] = gh_reasoning else: _effort = "medium" + _enabled = True if reasoning_config and isinstance(reasoning_config, dict): _effort = reasoning_config.get("effort", "medium") or "medium" - extra_body["reasoning"] = {"enabled": True, "effort": _effort} + # Honor an explicit "thinking off" (agent.reasoning_effort: + # none / the one-shot length-continuation override) the same + # way the provider-profile path does — never re-enable it. + if reasoning_config.get("enabled") is False or _effort == "none": + _enabled = False + if _enabled: + extra_body["reasoning"] = {"enabled": True, "effort": _effort} + else: + extra_body["reasoning"] = {"enabled": False, "effort": "none"} if provider_name == "gemini": raw_thinking_config = _build_gemini_thinking_config(model, reasoning_config) diff --git a/agent/tts_provider.py b/agent/tts_provider.py index c19166a702..075cab3f40 100644 --- a/agent/tts_provider.py +++ b/agent/tts_provider.py @@ -241,6 +241,22 @@ class TTSProvider(abc.ABC): "if your backend supports it." ) + def warm(self) -> None: + """Speech output was just turned on; pre-load so the first reply is hot. + + Optional. Called from the TTS lease path (Desktop read-aloud / voice + conversation, ``/voice tts``) when this provider is the configured + ``tts.provider`` — e.g. ask a local model server to load its model. + Best-effort: exceptions are logged at debug and ignored. Default: no-op. + """ + + def release(self) -> None: + """The last speech-output lease was released; free resident resources. + + Optional counterpart of :meth:`warm` — e.g. tell a local model server + to unload. Best-effort; default: no-op. + """ + @property def voice_compatible(self) -> bool: """Whether output is suitable for voice-bubble delivery. diff --git a/agent/turn_context.py b/agent/turn_context.py index bff4a8f094..e0f06338dc 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -493,6 +493,7 @@ def _should_idle_compact( tokens: int, floor_tokens: int, cooldown_active: bool, + last_compaction_tokens: int = 0, ) -> bool: """Decide whether an idle-triggered compaction should run this turn. @@ -508,6 +509,23 @@ def _should_idle_compact( *to*), so a small idle thread never pays for a summarisation that saves nothing, and it defers to an active compression-failure cooldown. + ``floor_tokens`` alone is a *theoretical* target (``threshold_tokens × + summary_target_ratio``) that a real pass routinely misses: the system + prompt, the tool schemas and the protected head/tail are an + incompressible floor. A session that compacted to well above that target + therefore stays above it forever, so every later idle resume re-runs a + full summarisation over a transcript that has not grown — minutes of + silently blocked prompt on a slow route, reclaiming nothing (#97239). + + ``last_compaction_tokens`` is what the previous pass on this session + actually produced (``ContextCompressor.last_compression_rough_tokens``, + the same ``estimate_request_tokens_rough`` shape as ``tokens``). When it + is known, require the transcript to have accumulated at least one + ``floor_tokens`` worth of *new* content on top of it before paying for + another pass. ``0`` — no compaction recorded yet, or the counter reset by + a rebind/recalibration — keeps the original floor semantics exactly, so + the first idle compaction of any session is unaffected. + Pure predicate so the policy is unit-testable without a live agent. """ if not enabled or idle_after_seconds <= 0: @@ -516,7 +534,10 @@ def _should_idle_compact( return False if cooldown_active: return False - return tokens > floor_tokens + effective_floor = floor_tokens + if last_compaction_tokens > 0: + effective_floor = max(effective_floor, last_compaction_tokens + floor_tokens) + return tokens > effective_floor @dataclass @@ -635,13 +656,18 @@ def build_turn_context( # Between-turns MCP refresh: an MCP server that finished connecting since # the previous turn (slow HTTP/OAuth servers routinely take 2-6s on a cold # connect, missing the bounded startup wait) lands in THIS turn's tool - # snapshot. This is cache-safe by construction: it runs in the per-turn + # snapshot. Timing is cache-safe by construction: it runs in the per-turn # prologue, before this turn's first API call assembles ``tools=``, so it - # only ever extends a fresh request prefix — it never mutates the cached - # prefix of an in-flight turn. No-op when no MCP servers are registered - # (the common case, gated by the cheap ``has_registered_mcp_tools`` check) - # or when the tool set is unchanged (``refresh_agent_mcp_tools`` diffs by - # name and leaves the snapshot untouched on no-change). + # never mutates the prefix of an in-flight turn. ``preserve_prefix`` makes + # the *content* cache-safe too (#100336): a plain rebuild re-derives the + # array from live availability, so a flapping ``check_fn`` silently drops a + # tool and a late arrival splices into sorted position — either one forks + # the tool block and re-prefills the whole history behind it, every turn it + # happens. With the flag the live order is authoritative and the array + # only ever grows. No-op when no MCP servers are registered (the common + # case, gated by the cheap ``has_registered_mcp_tools`` check) or when the + # tool set is unchanged (``refresh_agent_mcp_tools`` diffs by name and + # leaves the snapshot untouched on no-change). try: if not getattr(agent, "_skip_mcp_refresh", False): # Import-cost gate: ``tools.mcp_tool`` pulls in the whole ``mcp`` @@ -656,7 +682,9 @@ def build_turn_context( if "tools.mcp_tool" in _sys.modules: from tools.mcp_tool import has_registered_mcp_tools, refresh_agent_mcp_tools if has_registered_mcp_tools(): - refresh_agent_mcp_tools(agent, quiet_mode=True) + refresh_agent_mcp_tools( + agent, quiet_mode=True, preserve_prefix=True, + ) except Exception: logger.debug("between-turns MCP tool refresh skipped", exc_info=True) @@ -953,6 +981,18 @@ def build_turn_context( _idle_cooldown = getattr( _compressor, "get_active_compression_failure_cooldown", lambda: None )() + # What the previous pass on this session actually produced — the + # honest floor, versus the theoretical ``_idle_floor`` above. Type + # pin: minimal compressor doubles (SimpleNamespace / MagicMock) + # expose truthy non-ints here, and only a real int may raise the + # floor. Anything else falls back to 0 = original semantics. + _idle_last_compaction = getattr( + _compressor, "last_compression_rough_tokens", 0 + ) + if not isinstance(_idle_last_compaction, int) or isinstance( + _idle_last_compaction, bool + ): + _idle_last_compaction = 0 if _should_idle_compact( enabled=agent.compression_enabled, idle_after_seconds=_idle_after, @@ -960,14 +1000,16 @@ def build_turn_context( tokens=_idle_tokens, floor_tokens=_idle_floor, cooldown_active=bool(_idle_cooldown), + last_compaction_tokens=_idle_last_compaction, ): logger.info( "Idle compaction: %ss idle >= %ss, ~%s tokens > %s floor " - "(session %s)", + "(last compaction produced ~%s) (session %s)", int(_idle_gap), _idle_after, f"{_idle_tokens:,}", f"{_idle_floor:,}", + f"{_idle_last_compaction:,}" if _idle_last_compaction > 0 else "n/a", agent.session_id or "none", ) _idle_status = automatic_compaction_status_message( diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 193c56461d..93506f680c 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -126,6 +126,15 @@ def _drop_verification_continuation_scaffolding(messages) -> None: ] +def _clone_background_review_messages(messages): + """Copy the review input without aliasing the live transcript.""" + # Import lazily: conversation_loop imports this module during turn + # finalization, so a module-level import would create a cycle. + from agent.conversation_loop import _clone_message_for_send + + return [_clone_message_for_send(message) for message in messages] + + def finalize_turn( agent, *, @@ -810,6 +819,8 @@ def finalize_turn( and (_should_review_memory or _should_review_skills) ): try: + # _spawn_background_review clones the snapshot structurally so + # the fork's in-place sanitizers can't reach the live transcript. agent._spawn_background_review( messages_snapshot=list(messages), review_memory=_should_review_memory, diff --git a/agent/turn_liveness.py b/agent/turn_liveness.py index 939fc9b917..18550d6a6f 100644 --- a/agent/turn_liveness.py +++ b/agent/turn_liveness.py @@ -157,7 +157,8 @@ def resolve_turn_liveness_settings( class TurnLivenessWatchdog: - """Sampled-idle watchdog thread bound to one conversation turn. + """Sampled-idle watchdog bound to one conversation turn (polls on the + shared periodic scheduler thread). ``run_agent.py`` owns the turn-lease state (stop event, turn-active flag, interrupt plumbing); this class only reads the activity clock @@ -189,59 +190,53 @@ class TurnLivenessWatchdog: self._commit_abort = commit_abort self._deactivate_turn = deactivate_turn - def make_thread(self) -> threading.Thread: - """Build the (not yet started) watcher thread. + def schedule(self): + """Start polling on the shared periodic scheduler thread. ``run_agent.py`` creates the watchdog before the turn begins but - starts the thread at turn entry, right after the turn-active flag - and the activity clock are stamped. + schedules it at turn entry, right after the turn-active flag and + the activity clock are stamped. Returns the cancel handle. """ - return threading.Thread( - target=self._watch, - name="turn-liveness-watchdog", - daemon=True, - ) + from agent.periodic_scheduler import schedule - def start(self) -> threading.Thread: - """Spawn the watcher thread and return it (already running).""" - thread = self.make_thread() - thread.start() - return thread + return schedule(self._tick, self._poll_s) - def _watch(self) -> None: - while not self._stop_event.wait(self._poll_s): - snapshot = self._sample() - if snapshot is None: - # Turn is no longer active; nothing to watch. - return - if snapshot.idle_seconds < self._timeout_s: - continue - # Pre-commit surface is OBSERVATIONAL only: it reports the - # stall and that a recovery attempt is beginning. It must not - # claim the abort or the lease withdrawal has committed — the - # next operation can still veto the outcome. The definitive - # aborted/lease-stopped settlement is published by - # _surface_committed_abort only after _commit_abort succeeds - # and the turn is deactivated (#95663 review). - self._surface_stall(snapshot) - # Commit point: bind the abort to the sampled generation/ts - # and revalidate under the lock shared with `_touch_activity`. - # If progress resumed while the stall was being surfaced, the - # turn continues and this loop resumes sampling — the lease - # keeps renewing. The commit also carries the revalidated - # generation into the interrupt path, which reserves it as a - # claim, survives every blocking boundary (compression - # fence), and consumes it at the final mutation edge — progress - # landing anywhere in that window declines the abort. - if not self._commit_abort(snapshot, self._abort_message(snapshot)): - continue - # Stop renewing the durable lease: a wedge the hard interrupt - # cannot unwind must not keep the lease alive forever (the - # issue's "lease keeps renewing" masking). The TTL expiry then - # lets stale-turn cleanup reclaim the row. - self._deactivate_turn() - self._surface_committed_abort(snapshot) - return + def _tick(self): + """One poll. Returns False when the watchdog is finished.""" + if self._stop_event.is_set(): + return False + snapshot = self._sample() + if snapshot is None: + # Turn is no longer active; nothing to watch. + return False + if snapshot.idle_seconds < self._timeout_s: + return None + # Pre-commit surface is OBSERVATIONAL only: it reports the + # stall and that a recovery attempt is beginning. It must not + # claim the abort or the lease withdrawal has committed — the + # next operation can still veto the outcome. The definitive + # aborted/lease-stopped settlement is published by + # _surface_committed_abort only after _commit_abort succeeds + # and the turn is deactivated (#95663 review). + self._surface_stall(snapshot) + # Commit point: bind the abort to the sampled generation/ts + # and revalidate under the lock shared with `_touch_activity`. + # If progress resumed while the stall was being surfaced, the + # turn continues and this loop resumes sampling — the lease + # keeps renewing. The commit also carries the revalidated + # generation into the interrupt path, which reserves it as a + # claim, survives every blocking boundary (compression + # fence), and consumes it at the final mutation edge — progress + # landing anywhere in that window declines the abort. + if not self._commit_abort(snapshot, self._abort_message(snapshot)): + return None + # Stop renewing the durable lease: a wedge the hard interrupt + # cannot unwind must not keep the lease alive forever (the + # issue's "lease keeps renewing" masking). The TTL expiry then + # lets stale-turn cleanup reclaim the row. + self._deactivate_turn() + self._surface_committed_abort(snapshot) + return False def _sample(self) -> Optional[ActivitySnapshot]: with self._activity_lock: diff --git a/agent/turn_retry_state.py b/agent/turn_retry_state.py index 49790c6528..cfdf02b226 100644 --- a/agent/turn_retry_state.py +++ b/agent/turn_retry_state.py @@ -61,6 +61,7 @@ class TurnRetryState: native_compaction_reject_retry_attempted: bool = False image_shrink_retry_attempted: bool = False multimodal_tool_content_retry_attempted: bool = False + reasoning_mandatory_retry_attempted: bool = False oauth_1m_beta_retry_attempted: bool = False llama_cpp_grammar_retry_attempted: bool = False diff --git a/agent/turn_summary.py b/agent/turn_summary.py index f4440afb50..5953629eb9 100644 --- a/agent/turn_summary.py +++ b/agent/turn_summary.py @@ -69,7 +69,7 @@ _VERB_GROUPS: dict[str, tuple[str, str, str]] = { "skill_view": ("read", "skill", "skills"), "skill_manage": ("updated", "skill", "skills"), "skills_list": ("listed skills", "time", "times"), - "todo": ("updated", "task list", "task lists"), + "todo_list": ("updated", "task list", "task lists"), "delegate_task": ("delegated", "task", "tasks"), "memory": ("updated", "memory", "memories"), } diff --git a/agent/usage_pricing.py b/agent/usage_pricing.py index 9af08f6dc0..6c354bab7a 100644 --- a/agent/usage_pricing.py +++ b/agent/usage_pricing.py @@ -1278,6 +1278,10 @@ def get_pricing_entry( ) if route.provider == "openrouter": return _openrouter_pricing_entry(route) + + bundled_entry = _lookup_official_docs_pricing(route) + if bundled_entry: + return bundled_entry if route.base_url: entry = _pricing_entry_from_metadata( fetch_endpoint_model_metadata(route.base_url, api_key=api_key or ""), @@ -1287,7 +1291,7 @@ def get_pricing_entry( ) if entry: return entry - return _lookup_official_docs_pricing(route) + return None def normalize_usage( diff --git a/apps/desktop/e2e/bot-mode-closed-chat-stays-closed.spec.ts b/apps/desktop/e2e/bot-mode-closed-chat-stays-closed.spec.ts deleted file mode 100644 index da236ca6ff..0000000000 --- a/apps/desktop/e2e/bot-mode-closed-chat-stays-closed.spec.ts +++ /dev/null @@ -1,227 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' - -import { - buildAppEnv, - createSandbox, - launchDesktop, - type MockBackendFixture, - waitForAppReady, - writeEnvFile, - writeMockProviderConfig -} from './fixtures' -import { MOCK_REPLY, startMockServer } from './mock-server' -import { RealSessionBuilder } from './real-session-builder' -import { expect, test } from './test' - -// A bot row click is "go to this bot", not "open its Bot Chat". Before the -// fix, every click resolved the canonical chat by name and opened it as a tab -// again — a Bot Chat the user had closed came back beside every newer thread -// on every bot switch, because nothing records a close (the plugin keeps no -// closed set; core's tile bucket only forgets). Now a bot whose workspace -// already holds tabs comes back to the one the user left; the forever-chat is -// re-opened only by the explicit asks (row menu "Open Bot Chat"). -// -// UI note (post design-system rework): the canonical Bot Chat opens INTO the -// main workspace pane (`data-tree-tab="workspace"`), and a lone uncloseable -// workspace pane renders chromeless — its "Bot Chat" tab only exists once a -// second pane (e.g. a ⌘/Ctrl+T thread tile) shares the main zone. Assertions -// about the lone open therefore read the transcript, not a tab. - -type Page = MockBackendFixture['page'] - -let fixture: MockBackendFixture | null = null - -async function openBots(page: Page): Promise { - const tab = page - .getByRole('button', { name: 'Bots', exact: true }) - .or(page.getByRole('tab', { name: 'Bots', exact: true })) - .first() - - await tab.click() - await expect(page.getByRole('button', { name: 'New bot or group chat' })).toBeVisible() -} - -/** A bot's backend spawns on its first open; give the wake a real chance to - * clear before the next gesture races it. Tolerant: the mock backend can - * keep a tile's "Waking up…" notice around. */ -async function settle(page: Page, timeout = 90_000): Promise { - await page - .getByText(/Waking up/i) - .first() - .waitFor({ state: 'hidden', timeout }) - .catch(() => undefined) - await page.waitForTimeout(500) -} - -/** A first open right after a bot's backend spawned can strand on the - * profile socket (a separate, pre-existing reconnect race); a newer click - * supersedes it. Retry the gesture like a user would before giving up. */ -async function openUntil(action: () => Promise, expected: () => Promise, attempts = 3): Promise { - for (let attempt = 1; ; attempt += 1) { - await action() - - try { - await expected() - - return - } catch (error) { - if (attempt >= attempts) { - throw error - } - } - } -} - -const SCREENSHOT_DIR = process.env.BOT_MODE_SCREENSHOT_DIR - -async function snap(page: Page, name: string): Promise { - if (SCREENSHOT_DIR) { - await page.screenshot({ path: `${SCREENSHOT_DIR}/${name}.png` }) - } -} - -/** The session tabs on the main strip (the Bot Chat workspace tab may sit - * beside them). The strip itself auto-hides when the workspace pane is the - * only pane in the zone, so an empty result also covers "no strip at all". */ -const mainTabs = (page: Page) => - page.evaluate(() => - [...document.querySelectorAll('[data-zone-tabstrip="grp-main"] [data-tree-tab]')] - .map(element => element.getAttribute('data-tree-tab') ?? '') - .filter(id => id.startsWith('session-tile:')) - ) - -/** Bots are profiles. Seeding one on disk before launch — with the mock - * provider so its own backend can answer, and a real, durable "Bot Chat" - * row (the plugin's canonical forever-chat, found by exact title) — keeps - * in-app creation and the intro turn it fires out of a scenario that is - * about the row click. With the row present, the click takes the open-as- - * workspace path; without it, it would mint the chat into the pane. */ -async function seedBot(hermesHome: string, mockUrl: string, name: string): Promise { - const dir = path.join(hermesHome, 'profiles', name) - fs.mkdirSync(dir, { recursive: true }) - writeMockProviderConfig(dir, mockUrl) - writeEnvFile(dir) - - const builder = await RealSessionBuilder.start(dir) - - try { - await builder.createSession({ title: 'Bot Chat', turns: [`Hello ${name}`] }) - } finally { - await builder.close() - } -} - -test.beforeAll(async () => { - const mock = await startMockServer() - const sandbox = createSandbox('bots') - writeMockProviderConfig(sandbox.hermesHome, mock.url) - writeEnvFile(sandbox.hermesHome) - await seedBot(sandbox.hermesHome, mock.url, 'alpha') - await seedBot(sandbox.hermesHome, mock.url, 'beta') - - const { app, page } = await launchDesktop(buildAppEnv(sandbox)) - - fixture = { - app, - page, - mock, - mockUrl: mock.url, - sandbox, - cleanup: async () => { - await app.close().catch(() => undefined) - await mock.close() - sandbox.cleanup() - } - } - await waitForAppReady(fixture, 120_000) -}) - -test.afterAll(async () => { - await fixture?.cleanup() - fixture = null -}) - -test('a bot row click returns to the open thread and does not re-open a closed Bot Chat', async () => { - test.setTimeout(300_000) - const page = fixture!.page - - await openBots(page) - - const alphaRow = page.getByRole('button', { name: /^alpha\b/i }).filter({ visible: true }).first() - const betaRow = page.getByRole('button', { name: /^beta\b/i }).filter({ visible: true }).first() - await expect(alphaRow).toBeVisible({ timeout: 30_000 }) - await expect(betaRow).toBeVisible({ timeout: 30_000 }) - const botChatTab = page.getByRole('tab', { name: /Bot Chat/ }).filter({ visible: true }) - // The seeded forever-chat's first turn — visible only while the Bot Chat - // transcript is on screen. This is how a chromeless lone open is observed. - const seededTurn = page.getByText('Hello alpha', { exact: true }).filter({ visible: true }) - - // The first click on a bot with nothing open lands on its canonical chat. - // It fills the lone main workspace pane, which renders without a tab strip. - await openUntil( - () => alphaRow.click(), - () => expect(seededTurn.first()).toBeVisible({ timeout: 45_000 }) - ) - await settle(page, 15_000) - await snap(page, '01-first-click-opens-bot-chat') - - // Start a fresh thread for Alpha (⌘/Ctrl+T). The thread tile joins the main - // zone beside the Bot Chat workspace pane, which mounts the tab strip — the - // "Bot Chat" tab exists now, and the close affordance with it. - await page.keyboard.press('Control+t') - await expect(botChatTab.first()).toBeVisible({ timeout: 15_000 }) - await expect.poll(() => mainTabs(page), { timeout: 15_000 }).toHaveLength(1) - - const composer = page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first() - await expect(composer).toBeVisible({ timeout: 15_000 }) - await composer.click() - await composer.fill('hello alpha thread') - await page.keyboard.press('Enter') - await expect(page.getByText('hello alpha thread').filter({ visible: true }).first()).toBeVisible({ timeout: 15_000 }) - await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 }) - await snap(page, '02-new-thread-beside-bot-chat') - - const threadTabs = await mainTabs(page) - expect(threadTabs).toHaveLength(1) - const [threadTab] = threadTabs - expect(threadTab).toMatch(/^session-tile:/) - - // Close the Bot Chat. Its transcript leaves the screen; the thread stays. - await botChatTab.first().hover() - await botChatTab.first().getByRole('button', { name: 'Close' }).click({ force: true }) - await expect(botChatTab).toHaveCount(0) - await expect(seededTurn).toHaveCount(0) - await snap(page, '03-bot-chat-closed-thread-stays') - - // Switch to Beta: Alpha's thread leaves the strip (scoped away, not closed). - await betaRow.click() - await expect(page.locator(`[data-zone-tabstrip="grp-main"] [data-tree-tab="${threadTab}"]`)).toHaveCount(0, { - timeout: 60_000 - }) - await settle(page) - - // Back to Alpha: the workspace comes back to what the user left, and the - // closed Bot Chat STAYS closed. The regression this pins re-opened the - // canonical chat beside the thread on every switch — two panes in the main - // zone, which mounts the tab strip and puts the "Bot Chat" tab back on - // screen. Its absence (with the transcript present, so the click landed) is - // the observable "stays closed". - await alphaRow.click() - await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 30_000 }) - await page.waitForTimeout(3000) - await expect(botChatTab).toHaveCount(0) - await snap(page, '04-back-to-alpha-bot-chat-stays-closed') - - // The explicit ask still opens the forever-chat: its seeded first turn is - // back on screen. (As the surviving main-workspace pane it may render - // chromeless, so the transcript — not a tab — is the assertion.) - await openUntil( - async () => { - await alphaRow.click({ button: 'right' }) - await page.getByRole('menuitem', { name: 'Open Bot Chat' }).click() - }, - () => expect(seededTurn.first()).toBeVisible({ timeout: 45_000 }) - ) - await snap(page, '05-explicit-open-bot-chat') -}) diff --git a/apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts b/apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts new file mode 100644 index 0000000000..23354e7b77 --- /dev/null +++ b/apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts @@ -0,0 +1,158 @@ +import fs from 'node:fs' +import path from 'node:path' + +import { + buildAppEnv, + createSandbox, + launchDesktop, + type MockBackendFixture, + waitForAppReady, + writeEnvFile, + writeMockProviderConfig +} from './fixtures' +import { MOCK_REPLY, startMockServer } from './mock-server' +import { RealSessionBuilder } from './real-session-builder' +import { expect, test } from './test' + +// A bot row previews the bot's canonical Bot Chat (the gateway resolves it by +// name on every roster poll). Clicking the row must land on THAT conversation. +// Before this fix a plain click fronted whatever bots-workspace tile the user +// last had open for that bot — a `+` side thread outlived every restart in +// Local Storage and won every click forever, while the row kept previewing the +// Bot Chat. The user saw the sidebar and the center describe two different +// conversations ("sessions not in sync"; support thread 1544460286084391043). + +type Page = MockBackendFixture['page'] + +let fixture: MockBackendFixture | null = null + +async function openBots(page: Page): Promise { + const tab = page + .getByRole('button', { name: 'Bots', exact: true }) + .or(page.getByRole('tab', { name: 'Bots', exact: true })) + .first() + + await tab.click() + await expect(page.getByRole('button', { name: 'New bot or group chat' })).toBeVisible() +} + +async function settle(page: Page, timeout = 90_000): Promise { + await page + .getByText(/Waking up/i) + .first() + .waitFor({ state: 'hidden', timeout }) + .catch(() => undefined) + await page.waitForTimeout(500) +} + +async function openUntil(action: () => Promise, expected: () => Promise, attempts = 3): Promise { + for (let attempt = 1; ; attempt += 1) { + await action() + + try { + await expected() + + return + } catch (error) { + if (attempt >= attempts) { + throw error + } + } + } +} + +async function seedBot(hermesHome: string, mockUrl: string, name: string): Promise { + const dir = path.join(hermesHome, 'profiles', name) + fs.mkdirSync(dir, { recursive: true }) + writeMockProviderConfig(dir, mockUrl) + writeEnvFile(dir) + + const builder = await RealSessionBuilder.start(dir) + + try { + await builder.createSession({ title: 'Bot Chat', turns: [`Hello ${name}`] }) + } finally { + await builder.close() + } +} + +test.beforeAll(async () => { + const mock = await startMockServer() + const sandbox = createSandbox('bots-sync') + writeMockProviderConfig(sandbox.hermesHome, mock.url) + writeEnvFile(sandbox.hermesHome) + await seedBot(sandbox.hermesHome, mock.url, 'alpha') + await seedBot(sandbox.hermesHome, mock.url, 'beta') + + const { app, page } = await launchDesktop(buildAppEnv(sandbox)) + + fixture = { + app, + page, + mock, + mockUrl: mock.url, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + await mock.close() + sandbox.cleanup() + } + } + await waitForAppReady(fixture, 120_000) +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test('a bot row click lands on the Bot Chat the row previews, not a side thread', async () => { + test.setTimeout(300_000) + const page = fixture!.page + + await openBots(page) + + const alphaRow = page.getByRole('button', { name: /^alpha\b/i }).filter({ visible: true }).first() + const betaRow = page.getByRole('button', { name: /^beta\b/i }).filter({ visible: true }).first() + await expect(alphaRow).toBeVisible({ timeout: 30_000 }) + await expect(betaRow).toBeVisible({ timeout: 30_000 }) + const seededTurn = page.getByText('Hello alpha', { exact: true }).filter({ visible: true }) + + await openUntil( + () => alphaRow.click(), + () => expect(seededTurn.first()).toBeVisible({ timeout: 45_000 }) + ) + await settle(page, 15_000) + + // A `+` side thread for alpha, with a real turn so it is a persisted tile. + await page.keyboard.press('Control+t') + const composer = page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first() + await expect(composer).toBeVisible({ timeout: 15_000 }) + await composer.click() + await composer.fill('hello alpha thread') + await page.keyboard.press('Enter') + await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 }) + + // Leave alpha on the side thread, go to beta, come back via the row. + await betaRow.click() + await expect(page.getByText('Hello beta', { exact: true }).filter({ visible: true }).first()).toBeVisible({ + timeout: 60_000 + }) + await settle(page) + + await alphaRow.click() + // The row previews the Bot Chat; the click must front it. + await expect(seededTurn.first()).toBeVisible({ timeout: 45_000 }) + // The side thread is still open beside it (scoped to alpha), not closed. + await expect + .poll( + () => + page.evaluate(() => + [...document.querySelectorAll('[data-zone-tabstrip="grp-main"] [data-tree-tab]')] + .map(element => element.getAttribute('data-tree-tab') ?? '') + .filter(id => id.startsWith('session-tile:')).length + ), + { timeout: 15_000 } + ) + .toBe(1) +}) diff --git a/apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts b/apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts new file mode 100644 index 0000000000..c26cd18043 --- /dev/null +++ b/apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts @@ -0,0 +1,137 @@ +import fs from 'node:fs' +import path from 'node:path' + +import { + buildAppEnv, + createSandbox, + launchDesktop, + type MockBackendFixture, + waitForAppReady, + writeEnvFile, + writeMockProviderConfig +} from './fixtures' +import { MOCK_REPLY, startMockServer } from './mock-server' +import { RealSessionBuilder } from './real-session-builder' +import { expect, test } from './test' + +// Every bot's canonical chat is STORED under the same title ("Bot Chat" — the +// name the gateway resolves it by), so the main tab strip captioned every open +// bot chat identically and two bots' tabs were indistinguishable (#99152). The +// tab must read the bot's display name while the stored title stays canonical. + +type Page = MockBackendFixture['page'] + +let fixture: MockBackendFixture | null = null + +async function openBots(page: Page): Promise { + const tab = page + .getByRole('button', { name: 'Bots', exact: true }) + .or(page.getByRole('tab', { name: 'Bots', exact: true })) + .first() + + await tab.click() + await expect(page.getByRole('button', { name: 'New bot or group chat' })).toBeVisible() +} + +async function openUntil(action: () => Promise, expected: () => Promise, attempts = 3): Promise { + for (let attempt = 1; ; attempt += 1) { + await action() + + try { + await expected() + + return + } catch (error) { + if (attempt >= attempts) { + throw error + } + } + } +} + +async function seedBot(hermesHome: string, mockUrl: string, name: string): Promise { + const dir = path.join(hermesHome, 'profiles', name) + fs.mkdirSync(dir, { recursive: true }) + writeMockProviderConfig(dir, mockUrl) + writeEnvFile(dir) + + const builder = await RealSessionBuilder.start(dir) + + try { + await builder.createSession({ title: 'Bot Chat', turns: [`Hello ${name}`] }) + } finally { + await builder.close() + } +} + +/** Every tab caption in the main strip (the main `workspace` tab + tiles). */ +function mainStripTabTitles(page: Page): Promise { + return page.evaluate(() => + [...document.querySelectorAll('[data-zone-tabstrip="grp-main"] [data-tree-tab]')].map(element => + (element.textContent ?? '').trim() + ) + ) +} + +test.beforeAll(async () => { + const mock = await startMockServer() + const sandbox = createSandbox('bots-tabname') + writeMockProviderConfig(sandbox.hermesHome, mock.url) + writeEnvFile(sandbox.hermesHome) + await seedBot(sandbox.hermesHome, mock.url, 'alpha') + await seedBot(sandbox.hermesHome, mock.url, 'beta') + + const { app, page } = await launchDesktop(buildAppEnv(sandbox)) + + fixture = { + app, + page, + mock, + mockUrl: mock.url, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + await mock.close() + sandbox.cleanup() + } + } + await waitForAppReady(fixture, 120_000) +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test("an open Bot Chat's tab reads the bot's name, not the canonical 'Bot Chat' title", async () => { + test.setTimeout(300_000) + const page = fixture!.page + + await openBots(page) + + const alphaRow = page.getByRole('button', { name: /^alpha\b/i }).filter({ visible: true }).first() + await expect(alphaRow).toBeVisible({ timeout: 30_000 }) + + await openUntil( + () => alphaRow.click(), + () => + expect(page.getByText('Hello alpha', { exact: true }).filter({ visible: true }).first()).toBeVisible({ + timeout: 45_000 + }) + ) + + // A `+` side thread beside the Bot Chat gives the main zone a tab strip — + // the surface where every bot chat used to read "Bot Chat". + await page.keyboard.press('Control+t') + const composer = page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first() + await expect(composer).toBeVisible({ timeout: 15_000 }) + await composer.click() + await composer.fill('hello alpha thread') + await page.keyboard.press('Enter') + await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 }) + + await expect.poll(() => mainStripTabTitles(page), { timeout: 15_000 }).toHaveLength(2) + const captions = await mainStripTabTitles(page) + expect(captions.some(caption => /alpha/i.test(caption))).toBe(true) + expect(captions.some(caption => /bot chat/i.test(caption))).toBe(false) +}) diff --git a/apps/desktop/e2e/bot-roster-user-sections.spec.ts b/apps/desktop/e2e/bot-roster-user-sections.spec.ts new file mode 100644 index 0000000000..1b2e6753ab --- /dev/null +++ b/apps/desktop/e2e/bot-roster-user-sections.spec.ts @@ -0,0 +1,254 @@ +import fs from 'node:fs' +import path from 'node:path' + +import { + buildAppEnv, + createSandbox, + launchDesktop, + type MockBackendFixture, + waitForAppReady, + writeEnvFile, + writeMockProviderConfig +} from './fixtures' +import { startMockServer } from './mock-server' +import { RealSessionBuilder } from './real-session-builder' +import { expect, test } from './test' + +// User-made sections in the Bots roster: a bot is filed by dragging it onto a +// section or through its row menu, the section is renamed through the same +// dialog shape sessions use, and deleting a section returns its bots to +// Unassigned (with an Undo toast, no confirmation). With no sections created +// the roster is the plain list it always was. + +type Page = MockBackendFixture['page'] + +let fixture: MockBackendFixture | null = null + +// BOT_SECTIONS_SCREENSHOT_DIR= saves full-window captures at the key +// states — handy for design review; never part of the assertions. +async function capture(page: Page, name: string): Promise { + const dir = process.env.BOT_SECTIONS_SCREENSHOT_DIR + + if (!dir) { + return + } + + fs.mkdirSync(dir, { recursive: true }) + await page.screenshot({ path: path.join(dir, `${name}.png`) }) +} + +async function seedBot(hermesHome: string, mockUrl: string, name: string): Promise { + const dir = path.join(hermesHome, 'profiles', name) + fs.mkdirSync(dir, { recursive: true }) + writeMockProviderConfig(dir, mockUrl) + writeEnvFile(dir) + + const builder = await RealSessionBuilder.start(dir) + + try { + await builder.createSession({ title: 'Bot Chat', turns: [`Hello ${name}`] }) + } finally { + await builder.close() + } +} + +const roster = (page: Page) => page.locator('[data-slot="bots-roster"]') +const botRow = (page: Page, name: string) => roster(page).locator(`[data-roster-key="local::${name}"]`) + +/** A section's label span — the one node whose text is exactly the name. */ +const sectionLabel = (page: Page, name: string) => + page.locator('span.truncate', { hasText: new RegExp(`^${name}$`, 'i') }) + +/** The heading's fold button (label + count) — the ⋯ menu trigger is a sibling with no text. */ +const sectionHeading = (page: Page, name: string) => + roster(page).locator('[data-slot="bots-section"] button[aria-expanded]').filter({ has: sectionLabel(page, name) }) + +const sectionBlock = (page: Page, name: string) => + roster(page).locator('[data-slot="bots-section"]').filter({ has: sectionLabel(page, name) }) + +/** Section name → roster keys of the rows under it (the plain list has no sections). */ +async function layout(page: Page): Promise> { + return roster(page).locator('[data-slot="bots-section"]').evaluateAll(blocks => + blocks.map(block => [ + block.querySelector('button[aria-expanded] span.truncate')?.textContent?.trim() ?? '', + [...block.querySelectorAll('[data-roster-key]')].map(row => row.dataset.rosterKey ?? '') + ]) + ) +} + +test.beforeAll(async () => { + const mock = await startMockServer() + const sandbox = createSandbox('bots-sections') + writeMockProviderConfig(sandbox.hermesHome, mock.url) + writeEnvFile(sandbox.hermesHome) + + for (const name of ['alpha', 'beta', 'gamma']) { + await seedBot(sandbox.hermesHome, mock.url, name) + } + + const { app, page } = await launchDesktop(buildAppEnv(sandbox)) + + fixture = { + app, + page, + mock, + mockUrl: mock.url, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + await mock.close() + sandbox.cleanup() + } + } + await waitForAppReady(fixture, 120_000) +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test('file bots into user sections by menu and drag; rename; delete returns them to Unassigned', async () => { + test.setTimeout(300_000) + const page = fixture!.page + + const tab = page + .getByRole('button', { name: 'Bots', exact: true }) + .or(page.getByRole('tab', { name: 'Bots', exact: true })) + .first() + + await tab.click() + await expect(page.getByRole('button', { name: 'New bot or group chat' })).toBeVisible() + await expect(botRow(page, 'alpha')).toBeVisible({ timeout: 30_000 }) + await expect(botRow(page, 'beta')).toBeVisible({ timeout: 30_000 }) + + // No sections yet: the plain list, no section chrome at all. + await expect(roster(page).locator('[data-slot="bots-section"]')).toHaveCount(0) + await capture(page, '1-plain-roster') + + // Right-click alpha → Move to section → New section… → name it → alpha is filed. + await botRow(page, 'alpha').click({ button: 'right' }) + await page.getByRole('menuitem', { name: 'Move to section' }).hover() + await expect(page.getByRole('menuitem', { name: 'New section…' })).toBeVisible() + await capture(page, '2-row-menu-move-to-section') + await page.getByRole('menuitem', { name: 'New section…' }).click() + const nameField = page.getByRole('textbox', { name: 'Section name' }) + await expect(nameField).toBeVisible() + await nameField.fill('Clients') + await capture(page, '3-new-section-dialog') + await page.getByRole('button', { name: 'Create' }).click() + + await expect(sectionHeading(page, 'Clients')).toBeVisible() + await expect(sectionBlock(page, 'Clients').locator('[data-roster-key="local::alpha"]')).toBeVisible() + // The remainder is Unassigned, drawn last. + await expect + .poll(async () => (await layout(page)).map(([name, keys]) => [name, keys.length])) + .toEqual([ + ['Clients', 1], + ['Unassigned', 3] + ]) + await capture(page, '4-alpha-filed') + + // Drag beta over the Clients block: the target highlights while over it. + // Escape cancels — nothing moves, nothing stays highlighted or faded. + const target = sectionBlock(page, 'Clients') + const from = (await botRow(page, 'beta').boundingBox())! + const to = (await sectionHeading(page, 'Clients').boundingBox())! + + const dragBetaOverClients = async () => { + await page.mouse.move(from.x + from.width / 2, from.y + from.height / 2) + await page.mouse.down() + await page.mouse.move(from.x + from.width / 2, from.y + from.height / 2 - 10, { steps: 4 }) + await page.mouse.move(to.x + to.width / 2, to.y + to.height / 2, { steps: 12 }) + await expect(target).toHaveAttribute('data-drop-over', 'true') + } + + await dragBetaOverClients() + await page.keyboard.press('Escape') + await page.mouse.up() + await expect(target).not.toHaveAttribute('data-drop-over', 'true') + await expect(botRow(page, 'beta')).toHaveCSS('opacity', '1') + expect((await layout(page)).map(([name, keys]) => [name, keys.length])).toEqual([ + ['Clients', 1], + ['Unassigned', 3] + ]) + + // Drop it for real: the bot is filed. + await dragBetaOverClients() + await capture(page, '5-drag-over-clients') + await page.mouse.up() + + await expect(target.locator('[data-roster-key="local::beta"]')).toBeVisible() + await expect(target).not.toHaveAttribute('data-drop-over', 'true') + // The moved row remounts under its new section; it must not stay faded. + await expect(botRow(page, 'beta')).toHaveCSS('opacity', '1') + await expect + .poll(async () => (await layout(page)).map(([name, keys]) => [name, keys.length])) + .toEqual([ + ['Clients', 2], + ['Unassigned', 2] + ]) + await capture(page, '6-beta-dropped') + + // Rename through the heading's context menu — the same Dialog + Input + // + Save shape as a session rename. + await sectionHeading(page, 'Clients').click({ button: 'right' }) + await page.getByRole('menuitem', { name: 'Rename…' }).click() + await expect(nameField).toHaveValue('Clients') + await nameField.fill('Customers') + await page.getByRole('button', { name: 'Save' }).click() + await expect(sectionHeading(page, 'Customers')).toBeVisible() + await expect(sectionHeading(page, 'Clients')).toHaveCount(0) + await capture(page, '7-renamed') + + // A second, empty section from the + menu shows its drop hint; collapsing + // a section folds its rows like the gateway headings do. + await page.getByRole('button', { name: 'New bot or group chat' }).click() + await page.getByRole('menuitem', { name: 'New section' }).click() + await nameField.fill('Team') + await page.getByRole('button', { name: 'Create' }).click() + await expect(sectionBlock(page, 'Team').getByText('Drag bots here')).toBeVisible() + await sectionHeading(page, 'Customers').click() + await expect(sectionBlock(page, 'Customers').locator('[data-roster-key]')).toHaveCount(0) + await capture(page, '8-empty-section-and-collapsed') + await sectionHeading(page, 'Customers').click() + await expect(sectionBlock(page, 'Customers').locator('[data-roster-key]')).toHaveCount(2) + + // Delete Customers: no confirmation, its two bots return to Unassigned, + // and the toast offers Undo. + await sectionHeading(page, 'Customers').click({ button: 'right' }) + await page.getByRole('menuitem', { name: 'Delete' }).click() + await expect(sectionHeading(page, 'Customers')).toHaveCount(0) + const toast = page.getByRole('status').filter({ hasText: 'Deleted “Customers”' }) + await expect(toast).toBeVisible() + await expect + .poll(async () => (await layout(page)).map(([name, keys]) => [name, keys.length])) + .toEqual([ + ['Team', 0], + ['Unassigned', 4] + ]) + await capture(page, '9-deleted-with-undo-toast') + + await toast.getByRole('button', { name: 'Undo' }).click() + await expect(sectionHeading(page, 'Customers')).toBeVisible() + await expect + .poll(async () => (await layout(page)).map(([name, keys]) => [name, keys.length])) + .toEqual([ + ['Customers', 2], + ['Team', 0], + ['Unassigned', 2] + ]) + + // Membership rides the bot's profile ui_meta, so it follows profile sync. + const alphaProfile = path.join(fixture!.sandbox.hermesHome, 'profiles', 'alpha', 'profile.yaml') + await expect.poll(() => (fs.existsSync(alphaProfile) ? fs.readFileSync(alphaProfile, 'utf8') : '')).toMatch(/sectionId:\s*sec-/) + + // Delete both sections: the roster is the plain list again. + for (const name of ['Customers', 'Team']) { + await sectionHeading(page, name).click({ button: 'right' }) + await page.getByRole('menuitem', { name: 'Delete' }).click() + } + + await expect(roster(page).locator('[data-slot="bots-section"]')).toHaveCount(0) + await expect(botRow(page, 'alpha')).toBeVisible() +}) diff --git a/apps/desktop/e2e/group-to-local-bot-handoff.spec.ts b/apps/desktop/e2e/group-to-local-bot-handoff.spec.ts index f3710f6b0a..2f863d144f 100644 --- a/apps/desktop/e2e/group-to-local-bot-handoff.spec.ts +++ b/apps/desktop/e2e/group-to-local-bot-handoff.spec.ts @@ -73,7 +73,7 @@ test('local bot replaces an open group main workspace', async () => { await expect(groupTab).toHaveCount(0) await expect(groupComposer).toHaveCount(0) // No "Waking up…" assertion: the mock backend can keep a bot's wake notice - // around indefinitely (see bot-mode-closed-chat-stays-closed's settle()), + // around indefinitely (see bot-mode-row-click-mirrors-registry's settle()), // so its presence no longer distinguishes a stranded handoff. The splash // and composer above are the proof the bot's chat took the workspace. await expect(page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first()).toBeVisible() diff --git a/apps/desktop/electron/backend-claim.ts b/apps/desktop/electron/backend-claim.ts index f00a4d5bc9..16793d33ae 100644 --- a/apps/desktop/electron/backend-claim.ts +++ b/apps/desktop/electron/backend-claim.ts @@ -36,13 +36,26 @@ export function execText(command: string, args: string[], { timeout = 3000 } = { }) } +/** + * Probe budget for the ORPHAN-REAP path (matchesParent / matchesIdentity / + * stopOwnedBackend). The claim path keeps the full 30s headroom — a freshly + * spawned backend's marker is load-bearing and a slow probe must not kill a + * healthy child (#93608). Reap only needs to tell "same process" from "gone + * or reused" for OLD records, and the ownership file can accumulate dozens of + * them (one per profile per launch), so a 30s budget per record would let a + * cold PowerShell 5.1 stall boot for minutes (#87169). 5s is plenty for a + * warm probe; a timeout degrades to "unknown" and the record is preserved for + * the next launch instead of blocking boot. + */ +export const REAP_PROBE_TIMEOUT_MS = 5_000 + /** * Cross-platform process start marker: a value that changes when a PID is * reused, so `pid + marker` identifies one specific process incarnation. * Throws when the probe fails — callers decide what a failure means (see * `claimDecision` / `probeStartMarker`). */ -export async function processStartMarker(pid: number): Promise { +export async function processStartMarker(pid: number, timeoutMs: number = 30_000): Promise { // Cheap native dead-PID gate. Windows Get-Process / macOS `ps -p` exit 1 // on a missing PID (not ESRCH), so the identity matchers used to keep the // orphan and re-probe it every launch (#92875). ESRCH is the code those @@ -85,7 +98,9 @@ export async function processStartMarker(pid: number): Promise { ], // PowerShell 5.1 cold starts routinely exceed the default 3s execText // budget (2.4-8s observed in #87169); give the marker probe headroom. - { timeout: 30_000 } + // The claim path keeps this 30s budget; the orphan-reap path passes + // REAP_PROBE_TIMEOUT_MS so a slow probe cannot stall boot. + { timeout: timeoutMs } ) if (!/^\d+$/.test(ticks)) { diff --git a/apps/desktop/electron/backend-ownership.test.ts b/apps/desktop/electron/backend-ownership.test.ts index 7910478a67..77a6e46e2e 100644 --- a/apps/desktop/electron/backend-ownership.test.ts +++ b/apps/desktop/electron/backend-ownership.test.ts @@ -178,6 +178,51 @@ test('startup reap preserves failed stops for the next launch', async () => { assert.deepEqual(parseBackendOwnership(store.value()), [entry]) }) +test('startup reap stops at the deadline and preserves the unprocessed records', async () => { + const first = ownershipEntry({ pid: 60 }) + const second = ownershipEntry({ pid: 61 }) + const store = memoryStore(stored([first, second])) + const stop = vi.fn() + + const ownership = createOwnership(store, { + // Each probe is slow enough to blow a 1ms budget after the first entry. + matchesIdentity: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + + return false + }, + stop, + reapDeadlineMs: 1 + }) + + assert.deepEqual(await ownership.reapOrphans(), []) + // The first entry was processed (dropped); the second was preserved for the + // next launch instead of stalling boot on a slow identity probe. + assert.deepEqual(parseBackendOwnership(store.value()), [second]) +}) + +test('startup reap preserves would-be-reaped records when the budget runs out', async () => { + const first = ownershipEntry({ pid: 62 }) + const second = ownershipEntry({ pid: 63 }) + const store = memoryStore(stored([first, second])) + const stop = vi.fn() + + const ownership = createOwnership(store, { + matchesIdentity: async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + + return true + }, + stop, + reapDeadlineMs: 1 + }) + + assert.deepEqual(await ownership.reapOrphans(), [62]) + // The second would have been reaped too, but the budget ran out — it is + // preserved so a later launch retries it. + assert.deepEqual(parseBackendOwnership(store.value()), [second]) +}) + test('startup reap never stops a backend whose parent Electron is still alive', async () => { const entry = { ...ownershipEntry({ pid: 54 }), parentPid: 100, parentStartMarker: 'os-start-parent' } const store = memoryStore(stored([entry])) diff --git a/apps/desktop/electron/backend-ownership.ts b/apps/desktop/electron/backend-ownership.ts index 682dbe3f5b..3ef1080f0e 100644 --- a/apps/desktop/electron/backend-ownership.ts +++ b/apps/desktop/electron/backend-ownership.ts @@ -28,8 +28,22 @@ export interface BackendOwnershipDeps { matchesParent: (entry: BackendOwnershipEntry) => Promise stop: (identity: BackendIdentity) => Promise | void store: BackendOwnershipStore + /** + * Overall time budget for one reap sweep. The ownership file legitimately + * accumulates one record per profile per launch, and each record can cost + * up to two identity probes (parent + backend) plus a stop — on Windows + * those shell out to PowerShell, whose 5.1 cold starts are slow (#87169). + * Without a bound, a large roster could stall boot for minutes while the + * renderer's 45s backend-boot budget expires and the user stares at the + * connecting screen. When the budget is exhausted the sweep preserves the + * unprocessed records for the next launch and returns what it reaped. + */ + reapDeadlineMs?: number } +/** Default budget for one reap sweep (see `reapDeadlineMs`). */ +export const REAP_ORPHANS_DEADLINE_MS = 5_000 + export interface BackendClaim extends BackendIdentity { command?: string parentPid?: number @@ -222,8 +236,21 @@ export function createBackendOwnership(deps: BackendOwnershipDeps) { const survivors: BackendOwnershipEntry[] = [] const reaped: number[] = [] + const deadline = Date.now() + (deps.reapDeadlineMs ?? REAP_ORPHANS_DEADLINE_MS) + + for (let i = 0; i < entries.length; i += 1) { + // Budget exhausted: preserve the unprocessed records so a later launch + // can retry them. A slow identity probe must never stall boot — the + // renderer's backend-boot budget is 45s and the spawn itself needs + // most of it. + if (Date.now() >= deadline) { + survivors.push(...entries.slice(i)) + + break + } + + const entry = entries[i] - for (const entry of entries) { // A backend whose Electron parent is still running is NOT an orphan: // reaping it would kill a live instance's session. This is what stops // a second launch from SIGTERMing the running instance's backend even diff --git a/apps/desktop/electron/bundle-skew.test.ts b/apps/desktop/electron/bundle-skew.test.ts index a4de52905b..cbf3453f5a 100644 --- a/apps/desktop/electron/bundle-skew.test.ts +++ b/apps/desktop/electron/bundle-skew.test.ts @@ -1,6 +1,11 @@ -import { describe, expect, it } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' -import { detectBundleSkew, isFallbackCommit, type RunGit } from './bundle-skew' +import { afterAll, describe, expect, it } from 'vitest' + +import { detectBundleSkew, isFallbackCommit, type RunGit, RUNTIME_PATHS } from './bundle-skew' const REPO = '/repo' const STAMP = { commit: 'a'.repeat(40), source: 'ci' } @@ -9,6 +14,36 @@ function gitReturning(stdout: string, code = 0): RunGit { return async () => ({ code, stderr: '', stdout }) } +/** + * A git fake that answers per subcommand, so a test can say "ancestry fails, + * but the count would have claimed skew" — which is the shape of #92233. + */ +function gitAnswering(answers: Record): { + calls: string[][] + git: RunGit +} { + const calls: string[][] = [] + + const git: RunGit = async args => { + calls.push(args) + + const answer = answers[args[0]] ?? {} + + return { + code: answer.code ?? 0, + stderr: answer.stderr ?? '', + stdout: answer.stdout ?? '' + } + } + + return { calls, git } +} + +/** Every subcommand succeeds; rev-list reports `count`. */ +function gitCounting(count: string): RunGit { + return gitAnswering({ 'merge-base': { code: 0 }, 'rev-list': { stdout: count } }).git +} + describe('isFallbackCommit', () => { it('matches the all-zero placeholder at any stamp length', () => { expect(isFallbackCommit('0'.repeat(40))).toBe(true) @@ -19,27 +54,21 @@ describe('isFallbackCommit', () => { describe('detectBundleSkew', () => { it('reports stale when desktop commits landed after the stamp', async () => { - const result = await detectBundleSkew(STAMP, gitReturning('3\n'), REPO) + const result = await detectBundleSkew(STAMP, gitCounting('3\n'), REPO) expect(result).toEqual({ desktopCommitsBehind: 3, outOfSync: true }) }) - it('passes the stamp range scoped to apps/desktop', async () => { - let seen: string[] = [] - - const git: RunGit = async args => { - seen = args - - return { code: 0, stderr: '', stdout: '0' } - } + it('counts only commits that touch runtime desktop paths', async () => { + const { calls, git } = gitAnswering({ 'merge-base': { code: 0 }, 'rev-list': { stdout: '0' } }) await detectBundleSkew(STAMP, git, REPO) - expect(seen).toEqual(['rev-list', '--count', `${STAMP.commit}..HEAD`, '--', 'apps/desktop']) + expect(calls[1]).toEqual(['rev-list', '--count', `${STAMP.commit}..HEAD`, '--', ...RUNTIME_PATHS]) }) it('is quiet when no desktop commits follow the stamp', async () => { - const result = await detectBundleSkew(STAMP, gitReturning('0\n'), REPO) + const result = await detectBundleSkew(STAMP, gitCounting('0\n'), REPO) expect(result).toEqual({ desktopCommitsBehind: 0, outOfSync: false }) }) @@ -79,9 +108,213 @@ describe('detectBundleSkew', () => { }) it('is quiet on unparsable rev-list output', async () => { - expect(await detectBundleSkew(STAMP, gitReturning('fatal: bad object'), REPO)).toEqual({ + expect(await detectBundleSkew(STAMP, gitCounting('fatal: bad object'), REPO)).toEqual({ desktopCommitsBehind: null, outOfSync: false }) }) + + // #92233: a ZIP-fallback update rewrites the tree into a synthetic root, so + // the stamp commit still RESOLVES but is unreachable from HEAD. `A..HEAD` + // then counts HEAD's own history instead of measuring skew, and reports a + // permanent 1 even though apps/desktop is byte-identical. The user gets an + // "App build out of date" warning that cannot go off, so no remedy clears it. + it('is quiet when the stamp is not an ancestor of HEAD', async () => { + const { git } = gitAnswering({ + 'merge-base': { code: 1 }, + 'rev-list': { stdout: '1\n' } + }) + + expect(await detectBundleSkew(STAMP, git, REPO)).toEqual({ + desktopCommitsBehind: null, + outOfSync: false + }) + }) + + it('does not consult the commit count once ancestry is refused', async () => { + const { calls, git } = gitAnswering({ + 'merge-base': { code: 1 }, + 'rev-list': { stdout: '9999\n' } + }) + + await detectBundleSkew(STAMP, git, REPO) + + expect(calls.map(args => args[0])).toEqual(['merge-base']) + }) + + it('asks about ancestry before counting, against the same stamp', async () => { + const { calls, git } = gitAnswering({ + 'merge-base': { code: 0 }, + 'rev-list': { stdout: '2\n' } + }) + + const result = await detectBundleSkew(STAMP, git, REPO) + + expect(calls[0]).toEqual(['merge-base', '--is-ancestor', STAMP.commit, 'HEAD']) + expect(calls[1]?.[0]).toBe('rev-list') + expect(result).toEqual({ desktopCommitsBehind: 2, outOfSync: true }) + }) + + it('is quiet when git cannot answer the ancestry question at all', async () => { + const { git } = gitAnswering({ + 'merge-base': { code: 128 }, + 'rev-list': { stdout: '4\n' } + }) + + expect(await detectBundleSkew(STAMP, git, REPO)).toEqual({ + desktopCommitsBehind: null, + outOfSync: false + }) + }) + + // Shallow clones, measured against git 2.55 rather than assumed. A stamp + // commit from BEFORE the graft boundary is not an object the clone has, so + // `--is-ancestor` exits 128 with "Not a valid object name" — the same + // unknowable bucket as any other missing commit, not a shallow-specific + // failure. A stamp INSIDE the shallow graph is answered normally, so + // `--fetch-depth`-limited CI checkouts do not lose skew detection wholesale; + // only builds stamped deeper than the checkout goes do. + it('is quiet on a shallow clone whose stamp predates the graft boundary', async () => { + const { calls, git } = gitAnswering({ + 'merge-base': { + code: 128, + stderr: `fatal: Not a valid object name ${STAMP.commit}` + }, + 'rev-list': { stdout: '7\n' } + }) + + expect(await detectBundleSkew(STAMP, git, REPO)).toEqual({ + desktopCommitsBehind: null, + outOfSync: false + }) + expect(calls).toHaveLength(1) + }) + + it('still detects skew on a shallow clone when the stamp is in the graph', async () => { + const { git } = gitAnswering({ + 'merge-base': { code: 0 }, + 'rev-list': { stdout: '2\n' } + }) + + expect(await detectBundleSkew(STAMP, git, REPO)).toEqual({ + desktopCommitsBehind: 2, + outOfSync: true + }) + }) +}) + +// Real-git integration: proves the pathspec discriminates docs/e2e-only +// commits from runtime commits, and that a disconnected stamp goes quiet, in +// an actual repository rather than against a hand-written fake. +const scratchRepos: string[] = [] + +afterAll(() => { + for (const dir of scratchRepos) { + rmSync(dir, { force: true, recursive: true }) + } +}) + +function scratchGit(repoRoot: string) { + return (...args: string[]) => + execFileSync('git', ['-c', 'user.email=skew@test', '-c', 'user.name=skew', ...args], { + cwd: repoRoot, + stdio: ['ignore', 'pipe', 'pipe'] + }) + .toString() + .trim() +} + +function makeScratchRepo(): { base: string; repoRoot: string } { + const repoRoot = mkdtempSync(join(tmpdir(), 'bundle-skew-')) + scratchRepos.push(repoRoot) + + const git = scratchGit(repoRoot) + + git('init', '-q', '-b', 'main') + git('commit', '-q', '--allow-empty', '-m', 'base') + + return { base: git('rev-parse', 'HEAD'), repoRoot } +} + +function writeFiles(repoRoot: string, files: string[]) { + for (const file of files) { + const target = join(repoRoot, file) + + mkdirSync(dirname(target), { recursive: true }) + writeFileSync(target, '') + } +} + +function realGitRun(root: string): RunGit { + return async (args, options) => { + try { + const stdout = execFileSync('git', args, { + cwd: options.cwd || root, + stdio: ['ignore', 'pipe', 'pipe'] + }).toString() + + return { code: 0, stderr: '', stdout } + } catch (error) { + const e = error as { status?: number; stderr?: Buffer; stdout?: Buffer } + + return { + code: e.status ?? 1, + stderr: e.stderr?.toString() ?? '', + stdout: e.stdout?.toString() ?? '' + } + } + } +} + +describe('detectBundleSkew against a real git repo', () => { + it('is quiet when only docs and e2e specs changed under apps/desktop', async () => { + const { base, repoRoot } = makeScratchRepo() + const git = scratchGit(repoRoot) + + writeFiles(repoRoot, ['apps/desktop/AGENTS.md', 'apps/desktop/e2e/boot.spec.ts']) + git('add', '.') + git('commit', '-q', '-m', 'docs and e2e only') + + const result = await detectBundleSkew({ commit: base, source: 'local' }, realGitRun(repoRoot), repoRoot) + + expect(result).toEqual({ desktopCommitsBehind: 0, outOfSync: false }) + }) + + it('warns when a renderer file changed under apps/desktop', async () => { + const { base, repoRoot } = makeScratchRepo() + const git = scratchGit(repoRoot) + + writeFiles(repoRoot, ['apps/desktop/src/app/new-feature.tsx', 'apps/desktop/README.md']) + git('add', '.') + git('commit', '-q', '-m', 'renderer change') + + const result = await detectBundleSkew({ commit: base, source: 'local' }, realGitRun(repoRoot), repoRoot) + + expect(result).toEqual({ desktopCommitsBehind: 1, outOfSync: true }) + }) + + // The #92233 install, reproduced: the update rewrote the tree onto a fresh + // orphan root, so the stamp resolves but is unreachable. Real git answers + // `rev-list` with a positive count here — ancestry is the only thing that + // keeps the banner off. + it('is quiet when the stamp sits on a disconnected root', async () => { + const { base, repoRoot } = makeScratchRepo() + const git = scratchGit(repoRoot) + + git('checkout', '-q', '--orphan', 'rewritten') + writeFiles(repoRoot, ['apps/desktop/src/app/shell.tsx']) + git('add', '.') + git('commit', '-q', '-m', 'synthetic root after a ZIP-fallback update') + + const runGit = realGitRun(repoRoot) + + // Precondition: the raw count this function used to trust is nonzero. + const raw = await runGit(['rev-list', '--count', `${base}..HEAD`, '--', ...RUNTIME_PATHS], { cwd: repoRoot }) + + expect(Number.parseInt(raw.stdout.trim(), 10)).toBeGreaterThan(0) + + const result = await detectBundleSkew({ commit: base, source: 'local' }, runGit, repoRoot) + + expect(result).toEqual({ desktopCommitsBehind: null, outOfSync: false }) + }) }) diff --git a/apps/desktop/electron/bundle-skew.ts b/apps/desktop/electron/bundle-skew.ts index 7c99c006d5..9f872008bd 100644 --- a/apps/desktop/electron/bundle-skew.ts +++ b/apps/desktop/electron/bundle-skew.ts @@ -10,20 +10,33 @@ * Bot Mode update" reports). * * Detection: the packaged build carries install-stamp.json with the commit - * it was built from. If commits touching `apps/desktop/` exist in the source - * tree AFTER that stamp commit, the running renderer is provably missing - * desktop changes the installed runtime has: + * it was built from. If commits touching the RUNTIME paths of apps/desktop + * exist in the source tree AFTER that stamp commit, the running renderer is + * provably missing desktop changes the installed runtime has: * - * git rev-list --count ..HEAD -- apps/desktop + * git merge-base --is-ancestor HEAD + * git rev-list --count ..HEAD -- * - * Scoping to `apps/desktop/` keeps this quiet for the common case where the - * repo advances with agent-only changes — a shell built before those is not - * stale in any way the user can see. + * Ancestry has to come first, because `A..HEAD` only means "how far HEAD is + * ahead of A" when A is an ancestor of HEAD. When it is not, the range + * degenerates to HEAD's own history and the count stops describing skew at + * all: an update that rewrote the tree into a synthetic root leaves a stamp + * commit that still resolves but sits on a disconnected graph, so the count + * is a permanent >= 1 even when apps/desktop is byte-identical (#92233). + * Resolving the stamp is not enough — an unknown commit already exits + * non-zero below, but a merely *unrelated* one exits 0 with a positive count. + * + * Scoping to runtime paths keeps this quiet for the common cases where the + * repo advances without user-visible desktop changes: agent-only commits + * elsewhere in the repo, and docs / e2e spec / dev-script churn under + * apps/desktop that never reaches the shipped renderer or main process + * (#99832). * * Fail-quiet by design: no stamp (dev runs), a fallback all-zero stamp * (non-git build), an unknown commit (stamp predates a shallow clone's - * history), or any git failure all report "not stale". This warning must - * never false-positive — it tells users their install is torn. + * history), a stamp that is not an ancestor of HEAD, or any git failure all + * report "not stale". This warning must never false-positive — it tells + * users their install is torn. * * Pure + injectable so it is testable without booting Electron or git. */ @@ -35,7 +48,7 @@ export interface BundleSkewStamp { } export interface BundleSkewResult { - /** Commits under apps/desktop/ between the build stamp and HEAD (null = unknowable). */ + /** Runtime-path commits between the build stamp and HEAD (null = unknowable). */ desktopCommitsBehind: null | number /** True only on positive proof that the renderer predates desktop changes in the tree. */ outOfSync: boolean @@ -46,6 +59,23 @@ export type RunGit = ( options: { cwd: string } ) => Promise<{ code: number; stderr: string; stdout: string }> +/** + * The apps/desktop paths that actually reach the user: renderer sources, + * main-process sources, the HTML entry, the public/ assets Vite copies into + * the bundle, app icons, and the packaging config. Docs, e2e specs, scratch + * scripts, and dev tooling never reach the shipped app, so a delta confined + * to them is not a torn install in any way the user can see. + */ +export const RUNTIME_PATHS = [ + 'apps/desktop/src', + 'apps/desktop/electron', + 'apps/desktop/index.html', + 'apps/desktop/public', + 'apps/desktop/assets', + 'apps/desktop/package.json', + 'apps/desktop/vite.config.ts' +] as const + const NOT_STALE: BundleSkewResult = { desktopCommitsBehind: null, outOfSync: false } /** Matches write-build-stamp.mjs's all-zero placeholder for non-git builds. */ @@ -63,7 +93,26 @@ export async function detectBundleSkew( } try { - const result = await runGit(['rev-list', '--count', `${stamp.commit}..HEAD`, '--', 'apps/desktop'], { + // Exit 0 = ancestor, 1 = unrelated or diverged, anything else = git could + // not answer (unknown object, shallow clone, not a repo). Only the first + // makes the commit count below a statement about skew, and the other two + // are the same "unknowable" the branches above already answer quietly. + // + // Deliberately not falling back to comparing apps/desktop CONTENT here. + // Differing content would prove the build and the tree disagree, but not + // which way round: a user sitting on an older checkout than their build + // would be told "app build out of date" backwards. Ancestry is what makes + // this a proof that the renderer PREDATES the tree, which is the claim the + // warning actually makes. + const ancestry = await runGit(['merge-base', '--is-ancestor', stamp.commit, 'HEAD'], { + cwd: repoRoot + }) + + if (ancestry.code !== 0) { + return NOT_STALE + } + + const result = await runGit(['rev-list', '--count', `${stamp.commit}..HEAD`, '--', ...RUNTIME_PATHS], { cwd: repoRoot }) diff --git a/apps/desktop/electron/bundle-swap.test.ts b/apps/desktop/electron/bundle-swap.test.ts new file mode 100644 index 0000000000..8640b0657f --- /dev/null +++ b/apps/desktop/electron/bundle-swap.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' + +import { detectBundleSwap } from './bundle-swap' + +const RUNNING = { builtAt: '2026-08-29T04:00:00.000Z', commit: 'a'.repeat(40), source: 'local' } + +describe('detectBundleSwap', () => { + it('reports a swap when the on-disk stamp carries a different commit', () => { + const onDisk = { ...RUNNING, commit: 'b'.repeat(40) } + + expect(detectBundleSwap(RUNNING, onDisk)).toBe(true) + }) + + it('reports a swap when the same commit was rebuilt (builtAt moved)', () => { + const onDisk = { ...RUNNING, builtAt: '2026-08-31T23:55:41.149Z' } + + expect(detectBundleSwap(RUNNING, onDisk)).toBe(true) + }) + + // The Windows locked-binary case (#92233): the swap leg failed, so the + // bundle on disk is still the one we are running. A relaunch would repair + // nothing and cost the user their window. + it('is quiet when the on-disk stamp matches the running one', () => { + expect(detectBundleSwap(RUNNING, { ...RUNNING })).toBe(false) + }) + + it('is quiet without a running stamp (dev runs)', () => { + expect(detectBundleSwap(null, { ...RUNNING })).toBe(false) + }) + + it('is quiet without an on-disk stamp (unreadable resources)', () => { + expect(detectBundleSwap(RUNNING, null)).toBe(false) + }) + + it('is quiet on a fallback stamp on either side (non-git build)', () => { + const fallbackTagged = { ...RUNNING, source: 'fallback' } + const fallbackCommit = { ...RUNNING, commit: '0'.repeat(40) } + + expect(detectBundleSwap(fallbackTagged, { ...RUNNING, commit: 'b'.repeat(40) })).toBe(false) + expect(detectBundleSwap(RUNNING, fallbackCommit)).toBe(false) + }) + + it('treats a missing builtAt on either side as unprovable at the same commit', () => { + const noBuiltAt = { commit: RUNNING.commit, source: 'local' } + + expect(detectBundleSwap(noBuiltAt, { ...RUNNING })).toBe(false) + expect(detectBundleSwap(RUNNING, noBuiltAt)).toBe(false) + }) +}) diff --git a/apps/desktop/electron/bundle-swap.ts b/apps/desktop/electron/bundle-swap.ts new file mode 100644 index 0000000000..48f2b3af8a --- /dev/null +++ b/apps/desktop/electron/bundle-swap.ts @@ -0,0 +1,61 @@ +/** + * Swapped-bundle detection. + * + * The detached updater (scripts/desktop-update/posix.sh mac_swap / + * windows.ps1) rebuilds and swaps the packaged app on disk AFTER + * `hermes update` exits. An instance that was launched from the PRE-swap + * bundle — the user reopened Hermes mid-update, the #50238 gesture the boot + * gate exists for — would otherwise proceed to run the NEW runtime under the + * OLD renderer. The updater's own `open`/relaunch leg cannot rescue it: the + * single-instance lock turns that into a focus of the parked process, so no + * process ever loads the new build. + * + * That is the stale-renderer tail of a FULLY SUCCESSFUL update: the "App + * build out of date" banner appears right after the update, while the Updates + * card says "You're on the latest version" and so offers nothing that would + * clear it. + * + * Detection: compare the install stamp this process loaded at boot with the + * one on disk now. A different commit — or a different builtAt at the same + * commit (a dirty-tree or content-hash rebuild) — means the bundle under our + * feet is not the one we are running, and a plain relaunch loads it. + * + * Fail-quiet like bundle-skew: a missing stamp on either side (dev runs, + * unreadable resources) or a fallback all-zero commit reports "not swapped". + * This must never false-positive — a positive triggers an automatic relaunch. + * + * Pure so it is testable without booting Electron. + */ + +import { isFallbackCommit } from './bundle-skew' + +export interface BundleSwapStamp { + /** write-build-stamp.mjs build timestamp — differs on every rebuild. */ + builtAt?: null | string + commit: string + /** write-build-stamp.mjs source tag — 'fallback' means the commit is fake. */ + source?: null | string +} + +/** True only on positive proof that the bundle on disk is not the running one. */ +export function detectBundleSwap(running: BundleSwapStamp | null, onDisk: BundleSwapStamp | null): boolean { + if (!running?.commit || !onDisk?.commit) { + return false + } + + if (running.source === 'fallback' || isFallbackCommit(running.commit)) { + return false + } + + if (onDisk.source === 'fallback' || isFallbackCommit(onDisk.commit)) { + return false + } + + if (running.commit !== onDisk.commit) { + return true + } + + // Same commit: only a builtAt PRESENT ON BOTH sides can prove a rebuild — + // a missing timestamp (older stamp schema) proves nothing. + return Boolean(running.builtAt && onDisk.builtAt && running.builtAt !== onDisk.builtAt) +} diff --git a/apps/desktop/electron/connection-registry.test.ts b/apps/desktop/electron/connection-registry.test.ts index f0af94e759..261671b816 100644 --- a/apps/desktop/electron/connection-registry.test.ts +++ b/apps/desktop/electron/connection-registry.test.ts @@ -697,10 +697,23 @@ test('registry local route: v1 REMOTE global mode forces a genuinely-local backe assert.notEqual(route.poolKey, backendScopeKey(LOCAL_CONNECTION_ID, 'default')) }) -test('registry local route: a per-profile remote override also forces local', () => { +test('registry local route: a per-profile remote override delegates to the override (#90477)', () => { + // The per-profile SSH/remote override is the authoritative route for that + // profile. Forcing local here made the roster list the profile via its + // override but open the thread in a local child — which fails when the + // profile exists only on the remote. The override must win. const route = resolveRegistryLocalRoute('research', { profileRemoteOverride: true }) - assert.deepEqual(route, { delegate: false, poolKey: 'conn:local::research' }) + assert.deepEqual(route, { delegate: true, poolKey: 'research' }) +}) + +test('registry local route: per-profile override wins when global remote is also active', () => { + const route = resolveRegistryLocalRoute('research', { + globalRemote: true, + profileRemoteOverride: true + }) + + assert.deepEqual(route, { delegate: true, poolKey: 'research' }) }) // --- shouldDeferLocalEnumeration (roster's connect-on-demand for 'local') --- diff --git a/apps/desktop/electron/connection-registry.ts b/apps/desktop/electron/connection-registry.ts index 8f72948f65..2875f53c66 100644 --- a/apps/desktop/electron/connection-registry.ts +++ b/apps/desktop/electron/connection-registry.ts @@ -493,7 +493,17 @@ export function resolveRegistryLocalRoute( ): RegistryLocalRoute { const profileKey = String(profile ?? '').trim() || 'default' - if (opts.globalRemote || opts.profileRemoteOverride) { + // A per-profile SSH/remote override is an explicit per-profile routing + // decision: the override owns this profile's backend, so the 'local' entry + // must delegate to the legacy profile route (which resolves the override), + // not spawn a forced-local child. Forcing local here is the #90477 split: + // the roster lists the profile via its override, but opening the thread + // spawned a local backend that fails when the profile doesn't exist locally. + if (opts.profileRemoteOverride) { + return { delegate: true, poolKey: profileKey } + } + + if (opts.globalRemote) { return { delegate: false, poolKey: `${backendScopePrefix(LOCAL_CONNECTION_ID)}${profileKey}` } } diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index 931df474a6..22fa5eb40f 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -42,7 +42,8 @@ import { isPidOnlyStartMarker, pidOnlyStartMarker, probeStartMarker, - processStartMarker + processStartMarker, + REAP_PROBE_TIMEOUT_MS } from './backend-claim' import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command' import { createBackendConnectionState } from './backend-connection-state' @@ -88,6 +89,7 @@ import { buildBrowserWindowUrl } from './browser-windows' import { detectBundleSkew } from './bundle-skew' +import { detectBundleSwap } from './bundle-swap' import { applyConnectionChange, sshQuitShouldBlock, teardownSshState } from './connection-apply' import { apiRequestRegistryConnectionId, @@ -289,6 +291,12 @@ import { undialedSshRouteSeeds } from './plugin-profile-routes' import { selectPoolEvictions } from './pool-eviction' +import { clampPoolLimits, parsePoolLimits, POOL_LIMITS_DEFAULTS } from './pool-limits' +import { + LocalBackendSpawnCoordinator, + type LocalBackendSpawnRequest, + releaseLocalBackendSlotAfterExit +} from './pool-spawn-coordinator' import { createPoolStopper } from './pool-stop' import { poolTouchKeys } from './pool-touch-scope' import { createKeepAwake } from './power-save' @@ -1427,8 +1435,97 @@ const profileDeletionGate = new ProfileDeletionGate() // Keep the pool light: cap concurrent profile backends (LRU eviction) and reap // idle ones. A user idles at exactly the primary backend; pool backends only // exist while a non-primary profile is actively being chatted through. -const POOL_MAX_BACKENDS = Math.max(1, Number(process.env.HERMES_DESKTOP_POOL_MAX) || 3) -const POOL_IDLE_MS = Math.max(60_000, Number(process.env.HERMES_DESKTOP_POOL_IDLE_MS) || 10 * 60_000) +// Pool sizing is a device preference (Settings → Advanced → pool rows), not a +// launch constant: mutable at runtime, persisted in userData, applied live. +// The legacy HERMES_DESKTOP_POOL_* env vars remain the initial-value fallback +// for scripted/headless setups; after launch the stored preference wins. +const POOL_LIMITS_PATH = path.join(app.getPath('userData'), 'pool-limits.json') + +function readPersistedPoolLimits() { + try { + const limits = parsePoolLimits(fs.readFileSync(POOL_LIMITS_PATH, 'utf8')) + rememberLog( + `[pool-limits] loaded from ${POOL_LIMITS_PATH}: maxBackends=${limits.maxBackends}, idleMs=${limits.idleMs}` + ) + + return limits + } catch { + // No persisted file yet — fall back to the legacy env vars so scripted + // setups keep working. Log which source won: a silently-ignored env var + // here costs a scripted-setup user a debugging session. + const fromEnv = clampPoolLimits({ + maxBackends: Number(process.env.HERMES_DESKTOP_POOL_MAX) || undefined, + idleMs: Number(process.env.HERMES_DESKTOP_POOL_IDLE_MS) || undefined + }) + + if (fromEnv.maxBackends !== POOL_LIMITS_DEFAULTS.maxBackends || fromEnv.idleMs !== POOL_LIMITS_DEFAULTS.idleMs) { + rememberLog( + `[pool-limits] no saved file; using env-var overrides: maxBackends=${fromEnv.maxBackends}, idleMs=${fromEnv.idleMs}` + ) + } else { + rememberLog('[pool-limits] no saved file and no env overrides; using defaults') + } + + return fromEnv + } +} + +function persistPoolLimits(limits) { + try { + fs.mkdirSync(path.dirname(POOL_LIMITS_PATH), { recursive: true }) + // Atomic write: write to a temp file in the same directory, then rename. + // A crash mid-write would otherwise leave truncated JSON and silently + // lose the user's saved sizing. + const tmpPath = `${POOL_LIMITS_PATH}.tmp` + fs.writeFileSync(tmpPath, JSON.stringify(limits, null, 2), 'utf8') + fs.renameSync(tmpPath, POOL_LIMITS_PATH) + } catch (error) { + rememberLog(`[pool-limits] write failed: ${error.message}`) + } +} + +// rememberLog() state. Declared here, before the top-level +// readPersistedPoolLimits() call below, because that call logs during module +// evaluation; declaring these later crashed launch with `undefined.push` in +// the packaged build (esbuild lowers the TDZ to undefined instead of throwing). +const hermesLog = [] +let desktopLogBuffer = '' +let desktopLogFlushTimer = null +let desktopLogFlushPromise = Promise.resolve() + +let poolLimits = readPersistedPoolLimits() +// Hard cap on local backends that are starting OR running (the LRU eviction +// above is soft — it spares keepalive-fresh entries). Follows the live +// preference: setPoolLimits() pushes a new max into the coordinator. +const localBackendSpawnCoordinator = new LocalBackendSpawnCoordinator(poolLimits.maxBackends) +// How long a spawn may wait for a free local slot. Must stay under the +// renderer's BACKEND_BOOT_WAIT_TIMEOUT_MS (45s, src/lib/with-timeout.ts) so +// the queued ticket fails before the renderer does and the user sees why. +const POOL_SLOT_WAIT_MS = 30_000 + +function poolMaxBackends() { + return poolLimits.maxBackends +} + +function poolIdleMs() { + return poolLimits.idleMs +} + +/** + * Apply new limits live: persist, then converge the running pool — evict + * LRU backends down to the new max, and let the (already running) idle + * reaper handle a shortened idle window on its next tick. Returns the + * limits actually in force (post-clamp). + */ +function setPoolLimits(raw) { + poolLimits = clampPoolLimits(raw) + persistPoolLimits(poolLimits) + localBackendSpawnCoordinator.setLimit(poolLimits.maxBackends) + evictLruPoolBackends(poolMaxBackends()) + startPoolIdleReaper() + + return { ...poolLimits } +} // A backend touched within this window has a live renderer socket (the keepalive // pings every 60s for every open profile). LRU eviction must spare these — a @@ -1448,7 +1545,7 @@ const POOL_IDLE_MS = Math.max(60_000, Number(process.env.HERMES_DESKTOP_POOL_IDL // re-allocating pooled gateway secondaries ~700×/day). // * 3× ping + 60s headroom = ~4 min, comfortable margin for two missed // pings + WSL2 IPC stall. The hard ceiling for the cap-eligible set is -// POOL_IDLE_MS above (default 10 min) — this constant only governs the +// pool idle window above (default 10 min) — this constant only governs the // "is this backend plausibly still alive" question for LRU eviction, // not when the idle reaper definitively tears a backend down. const POOL_KEEPALIVE_FRESH_MS = Math.max( @@ -1506,12 +1603,8 @@ let connectionRegistryCache = null let connectionRegistryCacheMtime = null let remoteHeaderRulesInstalled = false const remoteWsHeaderStore = createRemoteWsHeaderStore() -const hermesLog = [] const previewWatchers = new Map() let previewShortcutActive = false -let desktopLogBuffer = '' -let desktopLogFlushTimer = null -let desktopLogFlushPromise = Promise.resolve() let nativeThemeListenerInstalled = false let bootProgressState = { @@ -2134,6 +2227,59 @@ function updateGateDeps() { } } +// One-shot guard for the automatic bundle-swap relaunch below: the relaunched +// instance carries this flag so a stamp that still mismatches (unreadable +// resources, exotic packaging) can never produce a relaunch loop. +const BUNDLE_SWAP_RELAUNCH_FLAG = '--hermes-bundle-swap-relaunched' + +// How long the parked instance waits for its own scheduled exit to land before +// giving up and booting the stale build anyway. Better a torn renderer with a +// banner than a window that never comes back. +const BUNDLE_SWAP_RELAUNCH_FAILSAFE_MS = 15_000 + +// The detached updater swaps the packaged bundle on disk AFTER `hermes update` +// exits (posix.sh mac_swap / windows.ps1). An instance reopened mid-update — +// the #50238 gesture the gate above exists for — was launched from the +// PRE-swap bundle, and the updater's `open` leg then merely focuses us (single +// instance), so no process ever loads the new build. Letting boot proceed here +// runs the new runtime under the old renderer: exactly the skew +// detectRendererSkew() warns about, except the Updates card already says +// "latest", so the warning's own remedy has nothing to run. +// +// This is the earliest point where the swap is PROVABLE — it happens while we +// are parked on the gate, so checking any sooner (at `ready`, before the gate) +// only ever compares a stamp with itself. Relaunching here also keeps the +// boot-progress window up for the whole wait instead of leaving the user with +// no window at all. +// +// Returns true when the relaunch was scheduled; the caller must park rather +// than continue booting, because the process exits underneath it. +function relaunchIntoSwappedBundle() { + if (!IS_PACKAGED || process.argv.includes(BUNDLE_SWAP_RELAUNCH_FLAG)) { + return false + } + + if (!detectBundleSwap(INSTALL_STAMP, loadInstallStamp())) { + return false + } + + rememberLog('[updates] app bundle was swapped during the update; relaunching into the new build') + + try { + app.relaunch({ + args: [...buildNoSandboxRelaunchArgs(process.argv.slice(1)), BUNDLE_SWAP_RELAUNCH_FLAG] + }) + } catch (err) { + rememberLog(`[updates] bundle-swap relaunch failed: ${err?.message || err}; continuing with the current build`) + + return false + } + + void exitAfterBackendShutdown(0) + + return true +} + // Block until no live update is in progress (or we hit the wait timeout). // Emits a boot-progress phase so the renderer shows "Update in progress…" // rather than a frozen splash. Returns true if it parked at all. @@ -2196,6 +2342,14 @@ async function waitForUpdateToFinish() { if (outcome === 'timeout') { rememberLog('[updates] update still in progress after wait timeout; starting backend anyway') + } else if (relaunchIntoSwappedBundle()) { + await advanceBootProgress('backend.update-restart', 'Restarting Hermes to load the updated app…', 14) + // Park while the scheduled exit lands so this stale build never starts a + // backend; the failsafe below only runs if the exit somehow does not. + await new Promise(resolve => setTimeout(resolve, BUNDLE_SWAP_RELAUNCH_FAILSAFE_MS)) + rememberLog( + `[updates] relaunch did not land within ${BUNDLE_SWAP_RELAUNCH_FAILSAFE_MS}ms; continuing with the current build` + ) } else { rememberLog('[updates] update finished; proceeding with backend start') } @@ -3480,7 +3634,7 @@ async function backendCommandForPid(pid) { } } -async function processIdentityMatches(identity) { +async function processIdentityMatches(identity, timeoutMs: number = 30_000) { // Degraded PID-only identity (#93608): the start-marker probe failed while // the child was verifiably alive, so only PID liveness can be checked here. // backendIdentityMatches layers the command-line check on top before @@ -3498,14 +3652,14 @@ async function processIdentityMatches(identity) { } try { - return (await processStartMarker(identity.pid)) === identity.startMarker + return (await processStartMarker(identity.pid, timeoutMs)) === identity.startMarker } catch (error) { return error?.code === 'ENOENT' || error?.code === 'ESRCH' ? false : undefined } } async function backendIdentityMatches(identity) { - const processMatches = await processIdentityMatches(identity) + const processMatches = await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS) if (processMatches !== true) { return processMatches @@ -3526,17 +3680,26 @@ async function backendParentMatches(entry) { } try { - return (await processStartMarker(entry.parentPid)) === entry.parentStartMarker + return (await processStartMarker(entry.parentPid, REAP_PROBE_TIMEOUT_MS)) === entry.parentStartMarker } catch (error) { return error?.code === 'ENOENT' || error?.code === 'ESRCH' ? false : undefined } } async function stopOwnedBackend(identity) { - if ((await processIdentityMatches(identity)) !== true) { + const matches = await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS) + + if (matches === false) { return } + if (matches !== true) { + // Identity probe failed (not confirmed gone): preserve the record so a + // later launch retries the stop instead of dropping it and leaking the + // backend. reapOrphans keeps the entry when stop() throws. + throw new Error(`Could not verify backend PID ${identity.pid} before stopping it.`) + } + if (IS_WINDOWS) { forceKillProcessTree(identity.pid) } else { @@ -3553,7 +3716,7 @@ async function stopOwnedBackend(identity) { const deadline = Date.now() + 1500 while (Date.now() < deadline) { - if ((await processIdentityMatches(identity)) !== true) { + if ((await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)) !== true) { return } @@ -3562,7 +3725,7 @@ async function stopOwnedBackend(identity) { // Revalidate immediately before escalation so PID reuse cannot target a // replacement process. - if ((await processIdentityMatches(identity)) === true) { + if ((await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)) === true) { try { process.kill(-identity.pid, 'SIGKILL') } catch { @@ -3572,7 +3735,7 @@ async function stopOwnedBackend(identity) { } await new Promise(resolve => setTimeout(resolve, 50)) - const remaining = await processIdentityMatches(identity) + const remaining = await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS) if (remaining !== false) { throw new Error(`Backend PID ${identity.pid} did not stop cleanly.`) @@ -11481,7 +11644,7 @@ async function ensureBackend(profile) { return connection } - evictLruPoolBackends(POOL_MAX_BACKENDS - 1) + evictLruPoolBackends(poolMaxBackends() - 1) const entry = { process: null, @@ -11489,7 +11652,10 @@ async function ensureBackend(profile) { token: null, connectionPromise: null, lastActiveAt: Date.now(), - remoteBaseUrl: null + remoteBaseUrl: null, + releaseLocalBackendSlot: null, + localBackendSlotKey: null, + localBackendSpawnRequest: null } entry.connectionPromise = spawnPoolBackend(key, entry).catch(async error => { @@ -11500,12 +11666,7 @@ async function ensureBackend(profile) { `Hermes backend for profile "${key}" failed to start: ${error instanceof Error ? error.message : String(error)}` ) - if (backendPool.get(key) === entry) { - backendPool.delete(key) - } - - stopBackendChild(entry.process) - await waitForBackendExit(entry.process) + await teardownFailedLocalBackend(key, entry) throw error }) backendPool.set(key, entry) @@ -11649,7 +11810,7 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela return existingLocal.connectionPromise } - evictLruPoolBackends(POOL_MAX_BACKENDS - 1) + evictLruPoolBackends(poolMaxBackends() - 1) const localEntry = { process: null, @@ -11657,7 +11818,10 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela token: null, connectionPromise: null, lastActiveAt: Date.now(), - remoteBaseUrl: null + remoteBaseUrl: null, + releaseLocalBackendSlot: null, + localBackendSlotKey: null, + localBackendSpawnRequest: null } localEntry.connectionPromise = spawnPoolBackend(profileKey, localEntry, { @@ -11670,12 +11834,7 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela `Hermes backend for profile "${profileKey}" (forced-local) failed to start: ${error instanceof Error ? error.message : String(error)}` ) - if (backendPool.get(localRoute.poolKey) === localEntry) { - backendPool.delete(localRoute.poolKey) - } - - stopBackendChild(localEntry.process) - await waitForBackendExit(localEntry.process) + await teardownFailedLocalBackend(localRoute.poolKey, localEntry) throw error }) backendPool.set(localRoute.poolKey, localEntry) @@ -11722,7 +11881,7 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela ) } - evictLruPoolBackends(POOL_MAX_BACKENDS - 1) + evictLruPoolBackends(poolMaxBackends() - 1) const entry = { process: null, @@ -12377,7 +12536,7 @@ function evictLruPoolBackends(keep) { const evictions = selectPoolEvictions(backendPool.entries(), Math.max(0, keep), Date.now(), POOL_KEEPALIVE_FRESH_MS) for (const profile of evictions) { - rememberLog(`Evicting idle profile backend "${profile}" (LRU cap ${POOL_MAX_BACKENDS})`) + rememberLog(`Evicting idle profile backend "${profile}" (LRU cap ${poolMaxBackends()})`) stopPoolBackend(profile) } } @@ -12391,8 +12550,8 @@ function startPoolIdleReaper() { const now = Date.now() for (const [profile, entry] of [...backendPool.entries()]) { - if (now - (entry.lastActiveAt || 0) > POOL_IDLE_MS) { - rememberLog(`Reaping idle profile backend "${profile}" (idle > ${Math.round(POOL_IDLE_MS / 1000)}s)`) + if (now - (entry.lastActiveAt || 0) > poolIdleMs()) { + rememberLog(`Reaping idle profile backend "${profile}" (idle > ${Math.round(poolIdleMs() / 1000)}s)`) stopPoolBackend(profile) } } @@ -12408,6 +12567,67 @@ function startPoolIdleReaper() { } } +function releaseLocalBackendSlot(entry: any) { + if (!entry) { + return + } + + const release = entry.releaseLocalBackendSlot + const request = entry.localBackendSpawnRequest as LocalBackendSpawnRequest | null + entry.releaseLocalBackendSlot = null + entry.localBackendSlotKey = null + entry.localBackendSpawnRequest = null + + if (release) { + release() + } else { + request?.cancel() + } +} + +function assertPoolEntryStillOwned(poolKey: string, entry: any) { + if (backendPool.get(poolKey) !== entry) { + releaseLocalBackendSlot(entry) + throw new Error(`Profile backend start for "${poolKey}" was cancelled before spawn.`) + } +} + +const failedLocalBackendTeardowns = new WeakMap>() + +function teardownFailedLocalBackend(poolKey: string, entry: any): Promise { + const existing = failedLocalBackendTeardowns.get(entry) + + if (existing) { + return existing + } + + if (backendPool.get(poolKey) === entry) { + backendPool.delete(poolKey) + } + + const child = entry.process + + const teardown = releaseLocalBackendSlotAfterExit( + () => releaseLocalBackendSlot(entry), + async () => { + stopBackendChild(child) + await waitForBackendExit(child) + + if (child && child.exitCode === null && child.signalCode === null) { + throw new Error(`Profile backend for "${poolKey}" did not exit; keeping the local slot occupied.`) + } + + releaseBackendChild(child) + } + ) + + // Keep the settled promise in the WeakMap for the lifetime of this entry. + // Error + exit + outer catch may all request cleanup; none may run it twice. + failedLocalBackendTeardowns.set(entry, teardown) + + return teardown +} + // Spawn an additional dashboard backend pinned to a named profile. Mirrors the // local-spawn portion of startHermes() but without the boot-progress UI, // bootstrap, or remote handling (those belong to the primary backend only). @@ -12445,6 +12665,29 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po } } + // Bound the slot wait BELOW the renderer's backend-boot budget (45s): once + // the renderer has given up on this spawn, a ticket still queued for the + // pool-idle window (10 min) would hold the pool key hostage and every + // later click on the profile would join that stale wait. Failing here + // surfaces the "all N slots busy" reason instead of a generic boot timeout. + const spawnRequest = localBackendSpawnCoordinator.request(poolKey, { timeoutMs: POOL_SLOT_WAIT_MS }) + entry.localBackendSlotKey = poolKey + entry.localBackendSpawnRequest = spawnRequest + + if (localBackendSpawnCoordinator.activeCount >= poolMaxBackends()) { + rememberLog( + `Profile backend "${profile}" waiting for a free local slot (${localBackendSpawnCoordinator.activeCount}/${poolMaxBackends()} busy, ${localBackendSpawnCoordinator.queuedCount} queued)` + ) + } + + entry.releaseLocalBackendSlot = await spawnRequest.acquired + + if (entry.localBackendSpawnRequest === spawnRequest) { + entry.localBackendSpawnRequest = null + } + + assertPoolEntryStillOwned(poolKey, entry) + const token = crypto.randomBytes(32).toString('base64url') // Same update mutual exclusion as the primary window's waitForLocalStart @@ -12488,12 +12731,12 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po assertLocalProfileCanStart(profile, profileDeletionGate, key => directoryExists(path.join(HERMES_HOME, 'profiles', key)) ) - rememberLog(`Starting Hermes backend for profile "${profile}" via ${backend.label}`) const parentStartMarker = await desktopParentStartMarker() const backendNonce = crypto.randomBytes(16).toString('hex') const parentIdentityEnv = parentWatchdogEnv(process.pid, parentStartMarker, backendNonce) + assertPoolEntryStillOwned(poolKey, entry) const child = spawn( backend.command, @@ -12548,6 +12791,7 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po // surface as an unhandled rejection before the Promise.race below attaches. portAnnouncement.catch(() => {}) await claimBackendChild(child, `${backend.command} ${backend.args.join(' ')}`, profile, backendNonce, outputTail) + assertPoolEntryStillOwned(poolKey, entry) child.stdout.on('data', rememberLog) child.stderr.on('data', rememberLog) @@ -12561,14 +12805,21 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po child.once('error', error => { rememberLog(`Hermes backend for profile "${profile}" failed to start: ${error.message}`) - releaseBackendChild(child) - backendPool.delete(poolKey) + void teardownFailedLocalBackend(poolKey, entry).catch(cleanupError => { + rememberLog( + `Hermes backend for profile "${profile}" cleanup failed: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}` + ) + }) rejectStart?.(error) }) child.once('exit', (code, signal) => { rememberLog(`Hermes backend for profile "${profile}" exited (${signal || code})`) + releaseLocalBackendSlot(entry) releaseBackendChild(child) - backendPool.delete(poolKey) + + if (backendPool.get(poolKey) === entry) { + backendPool.delete(poolKey) + } if (!ready) { rejectStart?.( @@ -12636,16 +12887,20 @@ const poolStopper = createPoolStopper({ waitForExit: child => waitForBackendExit(child) }) -function stopPoolBackend(profile) { - return poolStopper.stop(profile) +async function stopPoolBackend(profile: string) { + const entry = backendPool.get(profile) + await poolStopper.stop(profile) + releaseLocalBackendSlot(entry) } async function teardownPoolBackendAndWait(profile) { - await Promise.all(localProfilePoolKeys(profile).map(key => poolStopper.stop(key))) + await Promise.all(localProfilePoolKeys(profile).map(key => stopPoolBackend(key))) } -function stopAllPoolBackends() { - return poolStopper.stopAll() +async function stopAllPoolBackends() { + const entries = [...backendPool.values()] + await poolStopper.stopAll() + entries.forEach(releaseLocalBackendSlot) } /** @@ -13337,7 +13592,11 @@ function focusWindow(win) { win.focus() } -function spawnSecondaryWindow({ sessionId, watch }: { sessionId?: string; watch?: boolean } = {}) { +function spawnSecondaryWindow({ + sessionId, + profile, + watch +}: { sessionId?: string; profile?: null | string; watch?: boolean } = {}) { const icon = getAppIconPath() const win = new BrowserWindow({ @@ -13399,6 +13658,7 @@ function spawnSecondaryWindow({ sessionId, watch }: { sessionId?: string; watch? win, buildSessionWindowUrl(sessionId, { devServer: DEV_SERVER, + profile, rendererIndexPath: DEV_SERVER ? undefined : resolveRendererIndex(), watch }), @@ -13409,8 +13669,8 @@ function spawnSecondaryWindow({ sessionId, watch }: { sessionId?: string; watch? } // Open (or focus) a standalone window for a single chat session. -function createSessionWindow(sessionId, { watch = false } = {}) { - return sessionWindows.openOrFocus(sessionId, () => spawnSecondaryWindow({ sessionId, watch })) +function createSessionWindow(sessionId, { profile = null, watch = false } = {}) { + return sessionWindows.openOrFocus(sessionId, () => spawnSecondaryWindow({ sessionId, profile, watch })) } // Popped-out in-app Browser: same webview + address bar as a docked Browser @@ -14889,6 +15149,18 @@ ipcMain.handle('hermes:backend:touch', async (_event, profile) => { return { ok: true } }) +// Pool sizing (Settings → Advanced): device-local, live-applied. Main is +// authoritative (it owns the pool and the persisted copy); the returned +// limits are what actually took effect post-clamp. +ipcMain.handle('hermes:pool-limits:get', async () => ({ ...poolLimits })) +ipcMain.handle('hermes:pool-limits:set', async (_event, raw) => { + const next = setPoolLimits({ + maxBackends: typeof raw?.maxBackends === 'number' ? raw.maxBackends : poolLimits.maxBackends, + idleMs: typeof raw?.idleMs === 'number' ? raw.idleMs : poolLimits.idleMs + }) + + return { ok: true, limits: next } +}) ipcMain.handle('hermes:gateway:ws-url', async (_event, profile) => { return gatewayWsUrlIpcResult(() => freshGatewayWsUrl(profile)) }) @@ -14897,7 +15169,10 @@ ipcMain.handle('hermes:window:openSession', async (_event, sessionId, opts) => { return { ok: false, error: 'invalid-session-id' } } - createSessionWindow(sessionId.trim(), { watch: opts?.watch === true }) + createSessionWindow(sessionId.trim(), { + profile: typeof opts?.profile === 'string' ? opts.profile : null, + watch: opts?.watch === true + }) return { ok: true } }) @@ -17604,10 +17879,26 @@ ipcMain.handle('hermes:version', async () => { // Bundled artifacts always run their payload; light artifacts have no // runtime and only reach remote backends. External builds classify from // the install stamp (git/docker/nix), 'unknown' when it can't be told. - hermesRuntime: resolveHermesRuntime() + hermesRuntime: resolveHermesRuntime(), + // True when the bundle on disk is not the one this process loaded — a + // plain app restart (no rebuild, no installer) clears the skew above. + // Packaged only: a dev `--build-only` rewrites build/install-stamp.json + // under a running `npm start`, which is a rebuild the developer asked for, + // not a torn install to offer a restart for. + bundleSwapPending: IS_PACKAGED && detectBundleSwap(INSTALL_STAMP, loadInstallStamp()) } }) +// The About page's "Restart Hermes" button (shown when bundleSwapPending): +// load the already-swapped bundle without asking the user to quit manually. +// app.relaunch() re-executes by path, so the fresh process picks up whatever +// bundle now lives there. +ipcMain.handle('hermes:app:relaunch', async () => { + rememberLog('[updates] renderer requested an app relaunch (swapped bundle pending)') + app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) + void exitAfterBackendShutdown(0) +}) + /** The latest pm/venv/plugin-operation receipt — the machine-readable * surface every medium reads (CLI: `hermes pm status`). Returned as one * parsed JSON object: { kind, outcome, venv_rebuild, plugin_bisect, diff --git a/apps/desktop/electron/pool-limits.test.ts b/apps/desktop/electron/pool-limits.test.ts new file mode 100644 index 0000000000..5aff45f088 --- /dev/null +++ b/apps/desktop/electron/pool-limits.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' + +import { + clampPoolLimits, + parsePoolLimits, + POOL_LIMITS_BOUNDS, + POOL_LIMITS_DEFAULTS, + POOL_LIMITS_MIN +} from './pool-limits' + +describe('parsePoolLimits', () => { + it('falls back to defaults for null/empty/corrupt input', () => { + expect(parsePoolLimits(null)).toEqual(POOL_LIMITS_DEFAULTS) + expect(parsePoolLimits(undefined)).toEqual(POOL_LIMITS_DEFAULTS) + expect(parsePoolLimits('')).toEqual(POOL_LIMITS_DEFAULTS) + expect(parsePoolLimits('not json {')).toEqual(POOL_LIMITS_DEFAULTS) + }) + + it('parses a valid persisted blob', () => { + expect(parsePoolLimits(JSON.stringify({ maxBackends: 11, idleMs: 7_200_000 }))).toEqual({ + maxBackends: 11, + idleMs: 7_200_000 + }) + }) + + it('fills missing keys from defaults', () => { + expect(parsePoolLimits(JSON.stringify({ maxBackends: 5 }))).toEqual({ ...POOL_LIMITS_DEFAULTS, maxBackends: 5 }) + expect(parsePoolLimits('{}')).toEqual(POOL_LIMITS_DEFAULTS) + }) + + it('ignores non-numeric junk instead of NaN-poisoning the pool', () => { + expect(parsePoolLimits(JSON.stringify({ maxBackends: 'lots', idleMs: null }))).toEqual(POOL_LIMITS_DEFAULTS) + }) +}) + +describe('clampPoolLimits', () => { + it('clamps below the floors', () => { + expect(clampPoolLimits({ maxBackends: 0 }).maxBackends).toBe(POOL_LIMITS_MIN.maxBackends) + expect(clampPoolLimits({ idleMs: 100 }).idleMs).toBe(POOL_LIMITS_MIN.idleMs) + }) + + it('clamps absurdly high backend counts', () => { + expect(clampPoolLimits({ maxBackends: 10_000 }).maxBackends).toBeLessThanOrEqual(64) + }) + + it('clamps idleMs to the shared ceiling (7 days)', () => { + expect(clampPoolLimits({ idleMs: 999_000_000 }).idleMs).toBe(POOL_LIMITS_BOUNDS.idleMsMax) + }) + + it('floors fractional values', () => { + expect(clampPoolLimits({ maxBackends: 2.9 }).maxBackends).toBe(2) + }) +}) diff --git a/apps/desktop/electron/pool-limits.ts b/apps/desktop/electron/pool-limits.ts new file mode 100644 index 0000000000..ee7e5b8a8a --- /dev/null +++ b/apps/desktop/electron/pool-limits.ts @@ -0,0 +1,81 @@ +/** + * Pool limits — how many bot backends may stay spawned, and how long an + * unused one survives. + * + * A device-local preference (each machine trades RAM against switching + * speed for itself), stored in userData like keep-awake. The main process + * is authoritative: it owns the pool AND the persisted copy, and applies a + * new max IMMEDIATELY by evicting least-recently-used idle backends — no + * app restart. The renderer mirrors the values for its UI and prewarm + * guard over IPC. + * + * Defaults preserve the historical hard-coded behavior (3 backends, 10min + * idle) so machines that never open Settings behave exactly as before. + */ + +export interface PoolLimits { + /** Max concurrently spawned non-primary profile backends. */ + maxBackends: number + /** Idle lifetime of an unused pool backend, in milliseconds. */ + idleMs: number +} + +export const POOL_LIMITS_DEFAULTS: PoolLimits = { + maxBackends: 3, + idleMs: 10 * 60_000 +} + +/** Hard floors — match the clamps the env-var path always applied. */ +export const POOL_LIMITS_MIN: PoolLimits = { + maxBackends: 1, + idleMs: 60_000 +} + +/** Shared bounds for both pool knobs — imported by the Settings UI so the + * advertised input ranges can never drift from what main actually clamps + * to. idleMs has no ceiling: a user who wants backends kept warm all week + * may have exactly that. */ +export const POOL_LIMITS_BOUNDS = { + maxBackendsMax: 64, + /** 7 days, matching the UI's suggestion ceiling. */ + idleMsMax: 7 * 24 * 60 * 60_000 +} as const + +const MAX_BACKENDS_CEILING = POOL_LIMITS_BOUNDS.maxBackendsMax +const IDLE_MS_CEILING = POOL_LIMITS_BOUNDS.idleMsMax + +/** Clamp a raw partial to the floors/ceilings; missing keys fall to defaults. */ +export function clampPoolLimits(raw: Partial): PoolLimits { + const maxBackends = Number.isFinite(raw.maxBackends) + ? Math.min(MAX_BACKENDS_CEILING, Math.max(POOL_LIMITS_MIN.maxBackends, Math.floor(Number(raw.maxBackends)))) + : POOL_LIMITS_DEFAULTS.maxBackends + + const idleMs = Number.isFinite(raw.idleMs) + ? Math.min(IDLE_MS_CEILING, Math.max(POOL_LIMITS_MIN.idleMs, Math.floor(Number(raw.idleMs)))) + : POOL_LIMITS_DEFAULTS.idleMs + + return { maxBackends, idleMs } +} + +function clampLimits(raw: Partial): PoolLimits { + return clampPoolLimits(raw) +} + +/** Parse + clamp a persisted JSON blob; anything unreadable falls back to + * defaults so a corrupted file can never wedge the pool. */ +export function parsePoolLimits(json: string | null | undefined): PoolLimits { + if (!json) { + return { ...POOL_LIMITS_DEFAULTS } + } + + try { + const parsed = JSON.parse(json) + + return clampLimits({ + maxBackends: typeof parsed?.maxBackends === 'number' ? parsed.maxBackends : undefined, + idleMs: typeof parsed?.idleMs === 'number' ? parsed.idleMs : undefined + }) + } catch { + return { ...POOL_LIMITS_DEFAULTS } + } +} diff --git a/apps/desktop/electron/pool-spawn-coordinator.test.ts b/apps/desktop/electron/pool-spawn-coordinator.test.ts new file mode 100644 index 0000000000..73b2291fe1 --- /dev/null +++ b/apps/desktop/electron/pool-spawn-coordinator.test.ts @@ -0,0 +1,344 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { test } from 'vitest' + +import { LocalBackendSpawnCoordinator, releaseLocalBackendSlotAfterExit } from './pool-spawn-coordinator' + +const deferred = () => { + let resolve!: () => void + + const promise = new Promise(done => { + resolve = done + }) + + return { promise, resolve } +} + +const flush = () => new Promise(resolve => setImmediate(resolve)) + +test('100 concurrent local requests never hold more than the configured slots', async () => { + const limit = 12 + const coordinator = new LocalBackendSpawnCoordinator(limit) + const gates = Array.from({ length: 100 }, deferred) + let active = 0 + let maxActive = 0 + + const tasks = gates.map(async (gate, index) => { + const release = await coordinator.acquire(`profile-${index}`) + active += 1 + maxActive = Math.max(maxActive, active) + + await gate.promise + + active -= 1 + release() + }) + + await flush() + assert.equal(active, limit) + assert.equal(coordinator.activeCount, limit) + assert.equal(coordinator.queuedCount, 100 - limit) + + for (let start = 0; start < gates.length; start += limit) { + for (const gate of gates.slice(start, start + limit)) { + gate.resolve() + } + + await flush() + } + + await Promise.all(tasks) + assert.equal(maxActive, limit) + assert.equal(coordinator.activeCount, 0) + assert.equal(coordinator.queuedCount, 0) +}) + +test('a queued start can be cancelled without waiting for an active backend', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const releaseFirst = await coordinator.acquire('first') + const queued = coordinator.request('cancelled') + + assert.equal(coordinator.queuedCount, 1) + assert.equal(queued.cancel(), true) + await assert.rejects(queued.acquired, /cancelled while queued/) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 0) + + releaseFirst() + assert.equal(coordinator.activeCount, 0) +}) + +test('cancelling an old same-key request never rejects a newer waiter', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const blocker = coordinator.request('blocker') + const releaseBlocker = await blocker.acquired + const old = coordinator.request('same-profile') + + releaseBlocker() + const newer = coordinator.request('same-profile') + + assert.equal(old.cancel(), false, 'the old request was already granted') + assert.equal(coordinator.queuedCount, 1, 'the newer same-key waiter must remain queued') + + const releaseOld = await old.acquired + releaseOld() + const releaseNewer = await newer.acquired + releaseNewer() + + assert.equal(coordinator.activeCount, 0) + assert.equal(coordinator.queuedCount, 0) +}) + +test('a queued start times out with a clear error and frees its queue position', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const releaseFirst = await coordinator.acquire('first') + const queued = coordinator.request('timed-out', { timeoutMs: 10 }) + + await assert.rejects(queued.acquired, /timed out while waiting for a free slot/) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 0) + + releaseFirst() + assert.equal(coordinator.activeCount, 0) +}) + +test('100 real child processes never exceed twelve simultaneous local slots', async () => { + const limit = 12 + const coordinator = new LocalBackendSpawnCoordinator(limit) + const livePids = new Set() + const seenPids = new Set() + let maxLive = 0 + + await Promise.all( + Array.from({ length: 100 }, async (_, index) => { + const release = await coordinator.acquire(`real-profile-${index}`) + + try { + const child = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 40)'], { + stdio: 'ignore' + }) + + assert.ok(child.pid) + livePids.add(child.pid) + seenPids.add(child.pid) + maxLive = Math.max(maxLive, livePids.size) + + await new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', code => { + if (code === 0) { + resolve() + } else { + reject(new Error(`child ${child.pid} exited with ${code}`)) + } + }) + }) + + livePids.delete(child.pid) + } finally { + release() + } + }) + ) + + assert.equal(seenPids.size, 100) + assert.equal(maxLive, limit) + assert.equal(livePids.size, 0) + assert.equal(coordinator.activeCount, 0) + assert.equal(coordinator.queuedCount, 0) +}) + +test('failed start keeps its slot until the child has actually exited', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const childExit = deferred() + const releaseFailed = await coordinator.acquire('failed') + let successorEntered = false + + const successor = coordinator.acquire('successor').then(release => { + successorEntered = true + + return release + }) + + const cleanup = releaseLocalBackendSlotAfterExit(releaseFailed, () => childExit.promise) + await flush() + + assert.equal(successorEntered, false) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 1) + + childExit.resolve() + await cleanup + const releaseSuccessor = await successor + + assert.equal(successorEntered, true) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 0) + + releaseSuccessor() + assert.equal(coordinator.activeCount, 0) +}) + +test('a rejected wait keeps the slot occupied', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const releaseFailed = await coordinator.acquire('failed') + let successorEntered = false + + const successor = coordinator.acquire('successor').then(release => { + successorEntered = true + + return release + }) + + const cleanup = releaseLocalBackendSlotAfterExit(releaseFailed, async () => { + throw new Error('exit unproven') + }) + + await assert.rejects(cleanup, /exit unproven/) + await flush() + + assert.equal(successorEntered, false) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 1) + + releaseFailed() + const releaseSuccessor = await successor + assert.equal(successorEntered, true) + releaseSuccessor() + assert.equal(coordinator.activeCount, 0) + assert.equal(coordinator.queuedCount, 0) +}) + +test('an invalid timeout never enqueues a waiter', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const releaseFirst = await coordinator.acquire('first') + + assert.throws(() => coordinator.request('invalid', { timeoutMs: 0 }), /timeout must be a positive number/) + assert.throws(() => coordinator.request('invalid', { timeoutMs: Number.NaN }), /timeout must be a positive number/) + assert.throws(() => coordinator.request('invalid', { timeoutMs: -5 }), /timeout must be a positive number/) + + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 0) + + releaseFirst() + assert.equal(coordinator.activeCount, 0) +}) + +test('a failed or repeated cleanup releases exactly one slot', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const releaseFirst = await coordinator.acquire('first') + let secondEntered = false + + const second = coordinator.acquire('second').then(release => { + secondEntered = true + + return release + }) + + await flush() + assert.equal(secondEntered, false) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 1) + + releaseFirst() + releaseFirst() + const releaseSecond = await second + + assert.equal(secondEntered, true) + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 0) + + releaseSecond() + assert.equal(coordinator.activeCount, 0) +}) + +test('raising the limit at runtime drains queued waiters into the new slots', async () => { + const coordinator = new LocalBackendSpawnCoordinator(1) + const first = await coordinator.acquire('a') + const queuedB = coordinator.request('b') + const queuedC = coordinator.request('c') + await flush() + assert.equal(coordinator.activeCount, 1) + assert.equal(coordinator.queuedCount, 2) + + coordinator.setLimit(2) + const releaseB = await queuedB.acquired + assert.equal(coordinator.activeCount, 2) + assert.equal(coordinator.queuedCount, 1) + + first() + const releaseC = await queuedC.acquired + assert.equal(coordinator.activeCount, 2) + releaseB() + releaseC() + assert.equal(coordinator.activeCount, 0) +}) + +test('lowering the limit never revokes granted slots; new requests queue until under cap', async () => { + const coordinator = new LocalBackendSpawnCoordinator(3) + const releases = await Promise.all(['a', 'b', 'c'].map(key => coordinator.acquire(key))) + coordinator.setLimit(1) + assert.equal(coordinator.activeCount, 3, 'granted slots stay granted') + + const queued = coordinator.request('d') + await flush() + assert.equal(coordinator.queuedCount, 1) + + releases[0]() + releases[1]() + await flush() + assert.equal(coordinator.queuedCount, 1, 'still over the new cap of 1') + + releases[2]() + const releaseD = await queued.acquired + assert.equal(coordinator.activeCount, 1) + releaseD() +}) + +test('setLimit rejects a non-positive or fractional cap', () => { + const coordinator = new LocalBackendSpawnCoordinator(2) + assert.throws(() => coordinator.setLimit(0), RangeError) + assert.throws(() => coordinator.setLimit(1.5), RangeError) + assert.equal(coordinator.limit, 2) +}) + +// ── main.ts wiring ────────────────────────────────────────────────────────── +// The coordinator is only as good as the timeout main.ts hands it. A queued +// ticket that outlives the renderer's backend-boot budget holds the pool key +// hostage: the renderer has already reported "backend didn't come up", and +// every later click on that profile joins the stale wait instead of failing +// fast with a reason. +{ + const here = path.dirname(fileURLToPath(import.meta.url)) + const mainSource = fs.readFileSync(path.join(here, 'main.ts'), 'utf8').replace(/\r\n/g, '\n') + + const withTimeoutSource = fs + .readFileSync(path.join(here, '..', 'src', 'lib', 'with-timeout.ts'), 'utf8') + .replace(/\r\n/g, '\n') + + test('main.ts bounds the slot wait below the renderer backend-boot budget', () => { + const slotWait = Number(/const POOL_SLOT_WAIT_MS = ([\d_]+)/.exec(mainSource)?.[1]?.replace(/_/g, '')) + + const bootBudget = Number( + /export const BACKEND_BOOT_WAIT_TIMEOUT_MS = ([\d_]+)/.exec(withTimeoutSource)?.[1]?.replace(/_/g, '') + ) + + assert.ok(Number.isFinite(slotWait) && slotWait > 0, 'POOL_SLOT_WAIT_MS must be a literal in main.ts') + assert.ok(Number.isFinite(bootBudget), 'BACKEND_BOOT_WAIT_TIMEOUT_MS must be a literal') + assert.ok(slotWait < bootBudget, `slot wait ${slotWait}ms must be below the boot budget ${bootBudget}ms`) + assert.match(mainSource, /localBackendSpawnCoordinator\.request\(poolKey, \{ timeoutMs: POOL_SLOT_WAIT_MS \}\)/) + assert.doesNotMatch(mainSource, /request\(poolKey, \{ timeoutMs: POOL_IDLE_MS \}\)/) + }) + + test('main.ts pushes the live pool max into the coordinator when the preference changes', () => { + // Pool sizing is a live device preference (#92581); the hard cap must + // follow it, otherwise raising the max in Settings would leave spawns + // queued behind the launch-time value. + assert.match(mainSource, /new LocalBackendSpawnCoordinator\(poolLimits\.maxBackends\)/) + assert.match(mainSource, /localBackendSpawnCoordinator\.setLimit\(poolLimits\.maxBackends\)/) + }) +} diff --git a/apps/desktop/electron/pool-spawn-coordinator.ts b/apps/desktop/electron/pool-spawn-coordinator.ts new file mode 100644 index 0000000000..8e565ed133 --- /dev/null +++ b/apps/desktop/electron/pool-spawn-coordinator.ts @@ -0,0 +1,153 @@ +export type ReleaseLocalBackendSlot = () => void + +export type LocalBackendSpawnRequest = { + acquired: Promise + cancel: () => boolean +} + +type Waiter = { + key: string + resolve: (release: ReleaseLocalBackendSlot) => void + reject: (error: Error) => void + timer: ReturnType | null +} + +export async function releaseLocalBackendSlotAfterExit( + release: ReleaseLocalBackendSlot, + waitForExit: () => Promise +): Promise { + await waitForExit() + release() +} + +/** + * Bounds the number of local profile backends that are starting or running. + * + * A lease is acquired immediately before local start work and is held until + * the child exits or the start fails. Remote descriptors never call request(). + */ +export class LocalBackendSpawnCoordinator { + #limit: number + #active = 0 + #queue: Waiter[] = [] + + constructor(limit: number) { + if (!Number.isInteger(limit) || limit < 1) { + throw new RangeError('Local backend spawn limit must be a positive integer.') + } + + this.#limit = limit + } + + get activeCount(): number { + return this.#active + } + + get limit(): number { + return this.#limit + } + + /** + * Adopt a new cap at runtime (the pool size is a live device preference). + * Raising it drains waiters into the newly freed slots immediately; lowering + * it never revokes a granted slot — the running backends simply stay over + * the cap until they exit, and LRU eviction (main.ts) converges the pool. + */ + setLimit(limit: number): void { + if (!Number.isInteger(limit) || limit < 1) { + throw new RangeError('Local backend spawn limit must be a positive integer.') + } + + this.#limit = limit + this.#drain() + } + + get queuedCount(): number { + return this.#queue.length + } + + request(key: string, options: { timeoutMs?: number } = {}): LocalBackendSpawnRequest { + if (options.timeoutMs !== undefined && (!Number.isFinite(options.timeoutMs) || options.timeoutMs < 1)) { + throw new RangeError('Local backend spawn timeout must be a positive number.') + } + + if (this.#active < this.#limit) { + return { + acquired: Promise.resolve(this.#grant()), + cancel: () => false + } + } + + let waiter!: Waiter + + const acquired = new Promise((resolve, reject) => { + waiter = { key, resolve, reject, timer: null } + this.#queue.push(waiter) + + if (options.timeoutMs !== undefined) { + waiter.timer = setTimeout(() => { + this.#rejectWaiter( + waiter, + new Error(`Local backend start for "${key}" timed out while waiting for a free slot.`) + ) + }, options.timeoutMs) + waiter.timer.unref?.() + } + }) + + return { + acquired, + cancel: () => + this.#rejectWaiter(waiter, new Error(`Local backend start for "${key}" was cancelled while queued.`)) + } + } + + acquire(key: string): Promise { + return this.request(key).acquired + } + + #rejectWaiter(waiter: Waiter, error: Error): boolean { + const index = this.#queue.indexOf(waiter) + + if (index === -1) { + return false + } + + this.#queue.splice(index, 1) + this.#clearTimer(waiter) + waiter.reject(error) + + return true + } + + #clearTimer(waiter: Waiter): void { + if (waiter.timer) { + clearTimeout(waiter.timer) + waiter.timer = null + } + } + + #grant(): ReleaseLocalBackendSlot { + this.#active += 1 + let released = false + + return () => { + if (released) { + return + } + + released = true + this.#active -= 1 + this.#drain() + } + } + + /** Hand free slots to queued waiters while under the (possibly lowered) cap. */ + #drain(): void { + while (this.#active < this.#limit && this.#queue.length > 0) { + const next = this.#queue.shift()! + this.#clearTimer(next) + next.resolve(this.#grant()) + } + } +} diff --git a/apps/desktop/electron/preload.ts b/apps/desktop/electron/preload.ts index e1fec035e4..28cc94cd5c 100644 --- a/apps/desktop/electron/preload.ts +++ b/apps/desktop/electron/preload.ts @@ -22,6 +22,8 @@ contextBridge.exposeInMainWorld('hermesDesktop', { getProfileRoutes: profiles => ipcRenderer.invoke('hermes:plugin-profile-routes', profiles), revalidateConnection: () => ipcRenderer.invoke('hermes:connection:revalidate'), touchBackend: profile => ipcRenderer.invoke('hermes:backend:touch', profile), + getPoolLimits: () => ipcRenderer.invoke('hermes:pool-limits:get'), + setPoolLimits: limits => ipcRenderer.invoke('hermes:pool-limits:set', limits), getGatewayWsUrl: profile => ipcRenderer.invoke('hermes:gateway:ws-url', profile), // Registry-scoped fresh WS URL: { connectionId, profile } → result shape of // getGatewayWsUrl, minted against that connection's backend. @@ -497,6 +499,7 @@ contextBridge.exposeInMainWorld('hermesDesktop', { }, getVersion: () => ipcRenderer.invoke('hermes:version'), getSyncStatus: () => ipcRenderer.invoke('hermes:sync-status'), + relaunchApp: () => ipcRenderer.invoke('hermes:app:relaunch'), getRemoteDisplayReason: () => ipcRenderer.invoke('hermes:get-remote-display-reason'), uninstall: { summary: () => ipcRenderer.invoke('hermes:uninstall:summary'), diff --git a/apps/desktop/electron/session-windows.test.ts b/apps/desktop/electron/session-windows.test.ts index c0a1178429..64a9d6fb12 100644 --- a/apps/desktop/electron/session-windows.test.ts +++ b/apps/desktop/electron/session-windows.test.ts @@ -68,6 +68,12 @@ test('buildSessionWindowUrl avoids a double slash when the dev server has a trai assert.equal(url, 'http://localhost:5173/?win=secondary#/abc123') }) +test('buildSessionWindowUrl carries the owning profile in the query before the hash (#82768)', () => { + const url = buildSessionWindowUrl('abc123', { devServer: 'http://localhost:5173', profile: 'work', watch: true }) + + assert.equal(url, 'http://localhost:5173/?win=secondary&watch=1&profile=work#/abc123') +}) + test('buildSessionWindowUrl encodes the session id in the hash route', () => { const url = buildSessionWindowUrl('a b/c', { devServer: 'http://localhost:5173' }) diff --git a/apps/desktop/electron/session-windows.ts b/apps/desktop/electron/session-windows.ts index 19be87ff1c..5fbd456a5b 100644 --- a/apps/desktop/electron/session-windows.ts +++ b/apps/desktop/electron/session-windows.ts @@ -64,8 +64,13 @@ function chatWindowWebPreferences(preloadPath: string) { // onboarding overlays and the global session sidebar. `watch=1` marks a // spectator window (e.g. a running subagent's session): the renderer resumes it // lazily so the gateway never builds an agent just to stream into it. -function buildSessionWindowUrl(sessionId: string, { devServer, rendererIndexPath, watch }: any = {}) { - const query = `?win=secondary${watch ? '&watch=1' : ''}` +// `profile` names the backend the window must boot against (same carry as the +// HUD's buildHudWindowUrl): without it a pop-out/watch window adopts the +// PRIMARY profile and resolves the session id against the wrong backend +// (#82768, #61286). Absent → unchanged primary adoption. +function buildSessionWindowUrl(sessionId: string, { devServer, profile, rendererIndexPath, watch }: any = {}) { + const profileKey = typeof profile === 'string' ? profile.trim() : '' + const query = `?win=secondary${watch ? '&watch=1' : ''}${profileKey ? `&profile=${encodeURIComponent(profileKey)}` : ''}` const route = `#/${encodeURIComponent(sessionId)}` if (devServer) { diff --git a/apps/desktop/package.json b/apps/desktop/package.json index d841bf103f..006b941c26 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -27,7 +27,7 @@ "profile:main": "tsc --build tsconfig.electron.json && wait-on http://127.0.0.1:5174 && node scripts/bundle-electron-main.mjs --dev && cross-env XCURSOR_SIZE=24 HERMES_DESKTOP_DEV_SERVER=http://127.0.0.1:5174 electron --inspect=9229 .", "profile:main:cpu": "tsc --build tsconfig.electron.json && wait-on http://127.0.0.1:5174 && node scripts/bundle-electron-main.mjs --dev && cross-env XCURSOR_SIZE=24 NODE_OPTIONS=--cpu-prof HERMES_DESKTOP_DEV_SERVER=http://127.0.0.1:5174 electron .", "start": "npm run build && electron .", - "prebuild": "npm run clean", + "prebuild": "node scripts/assert-root-install.mjs && npm run clean", "build": "node scripts/assert-root-install.mjs && node scripts/write-build-stamp.mjs && vite build && node scripts/bundle-electron-main.mjs && node scripts/stage-native-deps.mjs", "postbuild": "node scripts/assert-dist-built.mjs", "prebuilder": "node scripts/patch-electron-builder-mac-binary.mjs", diff --git a/apps/desktop/scripts/assert-root-install.mjs b/apps/desktop/scripts/assert-root-install.mjs index 3a11031a3a..c3d388eabd 100644 --- a/apps/desktop/scripts/assert-root-install.mjs +++ b/apps/desktop/scripts/assert-root-install.mjs @@ -1,35 +1,146 @@ -import { accessSync, readFileSync } from "fs" +// Build-time guard: refuse to start a build the installed tree cannot finish. +// +// The desktop workspace's dependencies are hoisted to the repo-root +// `node_modules`, so a root install that only covers *part* of the workspace +// graph leaves this app importable-looking but unbuildable. The guard exists to +// turn that into one actionable line ("run npm ci from the repo root") instead +// of a failure deep inside vite. +// +// It runs from `prebuild`, ahead of `npm run clean`, so a tree that cannot +// build is rejected before the build starts deleting its own outputs. `build` +// re-runs it for anyone invoking the build steps directly; the check is pure +// filesystem lookups, so paying for it twice costs nothing. + +import { existsSync, readFileSync } from "fs" import { createRequire } from "module" -import { resolve, join } from "path" +import { resolve, join, dirname } from "path" +import { isMain } from "./utils.mjs" -const app = resolve(import.meta.dirname, "..") -const root = resolve(app, "..", "..") +// Packages the build *consumes*, as opposed to merely declares. Each one is +// load-bearing for a distinct build step, and each one has been observed +// missing from a partial root install: +// +// vite — bundles the renderer (`vite build`). +// katex — `src/styles.css` imports `katex/dist/katex.min.css`, so +// the CSS transform fails before a single chunk is emitted. +// electron — the runtime electron-builder packages; without it `pack` +// cannot produce an unpacked app at all. +// electron-builder — the packager `npm run builder` shells out to. +// +// Checking only `vite` (the original guard) passes a tree missing any of the +// others, which is how an incomplete install reached `vite build` and died on +// an unresolved `katex/dist/katex.min.css` with no hint that the install — not +// the source — was at fault (#86443). +// +// These four are the documented floor — always checked, even when the app's +// package.json cannot be read. The full class is wider: EVERY non-optional +// package the workspace manifest declares is something the build may import +// (`vite.config.ts` pulls `@rolldown/plugin-babel`, `@vitejs/plugin-react`, +// `@tailwindcss/vite`; `bundle-electron-main.mjs` pulls `esbuild`; the renderer +// imports the rest). A hand-maintained list drifts the moment a new import +// lands, so `checkRootInstall` unions the floor with the manifest's declared +// `dependencies` + `devDependencies` — a partial install is refused whichever +// package it happened to drop. `optionalDependencies` are excluded by design: +// npm legitimately skips them (platform-gated natives like `get-windows`). +const BUILD_CRITICAL_PACKAGES = ["vite", "katex", "electron", "electron-builder"] +export { BUILD_CRITICAL_PACKAGES } -try { - accessSync(join(root, "node_modules", "vite", "package.json")) -} catch { - console.error(`Run from repo root: cd ${root} && npm ci`) - process.exit(1) +// Resolve the way Node's own lookup does — walk `node_modules` upward — rather +// than through `require.resolve`. A package whose `exports` map does not expose +// `./package.json` is not resolvable by path even when correctly installed, and +// that must not read as "missing". Scoped names (`@scope/name`) are a nested +// directory under `node_modules`, which `join` handles. +function packageIsInstalled(name, fromDir) { + let dir = fromDir + for (;;) { + if (existsSync(join(dir, "node_modules", name, "package.json"))) return true + const parent = dirname(dir) + if (parent === dir) return false + dir = parent + } } -// `vite.config.ts` aliases react/react-dom to whatever this workspace resolves, -// and React refuses to run when the two come from different installed copies -// ("Minified React error #527" — it throws before the first paint, so the app -// window stays blank). npm stays silent about the split because the hoisted -// react still satisfies react-dom's caret peer range. Fail the build loudly -// instead of shipping a white screen. -const requireFromApp = createRequire(join(app, "package.json")) -const installedVersion = (pkg) => - JSON.parse(readFileSync(requireFromApp.resolve(`${pkg}/package.json`), "utf8")).version - -const react = installedVersion("react") -const reactDom = installedVersion("react-dom") - -if (react !== reactDom) { - console.error( - `react@${react} / react-dom@${reactDom} version mismatch — React would fail ` + - `with error #527 and render a blank window. Pin both to the same version ` + - `in ${join(app, "package.json")}, then reinstall: cd ${root} && npm ci` - ) - process.exit(1) +// Every package the workspace manifest at `appDir` declares as required +// (`dependencies` + `devDependencies`; never `optionalDependencies`). An +// unreadable or malformed manifest yields [] — the floor still applies, and +// the build's own manifest read fails loudly on its own. +export function requiredPackages(appDir) { + try { + const manifest = JSON.parse(readFileSync(join(appDir, "package.json"), "utf8")) + return [ + ...Object.keys(manifest.dependencies ?? {}), + ...Object.keys(manifest.devDependencies ?? {}), + ] + } catch { + return [] + } +} + +// Pure check — returns { ok: true } or { ok: false, error: "..." }. +// Kept side-effect-free so it can be unit tested without spawning a process. +export function checkRootInstall(appDir, rootDir) { + const wanted = [...new Set([...BUILD_CRITICAL_PACKAGES, ...requiredPackages(appDir)])] + const missing = wanted.filter(pkg => !packageIsInstalled(pkg, appDir)) + if (missing.length > 0) { + return { + ok: false, + error: + `the desktop build needs ${missing.join(", ")}, which the current install ` + + `does not provide. A partial root install leaves the workspace looking ` + + `present while the build cannot complete. Reinstall from the repo root: ` + + `cd ${rootDir} && npm ci` + } + } + + // `vite.config.ts` aliases react/react-dom to whatever this workspace resolves, + // and React refuses to run when the two come from different installed copies + // ("Minified React error #527" — it throws before the first paint, so the app + // window stays blank). npm stays silent about the split because the hoisted + // react still satisfies react-dom's caret peer range. Fail the build loudly + // instead of shipping a white screen. + const requireFromApp = createRequire(join(appDir, "package.json")) + const installedVersion = pkg => + JSON.parse(readFileSync(requireFromApp.resolve(`${pkg}/package.json`), "utf8")).version + + let react + let reactDom + try { + react = installedVersion("react") + reactDom = installedVersion("react-dom") + } catch (err) { + // Both are in BUILD_CRITICAL_PACKAGES' spirit but not its list: they are + // checked by version, and an unreadable package.json is a broken install + // rather than an absent one. Report it as such instead of throwing. + return { + ok: false, + error: `could not read the installed react/react-dom versions (${err.message}). Reinstall from the repo root: cd ${rootDir} && npm ci` + } + } + + if (react !== reactDom) { + return { + ok: false, + error: + `react@${react} / react-dom@${reactDom} version mismatch — React would fail ` + + `with error #527 and render a blank window. Pin both to the same version ` + + `in ${join(appDir, "package.json")}, then reinstall: cd ${rootDir} && npm ci` + } + } + + return { ok: true } +} + +function main() { + const app = resolve(import.meta.dirname, "..") + const root = resolve(app, "..", "..") + const result = checkRootInstall(app, root) + + if (!result.ok) { + console.error(`✗ assert-root-install: ${result.error}`) + process.exit(1) + } +} + +if (isMain(import.meta.url)) { + main() } diff --git a/apps/desktop/scripts/assert-root-install.test.mjs b/apps/desktop/scripts/assert-root-install.test.mjs new file mode 100644 index 0000000000..0d7ea4af6f --- /dev/null +++ b/apps/desktop/scripts/assert-root-install.test.mjs @@ -0,0 +1,197 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { test } from 'vitest' + +import { BUILD_CRITICAL_PACKAGES as BUILD_CRITICAL, checkRootInstall, requiredPackages } from '../scripts/assert-root-install.mjs' + +// Build a throwaway repo shaped like this one: an app workspace whose +// dependencies are hoisted to the repo root, which is what the guard walks. +// `manifest` is merged into the app's package.json so tests can declare +// dependencies the guard is expected to read. +function makeTree({ rootPackages = BUILD_CRITICAL, react = '19.2.7', reactDom = '19.2.7', manifest = {} } = {}) { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-assert-root-')) + const appDir = path.join(tempRoot, 'apps', 'desktop') + fs.mkdirSync(appDir, { recursive: true }) + fs.writeFileSync(path.join(appDir, 'package.json'), JSON.stringify({ name: 'desktop', ...manifest }), 'utf8') + + const writePackage = (name, version) => { + const dir = path.join(tempRoot, 'node_modules', name) + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name, version }), 'utf8') + } + for (const name of rootPackages) writePackage(name, '1.0.0') + if (react !== null) writePackage('react', react) + if (reactDom !== null) writePackage('react-dom', reactDom) + + return { tempRoot, appDir } +} + +test('checkRootInstall passes on a complete root install', () => { + const { tempRoot, appDir } = makeTree() + try { + assert.deepEqual(checkRootInstall(appDir, tempRoot), { ok: true }) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +// The regression this guard was widened for: the updater's partial `npm install` +// left katex out while vite was present, so the old vite-only check passed and +// the build died on an unresolved `katex/dist/katex.min.css` (#86443). +test('checkRootInstall fails when katex is missing but vite is present', () => { + const { tempRoot, appDir } = makeTree({ + rootPackages: BUILD_CRITICAL.filter(name => name !== 'katex') + }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /katex/) + assert.match(result.error, /npm ci/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('checkRootInstall fails when electron is missing', () => { + const { tempRoot, appDir } = makeTree({ + rootPackages: BUILD_CRITICAL.filter(name => name !== 'electron') + }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /electron/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('checkRootInstall reports every missing package at once', () => { + const { tempRoot, appDir } = makeTree({ rootPackages: ['vite'] }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + for (const name of ['katex', 'electron', 'electron-builder']) { + assert.match(result.error, new RegExp(name)) + } + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +// The original guard's only check — kept, so widening coverage cannot silently +// drop the case it already handled. +test('checkRootInstall still fails when vite is missing', () => { + const { tempRoot, appDir } = makeTree({ + rootPackages: BUILD_CRITICAL.filter(name => name !== 'vite') + }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /vite/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('checkRootInstall fails on a react/react-dom version split', () => { + const { tempRoot, appDir } = makeTree({ react: '19.2.7', reactDom: '19.1.0' }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /#527/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +// A package installed into the app's own node_modules rather than hoisted to the +// root is still installed. The guard walks upward like Node does, so it must not +// insist on the hoisted location. +test('checkRootInstall accepts a package nested in the app workspace', () => { + const { tempRoot, appDir } = makeTree({ + rootPackages: BUILD_CRITICAL.filter(name => name !== 'katex') + }) + const nested = path.join(appDir, 'node_modules', 'katex') + fs.mkdirSync(nested, { recursive: true }) + fs.writeFileSync(path.join(nested, 'package.json'), JSON.stringify({ name: 'katex' }), 'utf8') + try { + assert.deepEqual(checkRootInstall(appDir, tempRoot), { ok: true }) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +// The class, not the four instances: the floor list is what a partial install +// has been *seen* to drop, but any declared non-optional package can be the one +// missing next (`vite.config.ts` imports `@rolldown/plugin-babel`, which the +// floor never named). The guard must read the manifest so the list cannot drift +// behind a new import. +test('checkRootInstall fails when a declared devDependency outside the floor is missing', () => { + const { tempRoot, appDir } = makeTree({ + manifest: { devDependencies: { '@rolldown/plugin-babel': '1.0.0', esbuild: '1.0.0' } }, + rootPackages: [...BUILD_CRITICAL, 'esbuild'] + }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /@rolldown\/plugin-babel/) + assert.doesNotMatch(result.error, /esbuild/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('checkRootInstall fails when a declared runtime dependency is missing', () => { + const { tempRoot, appDir } = makeTree({ + manifest: { dependencies: { '@vscode/codicons': '1.0.0' } } + }) + try { + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /@vscode\/codicons/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +// npm skips optionalDependencies legitimately (platform-gated natives), so an +// absent optional package is not a partial install. +test('checkRootInstall ignores missing optionalDependencies', () => { + const { tempRoot, appDir } = makeTree({ + manifest: { optionalDependencies: { 'get-windows': '9.3.0' } } + }) + try { + assert.deepEqual(checkRootInstall(appDir, tempRoot), { ok: true }) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +test('checkRootInstall passes when every declared package is installed', () => { + const { tempRoot, appDir } = makeTree({ + manifest: { dependencies: { '@scope/pkg': '1.0.0' }, devDependencies: { esbuild: '1.0.0' } }, + rootPackages: [...BUILD_CRITICAL, '@scope/pkg', 'esbuild'] + }) + try { + assert.deepEqual(checkRootInstall(appDir, tempRoot), { ok: true }) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) + +// The floor is unconditional: a manifest the guard cannot parse must not turn +// the check off. +test('checkRootInstall keeps the floor when the manifest is unreadable', () => { + const { tempRoot, appDir } = makeTree({ rootPackages: ['vite'] }) + fs.writeFileSync(path.join(appDir, 'package.json'), '{not json', 'utf8') + try { + assert.deepEqual(requiredPackages(appDir), []) + const result = checkRootInstall(appDir, tempRoot) + assert.equal(result.ok, false) + assert.match(result.error, /katex/) + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/src/api/local-models.ts b/apps/desktop/src/api/local-models.ts index d2cbce5f2c..39c883e75c 100644 --- a/apps/desktop/src/api/local-models.ts +++ b/apps/desktop/src/api/local-models.ts @@ -1,9 +1,4 @@ -import type { - LocalCatalogModel, - LocalHardware, - LocalModelsStatus, - LocalRuntimeJob -} from '@/types/hermes' +import type { LocalCatalogModel, LocalHardware, LocalModelsStatus, LocalRuntimeJob } from '@/types/hermes' import { hermesApi, profileScoped } from './client' @@ -147,7 +142,10 @@ export function listHFRepoFiles(repo: string): Promise<{ files: HFFileGroup[] }> }) } -export function downloadBrowsedModel(repo: string, paths: string[]): Promise<{ already_downloaded?: boolean; job_id: null | string; model_id: string }> { +export function downloadBrowsedModel( + repo: string, + paths: string[] +): Promise<{ already_downloaded?: boolean; job_id: null | string; model_id: string }> { return hermesApi<{ already_downloaded?: boolean; job_id: null | string; model_id: string }>({ ...profileScoped(), body: { paths, repo }, @@ -156,7 +154,9 @@ export function downloadBrowsedModel(repo: string, paths: string[]): Promise<{ a }) } -export function sideloadLocalModel(path: string): Promise<{ already_present?: boolean; model_id: string; ok: boolean }> { +export function sideloadLocalModel( + path: string +): Promise<{ already_present?: boolean; model_id: string; ok: boolean }> { return hermesApi<{ already_present?: boolean; model_id: string; ok: boolean }>({ ...profileScoped(), body: { path }, diff --git a/apps/desktop/src/api/sessions.ts b/apps/desktop/src/api/sessions.ts index 8c2f358a5d..61368921bc 100644 --- a/apps/desktop/src/api/sessions.ts +++ b/apps/desktop/src/api/sessions.ts @@ -148,6 +148,11 @@ export interface SidebarSessionSlice { /** Per-profile tokens and spend over every session, not just this window. * Absent from the legacy per-slice endpoint, which has no aggregate. */ profiles_usage?: Record + /** Profiles whose scan for THIS slice failed. Batched `/sidebar` stamps the + * same profile errors on every slice (one DB open). Legacy per-slice calls + * stamp only the slice that actually failed, so a cron I/O error cannot + * carry-forward recents. */ + errors?: Array<{ profile: string; error: string }> } /** Which profiles filled their per-profile window in a returned page. The @@ -216,16 +221,24 @@ async function listSidebarSessionsLegacy(req: SidebarSessionsRequest): Promise { }) } +// Acquiring a lease pre-loads the configured TTS engine. For local engines +// that is a model load and, on a fresh install, a voice download — well past +// the default 15s Electron backend timeout. +export const AUDIO_TTS_LEASE_REQUEST_TIMEOUT_MS = 180_000 + +/** + * Tell the backend a speech-output toggle flipped so it can warm the TTS engine + * (`active: true`) or release it once no surface needs it (`active: false`). + * `lease` names the toggle — `desktop:read-aloud`, `desktop:conversation`. + */ +export function setTtsLease(lease: string, active: boolean): Promise { + return hermesApi({ + ...profileScoped(), + path: '/api/audio/tts-lease', + method: 'POST', + body: { active, lease }, + timeoutMs: AUDIO_TTS_LEASE_REQUEST_TIMEOUT_MS + }) +} + export function getElevenLabsVoices(profile?: null | string): Promise { return hermesApi({ path: '/api/audio/elevenlabs/voices', diff --git a/apps/desktop/src/app/agents/index.tsx b/apps/desktop/src/app/agents/index.tsx index eb74a2d2a7..cd3f36f168 100644 --- a/apps/desktop/src/app/agents/index.tsx +++ b/apps/desktop/src/app/agents/index.tsx @@ -152,10 +152,36 @@ function groupDelegations(roots: readonly SubagentNode[]): RootGroup[] { let n = 0 for (const node of roots) { + // Exact grouping when the backend tags workers with their batch id — + // concurrent or nested fan-outs of the same shape must not merge. + if (node.delegationId) { + const byId = groups.find(g => g.id === `delegation:${node.delegationId}`) + + if (byId) { + byId.nodes.push(node) + + continue + } + + n += 1 + groups.push({ + id: `delegation:${node.delegationId}`, + delegationIndex: n, + nodes: [node], + taskCount: node.taskCount + }) + + continue + } + + // Older backends (no delegation_id): heuristic grouping by shape + time. const prev = groups.at(-1) const prevTail = prev?.nodes.at(-1) const closeInTime = prevTail ? Math.abs(node.startedAt - prevTail.startedAt) <= 5_000 : false - const sameShape = prev && node.taskCount > 1 && prev.taskCount === node.taskCount + + const sameShape = + prev && !prev.id.startsWith('delegation:') && node.taskCount > 1 && prev.taskCount === node.taskCount + const uniqueStep = prev ? !prev.nodes.some(item => item.taskIndex === node.taskIndex) : false if (prev && sameShape && closeInTime && uniqueStep) { diff --git a/apps/desktop/src/app/chat/composer/composer-utils.test.ts b/apps/desktop/src/app/chat/composer/composer-utils.test.ts index 746115002c..5f523e844f 100644 --- a/apps/desktop/src/app/chat/composer/composer-utils.test.ts +++ b/apps/desktop/src/app/chat/composer/composer-utils.test.ts @@ -7,6 +7,7 @@ import { isPendingDraftPersistCurrent, type PendingDraftPersist, pickPlaceholder, + shouldDisableComposerInput, slashArgStage, slashChipKindForItem, slashCommandToken, @@ -16,6 +17,26 @@ import { const item = (group: string): Unstable_TriggerItem => ({ id: 'x', type: 'slash', label: 'x', metadata: { group } }) as unknown as Unstable_TriggerItem +describe('shouldDisableComposerInput', () => { + it.each(['idle', 'connecting', 'closed', 'error'] as const)( + 'keeps the draft editable while the gateway is %s', + gatewayState => { + expect(shouldDisableComposerInput(true, gatewayState)).toBe(false) + } + ) + + it('fails closed when connection atoms disagree about an open gateway', () => { + expect(shouldDisableComposerInput(true, 'open')).toBe(true) + }) + + it.each(['idle', 'connecting', 'open', 'closed', 'error'] as const)( + 'never disables an otherwise enabled composer while the gateway is %s', + gatewayState => { + expect(shouldDisableComposerInput(false, gatewayState)).toBe(false) + } + ) +}) + describe('slashArgStage', () => { it('is true only once the query is past the command name', () => { expect(slashArgStage('personality')).toBe(false) diff --git a/apps/desktop/src/app/chat/composer/composer-utils.ts b/apps/desktop/src/app/chat/composer/composer-utils.ts index 65b141a0de..97852c3133 100644 --- a/apps/desktop/src/app/chat/composer/composer-utils.ts +++ b/apps/desktop/src/app/chat/composer/composer-utils.ts @@ -1,4 +1,5 @@ import type { Unstable_TriggerItem } from '@assistant-ui/core' +import type { ConnectionState } from '@hermes/shared' import type { SlashChipKind } from '@/components/assistant-ui/directive-text' import type { ComposerAttachment } from '@/store/composer' @@ -52,6 +53,18 @@ export const COMPOSER_FADE_BACKGROUND = // unmount/pagehide flushes bypass it. export const DRAFT_PERSIST_DEBOUNCE_MS = 400 +/** + * Keep a reconnecting draft editable so transient gateway dials cannot blur + * the editor and discard the user's caret. Submission still reads the + * independent `disabled` prop, so non-open states cannot send. + * + * An `open` state paired with `disabled=true` is a transient disagreement + * between the connection atoms; fail closed until they converge. + */ +export function shouldDisableComposerInput(disabled: boolean, gatewayState: ConnectionState): boolean { + return disabled && gatewayState === 'open' +} + export const pickPlaceholder = (pool: readonly string[]) => pool[Math.floor(Math.random() * pool.length)] /** Completion items can carry an `action` (set in use-slash-completions) that diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index ffe6c4707f..9a7db494ee 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -5,6 +5,7 @@ import { useI18n } from '@/i18n' import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages' import { triggerHaptic } from '@/lib/haptics' import { markAssistantIdSpoken, resolveSpokenReply } from '@/lib/spoken-reply' +import { CONVERSATION_LEASE, READ_ALOUD_LEASE, syncTtsLease } from '@/lib/tts-lease' import { clearWakeIndicator, syncWakeIndicatorWithVoice } from '@/lib/wake-indicator' import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' @@ -265,6 +266,26 @@ export function useComposerVoice({ useEffect(() => resumeWakeIfPaused, [resumeWakeIfPaused]) + // Speech-output toggles are TTS warm-up / release signals. Entering a voice + // conversation acquires this window's lease (pre-loads the engine so the + // first spoken reply doesn't start with dead air); ending it releases the + // lease, and the backend unloads resident local models once no surface holds + // one. Fire-and-forget — the toggle never waits on or fails from this. + useEffect(() => { + void syncTtsLease(CONVERSATION_LEASE, voiceConversationActive) + }, [voiceConversationActive]) + + useEffect(() => () => void syncTtsLease(CONVERSATION_LEASE, false), []) + + // "Read replies aloud" is the same signal, held for as long as the toggle is + // on (it mirrors voice.auto_tts, so this also warms at startup when the + // preference is already set). + const autoSpeakReplies = useStore($autoSpeakReplies) + + useEffect(() => { + void syncTtsLease(READ_ALOUD_LEASE, autoSpeakReplies) + }, [autoSpeakReplies]) + // Explicit start/end for the on-screen conversation controls (the hotkey uses // the gated toggle above). const startConversation = useCallback(() => setVoiceConversationActive(true), []) diff --git a/apps/desktop/src/app/chat/composer/index.tsx b/apps/desktop/src/app/chat/composer/index.tsx index 3764633db6..b01381e41d 100644 --- a/apps/desktop/src/app/chat/composer/index.tsx +++ b/apps/desktop/src/app/chat/composer/index.tsx @@ -35,6 +35,7 @@ import { COMPOSER_FADE_BACKGROUND, implicitSlashAcceptIndex, type QueueEditState, + shouldDisableComposerInput, slashArgStage } from './composer-utils' import { ContextMenu } from './context-menu' @@ -220,8 +221,8 @@ export function ChatBar({ const { t } = useI18n() const gatewayState = useStore($gatewayState) - const reconnecting = gatewayState === 'closed' || gatewayState === 'error' - const inputDisabled = disabled && !reconnecting + const reconnecting = gatewayState !== 'open' + const inputDisabled = shouldDisableComposerInput(disabled, gatewayState) // The draft engine — detached source of truth (DOM + draftRef + edge // selectors); typing never re-renders the chrome. ChatBar owns `queueEditRef` diff --git a/apps/desktop/src/app/chat/index.tsx b/apps/desktop/src/app/chat/index.tsx index 14651b7f72..1b95bc769a 100644 --- a/apps/desktop/src/app/chat/index.tsx +++ b/apps/desktop/src/app/chat/index.tsx @@ -78,7 +78,7 @@ import { mergeOlderTranscriptPage, transcriptBackfillAvailable } from './transcript-backfill' -import { advanceTranscriptWindow, type TranscriptWindowState } from './transcript-window' +import { advanceSessionTranscriptWindow, type SessionWindowMemo } from './transcript-window' interface ChatViewProps extends Omit, 'onSubmit'> { gateway: HermesGateway | null @@ -247,22 +247,37 @@ function ChatRuntimeBoundary({ const [windowPages, setWindowPages] = useState(1) const [windowSessionKey, setWindowSessionKey] = useState(runtimeId) - // Sticky-cut continuity across flushes (advanceTranscriptWindow). A ref, not - // state: it is derived from `messages` and must never trigger a render. - const windowStateRef = useRef(null) + // Per-session sticky-cut continuity (advanceSessionTranscriptWindow). A ref, + // not state: it is derived from `messages` and must never trigger a render. + // Keyed by runtime id so a warm switch back to a session whose transcript + // is unchanged reuses the previous windowed slice BY REFERENCE — no window + // re-index, no runtime-repository rebuild, no per-row re-parse/re-highlight + // (#95595). Bounded internally (oldest session evicted). + const windowStateRef = useRef(new Map()) + // The memo below intentionally skips `runtimeId` in its deps (a switch + // always changes the messages array too, which re-runs it), so the current + // value must come from a ref rather than the stale render closure. + const runtimeIdRef = useRef(runtimeId) + runtimeIdRef.current = runtimeId // Reset the window on session swap during RENDER, so a large expand from the - // previous chat can't leak into the next one's first paint (#55191). + // previous chat can't leak into the next one's first paint (#55191). The + // per-session map above keeps each session's own cut; only the page count + // resets on a switch. if (windowSessionKey !== runtimeId) { setWindowSessionKey(runtimeId) setWindowPages(1) - windowStateRef.current = null } const { messages: windowedMessages, windowed } = useMemo(() => { - const next = advanceTranscriptWindow(windowStateRef.current, messages, windowPages) - - windowStateRef.current = next + const next = advanceSessionTranscriptWindow( + windowStateRef.current, + // Draft state has no runtime id yet; a single shared slot is fine there + // (mirrors the old single-slot behaviour for the no-runtime case). + runtimeIdRef.current ?? '', + messages, + windowPages + ) return next.window }, [messages, windowPages]) @@ -296,7 +311,7 @@ function ChatRuntimeBoundary({ // something older to show. Fire-and-forget: the prepend lands through the // session-state write path and re-renders this boundary. if ( - !windowStateRef.current?.window.windowed && + !windowStateRef.current.get(runtimeIdRef.current ?? '')?.state.window.windowed && runtimeId && storedId && transcriptBackfillAvailable(storedId, tailProfile) diff --git a/apps/desktop/src/app/chat/right-rail/preview-annotate-host.test.ts b/apps/desktop/src/app/chat/right-rail/preview-annotate-host.test.ts index d399bfa9f1..7ab5d185a2 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-annotate-host.test.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-annotate-host.test.ts @@ -57,4 +57,56 @@ describe('preview annotate host', () => { y: 20 - ANNOTATE_CROP_PAD }) }) + + it('brackets the shot so the crop is taken with only this comment marked', async () => { + const order: string[] = [] + + const executeJavaScript = vi.fn(async (code: string) => { + order.push(code.includes('beginCapture') ? 'begin' : 'end') + + return true + }) + + const capture = vi.fn(async () => { + order.push('capture') + + return 'data:image/png;base64,AA==' + }) + + await captureAnnotateCrop({ capture, executeJavaScript }, { height: 16, width: 40, x: 10, y: 20 }) + + expect(order).toEqual(['begin', 'capture', 'end']) + }) + + it('restores saved pins even when the capture fails', async () => { + const executeJavaScript = vi.fn(async (code: string) => { + void code + + return true + }) + + const capture = vi.fn(async () => { + throw new Error('capture exploded') + }) + + await expect( + captureAnnotateCrop({ capture, executeJavaScript }, { height: 16, width: 40, x: 10, y: 20 }) + ).rejects.toThrow('capture exploded') + + expect(executeJavaScript.mock.calls.some(([code]) => String(code).includes('endCapture'))).toBe(true) + }) + + it('still captures when the overlay cannot be reached', async () => { + const capture = vi.fn(async () => 'data:image/png;base64,AA==') + + const executeJavaScript = vi.fn(async (code: string) => { + void code + + throw new Error('guest is gone') + }) + + await expect( + captureAnnotateCrop({ capture, executeJavaScript }, { height: 16, width: 40, x: 10, y: 20 }) + ).resolves.toContain('data:image/png') + }) }) diff --git a/apps/desktop/src/app/chat/right-rail/preview-annotate-host.ts b/apps/desktop/src/app/chat/right-rail/preview-annotate-host.ts index 1ce04b5bcb..1fdad4eb09 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-annotate-host.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-annotate-host.ts @@ -95,6 +95,15 @@ export async function hideAnnotateDraft(guest: PreviewAnnotateGuest): Promise { + try { + await guest.executeJavaScript(code) + } catch { + // The overlay may be mid-teardown or the guest gone. Shoot anyway. + } +} + export async function captureAnnotateCrop( guest: PreviewAnnotateGuest, rect: AnnotatePinChrome['rect'] @@ -103,5 +112,15 @@ export async function captureAnnotateCrop( throw new Error('preview capture is unavailable') } - return guest.capture(padRect(rect)) + // Bracket the shot: the overlay hides saved pins and waits for a paint, so + // the crop carries this comment's marker and no neighbour's. `endCapture` + // runs even when the capture throws, or one failed crop leaves every saved + // pin invisible on the page. + await tryGuest(guest, 'window.__hermesAnnotate ? window.__hermesAnnotate.beginCapture() : null') + + try { + return await guest.capture(padRect(rect)) + } finally { + await tryGuest(guest, 'window.__hermesAnnotate && window.__hermesAnnotate.endCapture()') + } } diff --git a/apps/desktop/src/app/chat/right-rail/preview-file.tsx b/apps/desktop/src/app/chat/right-rail/preview-file.tsx index f3723f19d7..a16d64f280 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-file.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-file.tsx @@ -347,19 +347,7 @@ function MarkdownCode({ className, children, ...props }: ComponentProps<'code'>) const code = String(children).replace(/\n$/, '') - const highlighted = ( - - {code} - - ) + const highlighted = // ```mermaid / ```svg fences route to the shared lazy renderers (same // registry the chat transcript uses); everything else stays on Shiki. @@ -661,17 +649,7 @@ export function SourceView({ filePath, language, text }: { filePath?: string; la })}
- - {chunk.text} - +
))} diff --git a/apps/desktop/src/app/chat/session-tile-owner-route.test.ts b/apps/desktop/src/app/chat/session-tile-owner-route.test.ts deleted file mode 100644 index 31058c578d..0000000000 --- a/apps/desktop/src/app/chat/session-tile-owner-route.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { readFileSync } from 'node:fs' -import { resolve } from 'node:path' - -import { describe, expect, it } from 'vitest' - -const source = readFileSync(resolve(process.cwd(), 'src/app/chat/session-tile.tsx'), 'utf8') - -describe('SessionTilePane owner-scoped listing', () => { - it('resolves a newly active tile on its persisted owner route', () => { - expect(source).toContain('void resolveStoredSession(storedSessionId, ownerRoute)') - expect(source).not.toMatch(/void resolveStoredSession\(storedSessionId\)\s*\n/) - }) -}) diff --git a/apps/desktop/src/app/chat/session-tile-owner.test.ts b/apps/desktop/src/app/chat/session-tile-owner.test.ts new file mode 100644 index 0000000000..6df849b63a --- /dev/null +++ b/apps/desktop/src/app/chat/session-tile-owner.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it } from 'vitest' + +import { _resetSessionOwnerHintsForTests, setSessionOwnerHint } from '@/store/session' +import type { SessionTile } from '@/store/session-states' +import type { SessionInfo } from '@/types/hermes' + +import { tileOwnerRoute } from './session-tile-owner' + +const row = (over: Partial): SessionInfo => over as SessionInfo + +const tile = (over: Partial & Pick): SessionTile => over as SessionTile + +describe('tileOwnerRoute', () => { + beforeEach(() => { + _resetSessionOwnerHintsForTests() + }) + + it('prefers the tile own explicit route', () => { + const route = tileOwnerRoute( + [tile({ ownerRoute: { connectionId: 'pandora', profile: 'work' }, storedSessionId: 's1' })], + [row({ connection_id: 'other-box', id: 's1', profile: 'default' })], + 's1' + ) + + expect(route).toEqual({ connectionId: 'pandora', profile: 'work' }) + }) + + it('falls back to the session row owner when the tile carries no route', () => { + // How a branch child is opened: openSessionTile with no workspaceScope, so + // the tile route alone leaves the owner undefined and every RPC drops to + // the ambient socket. + const route = tileOwnerRoute( + [tile({ storedSessionId: 's1' })], + [row({ connection_id: 'rigremote', id: 's1', profile: 'default' })], + 's1' + ) + + expect(route).toEqual({ connectionId: 'rigremote', profile: 'default' }) + }) + + it('falls back to the owner hint when neither tile nor row is tagged', () => { + setSessionOwnerHint('s1', { connectionId: 'pandora', profile: 'work' }) + + expect(tileOwnerRoute([tile({ storedSessionId: 's1' })], [], 's1')).toMatchObject({ connectionId: 'pandora' }) + }) + + it('carries a targetProfile through, and omits it when absent', () => { + const routed = tileOwnerRoute( + [tile({ ownerRoute: { connectionId: 'pandora', profile: 'work', targetProfile: 'ceo' }, storedSessionId: 's1' })], + [], + 's1' + ) + + expect(routed).toEqual({ connectionId: 'pandora', profile: 'work', targetProfile: 'ceo' }) + expect( + tileOwnerRoute([tile({ ownerRoute: { connectionId: 'p', profile: 'w' }, storedSessionId: 's1' })], [], 's1') + ).not.toHaveProperty('targetProfile') + }) + + it('narrows a bare profile owner away', () => { + // knownSessionOwner returns a bare profile string for a row that names a + // profile but no connection. It carries no backend identity, so handing it + // on as a route would resolve against whichever connection is active. + expect(tileOwnerRoute([], [row({ id: 's1', profile: 'work' })], 's1')).toBeUndefined() + }) + + it('is undefined for an untagged session, preserving ambient routing', () => { + expect(tileOwnerRoute([], [row({ id: 's1' })], 's1')).toBeUndefined() + expect(tileOwnerRoute([], [], 'missing')).toBeUndefined() + }) +}) diff --git a/apps/desktop/src/app/chat/session-tile-owner.ts b/apps/desktop/src/app/chat/session-tile-owner.ts new file mode 100644 index 0000000000..7b768fd023 --- /dev/null +++ b/apps/desktop/src/app/chat/session-tile-owner.ts @@ -0,0 +1,37 @@ +import { knownSessionOwner } from '@/store/session' +import type { SessionOwnerRoute, SessionOwnerScope } from '@/store/session-request-router' +import type { SessionTile } from '@/store/session-states' +import type { SessionInfo } from '@/types/hermes' + +/** + * The owner a session tile routes its own RPCs through — the tile's explicit + * route first, then the session row's `(connection, profile)` tag, with + * `knownSessionOwner` folding in the owner hint. + * + * A tile opened without an explicit route — a branch child, which + * `openSessionTile` creates with no `workspaceScope` — has no tile route, so + * the row/hint rung is the only thing keeping its model and composer RPCs on + * the backend that owns the session instead of the ambient one. + * + * A bare profile string carries no connection and is not a usable route: + * handing it to `requestForSessionProfile` would resolve it against whichever + * connection is active, which is the bug this ladder exists to avoid. + */ +export function tileOwnerRoute( + tiles: readonly SessionTile[], + rows: readonly SessionInfo[], + storedSessionId: string +): SessionOwnerRoute | undefined { + const owner: SessionOwnerScope = + tiles.find(tile => tile.storedSessionId === storedSessionId)?.ownerRoute ?? knownSessionOwner(rows, storedSessionId) + + if (!owner || typeof owner !== 'object' || !owner.connectionId) { + return undefined + } + + return { + connectionId: owner.connectionId, + profile: owner.profile, + ...(owner.targetProfile ? { targetProfile: owner.targetProfile } : {}) + } +} diff --git a/apps/desktop/src/app/chat/session-tile.test.ts b/apps/desktop/src/app/chat/session-tile.test.ts index 3b93213855..70f3bdea85 100644 --- a/apps/desktop/src/app/chat/session-tile.test.ts +++ b/apps/desktop/src/app/chat/session-tile.test.ts @@ -1,6 +1,37 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' -import { sessionTileResumeFailure } from './session-tile' +import { $gatewayState, $sessions, setSessions } from '@/store/session' +import { $sessionTiles } from '@/store/session-states' + +import { sessionTileResumeFailure, shouldResumeSessionTile, startUnrestoredTileTitleBackfill } from './session-tile' + +describe('shouldResumeSessionTile', () => { + const live = { + gatewayOpen: true, + removalPending: false, + resuming: false, + runtimeId: null, + tileError: undefined + } + + it('resumes an unbound tile once the gateway is open', () => { + expect(shouldResumeSessionTile(live)).toBe(true) + }) + + it('does not resume a session the user is deleting', () => { + // A 4001 racing the delete unbinds the tile runtime, re-arming the resume + // effect against an id that is already gone: the resume 404s and latches an + // error card for a chat that is on its way out. + expect(shouldResumeSessionTile({ ...live, removalPending: true })).toBe(false) + }) + + it('waits for the gateway, a free slot, and an unbound, unlatched tile', () => { + expect(shouldResumeSessionTile({ ...live, gatewayOpen: false })).toBe(false) + expect(shouldResumeSessionTile({ ...live, runtimeId: 'rt-1' })).toBe(false) + expect(shouldResumeSessionTile({ ...live, tileError: 'boom' })).toBe(false) + expect(shouldResumeSessionTile({ ...live, resuming: true })).toBe(false) + }) +}) describe('sessionTileResumeFailure', () => { it('keeps a confirmed durable session retryable instead of repeating a stale 404', () => { @@ -17,3 +48,43 @@ describe('sessionTileResumeFailure', () => { expect(sessionTileResumeFailure('session not found', true, false)).toBeUndefined() }) }) + +describe('startUnrestoredTileTitleBackfill (#94167)', () => { + afterEach(() => { + $gatewayState.set('idle') + $sessionTiles.set([]) + setSessions([]) + }) + + it('backfills unlisted unrestored tiles by id via their ownerRoute once the gateway opens', async () => { + const ownerRoute = { connectionId: 'conn-a', profile: 'writer' } + setSessions([{ id: 'listed', title: 'Already listed' } as never]) + $sessionTiles.set([ + { ownerRoute, storedSessionId: 'old-chat' }, + { storedSessionId: 'listed' }, + { runtimeId: 'rt-live', storedSessionId: 'live' }, + { storedSessionId: 'bot', workspaceTabTitle: 'Bot Chat' } + ]) + + const lookup = vi.fn(async (id: string) => { + const row = { id, title: 'Quarterly review' } as never + setSessions(prev => [row, ...prev]) + + return row + }) + + const stop = startUnrestoredTileTitleBackfill(lookup as never) + expect(lookup).not.toHaveBeenCalled() + + $gatewayState.set('open') + await vi.waitFor(() => expect(lookup).toHaveBeenCalledTimes(1)) + expect(lookup).toHaveBeenCalledWith('old-chat', ownerRoute) + expect($sessions.get().find(row => row.id === 'old-chat')?.title).toBe('Quarterly review') + + // One-shot: a later reconnect does not re-probe. + $gatewayState.set('idle') + $gatewayState.set('open') + expect(lookup).toHaveBeenCalledTimes(1) + stop() + }) +}) diff --git a/apps/desktop/src/app/chat/session-tile.tsx b/apps/desktop/src/app/chat/session-tile.tsx index 773263e3ff..899c026676 100644 --- a/apps/desktop/src/app/chat/session-tile.tsx +++ b/apps/desktop/src/app/chat/session-tile.tsx @@ -28,6 +28,7 @@ import { formatRefValue } from '@/components/assistant-ui/directive-text' import { CenteredThreadSpinner } from '@/components/assistant-ui/thread/status' import { findGroupOfPane } from '@/components/pane-shell/tree/model' import { $layoutTree, closeTreePane, moveTreePane, setTreeGroupTabStrip } from '@/components/pane-shell/tree/store' +import { $workspaceOwnerLabels, workspaceOwnerTitle } from '@/components/pane-shell/workspace-scope' import { Button } from '@/components/ui/button' import { ConfirmDialog } from '@/components/ui/confirm-dialog' import { transcribeAudio } from '@/hermes' @@ -41,12 +42,15 @@ import { $activeGatewayProfile } from '@/store/profile' import { $projectTree } from '@/store/projects' import { sessionAwaitingInput } from '@/store/prompts' import { + $cronSessions, $gatewayState, + $messagingSessions, $selectedStoredSessionId, $sessions, sessionMatchesStoredId, sessionPinId } from '@/store/session' +import { isSessionRemovalPending } from '@/store/session-removal' import { requestForSessionProfile } from '@/store/session-request-router' import { $sessionStates, @@ -55,8 +59,7 @@ import { closeSessionTile, patchSessionTile, type SessionTile, - sessionTileDelegate, - sessionTileOwnerRoute + sessionTileDelegate } from '@/store/session-states' import type { SessionInfo } from '@/types/hermes' @@ -68,6 +71,7 @@ import { SessionDraftTitle } from './session-draft-title' import { startSessionDrag } from './session-drag' import { SessionStatusDot } from './session-status-dot' import { useSessionTileActions } from './session-tile-actions' +import { tileOwnerRoute } from './session-tile-owner' import { type SessionView, SessionViewProvider } from './session-view' import { SessionContextMenu } from './sidebar/session-actions-menu' import { lastVisibleMessageIsUser } from './thread-loading' @@ -96,6 +100,28 @@ export function sessionTileResumeFailure( return 'Session unavailable — you can retry resuming it.' } +/** Should this tile dispatch a `session.resume`? + * + * - The gateway must be OPEN: persisted tiles mount at boot while it is still + * connecting, and an ungated resume rejected there latched every restored + * tile into the error card. + * - A bound runtime, a latched error, or an in-flight attempt means there is + * nothing to do. + * - A removal-pending session is skipped for the same reason the primary's + * `resumeSession` skips it: a 4001 racing a delete unbinds this tile's + * runtime and re-arms the effect against an id that is already gone. The + * resume would 404 and latch an error card for a chat the user deleted; + * `closeSessionTile` lands moments later. */ +export function shouldResumeSessionTile(opts: { + gatewayOpen: boolean + removalPending: boolean + resuming: boolean + runtimeId: null | string | undefined + tileError: string | undefined +}): boolean { + return !opts.removalPending && opts.gatewayOpen && !opts.runtimeId && !opts.tileError && !opts.resuming +} + /** The tile's SessionView: the same atom shape the primary chat renders * from, computed from this session's slice of `$sessionStates`. */ function buildTileView(storedSessionId: string): SessionView { @@ -157,7 +183,21 @@ function TileChat({ }) { const { gateway, requestGateway } = useGatewayRequest() const queryClient = useQueryClient() - const ownerRoute = sessionTileOwnerRoute(storedSessionId) + + // Owner ladder, same as useSessionTileActions (session-tile-actions.ts:99-103). + // Recomputed when the tile store or any owner-bearing session list changes, + // NOT on every render: this component re-renders per streamed token, and the + // lookup spreads three arrays before scanning them. + const tiles = useStore($sessionTiles) + const sessionRows = useStore($sessions) + const cronRows = useStore($cronSessions) + const messagingRows = useStore($messagingSessions) + + const ownerRoute = useMemo(() => { + const rows = cronRows.length || messagingRows.length ? [...sessionRows, ...cronRows, ...messagingRows] : sessionRows + + return tileOwnerRoute(tiles, rows, storedSessionId) + }, [cronRows, messagingRows, sessionRows, storedSessionId, tiles]) const requestTileGateway = useCallback( (method: string, params?: Record, timeoutMs?: number, signal?: AbortSignal): Promise => @@ -354,13 +394,18 @@ export function SessionTilePane({ storedSessionId }: { storedSessionId: string } } }, [hasMessages, ownerRoute, runtimeId, storedSessionId, storedSessionStillExists]) - // Same gating as the primary's route resume (use-route-resume): never fire - // session.resume before the gateway is OPEN. Persisted tiles mount at boot - // while it's still connecting — an ungated resume rejected there and - // latched every restored tile into the error card. + // Gating lives in shouldResumeSessionTile (unit-tested there). // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { - if (!gatewayOpen || runtimeId || tile?.error || resumingRef.current) { + if ( + !shouldResumeSessionTile({ + gatewayOpen, + removalPending: isSessionRemovalPending(storedSessionId), + resuming: resumingRef.current, + runtimeId, + tileError: tile?.error + }) + ) { return } @@ -461,6 +506,33 @@ export function tileStoredRow(storedSessionId: string): SessionInfo | undefined ) } +/** One-shot by-id title fill for restored tiles that never mount (#94167). + * A restored background tab has no runtimeId and does not mount its pane, so + * the resolution effect above never runs; when its row is outside the recents + * page and project tree, `tileTitle()` reads "New session" until first click. + * `resolveStoredSession` upserts the row into `$sessions`, which the tab strip + * already watches — nothing is persisted. Runs once the gateway can answer. */ +export function startUnrestoredTileTitleBackfill(lookup = resolveStoredSession): () => void { + const run = () => { + if ($gatewayState.get() !== 'open') { + return + } + + off() + + for (const tile of $sessionTiles.get()) { + if (!tile.runtimeId && !tile.workspaceTabTitle && !tileStoredRow(tile.storedSessionId)) { + void lookup(tile.storedSessionId, tile.ownerRoute).catch(() => undefined) + } + } + } + + const off = $gatewayState.listen(run) + run() + + return off +} + /** The tab's REGISTERED name. Deliberately the bare placeholder for a draft * rather than its live composer title (`tabTitle` renders that): re-registering * per keystroke would re-render the strip, and holding the draft's text here @@ -473,14 +545,26 @@ function tileTitle(storedSessionId: string): string { return stored ? sessionTitle(stored) : explicit || NEW_SESSION_TITLE } +/** The tab's CAPTION: a bot chat's owner name over the canonical stored title + * (#99152). The menu keeps `tileTitle` — rename/delete show the real row. */ +function tileCaption(storedSessionId: string): string { + return workspaceOwnerTitle( + tileTitle(storedSessionId), + $sessionTiles.get().find(tile => tile.storedSessionId === storedSessionId) + ) +} + /** The `@session` link payload for a tile tab drag — id + owning profile + title. * Resolved at drag time, so an unsent tab drags under its draft name. */ function tileDragPayload(storedSessionId: string): SessionDragPayload { const stored = tileStoredRow(storedSessionId) - const explicit = $sessionTiles.get().find(tile => tile.storedSessionId === storedSessionId)?.workspaceTabTitle - const title = stored ? sessionTitle(stored) : explicit || draftTitleFor(storedSessionId) || NEW_SESSION_TITLE + const tile = $sessionTiles.get().find(candidate => candidate.storedSessionId === storedSessionId) - return { id: storedSessionId, profile: stored?.profile ?? '', title } + const title = stored + ? sessionTitle(stored) + : tile?.workspaceTabTitle || draftTitleFor(storedSessionId) || NEW_SESSION_TITLE + + return { id: storedSessionId, profile: stored?.profile ?? '', title: workspaceOwnerTitle(title, tile) } } // --------------------------------------------------------------------------- @@ -667,14 +751,14 @@ export const watchSessionTiles = paneMirror({ // $projectTree: a tile whose session is older than the recents page resolves // its title through the tree, which loads after the tiles register. (The tab's // status dot subscribes to color/state itself, so it needs no `also` entry.) - also: [$sessions, $projectTree], + also: [$sessions, $projectTree, $workspaceOwnerLabels], key: t => t.storedSessionId, prefix: 'session-tile', dir: t => t.dir, anchor: t => t.anchor, before: t => t.before, minWidth: '20rem', - title: tileTitle, + title: tileCaption, // The tab's status dot — the SAME primitive the sidebar row renders, keyed by // the stored id, so a session's status/color can never disagree between the // two surfaces. Self-subscribing (live state + resolved color), so the strip diff --git a/apps/desktop/src/app/chat/sidebar/index.tsx b/apps/desktop/src/app/chat/sidebar/index.tsx index e6afa1247f..41459bd2e1 100644 --- a/apps/desktop/src/app/chat/sidebar/index.tsx +++ b/apps/desktop/src/app/chat/sidebar/index.tsx @@ -89,7 +89,6 @@ import { $projectScope, $projectTree, $projectTreeLoading, - $removedSessionIds, $reposScanning, ALL_PROJECTS, enterProject, @@ -127,6 +126,7 @@ import { } from '@/store/session' import { $sessionDotStateById, sessionStatusBucket } from '@/store/session-dot-state' import { $unconfirmedPinWrites } from '@/store/session-pin-sync' +import { $removedSessionIds } from '@/store/session-removal' import { $focusedStoredSessionId, $workingSessionIds, type SplitDir } from '@/store/session-states' import { ackAllSessionsRead } from '@/store/session-unread' import { markSessionUnread } from '@/store/session-unread-remote' diff --git a/apps/desktop/src/app/chat/transcript-window.test.ts b/apps/desktop/src/app/chat/transcript-window.test.ts index 600056e0b7..963b914a79 100644 --- a/apps/desktop/src/app/chat/transcript-window.test.ts +++ b/apps/desktop/src/app/chat/transcript-window.test.ts @@ -4,8 +4,10 @@ import type { ChatMessage } from '@/lib/chat-messages' import { RENDER_WEIGHT_CHARS } from '@/lib/render-weight' import { + advanceSessionTranscriptWindow, advanceTranscriptWindow, alignToBranchGroup, + MAX_SESSION_WINDOWS, selectTranscriptWindow, TRANSCRIPT_WINDOW_BUDGET, TRANSCRIPT_WINDOW_MIN_MESSAGES, @@ -214,6 +216,87 @@ describe('advanceTranscriptWindow', () => { }) }) +describe('advanceSessionTranscriptWindow', () => { + const heavyChars = RENDER_WEIGHT_CHARS * 40 + + it('matches a fresh walk on first visit', () => { + const memos = new Map() + const messages = transcript(400, heavyChars) + + const state = advanceSessionTranscriptWindow(memos, 'session-a', messages) + + expect(state.window).toEqual(selectTranscriptWindow(messages)) + expect(state.anchorId).toBe(state.window.messages[0].id) + }) + + it('returns the SAME windowed slice by reference on a warm re-visit with an unchanged transcript', () => { + const memos = new Map() + const sessionA = transcript(400, heavyChars) + const sessionB = transcript(300, heavyChars).map(m => ({ ...m, id: `b-${m.id}` })) + + // Visit B, then A, then B again — the exact warm-switch shape of #95595. + const firstB = advanceSessionTranscriptWindow(memos, 'session-b', sessionB) + advanceSessionTranscriptWindow(memos, 'session-a', sessionA) + const secondB = advanceSessionTranscriptWindow(memos, 'session-b', sessionB) + + expect(secondB.window.windowed).toBe(true) + // THE perf guard: same transcript array => same windowed slice reference, + // so the runtime repository and every row keep their identity. + expect(secondB.window.messages).toBe(firstB.window.messages) + expect(secondB.anchorId).toBe(firstB.anchorId) + }) + + it('holds the sticky cut when a re-visited session grew while away', () => { + const memos = new Map() + const messages = transcript(400, heavyChars) + const state = advanceSessionTranscriptWindow(memos, 'session-a', messages) + + // While away, the session streamed a few light turns (within slack). + const grown = [...messages, ...transcript(10, 100)] + const next = advanceSessionTranscriptWindow(memos, 'session-a', grown) + + // Sticky cut survived the switch-away: same anchor, no fresh re-walk. + expect(next.anchorId).toBe(state.anchorId) + expect(next.window.messages[0].id).toBe(state.window.messages[0].id) + // The anchored slice now simply includes the 10 new light turns. + expect(next.window.messages.length).toBe(state.window.messages.length + 10) + }) + + it('falls back to a fresh walk when the anchor vanished while away', () => { + const memos = new Map() + const messages = transcript(400, heavyChars) + advanceSessionTranscriptWindow(memos, 'session-a', messages) + + // Compression rewrite: disjoint ids while the user was elsewhere. + const rewritten = transcript(300, heavyChars).map(m => ({ ...m, id: `compressed-${m.id}` })) + const next = advanceSessionTranscriptWindow(memos, 'session-a', rewritten) + + expect(next.window).toEqual(selectTranscriptWindow(rewritten)) + }) + + it('re-walks when pages change on re-entry', () => { + const memos = new Map() + const messages = transcript(400, heavyChars) + + const one = advanceSessionTranscriptWindow(memos, 'session-a', messages, 1) + const two = advanceSessionTranscriptWindow(memos, 'session-a', messages, 2) + + expect(two.window.messages.length).toBeGreaterThan(one.window.messages.length) + }) + + it('keeps sessions independent and evicts the oldest memo past the cap', () => { + const memos = new Map() + + for (let i = 0; i < MAX_SESSION_WINDOWS + 5; i++) { + advanceSessionTranscriptWindow(memos, `session-${i}`, transcript(400, heavyChars)) + } + + expect(memos.size).toBeLessThanOrEqual(MAX_SESSION_WINDOWS) + expect(memos.has('session-0')).toBe(false) + expect(memos.has(`session-${MAX_SESSION_WINDOWS + 4}`)).toBe(true) + }) +}) + describe('alignToBranchGroup', () => { const messages = [message('u-1', 10), message('a-1', 10, 'g'), message('a-2', 10, 'g'), message('u-2', 10)] diff --git a/apps/desktop/src/app/chat/transcript-window.ts b/apps/desktop/src/app/chat/transcript-window.ts index b63e6d316c..6437331646 100644 --- a/apps/desktop/src/app/chat/transcript-window.ts +++ b/apps/desktop/src/app/chat/transcript-window.ts @@ -170,3 +170,65 @@ export function advanceTranscriptWindow( return { anchorId: window.windowed ? window.messages[0].id : null, pages, window } } + +/** How many sessions keep a sticky window before the oldest is evicted. */ +export const MAX_SESSION_WINDOWS = 12 + +/** + * A window state plus the exact message array it was computed from. + * The array identity is load-bearing: when a session is re-entered with the + * IDENTICAL transcript (the warm-switch path of #95595), the stored window — + * including the exact `window.messages` slice reference — is reused as-is. + * The reference reuse is what stops `useRuntimeMessageRepository` from + * rebuilding (and every row from re-rendering) on a warm switch. + */ +export interface SessionWindowMemo { + messages: readonly ChatMessage[] + state: TranscriptWindowState +} + +/** + * `advanceTranscriptWindow` with a STICKY cut that survives session switches. + * + * The previous single-slot state was nulled on every switch, so a warm + * re-entry always re-ran the weight walk and rebuilt the windowed slice — + * which re-indexed the whole windowed transcript (markdown re-parse + + * re-highlight per row) even though nothing had changed. This keeps one memo + * per session: + * + * - Re-entering a session with the same transcript array returns the cached + * windowed slice BY REFERENCE — the runtime repository and every message + * row stay mounted, so the switch is O(1). + * - Re-entering with a changed transcript keeps the sticky cut (anchor still + * present, tail within budget + slack) instead of re-walking from scratch. + * - The anchor vanishing (compression rewrite) or a pages change falls + * through to `advanceTranscriptWindow`'s existing fresh-walk behaviour. + * + * The map is bounded (oldest session evicted) so an unbounded session list + * cannot grow it without limit. + */ +export function advanceSessionTranscriptWindow( + memos: Map, + sessionKey: string, + messages: readonly ChatMessage[], + pages = 1 +): TranscriptWindowState { + const memo = memos.get(sessionKey) + + // Warm re-visit with the identical transcript and page count: reuse the + // cached state wholesale, preserving the windowed slice reference. + if (memo && memo.messages === messages && memo.state.pages === pages) { + return memo.state + } + + const state = advanceTranscriptWindow(memo?.state ?? null, messages, pages) + + memos.set(sessionKey, { messages, state }) + + if (memos.size > MAX_SESSION_WINDOWS) { + const oldest = memos.keys().next().value as string + memos.delete(oldest) + } + + return state +} diff --git a/apps/desktop/src/app/contrib/controller.tsx b/apps/desktop/src/app/contrib/controller.tsx index 5c626d91b4..a1fa39755b 100644 --- a/apps/desktop/src/app/contrib/controller.tsx +++ b/apps/desktop/src/app/contrib/controller.tsx @@ -34,6 +34,7 @@ import { toggleTargetZoneTabStrip, watchContributedPanes } from '@/components/pane-shell/tree/store' +import { $workspaceOwnerLabels, workspaceOwnerTitle } from '@/components/pane-shell/workspace-scope' import { SidebarProvider } from '@/components/ui/sidebar' import { discoverBundledPlugins } from '@/contrib/plugins' import { Slot } from '@/contrib/react/slot' @@ -70,6 +71,7 @@ import { } from '@/store/review' import { $currentCwd, $selectedStoredSessionId, $sessions, $yoloActive, sessionMatchesStoredId } from '@/store/session' import { watchSessionPins } from '@/store/session-pin-sync' +import { $botChatScopes } from '@/store/session-states' import { watchUnreadWriteGuard } from '@/store/session-unread-remote' import { $statusbarVisible } from '@/store/statusbar-prefs' import { isBrowserWindow, isHudWindow } from '@/store/windows' @@ -82,6 +84,7 @@ import { startSessionDrag } from '../chat/session-drag' import { SessionTileCloseConfirm, stackSessionTilesIntoMain, + startUnrestoredTileTitleBackfill, watchSessionTiles, WorkspaceTabMenu } from '../chat/session-tile' @@ -457,6 +460,7 @@ watchContributedPanes() // into the transparent overlay). if (!isBrowserWindow() && !isHudWindow()) { watchSessionTiles() + startUnrestoredTileTitleBackfill() watchRouteTiles() watchPreviewTiles() } @@ -490,7 +494,12 @@ const syncWorkspaceTitle = () => { area: 'panes', // The placeholder, not the draft's live name — `tabTitle` below renders // that. Keeping it here would re-register the pane on every keystroke. - title: stored ? storedSessionTitle(stored) : NEW_SESSION_TITLE, + // A bot chat reads as its BOT: every canonical Bot Chat is stored under + // the same name, which told two open bots apart by nothing (#99152). + title: workspaceOwnerTitle( + stored ? storedSessionTitle(stored) : NEW_SESSION_TITLE, + selected ? $botChatScopes.get()[selected] : undefined + ), data: { // The tab's status dot — the SAME primitive the sidebar row and session // tiles render, so the main tab never disagrees with its sidebar row. A @@ -515,6 +524,8 @@ const syncWorkspaceTitle = () => { $selectedStoredSessionId.listen(syncWorkspaceTitle) $sessions.listen(syncWorkspaceTitle) +$botChatScopes.listen(syncWorkspaceTitle) +$workspaceOwnerLabels.listen(syncWorkspaceTitle) $workspaceIsPage.listen(syncWorkspaceTitle) // Layout reset collapses every session tile into main as a tab (after the diff --git a/apps/desktop/src/app/contrib/hooks/use-background-sync.test.ts b/apps/desktop/src/app/contrib/hooks/use-background-sync.test.ts index 404f5b536d..10f978faa9 100644 --- a/apps/desktop/src/app/contrib/hooks/use-background-sync.test.ts +++ b/apps/desktop/src/app/contrib/hooks/use-background-sync.test.ts @@ -8,15 +8,18 @@ import { $activeSessionId, $selectedStoredSessionId, setBusy, + setCronSessions, setMessagingSessions, setSessionOwnerHint, setSessions } from '@/store/session' import { $attentionSessionIds, + $sessionTiles, $stalledSessionIds, $workingSessionIds, clearAllSessionStates, + publishSessionState, SESSION_WATCHDOG_TIMEOUT_MS } from '@/store/session-states' @@ -38,7 +41,13 @@ vi.mock('@/hermes', async importOriginal => ({ getLatestSessionMessages: vi.fn() })) +vi.mock('@/store/projects', async importOriginal => ({ + ...(await importOriginal()), + refreshProjectTree: vi.fn(async () => undefined) +})) + const { getLatestSessionMessages } = await import('@/hermes') +const { refreshProjectTree } = await import('@/store/projects') const ACTIVE_RUNTIME_ID = 'runtime-active' const ACTIVE_STORED_ID = 'stored-active' @@ -91,11 +100,13 @@ function useSyncHarness({ activeIsMessaging = false, activeSessionId, activeStoredSessionId, + gatewayState = 'open', refreshActiveTranscript }: { activeIsMessaging?: boolean activeSessionId: string | null activeStoredSessionId: string | null + gatewayState?: string refreshActiveTranscript: () => Promise }) { const updateSessionState: Parameters[0]['updateSessionState'] = vi.fn( @@ -113,7 +124,7 @@ function useSyncHarness({ activeSessionId, activeStoredSessionId, freshDraftReady: false, - gatewayState: 'open', + gatewayState, refreshActiveTranscript, refreshCronJobs: vi.fn(), refreshCurrentModel: vi.fn(), @@ -125,17 +136,23 @@ function useSyncHarness({ }) } -function renderSync( - refreshActiveTranscript: () => Promise, - options: { activeIsMessaging?: boolean; activeSessionId?: null | string; activeStoredSessionId?: null | string } = {} -) { - return renderHook(() => - useSyncHarness({ - activeSessionId: ACTIVE_RUNTIME_ID, - activeStoredSessionId: ACTIVE_STORED_ID, - refreshActiveTranscript, - ...options - }) +type SyncOptions = { + activeIsMessaging?: boolean + activeSessionId?: null | string + activeStoredSessionId?: null | string + gatewayState?: string +} + +function renderSync(refreshActiveTranscript: () => Promise, options: SyncOptions = {}) { + return renderHook( + (props: SyncOptions) => + useSyncHarness({ + activeSessionId: ACTIVE_RUNTIME_ID, + activeStoredSessionId: ACTIVE_STORED_ID, + refreshActiveTranscript, + ...props + }), + { initialProps: options } ) } @@ -153,11 +170,13 @@ afterEach(() => { $activeSessionId.set(null) $selectedStoredSessionId.set(null) setSessions([]) + setCronSessions([]) setMessagingSessions([]) setBusy(false) vi.clearAllMocks() vi.restoreAllMocks() clearAllSessionStates() + $sessionTiles.set([]) resetTypingActivityTracking() }) @@ -228,7 +247,6 @@ describe('active transcript refresh', () => { const signatureRef = { current: new Map() } const requestSequenceRef = { current: 0 } - const busyRef = { current: false } vi.mocked(getLatestSessionMessages).mockImplementation(async (storedId: string) => { if (storedId === TILE_STORED_ID) { @@ -250,7 +268,6 @@ describe('active transcript refresh', () => { await act(async () => { await reconcileTileTranscriptsForTest({ tiles: [{ storedSessionId: TILE_STORED_ID, runtimeId: TILE_RUNTIME_ID }], - busyRef, requestSequenceRef, signatureRef, updateSessionState @@ -259,7 +276,136 @@ describe('active transcript refresh', () => { // Behavior assertions: expect(updaterCallCount).toBeGreaterThan(0) - expect(getLatestSessionMessages).toHaveBeenCalledWith(TILE_STORED_ID) + expect(getLatestSessionMessages).toHaveBeenCalledWith(TILE_STORED_ID, undefined) + }) + + it('reconciles an idle tile while the main pane is busy', async () => { + const runtimeId = 'runtime-idle-tile' + const storedId = 'stored-idle-tile' + const idleState = createClientSessionState(storedId) + + setBusy(true) + publishSessionState(runtimeId, idleState) + vi.mocked(getLatestSessionMessages).mockResolvedValue(transcript('idle tile update', storedId) as never) + + const updateSessionState = vi.fn((sessionId: string, updater: (state: typeof idleState) => typeof idleState) => { + expect(sessionId).toBe(runtimeId) + + return updater(idleState) + }) + + await reconcileTileTranscriptsForTest({ + tiles: [{ runtimeId, storedSessionId: storedId }], + requestSequenceRef: { current: 0 }, + signatureRef: { current: new Map() }, + updateSessionState + }) + + expect(getLatestSessionMessages).toHaveBeenCalledWith(storedId, undefined) + expect(updateSessionState).toHaveBeenCalledTimes(1) + }) + + it('does not reconcile a busy tile when the main pane is idle', async () => { + const runtimeId = 'runtime-busy-tile' + const storedId = 'stored-busy-tile' + const liveState = createClientSessionState(storedId) + + liveState.busy = true + liveState.messages = [ + { + id: 'live-assistant', + parts: [{ text: 'streaming answer', type: 'text' }], + pending: true, + role: 'assistant' + } + ] + publishSessionState(runtimeId, liveState) + vi.mocked(getLatestSessionMessages).mockResolvedValue({ messages: [], session_id: storedId } as never) + + const updateSessionState = vi.fn() + + await reconcileTileTranscriptsForTest({ + tiles: [{ runtimeId, storedSessionId: storedId }], + requestSequenceRef: { current: 0 }, + signatureRef: { current: new Map() }, + updateSessionState + }) + + expect(getLatestSessionMessages).not.toHaveBeenCalled() + expect(updateSessionState).not.toHaveBeenCalled() + }) + + it('discards a tile snapshot when the tile closes during the read', async () => { + const runtimeId = 'runtime-closing-tile' + const storedId = 'stored-closing-tile' + let resolveRead: (value: unknown) => void = () => undefined + + $sessionTiles.set([{ runtimeId, storedSessionId: storedId }]) + publishSessionState(runtimeId, createClientSessionState(storedId)) + vi.mocked(getLatestSessionMessages).mockReturnValueOnce( + new Promise(resolve => { + resolveRead = resolve + }) as never + ) + + const updateSessionState = vi.fn() + + const reconcile = reconcileTileTranscriptsForTest({ + requestSequenceRef: { current: 0 }, + signatureRef: { current: new Map() }, + updateSessionState + }) + + $sessionTiles.set([]) + resolveRead(transcript('stale tile answer', storedId)) + await reconcile + + expect(updateSessionState).not.toHaveBeenCalled() + }) + + it('isolates tile transcript reads by connection and profile while preserving the legacy local path', async () => { + vi.mocked(getLatestSessionMessages).mockImplementation(async storedId => transcript(storedId, storedId) as never) + + const updateSessionState: Parameters[0]['updateSessionState'] = vi.fn( + (_sessionId, updater) => updater({} as Parameters[0]) + ) + + await reconcileTileTranscriptsForTest({ + tiles: [ + { + ownerRoute: { + connectionId: 'connection-a', + mode: 'remote', + profile: 'shared-profile', + targetProfile: 'target-a' + }, + runtimeId: 'runtime-a', + storedSessionId: 'stored-a' + }, + { + ownerRoute: { connectionId: 'connection-b', mode: 'remote', profile: 'shared-profile' }, + runtimeId: 'runtime-b', + storedSessionId: 'stored-b' + }, + { runtimeId: 'runtime-local', storedSessionId: 'stored-local' } + ], + requestSequenceRef: { current: 0 }, + signatureRef: { current: new Map() }, + updateSessionState + }) + + expect(getLatestSessionMessages).toHaveBeenCalledWith('stored-a', { + connectionId: 'connection-a', + profile: 'target-a' + }) + expect(getLatestSessionMessages).toHaveBeenCalledWith('stored-b', { + connectionId: 'connection-b', + profile: 'shared-profile' + }) + expect(getLatestSessionMessages).toHaveBeenCalledWith('stored-local', undefined) + expect(updateSessionState).toHaveBeenCalledWith('runtime-a', expect.any(Function), 'stored-a') + expect(updateSessionState).toHaveBeenCalledWith('runtime-b', expect.any(Function), 'stored-b') + expect(updateSessionState).toHaveBeenCalledWith('runtime-local', expect.any(Function), 'stored-local') }) it('skips the tile fetch entirely when nothing changed (signature-gated)', async () => { @@ -287,13 +433,11 @@ describe('active transcript refresh', () => { signatureRef.current.set(`tile:${TILE_STORED_ID}`, preSignature) const updateSessionState = vi.fn() - const busyRef = { current: false } const requestSequenceRef = { current: 0 } await act(async () => { await reconcileTileTranscriptsForTest({ tiles: [{ storedSessionId: TILE_STORED_ID, runtimeId: TILE_RUNTIME_ID }], - busyRef, requestSequenceRef, signatureRef, updateSessionState @@ -343,14 +487,15 @@ describe('active transcript refresh', () => { const refresh = vi.fn(async () => undefined) renderSync(refresh) - expect(refresh).not.toHaveBeenCalled() + // Exactly the one connect-time pull (#94779) — no timer after it. + expect(refresh).toHaveBeenCalledTimes(1) await act(async () => { vi.advanceTimersByTime(60_000) await Promise.resolve() }) - expect(refresh).not.toHaveBeenCalled() + expect(refresh).toHaveBeenCalledTimes(1) }) it('retains the existing periodic backstop for messaging sessions', async () => { @@ -373,11 +518,12 @@ describe('active transcript refresh', () => { it('only defers an external tick while busy, then refreshes once after idle', async () => { $changeEventsAvailable.set(true) - setBusy(true) const refresh = vi.fn(async () => undefined) renderSync(refresh) + refresh.mockClear() // drop the connect-time pull; this test is about busy transitions + act(() => setBusy(true)) act(() => setBusy(false)) expect(refresh).not.toHaveBeenCalled() act(() => setBusy(true)) @@ -392,12 +538,31 @@ describe('active transcript refresh', () => { await waitFor(() => expect(refresh).toHaveBeenCalledTimes(1)) }) + it('pulls the open transcript once per (re)connect, not on session switches (#94779)', () => { + $changeEventsAvailable.set(true) + const refresh = vi.fn(async () => undefined) + + const { rerender } = renderSync(refresh, { gatewayState: 'connecting' }) + expect(refresh).not.toHaveBeenCalled() + + rerender({ gatewayState: 'open' }) + expect(refresh).toHaveBeenCalledTimes(1) + + rerender({ activeSessionId: 'runtime-other', activeStoredSessionId: 'stored-other', gatewayState: 'open' }) + expect(refresh).toHaveBeenCalledTimes(1) + + rerender({ activeSessionId: 'runtime-other', activeStoredSessionId: 'stored-other', gatewayState: 'closed' }) + rerender({ activeSessionId: 'runtime-other', activeStoredSessionId: 'stored-other', gatewayState: 'open' }) + expect(refresh).toHaveBeenCalledTimes(2) + }) + it('coalesces a burst of global session-change ticks', async () => { vi.useFakeTimers() $changeEventsAvailable.set(true) const refresh = vi.fn(async () => undefined) renderSync(refresh) + refresh.mockClear() // drop the connect-time pull; this test is about tick coalescing act(() => { for (let index = 0; index < 20; index += 1) { @@ -413,6 +578,16 @@ describe('active transcript refresh', () => { expect(refresh).toHaveBeenCalledTimes(1) }) + + it('refreshes the project tree on a sessions.changed tick, alongside the sessions list (#100354)', async () => { + $changeEventsAvailable.set(true) + + renderSync(vi.fn(async () => undefined)) + + act(() => notifySessionsChanged()) + + await waitFor(() => expect(refreshProjectTree).toHaveBeenCalledTimes(1)) + }) }) describe('reconcileActiveTranscript', () => { @@ -435,6 +610,19 @@ describe('reconcileActiveTranscript', () => { }) }) + it('resolves and hydrates a cron session from the cron sessions store', async () => { + setCronSessions([{ id: ACTIVE_STORED_ID, profile: 'cron-profile', source: 'cron' } as never]) + const fixture = makeRefresh(resolveActiveTranscriptSession) + vi.mocked(getLatestSessionMessages).mockResolvedValue(transcript('cron progress') as never) + + await fixture.refresh() + + expect(getLatestSessionMessages).toHaveBeenCalledWith(ACTIVE_STORED_ID, 'cron-profile') + expect(fixture.states.get(ACTIVE_RUNTIME_ID)?.messages.at(-1)?.parts[0]).toMatchObject({ + text: 'cron progress' + }) + }) + it('fails closed when a hidden session id has multiple owner hints', async () => { const ambiguousStoredSessionId = 'ambiguous-hidden-chat' setSessionOwnerHint(ambiguousStoredSessionId, { diff --git a/apps/desktop/src/app/contrib/hooks/use-background-sync.ts b/apps/desktop/src/app/contrib/hooks/use-background-sync.ts index 452e1ddb67..9bd5e47403 100644 --- a/apps/desktop/src/app/contrib/hooks/use-background-sync.ts +++ b/apps/desktop/src/app/contrib/hooks/use-background-sync.ts @@ -9,14 +9,14 @@ import { sessionMessagesSignature } from '@/lib/session-signatures' import { $changeEventsAvailable, $cronChangeTick, $sessionsChangeTick } from '@/store/live-sync' import { $onBattery, batteryPollInterval } from '@/store/power' import { refreshActiveProfile } from '@/store/profile' +import { refreshProjectTree } from '@/store/projects' import { $activeSessionId, $busy, $currentCwd, - $messagingSessions, $selectedStoredSessionId, - $sessions, getSessionOwnerHint, + ownerLookupSessionRows, sessionMatchesStoredId, setCurrentCwd } from '@/store/session' @@ -39,9 +39,7 @@ interface ActiveTranscriptSession { /** Resolve an active transcript from visible rows or its unique hidden owner. */ export function resolveActiveTranscriptSession(storedSessionId: string): ActiveTranscriptSession | undefined { - const visible = - $sessions.get().find(session => sessionMatchesStoredId(session, storedSessionId)) ?? - $messagingSessions.get().find(session => sessionMatchesStoredId(session, storedSessionId)) + const visible = ownerLookupSessionRows().find(session => sessionMatchesStoredId(session, storedSessionId)) if (visible) { return { profile: visible.profile } @@ -66,6 +64,22 @@ export interface ActiveTranscriptRefreshDeps { ) => ClientSessionState } +function tileRuntimeOwnsLiveState(runtimeId: string): boolean { + const state = $sessionStates.get()[runtimeId] + + return Boolean(state && (state.busy || state.awaitingResponse || state.needsInput || state.turnLive)) +} + +type TileTranscriptTarget = { ownerRoute?: SessionProfileRoute; storedSessionId: string; runtimeId?: string } + +/** Signature key per tile — carries the owner route so two connections/profiles + * sharing a stored id (or a tile re-homed to another owner) never alias. */ +function tileTranscriptSignatureKey(tile: TileTranscriptTarget): string { + const route = tile.ownerRoute + + return `tile:${route ? `${route.connectionId}:${route.targetProfile ?? route.profile}:` : ''}${tile.storedSessionId}` +} + /** * Reconcile the persisted transcripts of every open WORKSPACE TILE (#93942 * slice 1). Bot canonical chats live here — never in $sessions / @@ -84,15 +98,13 @@ export interface ActiveTranscriptRefreshDeps { */ export async function reconcileTileTranscripts({ requestSequenceRef, - busyRef, signatureRef, updateSessionState, tiles: tilesOverride }: { - busyRef: MutableRefObject requestSequenceRef: MutableRefObject signatureRef: MutableRefObject> - tiles?: Array<{ storedSessionId: string; runtimeId?: string }> + tiles?: TileTranscriptTarget[] updateSessionState: ( sessionId: string, updater: (state: ClientSessionState) => ClientSessionState, @@ -100,6 +112,13 @@ export async function reconcileTileTranscripts({ ) => ClientSessionState }): Promise { const tiles = tilesOverride ?? $sessionTiles.get() + const openSignatureKeys = new Set(tiles.map(tileTranscriptSignatureKey)) + + for (const signatureKey of signatureRef.current.keys()) { + if (!openSignatureKeys.has(signatureKey)) { + signatureRef.current.delete(signatureKey) + } + } for (const tile of tiles) { const storedSessionId = tile.storedSessionId @@ -110,7 +129,7 @@ export async function reconcileTileTranscripts({ continue } - if (!storedSessionId || !runtimeSessionId || busyRef.current) { + if (!storedSessionId || !runtimeSessionId || tileRuntimeOwnsLiveState(runtimeSessionId)) { continue } @@ -123,23 +142,39 @@ export async function reconcileTileTranscripts({ // With a tiles override (test path), the live $sessionTiles check can't // see the synthetic tile — treat override tiles as present. - const stillPresent = tilesOverride - ? tilesOverride.some(t => t.storedSessionId === storedSessionId && t.runtimeId === runtimeSessionId) - : $sessionTiles.get().some(t => t.storedSessionId === storedSessionId && t.runtimeId === runtimeSessionId) + const tileStillPresent = () => + tilesOverride + ? tilesOverride.some(t => t.storedSessionId === storedSessionId && t.runtimeId === runtimeSessionId) + : $sessionTiles.get().some(t => t.storedSessionId === storedSessionId && t.runtimeId === runtimeSessionId) + + // Bot tiles are pinned to an exact owner (connection + target profile); + // read from that backend, not whichever profile is foreground. Tiles + // without a route keep the legacy local read. + const profileScope: ProfileScope = tile.ownerRoute + ? { + connectionId: tile.ownerRoute.connectionId, + profile: tile.ownerRoute.targetProfile ?? tile.ownerRoute.profile + } + : undefined + + const signatureKey = tileTranscriptSignatureKey(tile) try { - const latest = await getLatestSessionMessages(storedSessionId) + const latest = await getLatestSessionMessages(storedSessionId, profileScope) - if (requestId !== requestSequenceRef.current || busyRef.current || !stillPresent) { + if ( + requestId !== requestSequenceRef.current || + tileRuntimeOwnsLiveState(runtimeSessionId) || + !tileStillPresent() + ) { // Tile closed or superseded mid-read — discard AND prune its // signature so the map doesn't grow one entry per ever-opened tile // for the app's lifetime (#94255 review point 3). - signatureRef.current.delete(`tile:${storedSessionId}`) + signatureRef.current.delete(signatureKey) continue } - const signatureKey = `tile:${storedSessionId}` const signature = sessionMessagesSignature(latest.messages) if (signatureRef.current.get(signatureKey) === signature) { @@ -547,10 +582,8 @@ export function useBackgroundSync({ // transcript signatures, so no-change ticks and closed tiles cost nothing. const tileRequestSequenceRef = useRef(0) const tileSignatureRef = useRef(new Map()) - // Read $busy.get() directly inside the reconcile loop instead of mirroring - // the atom into a ref (lint: no-restricted-syntax — refs synced from atoms - // lag one render). The reconcile runs on tick, not render, so .get() is - // always current. + // Tile reconciliation reads each runtime's live state directly from + // $sessionStates; the primary chat's $busy atom has no authority over tiles. const requestActiveTranscriptRefresh = useCallback( (preservePending: boolean) => { @@ -624,6 +657,19 @@ export function useBackgroundSync({ } }, [activeConnectionId, activeGatewayProfile, gatewayState, refreshCurrentModel, refreshSessions, requestGateway]) + // Reconnect backstop (#94779): turns that finished while the socket was + // down never replay their sessions.changed tick, so the open transcript + // stayed stale until the user reopened it. Pull one signature-gated tail on + // every (re)connect — a no-change read costs nothing. Keyed on the + // connection, not the session, so a plain session switch adds no read; + // messaging transcripts already refresh on open in their own effect below. + useEffect(() => { + if (gatewayState === 'open' && !activeIsMessaging && activeSessionId && activeStoredSessionId) { + requestActiveTranscriptRefresh(true) + } + // eslint-disable-next-line react-hooks/exhaustive-deps -- connect-scoped: session deps would fire on every switch + }, [activeConnectionId, activeGatewayProfile, gatewayState]) + // A reconnect loses renderer-only working/attention atoms while the backend // keeps the actual turns alive. Re-seed from the gateway's in-memory session // registry immediately, then re-pull on every sessions.changed broadcast; a @@ -691,17 +737,17 @@ export function useBackgroundSync({ lastRunAt = Date.now() void refreshSessions() void refreshMessagingSessions() + // The project tree is a grouping of the same stored rows, so a session + // created/deleted/renamed/re-homed outside this window goes stale in the + // Projects sidebar without this (#100354). refreshProjectTree() keeps the + // cached tree on failure, so a not-yet-ready backend costs nothing. + void refreshProjectTree() requestActiveTranscriptRefresh(true) // Bot canonical chats live in workspace tiles, never in the main-pane // selection — without this they never see background deliveries // (#93942 scenario A). Signature-gated per tile, so no-change ticks // cost nothing. void reconcileTileTranscripts({ - busyRef: { - get current() { - return $busy.get() - } - }, requestSequenceRef: tileRequestSequenceRef, signatureRef: tileSignatureRef, updateSessionState diff --git a/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.test.tsx b/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.test.tsx index 5dd54083cf..8bc399e67f 100644 --- a/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.test.tsx +++ b/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.test.tsx @@ -80,6 +80,8 @@ describe('useDesktopIntegrations', () => { locationPathname = '/', profileReady = false, resumeExhaustedSessionId = null as string | null, + // null = config record still loading (the hook takes undefined; null dodges the destructuring default). + resumeLastSession = true as boolean | null, routedSessionId = null as string | null, sessions = [] as readonly SessionInfo[] } = {}) { @@ -89,6 +91,7 @@ describe('useDesktopIntegrations', () => { locationPathname, profileReady, resumeExhaustedSessionId, + resumeLastSession, routedSessionId, sessions }: { @@ -96,6 +99,7 @@ describe('useDesktopIntegrations', () => { locationPathname: string profileReady: boolean resumeExhaustedSessionId: string | null + resumeLastSession: boolean | null routedSessionId: string | null sessions: readonly SessionInfo[] }) => @@ -108,6 +112,7 @@ describe('useDesktopIntegrations', () => { profileReady, refreshSessions: vi.fn(), resumeExhaustedSessionId, + resumeLastSession: resumeLastSession ?? undefined, routedSessionId, runtimeIdByStoredSessionId: { current: new Map() }, sessions @@ -118,6 +123,7 @@ describe('useDesktopIntegrations', () => { locationPathname, profileReady, resumeExhaustedSessionId, + resumeLastSession, routedSessionId, sessions } @@ -171,6 +177,7 @@ describe('useDesktopIntegrations', () => { locationPathname: '/', profileReady: true, resumeExhaustedSessionId: null, + resumeLastSession: true, routedSessionId: null, sessions: [session({ id: 'remembered-session', profile: 'default' })] }) @@ -179,6 +186,53 @@ describe('useDesktopIntegrations', () => { }) }) + describe('display.resume_last_session', () => { + it('stays on the fresh chat when the setting is off, and keeps remembering the open chat', () => { + window.localStorage.setItem('hermes.desktop.lastRoute.profile.default', '/remembered-session') + window.localStorage.setItem('hermes.desktop.lastSessionId.profile.default', 'remembered-session') + + const sessions = [session({ id: 'remembered-session', profile: 'default' })] + const result = render({ profileReady: true, resumeLastSession: false, sessions }) + + expect(navigate).not.toHaveBeenCalled() + + // The user opens another chat: it is still remembered for the next launch + // (and for notifications), so flipping the switch back on resumes it. + result.rerender({ + activeProfile: 'default', + locationPathname: '/other-session', + profileReady: true, + resumeExhaustedSessionId: null, + resumeLastSession: false, + routedSessionId: 'other-session', + sessions: [...sessions, session({ id: 'other-session', profile: 'default' })] + }) + + expect(window.localStorage.getItem('hermes.desktop.lastSessionId.profile.default')).toBe('other-session') + }) + + it('holds the restore until the config record answers, then restores when on', () => { + window.localStorage.setItem('hermes.desktop.lastSessionId.profile.default', 'remembered-session') + + const sessions = [session({ id: 'remembered-session', profile: 'default' })] + const result = render({ profileReady: true, resumeLastSession: null, sessions }) + + expect(navigate).not.toHaveBeenCalled() + + result.rerender({ + activeProfile: 'default', + locationPathname: '/', + profileReady: true, + resumeExhaustedSessionId: null, + resumeLastSession: true, + routedSessionId: null, + sessions + }) + + expect(navigate).toHaveBeenCalledWith('/remembered-session', { replace: true }) + }) + }) + describe('ownership validation', () => { it('refuses to restore a session route owned by another profile', () => { window.localStorage.setItem('hermes.desktop.lastRoute.profile.default', '/ai-session') @@ -329,6 +383,7 @@ describe('useDesktopIntegrations', () => { locationPathname: '/ops-session', profileReady: true, resumeExhaustedSessionId: null, + resumeLastSession: true, routedSessionId: 'ops-session', sessions }) @@ -395,6 +450,7 @@ describe('useDesktopIntegrations', () => { locationPathname: '/settings', profileReady: true, resumeExhaustedSessionId: null, + resumeLastSession: true, routedSessionId: null, sessions: [] }) diff --git a/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.ts b/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.ts index 9cd069c253..6d8cef2ec1 100644 --- a/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.ts +++ b/apps/desktop/src/app/contrib/hooks/use-desktop-integrations.ts @@ -41,6 +41,8 @@ interface DesktopIntegrationsParams { navigate: (to: string, options?: { replace?: boolean }) => void profileReady: boolean refreshSessions: () => Promise | unknown + /** `display.resume_last_session`; `undefined` while the config record is still loading. */ + resumeLastSession: boolean | undefined resumeExhaustedSessionId: null | string routedSessionId: null | string runtimeIdByStoredSessionId: { readonly current: Map } @@ -60,6 +62,7 @@ export function useDesktopIntegrations({ navigate, profileReady, refreshSessions, + resumeLastSession, resumeExhaustedSessionId, routedSessionId, runtimeIdByStoredSessionId, @@ -73,7 +76,12 @@ export function useDesktopIntegrations({ // Background MCP health: HTTP/SSE servers only (never spawns stdio), // notifies on transitions into needs-auth/error with a Sign in action. startMcpHealthChecker() - const unsubscribe = window.hermesDesktop?.onOpenUpdatesRequested?.(() => openUpdatesWindow()) + // The native "Check for Updates…" menu item lives in the app menu next to + // "About Hermes" — it is the OS-standard affordance for updating THIS app, + // so it always opens the client overlay. Inheriting the connection-mode + // default pointed a Mac at its remote Linux backend and left the app itself + // silently stale (#70266). + const unsubscribe = window.hermesDesktop?.onOpenUpdatesRequested?.(() => openUpdatesWindow('client')) return () => { unsubscribe?.() @@ -105,6 +113,20 @@ export function useDesktopIntegrations({ // Only cold-start navigation at the default route is replaceable; a deep // link or hidden-then-shown window keeps its explicit destination. if (locationPathname === NEW_CHAT_ROUTE) { + // display.resume_last_session (#60812): hold the latch until the config + // record answers, then either restore below or stay on the fresh chat. + // Remembered ids keep being written either way, so flipping the switch + // back on resumes from the very next launch. + if (resumeLastSession === undefined) { + return + } + + if (!resumeLastSession) { + restoredRef.current = true + + return + } + const route = getRememberedRoute(activeProfile) const routeSession = route ? routeSessionId(route) : null const last = getRememberedSessionId(activeProfile) @@ -163,7 +185,7 @@ export function useDesktopIntegrations({ } else if (!routedSessionId && !isOverlayView(appViewForPath(locationPathname))) { setRememberedRoute(locationPathname, activeProfile) } - }, [activeProfile, locationPathname, navigate, profileReady, routedSessionId, sessions]) + }, [activeProfile, locationPathname, navigate, profileReady, resumeLastSession, routedSessionId, sessions]) useEffect(() => { if (!profileReady || !resumeExhaustedSessionId) { diff --git a/apps/desktop/src/app/contrib/session-rpc-dispatcher.test.ts b/apps/desktop/src/app/contrib/session-rpc-dispatcher.test.ts index 8c6e83dcca..cee4203f44 100644 --- a/apps/desktop/src/app/contrib/session-rpc-dispatcher.test.ts +++ b/apps/desktop/src/app/contrib/session-rpc-dispatcher.test.ts @@ -38,6 +38,7 @@ vi.mock('@/store/session', async importActual => ({ const { createSessionRpcDispatcher } = await import('./session-rpc-dispatcher') const { $connectionsRegistry } = await import('@/store/connection-registry-state') const { $profiles } = await import('@/store/profile') +const { $removedSessionIds, $sessionMutationsInFlight } = await import('@/store/session-removal') const { _resetSessionOwnerHintsForTests, setCronSessions, setMessagingSessions, setSessionOwnerHint, setSessions } = await import('@/store/session') @@ -75,6 +76,8 @@ afterEach(() => { setMessagingSessions([]) $sessionTiles.set([]) $profiles.set([]) + $removedSessionIds.set(new Set()) + $sessionMutationsInFlight.set(new Set()) _resetSessionOwnerHintsForTests({ storage: true }) sessionMocks.requestSessionResume.mockReset() vi.clearAllMocks() @@ -233,6 +236,23 @@ describe('createSessionRpcDispatcher: stale runtime recovery', () => { expect(sessionMocks.requestSessionResume).not.toHaveBeenCalled() }) + it.each([ + ['tombstoned', $removedSessionIds], + ['being deleted', $sessionMutationsInFlight] + ])('still reports the 4001 for a selected session that is %s', async (_state, sessions) => { + // The rebind decision moved to requestSessionResume (store/session-removal), + // which drops resume requests for a removal-pending id — this seam only has + // to keep surfacing the error to its caller. + setSessions([makeSessionInfo({ connection_id: 'local', id: 'stored-omar', profile: 'omar' })]) + sessions.set(new Set(['stored-omar'])) + gatewayMocks.requestGatewayForAgent.mockRejectedValueOnce( + Object.assign(new Error('session not found'), { code: 4001 }) + ) + const { request } = dispatcher(undefined, 'stored-omar') + + await expect(request('process.list', { session_id: 'rt-omar' })).rejects.toThrow('session not found') + }) + it('does not interpret an unrelated coded RPC failure as a stale runtime', async () => { setSessions([makeSessionInfo({ connection_id: 'local', id: 'stored-omar', profile: 'omar' })]) gatewayMocks.requestGatewayForAgent.mockRejectedValueOnce( diff --git a/apps/desktop/src/app/contrib/session-rpc-dispatcher.ts b/apps/desktop/src/app/contrib/session-rpc-dispatcher.ts index cf9bfd94b4..d57bcbb115 100644 --- a/apps/desktop/src/app/contrib/session-rpc-dispatcher.ts +++ b/apps/desktop/src/app/contrib/session-rpc-dispatcher.ts @@ -106,6 +106,8 @@ export function createSessionRpcDispatcher(deps: SessionRpcDispatcherDeps): Ambi // calls; this seam covers the other session-scoped callers and wakes // route-resume for the visible main session only. Do not retry the // failing RPC — it may be destructive, and a fresh binding is async. + // A session the user just deleted is filtered by requestSessionResume, + // which drops resume requests for a removal-pending id. if ( method !== 'session.resume' && method !== 'session.activate' && diff --git a/apps/desktop/src/app/contrib/wiring.tsx b/apps/desktop/src/app/contrib/wiring.tsx index 7cb1655ce8..f9f3be28df 100644 --- a/apps/desktop/src/app/contrib/wiring.tsx +++ b/apps/desktop/src/app/contrib/wiring.tsx @@ -94,6 +94,7 @@ import { CommandPalette } from '../command-palette' import { triggerAndRefreshCronJobs } from '../cron/cron-actions' import { useGatewayBoot } from '../gateway/hooks/use-gateway-boot' import { useGatewayRequest } from '../gateway/hooks/use-gateway-request' +import { useHermesConfigRecord } from '../hooks/use-config-record' import { useKeybinds } from '../hooks/use-keybinds' import { useHudHandoff } from '../hud/handoff' import { ModelPickerOverlay } from '../model-picker-overlay' @@ -843,6 +844,15 @@ export function ContribWiring({ children }: { children: ReactNode }) { // remembered-session restore, and cross-window session-list sync. const previewTarget = useStore($previewTarget) + // display.resume_last_session gates the cold-start restore. `undefined` while + // the record is still loading holds the restore latch open; a failed fetch + // falls back to the historical behavior (resume). + const configRecord = useHermesConfigRecord() + + const resumeLastSession = configRecord.isPending + ? undefined + : (configRecord.data?.display as { resume_last_session?: unknown } | undefined)?.resume_last_session !== false + useDesktopIntegrations({ activeProfile: normalizeProfileKey(activeGatewayProfile), chatOpen, @@ -851,6 +861,7 @@ export function ContribWiring({ children }: { children: ReactNode }) { navigate, profileReady: boot.phase === 'renderer.ready', refreshSessions, + resumeLastSession, resumeExhaustedSessionId, routedSessionId, runtimeIdByStoredSessionId: runtimeIdByStoredSessionIdRef, diff --git a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts index 29dd2c7a5a..f5ee2ca66c 100644 --- a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts +++ b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts @@ -46,6 +46,7 @@ import { } from '@/store/gateway-switch' import { checkLocalRuntimeUpdate, watchLocalRuntimeJobs } from '@/store/local-runtime-jobs' import { notify, notifyError } from '@/store/notifications' +import { loadPoolLimits } from '@/store/pool-limits' import { $activeGatewayProfile, normalizeProfileKey, @@ -900,6 +901,10 @@ export function useGatewayBoot({ // this a socket dropped during sleep sits closed until the user clicks. window.addEventListener('focus', onFocus) + // Pool limits are main-process state; mirror them once for the Settings + // rows and prewarmProfileBackend's saturation guard. + void loadPoolLimits() + // Keep live pool backends alive while this window is open (the main process // can't observe the direct renderer↔backend WS). No-op for the primary. const keepaliveTimer = setInterval(() => { diff --git a/apps/desktop/src/app/session/hooks/branch-owner-routing.integration.test.ts b/apps/desktop/src/app/session/hooks/branch-owner-routing.integration.test.ts new file mode 100644 index 0000000000..d4a8f0fb8a --- /dev/null +++ b/apps/desktop/src/app/session/hooks/branch-owner-routing.integration.test.ts @@ -0,0 +1,127 @@ +/** + * End-to-end owner routing for BRANCH (the #97764-adjacent strand). + * + * The unit tests in use-session-actions.test.tsx mock `@/store/gateway`, so + * they prove the branch path ASKS for the right route. They cannot prove the + * routing layer HONOURS it. This file mocks nothing inside the router: the real + * `requestGatewayForAgent` runs against a fake Electron bridge + transport, so + * a regression that re-collapses a registry route onto the ambient socket fails + * here even if the call-site assertions still pass. + * + * Reproduces the reported shape: a session owned by a remote connection + * ("pandora") is branched while a different backend is active. Before the fix + * the create rode the ambient socket and the child was created on the wrong + * backend (or nowhere), stranding an optimistic sidebar row on an id no backend + * owned — "Couldn't load this session". + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// Every socket the registry dials, and every RPC that travelled over one. +const dialed: { connectionId: string; profile: string }[] = [] +const sent: { method: string; params: Record; url: string }[] = [] + +class FakeHermesGateway { + connectionState = 'closed' + private url = '' + + async connect(wsUrl: string) { + if (typeof wsUrl !== 'string' || !wsUrl.startsWith('ws')) { + throw new Error(`bad ws url: ${String(wsUrl)}`) + } + + this.url = wsUrl + this.connectionState = 'open' + } + + async request(method: string, params: Record = {}): Promise { + sent.push({ method, params, url: this.url }) + + if (method === 'session.create' || method === 'session.branch') { + return { session_id: 'branch-runtime', stored_session_id: 'branch-stored' } as T + } + + return {} as T + } + + close() { + this.connectionState = 'closed' + } + + onEvent(_listener: (event: unknown) => void) { + return () => undefined + } + + onState(_listener: (state: unknown) => void) { + return () => undefined + } + + onStateChange(_listener: (state: unknown) => void) { + return () => undefined + } + + on() {} + off() {} + addEventListener() {} + removeEventListener() {} +} + +vi.mock('@/hermes', async importOriginal => ({ + ...(await importOriginal>()), + HermesGateway: FakeHermesGateway, + setApiRequestConnection: vi.fn() +})) + +describe('branch owner routing (real router, faked transport)', () => { + beforeEach(() => { + dialed.length = 0 + sent.length = 0 + vi.resetModules() + + // A registry with two backends exposing the SAME profile name — the exact + // ambiguity that makes profile-only routing wrong. + ;(window as unknown as { hermesDesktop: unknown }).hermesDesktop = { + getConnection: async () => ({ mode: 'local' }), + getConnectionFor: async ({ connectionId, profile }: { connectionId: string; profile: string }) => { + dialed.push({ connectionId, profile }) + + return { connectionId, mode: 'remote', profile } + }, + getGatewayWsUrlFor: async ({ connectionId, profile }: { connectionId: string; profile: string }) => + `ws://${connectionId}/gateway?profile=${profile}`, + touchBackend: async () => undefined + } + }) + + it('dials the parent connection and sends the create over that socket', async () => { + const { requestGatewayForAgent } = await import('@/store/gateway') + + await requestGatewayForAgent('pandora', 'default', 'session.create', { + parent_session_id: 'stored-parent', + source: 'desktop' + }) + + // The registry resolved a socket for the PARENT's connection... + expect(dialed).toContainEqual({ connectionId: 'pandora', profile: 'default' }) + + // ...and the create actually travelled over that socket. + const create = sent.find(entry => entry.method === 'session.create') + expect(create).toBeDefined() + expect(create!.url).toContain('pandora') + expect(create!.params).toMatchObject({ parent_session_id: 'stored-parent' }) + }) + + it('keeps two same-named profiles on separate sockets', async () => { + const { requestGatewayForAgent } = await import('@/store/gateway') + + await requestGatewayForAgent('pandora', 'default', 'session.create', { source: 'desktop' }) + await requestGatewayForAgent('other-box', 'default', 'session.create', { source: 'desktop' }) + + const urls = sent.filter(entry => entry.method === 'session.create').map(entry => entry.url) + + expect(urls).toHaveLength(2) + // Same profile name, different backends — they must NOT share a socket. + expect(new Set(urls).size).toBe(2) + expect(urls.some(url => url.includes('pandora'))).toBe(true) + expect(urls.some(url => url.includes('other-box'))).toBe(true) + }) +}) diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx b/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx index 7152d9eba5..6a72ea9705 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-message-stream/compaction-event.test.tsx @@ -1,6 +1,7 @@ import { act, cleanup } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createClientSessionState } from '@/lib/chat-runtime' import { $compactingSessions, setSessionCompacting } from '@/store/compaction' import type { RpcEvent } from '@/types/hermes' @@ -56,6 +57,31 @@ describe('useMessageStream compaction lifecycle', () => { expect($compactingSessions.get()).toEqual({ [OTHER_SID]: true }) }) + // #97948: a manual /compress whose RPC answered `pending` (the compute host + // outlived the gateway's wait) has no turn-end hydrate — the `compacted` + // edge is the only signal the transcript changed. + it('rehydrates the idle active session on the compacted edge', () => { + const hydrateFromStoredSession = vi.fn(async () => undefined) + const states = new Map([[SID, { ...createClientSessionState(), storedSessionId: 'stored-1' }]]) + + stream = renderMessageStream(SID, { hydrateFromStoredSession, states }) + + emit('status.update', { kind: 'compacted' }) + + expect(hydrateFromStoredSession).toHaveBeenCalledWith(3, 'stored-1', SID) + }) + + it('leaves the transcript to the turn settle path when compaction ends mid-turn', () => { + const hydrateFromStoredSession = vi.fn(async () => undefined) + const states = new Map([[SID, { ...createClientSessionState(), busy: true, storedSessionId: 'stored-1' }]]) + + stream = renderMessageStream(SID, { hydrateFromStoredSession, states }) + + emit('status.update', { kind: 'compacted' }) + + expect(hydrateFromStoredSession).not.toHaveBeenCalled() + }) + it('reconciles a reconnecting compaction only from trusted terminal server state', () => { mountStream() emit('status.update', { kind: 'compacting' }) diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/index.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/index.ts index df11da34be..8c86f65d81 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/index.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/index.ts @@ -12,6 +12,7 @@ import { $gateway, activeGatewayConnectionId } from '@/store/gateway' import { $activeGatewayProfile, normalizeProfileKey } from '@/store/profile' import { replayPendingApproval } from '@/store/prompts' import { setSessionProviderWait } from '@/store/provider-wait' +import { isSessionGone } from '@/store/session-gone-latch' import { setSessionDraftingTool } from '@/store/tool-drafting' import type { RpcEvent } from '@/types/hermes' @@ -195,7 +196,10 @@ export function useGatewayEventHandler(deps: GatewayEventDeps) { const isActiveEvent = !!sessionId && sessionId === activeSessionIdRef.current - const replaySessionId = approvalReplaySessionId(event.type, activeSessionIdRef.current, sessionId) + const replaySessionId = approvalReplaySessionId(event.type, activeSessionIdRef.current, sessionId, { + explicit: Boolean(explicitSid), + isGone: isSessionGone + }) if (replaySessionId) { void replayPendingApproval($gateway.get(), replaySessionId).catch(() => undefined) diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/status.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/status.ts index 76b1b40a20..3fb0ff7e75 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/status.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/status.ts @@ -21,7 +21,16 @@ import type { GatewayEventContext } from './types' * error — the status-and-notice tail of the dispatcher. */ export function handleStatusEvent(ctx: GatewayEventContext): boolean { const { deps, event, payload, sessionId, isActiveEvent, occurredAt } = ctx - const { compactedTurnRef, failAssistantMessage, flushQueuedDeltas, queryClient, updateSessionState } = deps + + const { + compactedTurnRef, + failAssistantMessage, + flushQueuedDeltas, + hydrateFromStoredSession, + queryClient, + sessionStateByRuntimeIdRef, + updateSessionState + } = deps if (event.type === 'status.update') { if (sessionId && payload?.kind === 'compacting') { @@ -30,6 +39,17 @@ export function handleStatusEvent(ctx: GatewayEventContext): boolean { } else if (sessionId && payload?.kind === 'compacted') { reconcileSessionCompacting(sessionId, 'terminal') compactedTurnRef.current.delete(sessionId) + + // A compress that finished with no live turn (manual /compress whose + // RPC answered `pending` because the compute host outlived the wait, + // #97948) has no turn-end hydrate to refresh the transcript — the + // summarized bubbles would stay on screen forever. Mid-turn compaction + // still defers to the turn's own settle path. + const state = sessionStateByRuntimeIdRef.current.get(sessionId) + + if (isActiveEvent && state && !state.busy && !state.awaitingResponse && !state.streamId) { + void hydrateFromStoredSession(3, state.storedSessionId, sessionId) + } } else if (sessionId && payload?.kind === 'process') { // The gateway's notification poller announces background process // completions / watch matches here — re-sync the status stack. diff --git a/apps/desktop/src/app/session/hooks/use-model-controls.test.tsx b/apps/desktop/src/app/session/hooks/use-model-controls.test.tsx index 7d6a610d16..42421589a1 100644 --- a/apps/desktop/src/app/session/hooks/use-model-controls.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-model-controls.test.tsx @@ -275,7 +275,7 @@ describe('useModelControls', () => { }) }) - it('persists an active primary-session picker change as the profile default via config.set --global', async () => { + it('sends an active primary-session picker change without a scope flag so the gateway decides persistence', async () => { $activeSessionId.set('session-1') const requestGateway = vi.fn(async () => ({ key: 'model', value: 'claude-sonnet-4.6' }) as never) let controls!: Controls @@ -289,13 +289,13 @@ describe('useModelControls', () => { }) ).resolves.toBe(true) - // The primary main agent's pick IS the profile default, so it persists to - // config.yaml (model.default + model.provider) — which is what lets a - // chosen subscription provider outrank a leftover OPENAI_API_KEY env var. + // No hardcoded --global (#90235): resolve_persist_behavior on the gateway + // owns the policy — session-only unless model.persist_switch_by_default + // is set or no default has ever been configured (#86414's first pick). expect(requestGateway).toHaveBeenCalledWith('config.set', { session_id: 'session-1', key: 'model', - value: 'claude-sonnet-4.6 --provider anthropic --global' + value: 'claude-sonnet-4.6 --provider anthropic' }) expect(requestGateway).not.toHaveBeenCalledWith('slash.exec', expect.anything()) }) @@ -376,7 +376,7 @@ describe('useModelControls', () => { confirm_expensive_model: true, key: 'model', session_id: 'session-1', - value: 'muse-spark-1.2-contributor --provider opencode-go --global' + value: 'muse-spark-1.2-contributor --provider opencode-go' }) expect($currentModel.get()).toBe('muse-spark-1.2-contributor') expect($currentProvider.get()).toBe('opencode-go') diff --git a/apps/desktop/src/app/session/hooks/use-model-controls.ts b/apps/desktop/src/app/session/hooks/use-model-controls.ts index 4d0d63b058..8f28e1cf8e 100644 --- a/apps/desktop/src/app/session/hooks/use-model-controls.ts +++ b/apps/desktop/src/app/session/hooks/use-model-controls.ts @@ -256,13 +256,13 @@ export function useModelControls({ return true } - // The PRIMARY profile's main agent is the profile's default — its - // model/provider choice IS the default, so persist it to config.yaml - // (model.default + model.provider) via --global. This is what makes - // the selection "stick": a set model.provider outranks a leftover - // OPENAI_API_KEY env var in resolve_provider(), so the main agent - // keeps the chosen (e.g. subscription) provider across restarts - // instead of silently falling back to an env key. + // The PRIMARY profile's main agent lets the gateway decide persistence + // (resolve_persist_behavior): session-only by default, persisted when + // model.persist_switch_by_default is true or when no default has ever + // been configured (the first-ever pick, so resolve_provider never falls + // through to a leftover OPENAI_API_KEY env var — #86414). A plain pick + // no longer silently rewrites config.yaml (#90235); Settings → Model + // remains the explicit "set as default" door. // // Two things stay --session, deliberately: // - a SECONDARY chat tile: picking a model there must not rewrite the @@ -270,14 +270,13 @@ export function useModelControls({ // - MoA (mixture-of-agents) presets: a transient orchestration choice // that must never become the persisted global gateway default. const isSessionOnlyPreset = (selection.provider || '').toLowerCase() === 'moa' - const persistsAsDefault = touchesPrimary && !isSessionOnlyPreset - const scope = persistsAsDefault ? '--global' : '--session' + const scope = touchesPrimary && !isSessionOnlyPreset ? '' : ' --session' const requestSwitch = (confirmExpensiveModel = false) => requestGateway('config.set', { session_id: liveSessionId, key: 'model', - value: `${selection.model} --provider ${selection.provider} ${scope}`, + value: `${selection.model} --provider ${selection.provider}${scope}`, ...(confirmExpensiveModel ? { confirm_expensive_model: true } : {}) }) diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx index 59e7182195..0673798c18 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/index.test.tsx @@ -31,6 +31,7 @@ import { $wakeWord, resetWakeWordState } from '@/store/wake-word' import type { SessionInfo } from '@/types/hermes' import { clearSingleFlightSessionResumeState } from './single-flight-resume' +import { SESSION_COMPRESS_TIMEOUT_MS } from './slash' import type { SubmitTextOptions } from './utils' import { uploadComposerAttachment, usePromptActions } from '.' @@ -692,7 +693,7 @@ describe('usePromptActions /compress', () => { vi.restoreAllMocks() }) - it('routes through session.compress (not slash.exec) with a 120s timeout and renders the summary', async () => { + it('routes through session.compress (not slash.exec) with the compute-host ceiling timeout and renders the summary', async () => { const seeds: Record[] = [] const requestGateway = vi.fn(async (method: string, _params?: Record, _timeoutMs?: number) => { @@ -728,7 +729,7 @@ describe('usePromptActions /compress', () => { expect(requestGateway).toHaveBeenCalledWith( 'session.compress', expect.objectContaining({ session_id: RUNTIME_SESSION_ID }), - 120_000 + SESSION_COMPRESS_TIMEOUT_MS ) expect(requestGateway).not.toHaveBeenCalledWith('slash.exec', expect.anything()) expect(requestGateway).not.toHaveBeenCalledWith('command.dispatch', expect.anything()) @@ -862,7 +863,7 @@ describe('usePromptActions /compress', () => { expect(requestGateway).toHaveBeenCalledWith( 'session.compress', expect.objectContaining({ focus_topic: 'the auth refactor' }), - 120_000 + SESSION_COMPRESS_TIMEOUT_MS ) }) @@ -969,7 +970,9 @@ describe('usePromptActions /compress', () => { act(() => { submitted = handle!.submitTextRaw('/compress') }) - await waitFor(() => expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), 120_000)) + await waitFor(() => + expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), SESSION_COMPRESS_TIMEOUT_MS) + ) // Switch to session B before compression resolves. activeSessionIdRef.current = RUNTIME_SESSION_B @@ -1028,7 +1031,9 @@ describe('usePromptActions /compress', () => { act(() => { submitted = handle!.submitTextRaw('/compress') }) - await waitFor(() => expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), 120_000)) + await waitFor(() => + expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), SESSION_COMPRESS_TIMEOUT_MS) + ) activeSessionIdRef.current = RUNTIME_SESSION_B storedSessionIdRef.current = 'stored-b' rejectCompress(new Error('compression failed')) diff --git a/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts b/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts index 770f02ae9b..a06b6b1643 100644 --- a/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts +++ b/apps/desktop/src/app/session/hooks/use-prompt-actions/slash.ts @@ -74,9 +74,13 @@ import { } from './utils' // Manual compression is LLM-bound and routinely outlives the desktop's 30s -// default WS request timeout on large sessions — give it the TUI client's -// 120s RPC budget (HERMES_TUI_RPC_TIMEOUT_MS default) instead. -const SESSION_COMPRESS_TIMEOUT_MS = 120_000 +// default WS request timeout on large sessions. The gateway blocks its own +// compute-host wait for up to compression.context_total_ceiling_seconds + 30s +// (capped at 630s, tui_gateway/server.py _COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS) +// and then answers `status: 'pending'` rather than an error, so this budget +// must sit above that cap or the desktop reports a false timeout while the +// host is still compressing (#97948). +export const SESSION_COMPRESS_TIMEOUT_MS = 660_000 const WAKE_START_TIMEOUT_MS = 180_000 const wakeDeviceLabel = (device?: WakeInputDeviceStatus): string => { @@ -666,6 +670,16 @@ export function useSlashCommand(deps: SlashCommandDeps) { sessionId = liveSessionId + // The gateway's compute-host wait expired but compression is still + // running there; it pushes session.info + a `compacted` status edge + // when the host finishes. Not an error (#97948). + if (result?.status === 'pending') { + const pendingMessage = result.message || 'compression still running in the background' + notify({ durationMs: 8_000, id: noticeId, kind: 'info', message: pendingMessage }) + + return + } + // Replace the transcript with the post-compress history so the // summarized bubbles actually disappear. `messages` is the same // shape session.resume returns (_history_to_messages), so diff --git a/apps/desktop/src/app/session/hooks/use-route-resume.test.tsx b/apps/desktop/src/app/session/hooks/use-route-resume.test.tsx index ac6f24b016..fae23a9800 100644 --- a/apps/desktop/src/app/session/hooks/use-route-resume.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-route-resume.test.tsx @@ -101,6 +101,45 @@ describe('useRouteResume', () => { expect(resumeSession).not.toHaveBeenCalled() }) + it('honors an explicit resume request on /:sid even while a fresh draft is staged', () => { + // A gateway/profile switch stages a fresh draft (wipeSessionListsForGatewaySwitch) + // but deliberately leaves the URL on /:sid. In that state pathnameChanged, + // gatewayBecameOpen and stuckOnRoutedSession are all false, so an explicit + // request (plugin/SDK reselect, 4001 recovery) is the ONLY lever left — it + // must still fire. Filtering doomed ids happens at requestSessionResume and + // resumeSession, not by guessing from freshDraftReady. + const resumeSession = vi.fn(async () => undefined) + const startFreshSessionDraft = vi.fn() + const activeSessionIdRef: MutableRefObject = { current: null } + const creatingSessionRef = { current: false } + const runtimeIdByStoredSessionIdRef = { current: new Map() } + const selectedStoredSessionIdRef: MutableRefObject = { current: null } + + const props = { + activeSessionId: null, + activeSessionIdRef, + creatingSessionRef, + currentView: 'chat', + freshDraftReady: true, + gatewayState: 'open', + locationPathname: '/session-1', + resumeSession, + routedSessionId: 'session-1', + runtimeIdByStoredSessionIdRef, + selectedStoredSessionId: null, + selectedStoredSessionIdRef, + startFreshSessionDraft + } + + const { rerender } = render() + + resumeSession.mockClear() + + rerender() + + expect(resumeSession).toHaveBeenCalledWith('session-1', true) + }) + it('self-heals a stranded routed session (null selected/active, same pathname, not a fresh draft)', () => { const resumeSession = vi.fn(async () => undefined) const startFreshSessionDraft = vi.fn() diff --git a/apps/desktop/src/app/session/hooks/use-session-actions.test.tsx b/apps/desktop/src/app/session/hooks/use-session-actions.test.tsx index 256975b652..e8550299c1 100644 --- a/apps/desktop/src/app/session/hooks/use-session-actions.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-session-actions.test.tsx @@ -25,13 +25,7 @@ import { clearSessionDraft, stashSessionDraft, takeSessionDraft } from '@/store/ import { requestGatewayForAgent, requestGatewayForProfile } from '@/store/gateway' import { $pinnedSessionIds } from '@/store/layout' import { $activeGatewayProfile, $newChatProfile, $newChatRoute, $profiles, ensureGatewayProfile } from '@/store/profile' -import { - $projectScope, - $projectTree, - $removedSessionIds, - $sessionMutationsInFlight, - ALL_PROJECTS -} from '@/store/projects' +import { $projectScope, $projectTree, ALL_PROJECTS } from '@/store/projects' import { $activeSessionId, $activeSessionStoredIdRotation, @@ -71,13 +65,14 @@ import { setSessions, setTurnStartedAt } from '@/store/session' +import { $removedSessionIds, $sessionMutationsInFlight } from '@/store/session-removal' import { requestForSessionProfile, type SessionProfileRoute } from '@/store/session-request-router' import { $sessionTiles, sessionTileOwnerRoute } from '@/store/session-states' import { $sessionSeenCounts, $unreadFinishedMarkers } from '@/store/session-unread' import sessionResumeActiveTurn from '../../../../../../tests/fixtures/session-resume-active-turn.json' import { deferred } from '../../../test/deferred' -import { sessionRoute } from '../../routes' +import { NEW_CHAT_ROUTE, sessionRoute } from '../../routes' import type { ClientSessionState } from '../../types' import { useSessionActions } from './use-session-actions' @@ -118,6 +113,7 @@ const RUNTIME_SESSION_ID = 'rt-new-001' type HarnessHandle = Pick< ReturnType, | 'archiveSession' + | 'branchStoredSession' | 'createBackendSessionForSend' | 'openNewSessionTile' | 'removeSession' @@ -146,22 +142,26 @@ function storedSession(overrides: Partial = {}): SessionInfo { function Harness({ activeSessionId = null, + activeSessionIdRef: activeSessionIdRefOverride, navigate = vi.fn(), onReady, requestGateway, - selectedStoredSessionId = null + selectedStoredSessionId = null, + selectedStoredSessionIdRef: selectedStoredSessionIdRefOverride }: { activeSessionId?: null | string + activeSessionIdRef?: MutableRefObject navigate?: ReturnType onReady: (handle: HarnessHandle) => void requestGateway: (method: string, params?: Record) => Promise selectedStoredSessionId?: null | string + selectedStoredSessionIdRef?: MutableRefObject }) { const ref = (value: T): MutableRefObject => ({ current: value }) const actions = useSessionActions({ activeSessionId, - activeSessionIdRef: ref(activeSessionId), + activeSessionIdRef: activeSessionIdRefOverride ?? ref(activeSessionId), busyRef: ref(false), creatingSessionRef: ref(false), ensureSessionState: () => ({}) as ClientSessionState, @@ -172,7 +172,7 @@ function Harness({ resetViewSync: vi.fn(), runtimeIdByStoredSessionIdRef: ref(new Map()), selectedStoredSessionId, - selectedStoredSessionIdRef: ref(selectedStoredSessionId), + selectedStoredSessionIdRef: selectedStoredSessionIdRefOverride ?? ref(selectedStoredSessionId), sessionStateByRuntimeIdRef: ref(new Map()), syncSessionStateToView: vi.fn(), updateSessionState: () => ({}) as ClientSessionState @@ -185,6 +185,134 @@ function Harness({ return null } +describe('desktop branch creation idempotency', () => { + afterEach(() => { + cleanup() + setSessions([]) + vi.clearAllMocks() + }) + + it('coalesces duplicate stored-session branch attempts onto one backend child', async () => { + const createReady = deferred<{ session_id: string; stored_session_id: string }>() + + const requestGateway = vi.fn(async (method: string, params?: Record) => { + if (method === 'session.create') { + return createReady.promise as never + } + + return {} as never + }) + + let actions: HarnessHandle | null = null + + setSessions([storedSession({ id: 'parent', message_count: 2, title: 'Parent' })]) + vi.mocked(getAllSessionMessages).mockResolvedValue({ + messages: [ + { content: 'question', role: 'user', timestamp: 1 }, + { content: 'answer', role: 'assistant', timestamp: 2 } + ], + session_id: 'parent' + } as never) + + render( (actions = value)} requestGateway={requestGateway} />) + await waitFor(() => expect(actions).not.toBeNull()) + + let first!: Promise + let second!: Promise + + act(() => { + first = actions!.branchStoredSession('parent') + second = actions!.branchStoredSession('parent') + }) + + await waitFor(() => + expect(requestGateway.mock.calls.filter(([method]) => method === 'session.create')).toHaveLength(1) + ) + + await act(async () => { + createReady.resolve({ session_id: 'runtime-branch', stored_session_id: 'stored-branch' }) + await expect(Promise.all([first, second])).resolves.toEqual([true, true]) + }) + + expect(requestGateway.mock.calls.filter(([method]) => method === 'session.create')).toHaveLength(1) + expect(requestGateway).toHaveBeenCalledWith( + 'session.create', + expect.objectContaining({ + messages: [ + { content: 'question', role: 'user' }, + { content: 'answer', role: 'assistant' } + ], + parent_session_id: 'parent', + source: 'desktop' + }) + ) + expect($sessions.get().filter(session => session.id === 'stored-branch')).toHaveLength(1) + }) + + it('does not coalesce two same-id parents that live on different connections', async () => { + // Two backends each expose a session called `parent`. They are different + // conversations, so a route-blind flight key would collapse both branch + // actions onto ONE create and hand the second caller the other backend's + // child. Both creates are held open so the second call sees the first's + // flight still in the map — that is the only state the key guards. + const routedCreate = vi.mocked(requestGatewayForAgent) + const pandoraCreate = deferred<{ session_id: string; stored_session_id: string }>() + const otherCreate = deferred<{ session_id: string; stored_session_id: string }>() + + routedCreate.mockImplementation((async (connectionId: string, _profile: string, method: string) => { + if (method !== 'session.create') { + return {} as never + } + + return connectionId === 'pandora' ? pandoraCreate.promise : otherCreate.promise + }) as never) + + let actions: HarnessHandle | null = null + + vi.mocked(getAllSessionMessages).mockResolvedValue({ + messages: [{ content: 'question', role: 'user', timestamp: 1 }], + session_id: 'parent' + } as never) + + render( (actions = value)} requestGateway={vi.fn(async () => ({}) as never)} />) + await waitFor(() => expect(actions).not.toBeNull()) + + // Same stored id, one owner at a time in the row cache — the branch resolves + // its owner from the row, so this is how the two owners reach forkBranch. + setSessions([storedSession({ connection_id: 'pandora', id: 'parent', message_count: 2, profile: 'default' })]) + + let first!: Promise + let second!: Promise + + await act(async () => { + first = actions!.branchStoredSession('parent') + await waitFor(() => expect(routedCreate).toHaveBeenCalled()) + }) + + setSessions([storedSession({ connection_id: 'other-box', id: 'parent', message_count: 2, profile: 'default' })]) + + await act(async () => { + second = actions!.branchStoredSession('parent') + await waitFor(() => + expect(routedCreate.mock.calls.filter(([, , method]) => method === 'session.create')).toHaveLength(2) + ) + }) + + await act(async () => { + pandoraCreate.resolve({ session_id: 'rt-pandora', stored_session_id: 'stored-pandora' }) + otherCreate.resolve({ session_id: 'rt-other', stored_session_id: 'stored-other-box' }) + await expect(Promise.all([first, second])).resolves.toEqual([true, true]) + }) + + const creates = routedCreate.mock.calls.filter(([, , method]) => method === 'session.create') + + expect(creates.map(([connectionId]) => connectionId)).toEqual(['pandora', 'other-box']) + // Two distinct children, not one child claimed twice. + expect($sessions.get().filter(session => session.id === 'stored-pandora')).toHaveLength(1) + expect($sessions.get().filter(session => session.id === 'stored-other-box')).toHaveLength(1) + }) +}) + describe('connection-qualified session deletion', () => { afterEach(() => { cleanup() @@ -231,6 +359,44 @@ describe('connection-qualified session deletion', () => { }) expect(requestGateway).not.toHaveBeenCalledWith('session.close', expect.anything()) }) + + it('tears down the selected session from synchronous refs when render state is stale', async () => { + const navigate = vi.fn() + const requestGateway = vi.fn().mockResolvedValue({}) + const activeSessionIdRef: MutableRefObject = { current: 'runtime-shared' } + const selectedStoredSessionIdRef: MutableRefObject = { current: 'shared-session' } + let actions: HarnessHandle | null = null + + setSessions([storedSession({ connection_id: 'source-a', id: 'shared-session', profile: 'worker' })]) + vi.mocked(deleteSession).mockResolvedValue({ ok: true }) + vi.mocked(requestGatewayForAgent).mockResolvedValue({} as never) + + render( + { + actions = value + }} + requestGateway={requestGateway} + selectedStoredSessionId={null} + selectedStoredSessionIdRef={selectedStoredSessionIdRef} + /> + ) + await waitFor(() => expect(actions).not.toBeNull()) + + await act(async () => { + await actions?.removeSession('shared-session') + }) + + expect(navigate).toHaveBeenCalledWith(NEW_CHAT_ROUTE, { replace: true }) + expect(requestGatewayForAgent).toHaveBeenCalledWith('source-a', 'worker', 'session.close', { + session_id: 'runtime-shared' + }) + expect(selectedStoredSessionIdRef.current).toBeNull() + expect(activeSessionIdRef.current).toBeNull() + }) }) function StoredIdRotationHarness({ @@ -906,6 +1072,8 @@ describe('resumeSession failure recovery', () => { setResumeFailedSessionId(null) setMessages([]) setSessions([]) + $removedSessionIds.set(new Set()) + $sessionMutationsInFlight.set(new Set()) clearClarifyRequest() vi.restoreAllMocks() }) @@ -923,6 +1091,20 @@ describe('resumeSession failure recovery', () => { await resume!('stored-1', true) } + it('does not resume a tombstoned session after delete', async () => { + $removedSessionIds.set(new Set(['stored-1'])) + + const requestGateway = vi.fn(async () => { + throw new Error('404: Session not found') + }) + + await runResume(requestGateway) + + expect(requestGateway).not.toHaveBeenCalled() + expect($resumeFailedSessionId.get()).toBeNull() + expect($selectedStoredSessionId.get()).toBeNull() + }) + it.each([ ['Codex tool-only', ''], ['DeepSeek text-plus-tool', 'I found two paths; choose one.'] @@ -1098,6 +1280,44 @@ describe('resumeSession failure recovery', () => { expect($messages.get().length).toBeGreaterThan(0) }) + it('paints the REST transcript before a cold session.resume settles and keeps it when resume rejects', async () => { + // A cold profile build (skills/MCP/memory) can keep session.resume pending + // past the hydration budget; the already-available REST history must not + // wait for it (#90130), and a later resume failure must not blank it. + const runtimeResume = deferred() + + const requestGateway = vi.fn((method: string) => + method === 'session.resume' ? runtimeResume.promise : Promise.resolve({} as never) + ) as (method: string, params?: Record) => Promise + + vi.mocked(getLatestSessionMessages).mockResolvedValue({ + messages: [ + { content: 'older question', role: 'user', timestamp: 1 }, + { content: 'history visible before runtime', role: 'assistant', timestamp: 2 } + ], + session_id: 'stored-1' + } as never) + + let resume: ((storedSessionId: string, replaceRoute?: boolean) => Promise) | null = null + render( (resume = r)} requestGateway={requestGateway} />) + await waitFor(() => expect(resume).not.toBeNull()) + + let settled = false + const pending = resume!('stored-1', true).finally(() => (settled = true)) + + await waitFor(() => expect(JSON.stringify($messages.get())).toContain('history visible before runtime')) + expect(settled).toBe(false) + const painted = $messages.get() + + await act(async () => { + runtimeResume.reject(new Error('request timed out: session.resume')) + await pending + }) + + expect($messages.get()).toBe(painted) + expect($resumeFailedSessionId.get()).toBeNull() + }) + it('preserves an optimistic user message during a same-session reconnect', async () => { setMessages([ { @@ -1672,8 +1892,11 @@ describe('branchStoredSession desktop source tagging', () => { await expect(branchStoredSession!('stored-parent')).resolves.toBe(true) // The branch becomes the primary session — this is what routes the main - // workspace area to it, not just a new sidebar row. + // workspace area to it, not just a new sidebar row. Selection alone is not + // enough: leaving the URL on the parent makes chat/index see a permanent + // routeSessionMismatch and keeps the central loader mounted. expect($selectedStoredSessionId.get()).toBe('branch-stored') + expect(navigate).toHaveBeenCalledWith(sessionRoute('branch-stored'), { replace: true }) // It must not ALSO exist as a tile: a session is either the main thread or // a tile, never both (resumeSession closes any tile with the same id). expect($sessionTiles.get().some(tile => tile.storedSessionId === 'branch-stored')).toBe(false) @@ -1725,6 +1948,7 @@ describe('branchStoredSession desktop source tagging', () => { // Branching a session that is not the one currently open must not steal // the user's active view — "stored-other" stays selected. expect($selectedStoredSessionId.get()).toBe('stored-other') + expect(navigate).not.toHaveBeenCalled() // The branch instead opens as its own tile. expect($sessionTiles.get().some(tile => tile.storedSessionId === 'branch-stored')).toBe(true) }) @@ -1760,6 +1984,162 @@ describe('branchStoredSession desktop source tagging', () => { }) }) + // A branch belongs to the backend that OWNS its parent. Routing on profile + // alone silently sends session.create to whatever socket is active, so a + // remote-owned parent branched while another connection is active creates + // the child on the wrong backend — or nowhere — while the sidebar still + // paints an optimistic row that can never hydrate ("Couldn't load this + // session"). Same ownership contract removeSession already honours. + it('routes a connection-tagged parent branch through its owning connection', async () => { + const ambientRequest = vi.fn(async () => ({}) as never) + + vi.mocked(requestGatewayForAgent).mockImplementation((async ( + _connectionId: null | string, + _profile: string, + method: string + ) => { + if (method === 'session.create') { + return { session_id: 'branch-runtime', stored_session_id: 'branch-stored' } as never + } + + return {} as never + }) as never) + + setSessions([ + storedSession({ connection_id: 'pandora', id: 'stored-parent', message_count: 1, profile: 'default' }) + ]) + vi.mocked(getAllSessionMessages).mockResolvedValue({ + messages: [{ content: 'branch me', role: 'user', timestamp: 1 }], + session_id: 'stored-parent' + } as never) + + let branchStoredSession: ((storedSessionId: string) => Promise) | null = null + render( (branchStoredSession = branch)} requestGateway={ambientRequest} />) + await waitFor(() => expect(branchStoredSession).not.toBeNull()) + + await expect(branchStoredSession!('stored-parent')).resolves.toBe(true) + + // The create must ride the parent's own (connection, profile) socket... + expect(requestGatewayForAgent).toHaveBeenCalledWith( + 'pandora', + 'default', + 'session.create', + expect.objectContaining({ parent_session_id: 'stored-parent', source: 'desktop' }) + ) + // ...and never the ambient socket, which may serve a different machine. + expect(ambientRequest).not.toHaveBeenCalledWith('session.create', expect.anything()) + }) + + // The parent transcript read must land on the owning backend too: reading it + // from the ambient socket returns nothing for a foreign-owned parent, which + // aborts the branch as "nothing to branch" before any create is attempted. + it('reads a connection-tagged parent transcript from its owning connection', async () => { + const ambientRequest = vi.fn(async () => ({}) as never) + + vi.mocked(requestGatewayForAgent).mockImplementation((async ( + _connectionId: null | string, + _profile: string, + method: string + ) => { + if (method === 'session.create') { + return { session_id: 'branch-runtime', stored_session_id: 'branch-stored' } as never + } + + return {} as never + }) as never) + + setSessions([ + storedSession({ connection_id: 'pandora', id: 'stored-parent', message_count: 1, profile: 'default' }) + ]) + vi.mocked(getAllSessionMessages).mockResolvedValue({ + messages: [{ content: 'branch me', role: 'user', timestamp: 1 }], + session_id: 'stored-parent' + } as never) + + let branchStoredSession: ((storedSessionId: string) => Promise) | null = null + render( (branchStoredSession = branch)} requestGateway={ambientRequest} />) + await waitFor(() => expect(branchStoredSession).not.toBeNull()) + + await expect(branchStoredSession!('stored-parent')).resolves.toBe(true) + + expect(getAllSessionMessages).toHaveBeenCalledWith('stored-parent', { + connectionId: 'pandora', + profile: 'default' + }) + }) + + // The create landing on the right backend is only half the job: the sidebar + // row must be TAGGED with that owner too. An untagged optimistic row inherits + // the ambient profile, so every later owner lookup (resume/hydrate/prompt) + // routes to the wrong backend and the chat pane spins forever on a session + // that backend never had. + it('tags the optimistic branch row with the parent connection owner', async () => { + const ambientRequest = vi.fn(async () => ({}) as never) + + vi.mocked(requestGatewayForAgent).mockImplementation((async ( + _connectionId: null | string, + _profile: string, + method: string + ) => { + if (method === 'session.create') { + return { session_id: 'branch-runtime', stored_session_id: 'branch-stored' } as never + } + + return {} as never + }) as never) + + setSessions([ + storedSession({ connection_id: 'pandora', id: 'stored-parent', message_count: 1, profile: 'default' }) + ]) + vi.mocked(getAllSessionMessages).mockResolvedValue({ + messages: [{ content: 'branch me', role: 'user', timestamp: 1 }], + session_id: 'stored-parent' + } as never) + + let branchStoredSession: ((storedSessionId: string) => Promise) | null = null + render( (branchStoredSession = branch)} requestGateway={ambientRequest} />) + await waitFor(() => expect(branchStoredSession).not.toBeNull()) + + await expect(branchStoredSession!('stored-parent')).resolves.toBe(true) + + const row = $sessions.get().find(session => session.id === 'branch-stored') + expect(row).toBeDefined() + expect(row!.connection_id).toBe('pandora') + expect(row!.profile).toBe('default') + }) + + // An untagged row (single-backend users, the overwhelmingly common case) + // must keep the ambient path exactly as before — no behaviour change. + it('keeps an untagged parent branch on the ambient socket', async () => { + let createParams: Record | undefined + + const ambientRequest = vi.fn(async (method: string, params?: Record) => { + if (method === 'session.create') { + createParams = params + + return { session_id: 'branch-runtime', stored_session_id: 'branch-stored' } as never + } + + return {} as never + }) + + vi.mocked(requestGatewayForAgent).mockClear() + setSessions([storedSession({ id: 'stored-parent', message_count: 1 })]) + vi.mocked(getAllSessionMessages).mockResolvedValue({ + messages: [{ content: 'branch me', role: 'user', timestamp: 1 }], + session_id: 'stored-parent' + } as never) + + let branchStoredSession: ((storedSessionId: string) => Promise) | null = null + render( (branchStoredSession = branch)} requestGateway={ambientRequest} />) + await waitFor(() => expect(branchStoredSession).not.toBeNull()) + + await expect(branchStoredSession!('stored-parent')).resolves.toBe(true) + + expect(createParams).toMatchObject({ parent_session_id: 'stored-parent', source: 'desktop' }) + expect(requestGatewayForAgent).not.toHaveBeenCalled() + }) + it('branches an open live chat via session.branch with a trimmed message count (bug #1/#3 fix)', async () => { let branchParams: Record | undefined @@ -2347,7 +2727,7 @@ describe('resumeSession warm-cache mapping integrity', () => { expect(sessionStateByRuntimeIdRef.current.has('rt-recycled')).toBe(false) }) - it('paints the bounded latest transcript after the deferred resume acknowledgement', async () => { + it('paints the bounded latest transcript before the deferred resume acknowledgement without rebuilding it', async () => { const latestPage = Array.from({ length: 500 }, (_, index) => ({ content: `message-${index}`, role: index % 2 === 0 ? ('user' as const) : ('assistant' as const), @@ -2382,7 +2762,8 @@ describe('resumeSession warm-cache mapping integrity', () => { await waitFor(() => expect(getLatestSessionMessages).toHaveBeenCalledTimes(1)) expect(getLatestSessionMessages).toHaveBeenCalledWith('stored-A', undefined) - expect($messages.get()).toHaveLength(0) + await waitFor(() => expect($messages.get()).toHaveLength(500)) + const paintedTranscript = $messages.get() expect(requestGatewayMock).toHaveBeenCalledWith( 'session.resume', expect.objectContaining({ @@ -2400,7 +2781,7 @@ describe('resumeSession warm-cache mapping integrity', () => { info: {} }) await resumePromise - expect($messages.get()).toHaveLength(500) + expect($messages.get()).toBe(paintedTranscript) }) it('honours a warm cache entry whose stored id matches and refreshes its persisted transcript', async () => { diff --git a/apps/desktop/src/app/session/hooks/use-session-actions/index.ts b/apps/desktop/src/app/session/hooks/use-session-actions/index.ts index 635880cae6..cec1ebf521 100644 --- a/apps/desktop/src/app/session/hooks/use-session-actions/index.ts +++ b/apps/desktop/src/app/session/hooks/use-session-actions/index.ts @@ -49,14 +49,7 @@ import { normalizeProfileKey, resolveNewChatOwnerRoute } from '@/store/profile' -import { - $projectScope, - beginSessionMutation, - endSessionMutation, - resolveNewSessionCwd, - tombstoneSessions, - untombstoneSessions -} from '@/store/projects' +import { $projectScope, resolveNewSessionCwd } from '@/store/projects' import { setApprovalRequest } from '@/store/prompts' import { clearStoredTranscriptReadOnly, markStoredTranscriptReadOnly } from '@/store/read-only-transcript' import { @@ -99,8 +92,17 @@ import { setYoloActive } from '@/store/session' import { isSessionOwnerResolutionError } from '@/store/session-owner-resolution' +import { + beginSessionMutation, + endSessionMutation, + isSessionRemovalPending, + tombstoneSessions, + untombstoneSessions +} from '@/store/session-removal' import { requestForSessionProfile, + type SessionOwnerRoute, + sessionOwnerRouteFromRow, type SessionOwnerScope, type SessionProfileRoute } from '@/store/session-request-router' @@ -146,6 +148,7 @@ import { applyRuntimeInfo, applyStoredSessionPreviewRuntimeInfo, type BranchMessage, + cachedSessionRow, chatMessageArraysEquivalent, dedupeInflightUserAgainstTranscript, dropListedSession, @@ -154,6 +157,7 @@ import { isSessionGoneError, overlayConcurrentMessageChanges, patchSessionWorkspace, + preserveEquivalentTranscript, preserveLocalPendingTurnMessages, reconcileResumeMessages, removeRepresentedLocalLiveProjection, @@ -202,6 +206,42 @@ interface SessionActionsOptions { // (NOT in this set) still legitimately drops to a draft. const createdThisRun = new Set() +const branchMessagesFingerprint = (messages: BranchMessage[]): string => + JSON.stringify(messages.map(({ content, role }) => [role, content])) + +// Identity of one branch create, so a re-entered branch action (a retried +// renderer transition, a double right-click) rides the create already in +// flight instead of minting a second child. The OWNER is part of the identity: +// the same parent id served by two connections is two different sessions. +function branchCreateKey({ + branchCount, + branchMessages, + cwd, + ownerRoute, + parentStoredId, + profile, + sourceSessionId +}: { + branchCount?: number + branchMessages: BranchMessage[] + cwd?: string + ownerRoute?: SessionOwnerRoute + parentStoredId: null | string + profile?: null | string + sourceSessionId: null | string +}): string { + return JSON.stringify({ + branchCount: branchCount ?? null, + connectionId: ownerRoute?.connectionId || null, + cwd: cwd?.trim() || null, + messages: sourceSessionId ? null : branchMessagesFingerprint(branchMessages), + ownerProfile: ownerRoute?.profile || null, + parentStoredId, + profile: profile?.trim() || null, + sourceSessionId + }) +} + // Reflect a stored row's persisted token counts into the live usage atom // (total is derived, so callers can't drift it out of sync with input/output). function applyStoredUsage(stored: { input_tokens?: number | null; output_tokens?: number | null }) { @@ -341,6 +381,7 @@ export function useSessionActions({ const { t } = useI18n() const copy = t.desktop const resumeRequestRef = useRef(0) + const branchCreateFlightsRef = useRef(new Map>()) // Follow auto-compression's stored-id rotation only while the exact runtime, // selection, and route intent still belong to the rotating conversation. @@ -818,6 +859,15 @@ export function useSessionActions({ const resumeSession = useCallback( async (storedSessionId: string, replaceRoute = false, capturedOwner?: SessionProfileRoute) => { + // Delete/archive tombstones the durable id before the route flips, and + // requestSessionResume already refuses to queue for a doomed id. This is + // the actuator-side half of the same rule: a resume that was queued + // BEFORE the tombstone (an idle-reap 4001 racing the delete) must not + // re-select the chat and toast "Resume failed / Session not found". + if (isSessionRemovalPending(storedSessionId)) { + return + } + const requestId = resumeRequestRef.current + 1 resumeRequestRef.current = requestId const resumedSameSelectedSession = selectedStoredSessionIdRef.current === storedSessionId @@ -1323,26 +1373,37 @@ export function useSessionActions({ const activatedState = updateSessionState( cachedRuntimeId, - state => ({ - ...state, - messages: visibleActivatedMessages, - transcriptProvenance: - acceptedPersistedDisplayTranscript || hasValidProvenance - ? (expectedProvenance ?? undefined) - : undefined, - ...(pendingClarifyProjection - ? { - awaitingResponse: false, - sawAssistantPayload: true, - streamId: pendingClarifyProjection.streamId - } - : {}), - ...(clearedClarifyProjection - ? { - streamId: state.busy ? (clearedClarifyProjection.streamId ?? state.streamId) : null - } - : {}) - }), + state => { + // #95595: the reconcilers above always produce fresh + // message objects, so an unconditional publish replaces the + // warm-cached array with new-object equivalents and every + // visible row re-normalizes + remounts (markdown re-parse + + // shiki re-highlight per row, seconds of main-thread work). + // Keep the existing array when the content is unchanged — + // same guard the cold-resume path uses below. + const messages = preserveEquivalentTranscript(state.messages, visibleActivatedMessages) + + return { + ...state, + messages, + transcriptProvenance: + acceptedPersistedDisplayTranscript || hasValidProvenance + ? (expectedProvenance ?? undefined) + : undefined, + ...(pendingClarifyProjection + ? { + awaitingResponse: false, + sawAssistantPayload: true, + streamId: pendingClarifyProjection.streamId + } + : {}), + ...(clearedClarifyProjection + ? { + streamId: state.busy ? (clearedClarifyProjection.streamId ?? state.streamId) : null + } + : {}) + } + }, storedSessionId ) @@ -1507,9 +1568,6 @@ export function useSessionActions({ // keeps it from surfacing as unhandled while the prefetch settles. resumePromise.catch(() => undefined) - // Keep both requests concurrent, but do not paint the REST result until - // the runtime resume has also settled. An eager prefetch paint followed - // by the runtime projection rebuilds large transcripts during resume. let prefetchedResult: { messages: SessionMessage[]; session_id?: string } | null = null try { @@ -1520,13 +1578,15 @@ export function useSessionActions({ // Non-fatal: gateway resume below can still hydrate the session. } - const resumed = await resumePromise - - if (!isCurrentResume()) { - return - } - - if (prefetchedResult) { + // Paint the persisted transcript as soon as REST returns instead of + // holding it until the runtime resume settles. A cold profile build + // (skills, MCP, memory) can keep `session.resume` pending far longer + // than the hydration budget while the complete history is already in + // hand — holding it stranded Bot Chats on the loader (#90130). The + // runtime path below grafts only its live projection onto this same + // snapshot, so an unchanged acknowledgement keeps reference identity + // and never rebuilds the transcript a second time. + if (prefetchedResult && isCurrentResume()) { const previousMessages = resumedSameSelectedSession ? preserveLocalPendingTurnMessages(viewMessagesForReconcile(), resumeStartMessages) : viewMessagesForReconcile() @@ -1541,6 +1601,16 @@ export function useSessionActions({ localSnapshot = reconcileAuthoritativeChatMessages(graftedPrefetch, previousMessages) prefetchApplied = true prefetchedStoredSessionId = prefetchedResult.session_id || storedSessionId + + if (!chatMessageArraysEquivalent($messages.get(), localSnapshot)) { + setMessages(localSnapshot) + } + } + + const resumed = await resumePromise + + if (!isCurrentResume()) { + return } const currentMessages = viewMessagesForReconcile() @@ -1707,12 +1777,25 @@ export function useSessionActions({ const visibleMessagesForView = pendingClarifyProjection?.messages ?? clearedClarifyProjection?.messages ?? messagesForView + // The eagerly painted REST page is persisted-display authority: stamp + // its provenance so the next warm switch to this session paints it + // immediately instead of holding it as an unproven runtime tail. + const transcriptProvenance = + prefetchApplied && prefetchMatchesResumedSession && stored + ? createPersistedDisplayTranscriptProvenance({ + lineageRootId: stored._lineage_root_id ?? null, + scope: sessionRestScope, + storedSessionId + }) + : undefined + updateSessionState( resumed.session_id, state => ({ ...state, ...(runtimeInfo ?? {}), messages: visibleMessagesForView, + transcriptProvenance, busy: resumedRunning, awaitingResponse: resumedRunning && !recoveredInFlightTail, // Backend reported this turn running at resume time — live proof. @@ -1800,7 +1883,10 @@ export function useSessionActions({ reconcileAuthoritativeMessages(fallback.messages, previousMessages) ) - setMessages(fallbackRecovery.messages) + // The eager prefetch paint above may already show this transcript. + if (!chatMessageArraysEquivalent($messages.get(), fallbackRecovery.messages)) { + setMessages(fallbackRecovery.messages) + } } catch (e) { // Fallback also failed: nothing to paint. Leave whatever messages are // already shown and fall through to arm the resume-failure latch so @@ -1954,34 +2040,82 @@ export function useSessionActions({ parentStoredId: null | string, cwd?: string, profile?: null | string, - branchCount?: number + branchCount?: number, + ownerRoute?: SessionOwnerRoute ): Promise => { creatingSessionRef.current = true try { - // A branch belongs to its parent's OWNING profile. Swapping the live - // gateway first AND passing `profile` on the create mirrors - // desktopSessionCreateParams/resumeSession: in app-global remote mode - // one backend serves every profile, so an omitted profile silently - // lands the branch on the launch (default) profile — the "session - // jumps between profiles after branching" bug. The swap also makes - // upsertOptimisticSession's $activeGatewayProfile stamp correct. - await ensureGatewayProfile(profile) + // A branch belongs to its parent's OWNING backend. Two facets, and both + // matter once more than one connection is configured: + // + // 1. PROFILE — passing `profile` on the create mirrors + // desktopSessionCreateParams/resumeSession: in app-global remote mode + // one backend serves every profile, so an omitted profile silently + // lands the branch on the launch (default) profile — the "session + // jumps between profiles after branching" bug. + // 2. CONNECTION — a profile name alone does not identify a backend when + // several connections expose the same name. Routing on profile only + // sends session.create to whatever socket happens to be active, so + // branching a remote-owned parent from another connection creates the + // child on the wrong backend (or nowhere), while the optimistic + // sidebar row below still points at an id no backend owns — the + // "Couldn't load this session" strand. removeSession already routes + // by (connection, profile); this is the same ownership contract. + // + // An untagged parent keeps the historic profile-only path exactly. + if (ownerRoute) { + await ensureGatewayAgent(ownerRoute.connectionId, ownerRoute.profile) + } else { + await ensureGatewayProfile(profile) + } + + const requestBranchGateway = (method: string, params: Record): Promise => + ownerRoute + ? requestGatewayForAgent(ownerRoute.connectionId, ownerRoute.profile, method, params) + : requestGateway(method, params) + + // The owner is part of the identity: the same parent id on two + // connections is two different sessions, so a route-blind key would + // coalesce them onto one create. + const createKey = branchCreateKey({ + branchCount, + branchMessages, + cwd, + ownerRoute, + parentStoredId, + profile, + sourceSessionId + }) + + let createFlight = branchCreateFlightsRef.current.get(createKey) // No title: the backend auto-names the branch from its parent's lineage. - const branched = sourceSessionId - ? await requestGateway('session.branch', { - session_id: sourceSessionId, - ...(branchCount !== undefined ? { count: branchCount } : {}) - }) - : await requestGateway('session.create', { - cols: 96, - source: 'desktop', - ...(cwd && { cwd }), - ...(profile ? { profile } : {}), - messages: branchMessages.map(({ content, role }) => ({ content, role })), - ...(parentStoredId && { parent_session_id: parentStoredId }) - }) + if (!createFlight) { + createFlight = ( + sourceSessionId + ? requestBranchGateway('session.branch', { + session_id: sourceSessionId, + ...(branchCount !== undefined ? { count: branchCount } : {}) + }) + : requestBranchGateway('session.create', { + cols: 96, + source: 'desktop', + ...(cwd && { cwd }), + ...(profile ? { profile } : {}), + messages: branchMessages.map(({ content, role }) => ({ content, role })), + ...(parentStoredId && { parent_session_id: parentStoredId }) + }) + ).catch(err => { + // Drop the flight so a genuine retry re-issues the create; a + // resolved flight is cleared once the child is fully published. + branchCreateFlightsRef.current.delete(createKey) + throw err + }) + branchCreateFlightsRef.current.set(createKey, createFlight) + } + + const branched = await createFlight const responseBranchMessages = sourceSessionId && branched.messages?.length ? toBranchMessages(toChatMessages(branched.messages)) : [] @@ -1989,6 +2123,19 @@ export function useSessionActions({ const effectiveBranchMessages = responseBranchMessages.length ? responseBranchMessages : branchMessages const routedSessionId = branched.stored_session_id ?? branched.session_id const preview = effectiveBranchMessages.map(({ content }) => content).find(Boolean) ?? null + + // Record the exact owner and pin its socket THE MOMENT the create + // returns, before the optimistic row / tile publication can lose a + // race with the gateway pruner. A draft branch child exists only as a + // runtime on the owning backend (the stored row lands on first turn), + // so a prune in this gap orphan-reaps it and the tile enters the + // resume→reclaim flicker loop (#93892 shape). Mirrors the two routed + // creates at the top of this file. + if (ownerRoute) { + setSessionOwnerHint(routedSessionId, ownerRoute) + holdSessionOwnerUntilForeground(routedSessionId, ownerRoute) + } + // Draft until submit: nest under the parent at the parent's recency so it // doesn't bubble to the top until a real message lands (backend persists // + auto-names it then). The selected row survives refreshes (sessionsToKeep). @@ -2000,13 +2147,22 @@ export function useSessionActions({ : 0 setFreshDraftReady(false) + // Stamp the optimistic row with the branch's EXACT owner. Without it the + // row inherits $activeGatewayProfile and carries no connection_id, so a + // child correctly created on the parent's remote backend is listed as + // belonging to whichever backend happens to be active. Every later + // owner lookup off that row (resume, hydrate, prompt) then routes to the + // wrong machine and the chat pane spins on a session that backend never + // had — the create is right, the row is a lie. Mirrors the routed + // creates at the top of this file, which already pass their route here. upsertOptimisticSession( branched, routedSessionId, copy.branchTitle(siblings + 1).toLowerCase(), preview, parentStoredId, - parent ? parent.last_active || parent.started_at : undefined + parent ? parent.last_active || parent.started_at : undefined, + ownerRoute ?? null ) ensureSessionState(branched.session_id, routedSessionId) updateSessionState( @@ -2034,13 +2190,25 @@ export function useSessionActions({ // unconditionally). resumeSession reuses the runtime warm-cached above // (ensureSessionState/updateSessionState) instead of an extra resume RPC. if (parentStoredId !== null && selectedStoredSessionIdRef.current === parentStoredId) { + navigate(sessionRoute(routedSessionId), { replace: true }) await resumeSession(routedSessionId) } else { - openSessionTile(routedSessionId, 'center') + // Carry the exact owner onto the tile: its persisted ownerRoute is + // what pins the owning backend's socket in the gateway keep-set + // (openTileGatewayScopes) for the tile's whole lifetime. Without it + // a remote-owned branch child's tile pinned nothing, the pruner + // closed the owner socket, the backend reaped the draft runtime, + // and the tile looped resume→reclaim until the storm breaker + // latched "Couldn't open this session". + openSessionTile(routedSessionId, 'center', undefined, null, { + ownerRoute, + workspaceMode: 'sessions' + }) patchSessionTile(routedSessionId, { runtimeId: branched.session_id }) revealTreePane(`session-tile:${routedSessionId}`) } + branchCreateFlightsRef.current.delete(createKey) broadcastSessionsChanged() return true @@ -2058,6 +2226,7 @@ export function useSessionActions({ copy, creatingSessionRef, ensureSessionState, + navigate, requestGateway, resumeSession, selectedStoredSessionIdRef, @@ -2094,9 +2263,15 @@ export function useSessionActions({ let authoritativeMessages: ChatMessage[] | null = null const profile = await resolveSessionProfile(storedSessionId) + // The open chat's exact owner, when its row carries a connection tag. + // Same contract as branchStoredSession: the transcript read and the + // branch RPC must both land on the backend that owns the parent, not on + // whichever socket is active. + const ownerRoute = storedSessionId ? sessionOwnerRouteFromRow(cachedSessionRow(storedSessionId)) : undefined + if (storedSessionId) { try { - const persisted = await getAllSessionMessages(storedSessionId, profile) + const persisted = await getAllSessionMessages(storedSessionId, ownerRoute ?? profile) const hydrated = toChatMessages(persisted.messages) if (hydrated.length) { @@ -2144,7 +2319,8 @@ export function useSessionActions({ storedSessionId, startingCwd, profile, - messageId ? branchMessages.length : undefined + messageId ? branchMessages.length : undefined, + ownerRoute ) }, [activeSessionIdRef, busyRef, copy, forkBranch, getRouteToken, selectedStoredSessionIdRef] @@ -2160,15 +2336,30 @@ export function useSessionActions({ // Right-clicking a session outside the paginated sidebar window is a cache // miss: resolve it (cache → active backend → cross-profile) so the branch // is created on the parent's OWNING profile, not whichever is live (#67603). + // cachedSessionRow spans Recents, cron/messaging and the profile-scoped + // project tree, and prefers the self-describing row — an ownerless legacy + // Recents copy of the same id must not mask the row carrying the owner. const stored = - $sessions.get().find(session => sessionMatchesStoredId(session, storedSessionId)) ?? - (sessionProfile ? undefined : await resolveStoredSession(storedSessionId)) + cachedSessionRow(storedSessionId) ?? (sessionProfile ? undefined : await resolveStoredSession(storedSessionId)) const profile = sessionProfile ?? stored?.profile + // An exact owner from the parent row — connection AND profile. Undefined + // for an untagged row, which keeps the ambient/profile-only path. + const ownerRoute = sessionOwnerRouteFromRow(stored) + try { - await ensureGatewayProfile(profile) - const { messages } = await getAllSessionMessages(storedSessionId, profile) + if (ownerRoute) { + await ensureGatewayAgent(ownerRoute.connectionId, ownerRoute.profile) + } else { + await ensureGatewayProfile(profile) + } + + // Read the parent transcript from the backend that OWNS it. A bare + // profile scope resolves against the active connection, which for a + // foreign-owned parent holds no such session: the read comes back empty + // and the branch aborts as "nothing to branch" before any create. + const { messages } = await getAllSessionMessages(storedSessionId, ownerRoute ?? profile) const branchMessages = toBranchMessages(toChatMessages(messages)) if (!branchMessages.length) { @@ -2177,7 +2368,15 @@ export function useSessionActions({ return false } - return await forkBranch(branchMessages, null, stored?.id ?? storedSessionId, stored?.cwd?.trim(), profile) + return await forkBranch( + branchMessages, + null, + stored?.id ?? storedSessionId, + stored?.cwd?.trim(), + profile, + undefined, + ownerRoute + ) } catch (err) { notifyError(err, copy.branchFailed) @@ -2220,8 +2419,12 @@ export function useSessionActions({ return } - const wasSelected = selectedStoredSessionId === storedSessionId - const closingRuntimeId = wasSelected ? activeSessionId : null + // Selection and runtime refs are updated synchronously at routing + // boundaries. React props can still describe the previous render when a + // delete lands in the same tick, which used to leave the doomed route in + // place and let the generic 4001 recovery rebind it. + const wasSelected = selectedStoredSessionIdRef.current === storedSessionId + const closingRuntimeId = wasSelected ? activeSessionIdRef.current : null const previousMessages = $messages.get() const previousPinned = $pinnedSessionIds.get() @@ -2323,13 +2526,11 @@ export function useSessionActions({ } }, [ - activeSessionId, activeSessionIdRef, copy, navigate, requestGateway, runtimeIdByStoredSessionIdRef, - selectedStoredSessionId, selectedStoredSessionIdRef, sessionStateByRuntimeIdRef, startFreshSessionDraft @@ -2356,7 +2557,7 @@ export function useSessionActions({ return } - const wasSelected = selectedStoredSessionId === storedSessionId + const wasSelected = selectedStoredSessionIdRef.current === storedSessionId const previousPinned = $pinnedSessionIds.get() // Pins are keyed on the durable lineage-root id; the stored id may be the // live tip after compression. Drop both so the pin can't linger. @@ -2401,7 +2602,13 @@ export function useSessionActions({ endSessionMutation(archivedIds) } }, - [copy, runtimeIdByStoredSessionIdRef, selectedStoredSessionId, sessionStateByRuntimeIdRef, startFreshSessionDraft] + [ + copy, + runtimeIdByStoredSessionIdRef, + selectedStoredSessionIdRef, + sessionStateByRuntimeIdRef, + startFreshSessionDraft + ] ) return { diff --git a/apps/desktop/src/app/session/hooks/use-session-actions/resolve-stored-session.test.ts b/apps/desktop/src/app/session/hooks/use-session-actions/resolve-stored-session.test.ts index 468936f5b6..cccb323317 100644 --- a/apps/desktop/src/app/session/hooks/use-session-actions/resolve-stored-session.test.ts +++ b/apps/desktop/src/app/session/hooks/use-session-actions/resolve-stored-session.test.ts @@ -3,10 +3,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as HermesModule from '@/hermes' import { getSession } from '@/hermes' import { $activeGatewayProfile, $profiles } from '@/store/profile' +import { $projectTree } from '@/store/projects' import { $cronSessions, $messagingSessions, $sessions } from '@/store/session' import type { SessionInfo } from '@/types/hermes' -import { resolveSessionProfile, resolveStoredSession } from './utils' +import { cachedSessionRow, resolveSessionProfile, resolveStoredSession } from './utils' vi.mock('@/hermes', async importActual => ({ ...(await importActual()), @@ -24,6 +25,7 @@ describe('resolveStoredSession profile ownership', () => { $cronSessions.set([]) $messagingSessions.set([]) $sessions.set([]) + $projectTree.set([]) $profiles.set(profiles('default', 'meta')) $activeGatewayProfile.set('meta') mockGetSession.mockReset() @@ -33,6 +35,7 @@ describe('resolveStoredSession profile ownership', () => { $cronSessions.set([]) $messagingSessions.set([]) $sessions.set([]) + $projectTree.set([]) $profiles.set([]) $activeGatewayProfile.set('default') }) @@ -144,3 +147,50 @@ describe('resolveStoredSession profile ownership', () => { await expect(resolveSessionProfile('s1')).resolves.toBe('default') }) }) + +describe('cachedSessionRow owner preference', () => { + const projectNode = (sessions: SessionInfo[], preview: SessionInfo[] = []) => + ({ + previewSessions: preview, + repos: [{ groups: [{ sessions }] }] + }) as never + + beforeEach(() => { + $cronSessions.set([]) + $messagingSessions.set([]) + $sessions.set([]) + $projectTree.set([]) + mockGetSession.mockReset() + }) + + afterEach(() => { + $cronSessions.set([]) + $messagingSessions.set([]) + $sessions.set([]) + $projectTree.set([]) + }) + + it('prefers a self-describing project-tree row over an ownerless Recents duplicate', () => { + // The same conversation, listed twice: a legacy Recents row with no owner + // and the profile-scoped project-tree row the gateway stamped. Picking the + // Recents copy throws away the only routing information there is, and the + // branch then creates its child on whichever backend is active. + $sessions.set([session({ cwd: '/wrong', id: 's1' })]) + $projectTree.set([projectNode([session({ connection_id: 'pandora', cwd: '/right', id: 's1', profile: 'work' })])]) + + expect(cachedSessionRow('s1')).toMatchObject({ connection_id: 'pandora', cwd: '/right', profile: 'work' }) + }) + + it('finds a project-tree preview row when the session is in no other list', () => { + $projectTree.set([projectNode([], [session({ connection_id: 'rigremote', id: 's1', profile: 'default' })])]) + + expect(cachedSessionRow('s1')).toMatchObject({ connection_id: 'rigremote' }) + }) + + it('keeps the plain Recents row when nothing carries an owner', () => { + $sessions.set([session({ cwd: '/only', id: 's1' })]) + + expect(cachedSessionRow('s1')).toMatchObject({ cwd: '/only' }) + expect(cachedSessionRow('missing')).toBeUndefined() + }) +}) diff --git a/apps/desktop/src/app/session/hooks/use-session-actions/utils.test.ts b/apps/desktop/src/app/session/hooks/use-session-actions/utils.test.ts index d5f76b1423..19bff08385 100644 --- a/apps/desktop/src/app/session/hooks/use-session-actions/utils.test.ts +++ b/apps/desktop/src/app/session/hooks/use-session-actions/utils.test.ts @@ -26,6 +26,7 @@ import { goneSessionVerdict, isSessionGoneError, overlayConcurrentMessageChanges, + preserveEquivalentTranscript, preserveLocalPendingTurnMessages, reconcileResumeMessages, removeRepresentedLocalLiveProjection, @@ -1717,3 +1718,43 @@ describe('overlayConcurrentMessageChanges', () => { ]) }) }) + +describe('preserveEquivalentTranscript', () => { + it('keeps the current array BY REFERENCE when the replacement is content-equivalent', () => { + // The exact warm-resume shape of #95595: fresh objects, identical content. + const current = [msg('u-1', 'user', 'hello'), msg('a-1', 'assistant', 'const x = 1')] + const freshObjects = current.map(message => ({ ...message, parts: [...message.parts] })) + + const preserved = preserveEquivalentTranscript(current, freshObjects) + + expect(preserved).toBe(current) + expect(preserved[0]).toBe(current[0]) + }) + + it('keeps the current array when the arrays are the same reference', () => { + const current = [msg('u-1', 'user', 'hello')] + + expect(preserveEquivalentTranscript(current, current)).toBe(current) + }) + + it('accepts the replacement when anything changed', () => { + const current = [msg('u-1', 'user', 'hello')] + const next = [msg('u-1', 'user', 'hello'), msg('a-1', 'assistant', 'new turn')] + + expect(preserveEquivalentTranscript(current, next)).toBe(next) + }) + + it('rejects the replacement when a message diverges in content', () => { + const current = [msg('u-1', 'user', 'hello')] + const next = [msg('u-1', 'user', 'hello world')] + + expect(preserveEquivalentTranscript(current, next)).toBe(next) + }) + + it('rejects the replacement when metadata a row renders diverges', () => { + const current = [msg('u-1', 'user', 'hello')] + const next = [msg('u-1', 'user', 'hello', { pending: true })] + + expect(preserveEquivalentTranscript(current, next)).toBe(next) + }) +}) diff --git a/apps/desktop/src/app/session/hooks/use-session-actions/utils.ts b/apps/desktop/src/app/session/hooks/use-session-actions/utils.ts index 83e599b589..875b059ed2 100644 --- a/apps/desktop/src/app/session/hooks/use-session-actions/utils.ts +++ b/apps/desktop/src/app/session/hooks/use-session-actions/utils.ts @@ -8,6 +8,7 @@ import { isMessagingSource, normalizeSessionSource } from '@/lib/session-source' import { reconcileApprovalModeForProfile } from '@/store/approval-mode' import { requestDesktopOnboardingForCredentialWarning } from '@/store/onboarding' import { $activeGatewayProfile, $profiles, normalizeProfileKey } from '@/store/profile' +import { $projectTree } from '@/store/projects' import { $cronSessions, $currentCwd, @@ -297,6 +298,23 @@ export function chatMessageArraysEquivalent(a: ChatMessage[], b: ChatMessage[]): return a.length === b.length && a.every((message, index) => chatMessagesEquivalent(message, b[index])) } +/** + * Keep the CURRENT array when the replacement is content-equivalent. + * + * The resume reconcilers create fresh `ChatMessage` objects via + * `toChatMessages` even when nothing changed. Publishing those unconditionally + * replaces the `$messages`/session-slice array with a new reference of fresh + * objects — and because `useRuntimeMessageRepository` keys its normalization + * cache (and React keys its rows) by object identity, every message in the + * window re-normalizes and remounts: full markdown re-parse + shiki + * re-highlight per row, on the main thread, per warm session switch (#95595). + * Returning `current` when the content is equivalent keeps array AND object + * identity, so the warm switch is O(1) paint. + */ +export function preserveEquivalentTranscript(current: ChatMessage[], next: ChatMessage[]): ChatMessage[] { + return chatMessageArraysEquivalent(current, next) ? current : next +} + export function reconcileResumeMessages(nextMessages: ChatMessage[], previousMessages: ChatMessage[]): ChatMessage[] { if (!previousMessages.length) { return nextMessages @@ -1387,13 +1405,43 @@ function upsertResolvedSession(session: SessionInfo, storedSessionId: string) { ]) } +// Every session row reachable through the profile-scoped project tree — +// preview rows on a collapsed project plus the drill-in lane rows. These are +// the only rows guaranteed to name their owning profile (the gateway stamps +// the request scope onto them), so owner resolution has to see them. +function projectTreeSessions(): SessionInfo[] { + return $projectTree + .get() + .flatMap(project => [ + ...(project.previewSessions ?? []), + ...project.repos.flatMap(repo => repo.groups.flatMap(group => group.sessions)) + ]) +} + +// The best cached row for a stored id, across every list that can hold one. +// "Best" means self-describing: the same conversation can appear both as an +// ownerless legacy Recents copy and as a profile-stamped project-tree row, and +// picking the ownerless one throws away the only routing information we have. +export function cachedSessionRow(storedSessionId: string): SessionInfo | undefined { + const candidates = [ + ...$sessions.get(), + ...$cronSessions.get(), + ...$messagingSessions.get(), + ...projectTreeSessions() + ].filter(session => sessionMatchesStoredId(session, storedSessionId)) + + return ( + candidates.find(session => session.connection_id?.trim()) ?? + candidates.find(session => session.profile?.trim()) ?? + candidates[0] + ) +} + export async function resolveStoredSession( storedSessionId: string, ownerRoute?: SessionProfileRoute ): Promise { - const cached = [...$sessions.get(), ...$cronSessions.get(), ...$messagingSessions.get()].find(session => - sessionMatchesStoredId(session, storedSessionId) - ) + const cached = cachedSessionRow(storedSessionId) if (ownerRoute) { const scope = { diff --git a/apps/desktop/src/app/session/hooks/use-session-list-actions.test.tsx b/apps/desktop/src/app/session/hooks/use-session-list-actions.test.tsx index 8a953a043a..eb7f96ee88 100644 --- a/apps/desktop/src/app/session/hooks/use-session-list-actions.test.tsx +++ b/apps/desktop/src/app/session/hooks/use-session-list-actions.test.tsx @@ -98,7 +98,8 @@ vi.mock('@/store/gateway', async importOriginal => ({ // the whole projects store (gateway / fs / git) into this hook's test. const removed = vi.hoisted(() => ({ ids: new Set() })) -vi.mock('@/store/projects', () => ({ +vi.mock('@/store/session-removal', async importActual => ({ + ...(await importActual>()), $removedSessionIds: { get: () => removed.ids } })) @@ -216,6 +217,57 @@ describe('refreshSessions identity + loading hygiene', () => { expect($sessions.get().map(s => s.id)).toEqual(['a']) }) + it('keeps idle recents when the sidebar returns an empty page plus profile errors', async () => { + // Backend contract on disk I/O / lock: HTTP 200, recents=[], errors=[{profile}]. + // mergeSessionPage only keeps working/pinned/selected, so Yesterday/This-week + // idle rows must be carried forward from the previous list — not clobbered. + const idle = [row('yesterday'), row('week')] + listSidebarSessions.mockResolvedValue(sidebar({ sessions: idle })) + + const { result } = renderHook(() => useSessionListActions({ profileScope: 'default' })) + + await act(async () => { + await result.current.refreshSessions() + }) + + expect($sessions.get().map(s => s.id)).toEqual(['yesterday', 'week']) + + setSessionProfilesTruncated({ default: true }) + setSessionProfilesUsage({ default: { cost_usd: 3, tokens: 30 } }) + setMessagingTruncated(true) + + listSidebarSessions.mockResolvedValue({ + ...sidebar({ sessions: [] }), + errors: [{ error: 'disk I/O error', profile: 'default' }] + }) + + await act(async () => { + await result.current.refreshSessions() + }) + + expect($sessions.get().map(s => s.id)).toEqual(['yesterday', 'week']) + expect($sessionProfilesTruncated.get()).toEqual({ default: true }) + expect($sessionProfilesUsage.get()).toEqual({ default: { cost_usd: 3, tokens: 30 } }) + expect($messagingTruncated.get()).toBe(true) + }) + + it('still accepts a genuine empty recents page when the backend reported no errors', async () => { + listSidebarSessions.mockResolvedValue(sidebar({ sessions: [row('a')] })) + const { result } = renderHook(() => useSessionListActions({ profileScope: 'default' })) + + await act(async () => { + await result.current.refreshSessions() + }) + + listSidebarSessions.mockResolvedValue(sidebar({ sessions: [] })) + + await act(async () => { + await result.current.refreshSessions() + }) + + expect($sessions.get()).toEqual([]) + }) + it('drops tombstoned rows from the messaging slice and per-platform paging too (#50928)', async () => { // The same delete race exists on every ingestion point: the batched // refresh's messaging slice and the per-platform "load more" pager must diff --git a/apps/desktop/src/app/session/hooks/use-session-list-actions.ts b/apps/desktop/src/app/session/hooks/use-session-list-actions.ts index 255e85189b..79ed487ea2 100644 --- a/apps/desktop/src/app/session/hooks/use-session-list-actions.ts +++ b/apps/desktop/src/app/session/hooks/use-session-list-actions.ts @@ -19,12 +19,13 @@ import { SIDEBAR_SESSIONS_PAGE_SIZE } from '@/store/layout' import { messagingTotalsKey, normalizeProfileKey, sidebarProfileForScope } from '@/store/profile' -import { $removedSessionIds } from '@/store/projects' import { $messagingSessions, $selectedStoredSessionId, $sessions, + carryForwardFailedProfileSessions, CRON_SECTION_LIMIT, + keepFailedProfileMeta, mergeSessionPage, MESSAGING_SECTION_LIMIT, setCronSessions, @@ -36,7 +37,8 @@ import { setSessions, setSessionsLoading } from '@/store/session' -import { $workingSessionIds, getRecentlySettledSessionIds } from '@/store/session-states' +import { $removedSessionIds } from '@/store/session-removal' +import { $sessionTiles, $workingSessionIds, getRecentlySettledSessionIds } from '@/store/session-states' import { refreshCronJobs as refreshCronJobsStore } from '../../cron/cron-actions' @@ -80,6 +82,15 @@ function sessionsToKeep(scope?: string): Set { ...getRecentlySettledSessionIds() ]) + // Open tiles are user-visible state exactly like the selected row: a branch + // child is a DRAFT until its first real turn, so the aggregator can't return + // it — without this the next background refresh silently dropped the + // optimistic `draft: branch #N` row while its tab was open, and the sidebar + // showed no trace of the branch until first send. + for (const tile of $sessionTiles.get()) { + keep.add(tile.storedSessionId) + } + const active = $selectedStoredSessionId.get() if (active) { @@ -198,7 +209,11 @@ export function useSessionListActions({ profileScope }: UseSessionListActionsArg setMessagingSessions(prev => [ ...prev.filter(s => !inPlatform(s)), - ...mergeSessionPage(prev.filter(inPlatform), incoming, sessionsToKeep()) + ...mergeSessionPage( + prev.filter(inPlatform), + carryForwardFailedProfileSessions(prev.filter(inPlatform), incoming, result.errors), + sessionsToKeep() + ) ]) const total = result.total ?? incoming.length @@ -282,13 +297,15 @@ export function useSessionListActions({ profileScope }: UseSessionListActionsArg // in-flight mutation and the backend page still carries the doomed row. // Honoring the optimistic tombstone keeps the removal from flashing back // (the tombstone self-clears once projects.tree confirms the delete). - const incoming = dropTombstoned(recents.sessions) - // Signature-gate the swap (same pattern as cron/messaging): a refresh // that returns content-identical rows must keep the previous array // identity, or every sidebar memo keyed on $sessions recomputes and the // whole list re-renders once per turn/broadcast for nothing. setSessions(prev => { + const incoming = dropTombstoned( + carryForwardFailedProfileSessions(prev, recents.sessions ?? [], recents.errors ?? result.errors) + ) + const next = mergeSessionPage(prev, incoming, sessionsToKeep()) return sameCronSignature(prev, next) ? prev : next @@ -298,8 +315,9 @@ export function useSessionListActions({ profileScope }: UseSessionListActionsArg // top of the rows it already read (the old exact totals ran a COUNT(*) // per profile DB on every refresh). Reference-stable when unchanged so // the sidebar's group memos don't recompute per refresh. + const recentsErrors = recents.errors ?? result.errors setSessionProfilesTruncated(prev => { - const next = recents.profiles_truncated ?? {} + const next = keepFailedProfileMeta(prev, recents.profiles_truncated ?? {}, recentsErrors) const prevKeys = Object.keys(prev) return prevKeys.length === Object.keys(next).length && prevKeys.every(key => prev[key] === next[key]) @@ -309,7 +327,7 @@ export function useSessionListActions({ profileScope }: UseSessionListActionsArg // Same identity gate: these totals only move when a session bills, and // a fresh object every refresh would repaint every profile header. setSessionProfilesUsage(prev => { - const next = recents.profiles_usage ?? {} + const next = keepFailedProfileMeta(prev, recents.profiles_usage ?? {}, recentsErrors) const prevKeys = Object.keys(prev) return prevKeys.length === Object.keys(next).length && @@ -322,16 +340,35 @@ export function useSessionListActions({ profileScope }: UseSessionListActionsArg // Cron section: latest N cron sessions (kept so a pinned cron run still // resolves via sessionByAnyId), signature-gated like above. - setCronSessions(prev => (sameCronSignature(prev, result.cron.sessions) ? prev : result.cron.sessions)) + setCronSessions(prev => { + const incoming = carryForwardFailedProfileSessions( + prev, + result.cron.sessions ?? [], + result.cron.errors ?? result.errors + ) + + return sameCronSignature(prev, incoming) ? prev : incoming + }) // Messaging sections: drop any non-messaging source the broad exclude // didn't catch (custom sources stay in local recents), then split per // platform in the UI. - const messagingRows = dropTombstoned(result.messaging.sessions.filter(s => isMessagingSource(s.source))) + const messagingErrors = result.messaging.errors ?? result.errors + setMessagingSessions(prev => { + const messagingRows = dropTombstoned( + carryForwardFailedProfileSessions( + prev, + (result.messaging.sessions ?? []).filter(s => isMessagingSource(s.source)), + messagingErrors + ) + ) - setMessagingSessions(prev => (sameCronSignature(prev, messagingRows) ? prev : messagingRows)) + return sameCronSignature(prev, messagingRows) ? prev : messagingRows + }) // Hit the cap → at least one platform may have more on disk than loaded. - setMessagingTruncated(result.messaging.sessions.length >= MESSAGING_SECTION_LIMIT) + setMessagingTruncated(prev => + messagingErrors?.length ? prev : result.messaging.sessions.length >= MESSAGING_SECTION_LIMIT + ) } } finally { // Request identity preserves the zero-argument refresh contract across a diff --git a/apps/desktop/src/app/session/workspace-session-target.test.ts b/apps/desktop/src/app/session/workspace-session-target.test.ts index 852f83ab97..b50f91a61c 100644 --- a/apps/desktop/src/app/session/workspace-session-target.test.ts +++ b/apps/desktop/src/app/session/workspace-session-target.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' +import { $activeGatewayProfile, $newChatProfile } from '@/store/profile' import { $projectScope, $projectTree, ALL_PROJECTS } from '@/store/projects' import { $currentBranch, @@ -22,6 +23,8 @@ describe('startWorkspaceSession', () => { setNewChatWorkspaceTarget(undefined) $projectScope.set(ALL_PROJECTS) $projectTree.set([]) + $activeGatewayProfile.set('default') + $newChatProfile.set(null) vi.restoreAllMocks() }) @@ -107,4 +110,22 @@ describe('startWorkspaceSession', () => { expect($newChatWorkspaceTarget.get()).toBeNull() expect($currentCwd.get()).toBe('') }) + + // #79005 flaw 3: the project "+" must pin the profile the tree is shown + // under; otherwise session.create reads $activeGatewayProfile after a swap. + it('pins the new chat to the profile the project tree is displayed under', () => { + $activeGatewayProfile.set('work') + $newChatProfile.set(null) + + startWorkspaceSession({ + activeSessionIdRef: { current: null }, + path: '/workspace-work', + requestGateway: vi.fn(() => new Promise(() => {})), + startFreshSessionDraft: vi.fn() + }) + + $activeGatewayProfile.set('personal') + + expect($newChatProfile.get()).toBe('work') + }) }) diff --git a/apps/desktop/src/app/session/workspace-session-target.ts b/apps/desktop/src/app/session/workspace-session-target.ts index 868b7c404a..fb0dae1591 100644 --- a/apps/desktop/src/app/session/workspace-session-target.ts +++ b/apps/desktop/src/app/session/workspace-session-target.ts @@ -1,6 +1,7 @@ import type { MutableRefObject } from 'react' -import { followActiveSessionCwd, resolveNewSessionCwd } from '@/store/projects' +import { pinNewChatProfile } from '@/store/profile' +import { followActiveSessionCwd, projectProfile, resolveNewSessionCwd } from '@/store/projects' import { $newChatWorkspaceTargetGeneration, type NewChatWorkspaceTarget, @@ -26,6 +27,16 @@ export function startWorkspaceSession({ requestGateway, startFreshSessionDraft }: WorkspaceSessionOptions): void { + // The project tree is rendered under one profile; the "+" belongs to it. + // Pin that intent now — otherwise desktopSessionCreateParams falls back to + // $activeGatewayProfile, which a still-settling profile swap can move + // between this click and Send (#79005). All-profiles view has no owner. + const profile = projectProfile() + + if (profile) { + pinNewChatProfile(profile) + } + // Home's "+" passes path=null on purpose ("no folder"). That must stay // detached — do NOT fall through to resolveNewSessionCwd(), which can still // return a default/remembered project folder and re-attach the last repo diff --git a/apps/desktop/src/app/settings/appearance-settings.tsx b/apps/desktop/src/app/settings/appearance-settings.tsx index c007ea937b..64c3be3245 100644 --- a/apps/desktop/src/app/settings/appearance-settings.tsx +++ b/apps/desktop/src/app/settings/appearance-settings.tsx @@ -7,6 +7,7 @@ import { LanguageSwitcher } from '@/components/language-switcher' import { Button } from '@/components/ui/button' import { SegmentedControl } from '@/components/ui/segmented-control' import type { DesktopMarketplaceSearchItem } from '@/global' +import { saveHermesConfig } from '@/hermes' import { useI18n } from '@/i18n' import { triggerHaptic } from '@/lib/haptics' import { Check, Download, Loader2, Palette, Trash2 } from '@/lib/icons' @@ -17,6 +18,7 @@ import { $backdrop, setBackdrop } from '@/store/backdrop' import { $composerPopoutGesturesEnabled, setComposerPopoutGesturesEnabled } from '@/store/composer-popout' import { $embedAllowed, $embedMode, clearEmbedAllowed, type EmbedMode, setEmbedMode } from '@/store/embed-consent' import { $introSplash, setIntroSplash } from '@/store/intro-splash' +import { notifyError } from '@/store/notifications' import { $activeGatewayProfile, $profiles, normalizeProfileKey } from '@/store/profile' import { $reactionsEnabled, setReactionsEnabled } from '@/store/reactions-enabled' import { $reasoningCollapsedByDefault, setReasoningCollapsedByDefault } from '@/store/reasoning-disclosure' @@ -53,13 +55,57 @@ import { installVscodeThemeFromMarketplace } from '@/themes/install' import type { DesktopTheme } from '@/themes/types' import { $marketplaceInstalls, isUserTheme, removeUserTheme } from '@/themes/user-themes' +import { setHermesConfigCache, useHermesConfigRecord } from '../hooks/use-config-record' + import { MODE_OPTIONS } from './constants' +import { setNested } from './helpers' import { PetSettings } from './pet-settings' import { ListRow, SectionHeading, SettingsContent, ToggleRow } from './primitives' import { APPEARANCE_SETTING_IDS } from './settings-search' import { TerminalFontSetting } from './terminal-font-setting' import { useDeepLinkHighlight } from './use-deep-link-highlight' +// display.resume_last_session lives in the backend config record (shared with +// config.yaml and the cold-start restore in use-desktop-integrations), not a +// renderer store. Saves write through the shared react-query cache so the +// restore gate sees the new value on the next launch. +function ResumeLastSessionSetting() { + const { t } = useI18n() + const a = t.settings.appearance + const configQuery = useHermesConfigRecord() + const config = configQuery.data + const checked = (config?.display as { resume_last_session?: unknown } | undefined)?.resume_last_session !== false + + const update = (on: boolean) => { + if (!config) { + return + } + + const next = setNested(config, 'display.resume_last_session', on) + setHermesConfigCache(next) + void saveHermesConfig(next) + .then(result => { + if (!result.ok) { + throw new Error(t.settings.config.autosaveFailed) + } + }) + .catch(error => { + setHermesConfigCache(config) + notifyError(error, t.settings.config.autosaveFailed) + }) + } + + return ( + + ) +} + function ThemePreview({ name, mode }: { name: string; mode: 'light' | 'dark' }) { // Preview in the *current* mode: the dark palette in Dark, and the light // palette in Light — synthesizing one for dark-only themes — so every card @@ -742,6 +788,8 @@ export function AppearanceSettings() { onChange={setComposerPopoutGesturesEnabled} /> + + + )} diff --git a/apps/desktop/src/app/settings/local-models-settings.tsx b/apps/desktop/src/app/settings/local-models-settings.tsx index 840e736aae..f845bfeacd 100644 --- a/apps/desktop/src/app/settings/local-models-settings.tsx +++ b/apps/desktop/src/app/settings/local-models-settings.tsx @@ -24,7 +24,21 @@ import { sideloadLocalModel } from '@/hermes' import { useI18n } from '@/i18n' -import { Check, CheckCircle2, Cpu, Download, Eject, FolderOpen, Loader2, Monitor, Package, Search, StopFilled, Trash2, Zap } from '@/lib/icons' +import { + Check, + CheckCircle2, + Cpu, + Download, + Eject, + FolderOpen, + Loader2, + Monitor, + Package, + Search, + StopFilled, + Trash2, + Zap +} from '@/lib/icons' import { cn } from '@/lib/utils' import { $localRuntimeJobs, @@ -251,9 +265,7 @@ export function LocalModelsSettings() { const navigate = useNavigate() const seenQuickstarts = useRef(new Set()) - const runningQuickstart = jobs.find( - j => j.kind === 'quickstart' && j.status === 'running' - ) + const runningQuickstart = jobs.find(j => j.kind === 'quickstart' && j.status === 'running') useEffect(() => { // Event detection, not value mirroring: the ref only remembers which @@ -298,11 +310,7 @@ export function LocalModelsSettings() { // Stage rail derived from the job phase: engine -> model -> finish. const phase = qJob?.phase ?? '' - const stageIndex = ['starting-server', 'setting-default'].includes(phase) - ? 2 - : phase === 'downloading' - ? 1 - : 0 + const stageIndex = ['starting-server', 'setting-default'].includes(phase) ? 2 : phase === 'downloading' ? 1 : 0 const stages = [copy.quickstartStageEngine, copy.quickstartStageModel, copy.quickstartStageFinish] @@ -334,9 +342,7 @@ export function LocalModelsSettings() { {qJob ? ( <> -

- {liveDetail} -

+

{liveDetail}

@@ -397,8 +403,7 @@ export function LocalModelsSettings() { // Up to date = the authority (status) says the configured tag is what's // serving. Shown whenever true — not only right after an update. - const updateApplied = - status.runtime_installed && !status.update_available && status.tag === status.configured_tag + const updateApplied = status.runtime_installed && !status.update_available && status.tag === status.configured_tag return ( @@ -511,9 +516,7 @@ export function LocalModelsSettings() { /> )} - {lastError?.kind === 'runtime-install' && ( -

{lastError.error}

- )} + {lastError?.kind === 'runtime-install' &&

{lastError.error}

} {/* ── This machine ── */} @@ -571,13 +574,7 @@ export function LocalModelsSettings() { model.downloaded ? (
{isLoaded && livePlacement && ( - + {livePlacement.granted_window_label ?? livePlacement.window_label ?? ''} @@ -701,8 +698,9 @@ export function LocalModelsSettings() { model, so a spilled full window goes gray. Anything starting below its native window gets one quiet 'Up to' pill instead of a start/grow pair. */} - {model.fits && model.start_window_label && ( - model.start_window && model.start_window >= model.native_context ? ( + {model.fits && + model.start_window_label && + (model.start_window && model.start_window >= model.native_context ? ( {copy.pillFullContext(model.native_context_label)} @@ -712,12 +710,9 @@ export function LocalModelsSettings() { {copy.pillUpTo(model.native_context_label)} - ) - )} + ))} - {!model.fits && ( - {copy.pillUpTo(model.native_context_label)} - )} + {!model.fits && {copy.pillUpTo(model.native_context_label)}} {model.vision && {copy.pillVision}} @@ -758,9 +753,7 @@ export function LocalModelsSettings() { const isLoadingNow = residency === 'loading' const livePlacement = status.placement?.[m.id] - const aJob = jobs.find( - j => j.kind === 'model-activate' && j.status === 'running' && j.model_id === m.id - ) + const aJob = jobs.find(j => j.kind === 'model-activate' && j.status === 'running' && j.model_id === m.id) const anyActivateRunning = jobs.some(j => j.kind === 'model-activate' && j.status === 'running') @@ -769,9 +762,7 @@ export function LocalModelsSettings() { action={
{isLoaded && livePlacement && ( - + {livePlacement.granted_window_label ?? livePlacement.window_label ?? ''} @@ -840,9 +831,7 @@ export function LocalModelsSettings() { })}
- {lastError?.kind === 'model-download' && ( -

{lastError.error}

- )} + {lastError?.kind === 'model-download' &&

{lastError.error}

} @@ -1099,7 +1088,9 @@ function BrowseSection({ onChanged }: { onChanged: () => void }) { : copy.browseFitUnknown} - {gbLabel(group.total_bytes)} + + {gbLabel(group.total_bytes)} + )}
diff --git a/apps/desktop/src/app/settings/pool-limits-setting.tsx b/apps/desktop/src/app/settings/pool-limits-setting.tsx new file mode 100644 index 0000000000..4db5ee8824 --- /dev/null +++ b/apps/desktop/src/app/settings/pool-limits-setting.tsx @@ -0,0 +1,113 @@ +import { useStore } from '@nanostores/react' +import { useEffect, useState } from 'react' + +import { ListRow } from '@/app/settings/primitives' +import { Input } from '@/components/ui/input' +import { $poolLimits, loadPoolLimits, savePoolLimits } from '@/store/pool-limits' + +// Bounds imported from main's clamp module so the advertised input ranges +// can never drift from what the pool actually enforces (review note on #92581). +import { POOL_LIMITS_BOUNDS } from '../../../electron/pool-limits' + +const MAX_BACKENDS_MAX = POOL_LIMITS_BOUNDS.maxBackendsMax +const IDLE_MS_MAX = POOL_LIMITS_BOUNDS.idleMsMax + +/** Settings → Advanced: warm-bot-backends count + backend idle timeout. + * Device-local (not profile-scoped): the pool is sized once per machine and + * changes apply live — main evicts/reaps to converge without a restart. */ +export function PoolLimitsSetting() { + const limits = useStore($poolLimits) + const [maxDraft, setMaxDraft] = useState(String(limits.maxBackends)) + const [idleDraft, setIdleDraft] = useState(String(limits.idleMs)) + + useEffect(() => { + void loadPoolLimits() + }, []) + + useEffect(() => { + setMaxDraft(String(limits.maxBackends)) + setIdleDraft(String(limits.idleMs)) + }, [limits]) + + const commitMax = () => { + const parsed = Number(maxDraft) + + if (!Number.isFinite(parsed) || parsed === limits.maxBackends) { + setMaxDraft(String(limits.maxBackends)) + + return + } + + void savePoolLimits({ maxBackends: parsed }) + .then(() => undefined) + .catch(() => setMaxDraft(String($poolLimits.get().maxBackends))) + } + + const commitIdle = () => { + const parsed = Number(idleDraft) + + if (!Number.isFinite(parsed) || parsed === limits.idleMs) { + setIdleDraft(String(limits.idleMs)) + + return + } + + void savePoolLimits({ idleMs: parsed }) + .then(() => undefined) + .catch(() => setIdleDraft(String($poolLimits.get().idleMs))) + } + + return ( + <> + + setMaxDraft(event.target.value)} + onKeyDown={event => { + if (event.key === 'Enter') { + event.currentTarget.blur() + } + }} + type="number" + value={maxDraft} + /> +
+ } + description="How many bot backends stay running for instant switching. Higher = faster switches, more memory (~60MB per backend). Applies immediately." + title="Warm Bot Backends" + /> + + setIdleDraft(event.target.value)} + onKeyDown={event => { + if (event.key === 'Enter') { + event.currentTarget.blur() + } + }} + type="number" + value={idleDraft} + /> + ms + + } + description="How long an unused bot backend stays warm before it is shut down. Raise this so bots you revisit every few minutes never pay a cold start." + title="Backend Idle Timeout" + /> + + ) +} diff --git a/apps/desktop/src/app/settings/sessions-settings.tsx b/apps/desktop/src/app/settings/sessions-settings.tsx index e990557e7a..a194a6f0a6 100644 --- a/apps/desktop/src/app/settings/sessions-settings.tsx +++ b/apps/desktop/src/app/settings/sessions-settings.tsx @@ -17,8 +17,8 @@ import { triggerHaptic } from '@/lib/haptics' import { Archive, ArchiveOff, FolderOpen, Loader2, Trash2 } from '@/lib/icons' import { confirm } from '@/store/confirm' import { notify, notifyError } from '@/store/notifications' -import { untombstoneSessions } from '@/store/projects' import { applyConfiguredDefaultProjectDir, ensureDefaultWorkspaceCwd, setSessions } from '@/store/session' +import { untombstoneSessions } from '@/store/session-removal' import { forgetSessionUnread } from '@/store/session-unread' import type { HermesConfigRecord, SessionInfo } from '@/types/hermes' diff --git a/apps/desktop/src/app/shell/hooks/use-statusbar-items.tsx b/apps/desktop/src/app/shell/hooks/use-statusbar-items.tsx index 7f5f50027a..f2bcf39fd2 100644 --- a/apps/desktop/src/app/shell/hooks/use-statusbar-items.tsx +++ b/apps/desktop/src/app/shell/hooks/use-statusbar-items.tsx @@ -14,9 +14,21 @@ import { Codicon } from '@/components/ui/codicon' import { GlyphSpinner } from '@/components/ui/glyph-spinner' import { useI18n } from '@/i18n' import { displayPath, pathLeaf } from '@/lib/display-path' -import { Activity, AlertCircle, Clock, Command, FolderOpen, Globe, Hash, Loader2, Terminal } from '@/lib/icons' +import { + Activity, + AlertCircle, + Clock, + Command, + FolderOpen, + Globe, + Hash, + Layers3, + Loader2, + Terminal, + Zap +} from '@/lib/icons' import { runtimeReadinessDisplay, type RuntimeReadinessResult } from '@/lib/runtime-readiness' -import { contextBarLabel, LiveDuration, usageContextLabel } from '@/lib/statusbar' +import { cacheHitLabel, contextBarLabel, LiveDuration, tokensPerSecondLabel, usageContextLabel } from '@/lib/statusbar' import { useStoreSelector } from '@/lib/use-session-slice' import { cn } from '@/lib/utils' import { resolveVersionStatus } from '@/lib/version-status' @@ -267,6 +279,10 @@ export function useStatusbarItems({ const contextUsage = useMemo(() => usageContextLabel(gaugeUsage), [gaugeUsage]) const contextBar = useMemo(() => contextBarLabel(gaugeUsage), [gaugeUsage]) + // Both ride the same usage payload the context meter does (session.usage + // ticks mid-turn, message.complete after) — no extra RPC, no polling. + const cacheHit = cacheHitLabel(currentUsage) + const tokensPerSecond = tokensPerSecondLabel(currentUsage) const approvalModeItem = useApprovalModeStatusbarItem(activeGatewayProfile, requestGateway) const systemResourcesItem = useSystemResourcesStatusbarItem() @@ -562,6 +578,24 @@ export function useStatusbarItems({ toggleLabel: copy.toggleContextUsage, variant: 'menu' }, + { + icon: , + id: 'cache-hit-rate', + // Same never-self-hide rule as the context meter: opted in means a + // placeholder until the first cached turn reports, not a vanished item. + label: cacheHit || '—', + title: copy.cacheHitRateTitle, + toggleLabel: copy.toggleCacheHitRate, + variant: 'text' + }, + { + icon: , + id: 'tokens-per-second', + label: tokensPerSecond || '—', + title: copy.tokensPerSecondTitle, + toggleLabel: copy.toggleTokensPerSecond, + variant: 'text' + }, { detail: , hidden: !sessionStartedAt, @@ -594,6 +628,7 @@ export function useStatusbarItems({ approvalModeItem, backendVersionItem, busy, + cacheHit, chatOpen, clientVersionItem, contextBar, @@ -606,6 +641,7 @@ export function useStatusbarItems({ gatewayState, systemResourcesItem, terminalShowing, + tokensPerSecond, turnStartedAt ] ) diff --git a/apps/desktop/src/app/shell/model-catalog-menu.tsx b/apps/desktop/src/app/shell/model-catalog-menu.tsx index 351fa24c54..3cde3cd229 100644 --- a/apps/desktop/src/app/shell/model-catalog-menu.tsx +++ b/apps/desktop/src/app/shell/model-catalog-menu.tsx @@ -502,10 +502,12 @@ export function ModelCatalogMenu({ const isCurrent = activeId !== null const name = modelDisplayParts(family.id).name const caps = group.provider.capabilities?.[family.id] + // Managed local model loading into memory right now: // real load percent, keyed by exact model id (remote // providers never collide with GGUF stems). - const loadProgress = loadingModels[family.id] ?? (family.fastId ? loadingModels[family.fastId] : undefined) + const loadProgress = + loadingModels[family.id] ?? (family.fastId ? loadingModels[family.fastId] : undefined) // Effective settings for this row: the live choice when it's // the active model, otherwise its remembered preset. Row @@ -602,7 +604,9 @@ export function ModelCatalogMenu({ })} {!collapsed && slug === LOCAL_PROVIDER_SLUG && - shownDownloads.map(job => )} + shownDownloads.map(job => ( + + ))} ) })} @@ -680,10 +684,7 @@ function DownloadingModelRow({ jobId, target }: { jobId: string; target: string const { t } = useI18n() const copy = t.modelPicker - const percent = useStoreSelector( - $localRuntimeJobs, - jobs => jobs.find(job => job.job_id === jobId)?.percent ?? null - ) + const percent = useStoreSelector($localRuntimeJobs, jobs => jobs.find(job => job.job_id === jobId)?.percent ?? null) return ( { const statusbar = bar([ item('running-timer', 'Turn timer', { variant: 'text' }), item('context-usage', 'Context meter', { variant: 'menu' }), + item('cache-hit-rate', 'Cache hit rate', { variant: 'text' }), + item('tokens-per-second', 'Tokens per second', { variant: 'text' }), item('session-timer', 'Session timer', { variant: 'text' }), item('gateway-health', 'Gateway') ]) - for (const label of ['Turn timer', 'Context meter', 'Session timer']) { + for (const label of ['Turn timer', 'Context meter', 'Cache hit rate', 'Tokens per second', 'Session timer']) { expect(screen.queryByText(label)).toBeNull() } openContextMenu(statusbar) - const row = await screen.findByRole('menuitemcheckbox', { name: 'Session timer' }) - fireEvent.click(row) + for (const [id, label] of [ + ['session-timer', 'Session timer'], + ['cache-hit-rate', 'Cache hit rate'] + ]) { + fireEvent.click(await screen.findByRole('menuitemcheckbox', { name: label })) - expect($statusbarHiddenIds.get()).not.toContain('session-timer') - expect(within(statusbar).getByText('Session timer')).toBeTruthy() + expect($statusbarHiddenIds.get()).not.toContain(id) + expect(within(statusbar).getByText(label)).toBeTruthy() + } }) }) diff --git a/apps/desktop/src/app/shell/system-resources-statusbar.tsx b/apps/desktop/src/app/shell/system-resources-statusbar.tsx index d7ea1aeebf..b4de162106 100644 --- a/apps/desktop/src/app/shell/system-resources-statusbar.tsx +++ b/apps/desktop/src/app/shell/system-resources-statusbar.tsx @@ -103,7 +103,9 @@ export function useSystemResourcesStatusbarItem(): StatusbarItem { : null const ramUsed = hardware ? hardware.ram_total_bytes - hardware.ram_available_bytes : null - const ramPercent = hardware?.ram_total_bytes && ramUsed != null ? Math.round((ramUsed / hardware.ram_total_bytes) * 100) : null + + const ramPercent = + hardware?.ram_total_bytes && ramUsed != null ? Math.round((ramUsed / hardware.ram_total_bytes) * 100) : null // Compact bar label: the numbers a local-inference user glances at. // "GPU 34% · 18G/32G" with a GPU; "RAM 41G/256G" without. @@ -124,10 +126,7 @@ export function useSystemResourcesStatusbarItem(): StatusbarItem { menuAlign: 'end', menuClassName: 'w-64 p-0', menuContent: ( -
+
{/* min-w-0 everywhere a flex/grid child must shrink: grid items default min-width:auto, so a long GPU name's nowrap min-content props the track open past the w-64 box and overflow-x:hidden diff --git a/apps/desktop/src/app/types.ts b/apps/desktop/src/app/types.ts index 0931ecf879..57e85a0a9c 100644 --- a/apps/desktop/src/app/types.ts +++ b/apps/desktop/src/app/types.ts @@ -66,6 +66,10 @@ export interface SessionCompressResponse { usage?: Partial } messages?: SessionMessage[] + /** Set with `status: 'pending'` when the gateway's compute-host wait expired + * while compression is still running; the transcript refreshes from the + * pushed session.info / `compacted` status edge (#97948). */ + message?: string removed?: number status?: string summary?: { diff --git a/apps/desktop/src/components/assistant-ui/thread/index.test.tsx b/apps/desktop/src/components/assistant-ui/thread/index.test.tsx new file mode 100644 index 0000000000..4bf2c82232 --- /dev/null +++ b/apps/desktop/src/components/assistant-ui/thread/index.test.tsx @@ -0,0 +1,82 @@ +import { render } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' + +/** + * Issue #95595 proposed-fix #3: the `messageComponents` map handed to + * ThreadMessageList must keep its REFERENCE IDENTITY across a session switch. + * If it re-minted, React would unmount/remount every visible message — async + * re-rendered parts (shiki code blocks) collapse and re-expand, and the whole + * thread visibly jumps on every tab switch. + * + * The memo deps are deliberately only the boolean "definedness" gates (the + * callbacks themselves reach the composer through a ref), so a plain switch + * — sessionId changing, callbacks unchanged — must not change the map. + */ +let lastComponents: unknown + +vi.mock('@/components/assistant-ui/thread/list', () => ({ + ThreadMessageList: (props: { components: unknown }) => { + lastComponents = props.components + + return null + } +})) + +vi.mock('@/components/assistant-ui/thread/timeline', () => ({ + ThreadTimeline: () => null +})) + +vi.mock('@/components/assistant-ui/thread/status', () => ({ + BackgroundResumeNotice: () => null, + CenteredThreadSpinner: () => null +})) + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + assistant: { + thread: { + restoreBody: 'restore body', + restoreConfirm: 'Restore', + restoreTitle: 'Restore this turn?' + } + }, + common: { + cancel: 'Cancel', + confirm: 'Confirm', + done: 'Done', + loading: 'Loading' + } + } + }) +})) + +import { Thread } from './index' + +describe('Thread messageComponents identity across session switches', () => { + it('does not re-mint messageComponents when only the session changes', () => { + const { rerender } = render() + const first = lastComponents + + expect(first).toBeDefined() + + rerender() + + // THE guard: a switch must keep the component map reference, so the + // incoming transcript reconciles instead of remounting. + expect(lastComponents).toBe(first) + + rerender() + + expect(lastComponents).toBe(first) + }) + + it('keeps the map stable across a plain parent re-render', () => { + const { rerender } = render() + const first = lastComponents + + rerender() + + expect(lastComponents).toBe(first) + }) +}) diff --git a/apps/desktop/src/components/assistant-ui/thread/status.tsx b/apps/desktop/src/components/assistant-ui/thread/status.tsx index 22205b5dad..c3f40ac255 100644 --- a/apps/desktop/src/components/assistant-ui/thread/status.tsx +++ b/apps/desktop/src/components/assistant-ui/thread/status.tsx @@ -62,7 +62,7 @@ const HintText: FC<{ children: ReactNode }> = ({ children }) => ( * call (title generation autoloads the same model), never gets a frame, * and the load looked like nothing was happening. The status route reads * the same SSE snapshot, so this bar carries the identical percent. */ -function useLocalModelLoad(active: boolean): LocalModelLoadProgress & { model: string } | null { +function useLocalModelLoad(active: boolean): (LocalModelLoadProgress & { model: string }) | null { const model = useStore($currentModel) const [progress, setProgress] = useState<(LocalModelLoadProgress & { model: string }) | null>(null) diff --git a/apps/desktop/src/components/chat/shiki-block.test.tsx b/apps/desktop/src/components/chat/shiki-block.test.tsx new file mode 100644 index 0000000000..26273a73e7 --- /dev/null +++ b/apps/desktop/src/components/chat/shiki-block.test.tsx @@ -0,0 +1,122 @@ +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * Perf regression guard for #95595: switching to a warm session remounts the + * incoming transcript, and every mounted code block used to re-tokenize from + * scratch on the main thread. The content-keyed cache must make a remount of + * an unchanged block a cache hit — ZERO highlighter calls. + * + * shiki itself is mocked (jsdom cannot run the oniguruma wasm engine); the + * mock counts `codeToHtml` invocations, which is the cost we are guarding. + */ +const { codeToHtml } = vi.hoisted(() => ({ + codeToHtml: vi.fn((code: string) => { + const escaped = String(code).replace(/&/g, '&').replace(//g, '>') + + return `
${escaped}
` + }) +})) + +vi.mock('shiki', () => ({ + bundledLanguages: { typescript: 'typescript-loader', text: 'text-loader' }, + getSingletonHighlighter: vi.fn(async () => ({ + codeToHtml: (code: string) => codeToHtml(code), + getLoadedLanguages: () => ['text', 'typescript'], + loadLanguage: vi.fn(async () => undefined) + })) +})) + +vi.mock('shiki/engine/oniguruma', () => ({ + createOnigurumaEngine: vi.fn(() => ({}) as never) +})) + +import CachedShikiBlock from '@/components/chat/shiki-block' +import { highlightCache } from '@/components/chat/shiki-highlight-cache' + +const TS_BLOCK = { language: 'typescript', code: 'const answer: number = 42\n' } + +async function waitForHighlighted(): Promise { + await screen.findByTestId('shiki-container', undefined, { timeout: 2_000 }) +} + +beforeEach(() => { + codeToHtml.mockClear() + highlightCache.clear() +}) + +afterEach(() => { + cleanup() +}) + +describe('CachedShikiBlock (warm-switch perf guard)', () => { + it('highlights on first mount and reuses the cached HTML on remount', async () => { + const { unmount } = render() + await waitForHighlighted() + + expect(codeToHtml).toHaveBeenCalledTimes(1) + expect(screen.getByTestId('shiki-container').innerHTML).toContain('const answer') + + // Warm session switch: the row unmounts and the SAME block mounts again. + unmount() + render() + await waitForHighlighted() + + // The guard: remounting an unchanged block must NOT re-tokenize. + expect(codeToHtml).toHaveBeenCalledTimes(1) + }) + + it('re-highlights a block whose code changed (cache miss)', async () => { + const { unmount } = render() + await waitForHighlighted() + + unmount() + render() + await waitForHighlighted() + + expect(codeToHtml).toHaveBeenCalledTimes(2) + }) + + it('keeps blocks independent: N blocks highlight N times across two mounts', async () => { + const { unmount } = render( + <> + + + + ) + + await screen.findAllByTestId('shiki-container', undefined, { timeout: 2_000 }) + + expect(codeToHtml).toHaveBeenCalledTimes(2) + + unmount() + render( + <> + + + + ) + await screen.findAllByTestId('shiki-container', undefined, { timeout: 2_000 }) + + // Two mounts of the same two blocks: exactly two tokenizations total. + expect(codeToHtml).toHaveBeenCalledTimes(2) + }) + + it('does not cache a failed highlight, so a retry can succeed', async () => { + codeToHtml.mockRejectedValueOnce(new Error('boom')) + + const { unmount } = render() + await waitForHighlighted() + + // The failure degrades to escaped plain text (and is NOT cached). + expect(screen.getByTestId('shiki-container').innerHTML).toContain('const answer') + expect(codeToHtml).toHaveBeenCalledTimes(1) + + unmount() + render() + await waitForHighlighted() + + // Second mount tries the highlighter again instead of serving stale HTML. + expect(codeToHtml).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/desktop/src/components/chat/shiki-block.tsx b/apps/desktop/src/components/chat/shiki-block.tsx index 4279d21835..3704185caf 100644 --- a/apps/desktop/src/components/chat/shiki-block.tsx +++ b/apps/desktop/src/components/chat/shiki-block.tsx @@ -1,15 +1,173 @@ 'use client' /** - * The ONLY static importer of `react-shiki` (and through it the multi-MB - * shiki language/theme bundle). Every consumer reaches this module through + * The ONLY static importer of shiki (and through it the multi-MB shiki + * language/theme/wasm bundle). Every consumer reaches this module through * `React.lazy(() => import('./shiki-block'))` — see `LazyShiki` in * shiki-highlighter.tsx — so the shiki chunk stays entirely off the * cold-start path and loads on the first highlighted code block instead. * * Do NOT import this module statically from anything the entry graph * reaches, or the chunk moves back into boot. + * + * Unlike the previous pass-through of `react-shiki`'s component, this module + * is cache-aware: highlighted output is stored in a module-level LRU cache + * keyed by (theme scope, language, code), so a REMOUNT of an unchanged code + * block (the warm-session-switch path of #95595) paints the cached HTML + * synchronously and never re-tokenizes. Only cache misses run shiki, and + * misses are debounced so a streaming block settles before the heavy work + * starts. */ -import ShikiHighlighter from 'react-shiki' +import { useEffect, useMemo, useState } from 'react' +import { bundledLanguages, getSingletonHighlighter } from 'shiki' +import type { BundledLanguage, BundledTheme, Highlighter } from 'shiki' +import { createOnigurumaEngine } from 'shiki/engine/oniguruma' -export default ShikiHighlighter +import { SHIKI_HIGHLIGHT_SCOPE, SHIKI_THEME } from '@/components/chat/shiki-config' +import { highlightCache, highlightCacheKey } from '@/components/chat/shiki-highlight-cache' + +/** Same debounce react-shiki's `delay` used to throttle highlight work with. */ +const HIGHLIGHT_DELAY_MS = 120 + +// Stable identity for "no color replacements" so the memo/effect deps below +// never churn on renders that don't pass the prop. +const NO_COLOR_REPLACEMENTS: Record> = {} + +export interface CachedShikiBlockProps { + language: string + code: string + /** Theme override; defaults to the shared SHIKI_THEME. */ + theme?: { dark: string; light: string } + /** Color replacements; defaults to none (the chat passes its own). */ + colorReplacements?: Record> +} + +function isLoadableLanguage(language: string): boolean { + return language === 'text' || language in bundledLanguages +} + +/** + * Cache scope for one theme configuration. Part of the cache key: a block + * highlighted under a different theme is a different render. + */ +function highlightScope( + theme: { dark: string; light: string }, + colorReplacements: Record> +): string { + return `${SHIKI_HIGHLIGHT_SCOPE}:${theme.dark}:${theme.light}:${JSON.stringify(colorReplacements)}` +} + +let highlighterPromise: Promise | null = null +let loadedThemes = new Set([SHIKI_THEME.dark, SHIKI_THEME.light]) + +/** + * Lazily-created shiki singleton, mirroring react-shiki's full bundle: only + * the languages actually seen are loaded into it (the singleton is created + * with the first block's language, later ones are `loadLanguage`d on demand; + * override themes are `loadTheme`d the same way). Unknown languages are left + * unloaded and fall through to shiki's plain-text handling, as before. + */ +async function highlightToHtml( + language: string, + code: string, + theme: { dark: string; light: string }, + colorReplacements: Record> +): Promise { + if (!highlighterPromise) { + highlighterPromise = getSingletonHighlighter({ + // Only bundled languages are ever passed here (isLoadableLanguage + // guards both call sites), so the cast is safe. + langs: isLoadableLanguage(language) ? [language as BundledLanguage] : [], + themes: [SHIKI_THEME.dark, SHIKI_THEME.light], + engine: createOnigurumaEngine(import('shiki/wasm')) + }) + } + + const highlighter = await highlighterPromise + + if (isLoadableLanguage(language) && !highlighter.getLoadedLanguages().includes(language)) { + await highlighter.loadLanguage(language as BundledLanguage) + } + + const missingThemes = [theme.dark, theme.light].filter(name => !loadedThemes.has(name)) + + if (missingThemes.length > 0) { + await highlighter.loadTheme(...(missingThemes as BundledTheme[])) + missingThemes.forEach(name => loadedThemes.add(name)) + } + + return highlighter.codeToHtml(code, { + lang: language, + themes: { dark: theme.dark, light: theme.light }, + defaultColor: 'light-dark()', + colorReplacements + }) +} + +function escapeHtml(text: string): string { + return text.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') +} + +/** Never let a highlight failure blank a block — degrade to escaped plain text. */ +function plainTextHtml(code: string): string { + return `
${escapeHtml(code)}
` +} + +export default function CachedShikiBlock({ language, code, theme, colorReplacements }: CachedShikiBlockProps) { + const themeConfig = theme ?? SHIKI_THEME + const replacements = colorReplacements ?? NO_COLOR_REPLACEMENTS + + const cacheKey = useMemo( + () => highlightCacheKey(highlightScope(themeConfig, replacements), language, code), + [language, code, replacements, themeConfig] + ) + + const [html, setHtml] = useState(() => highlightCache.get(cacheKey) ?? null) + + useEffect(() => { + let cancelled = false + + // Cache hit — no highlighter work at all. This is the warm-switch path: + // the previous visit already rendered this block, so paint it again. + const cached = highlightCache.get(cacheKey) + + if (cached !== undefined) { + setHtml(cached) + + return + } + + const timer = window.setTimeout(() => { + highlightToHtml(language, code, themeConfig, replacements) + .then(result => { + if (cancelled) { + return + } + + highlightCache.set(cacheKey, result) + setHtml(result) + }) + .catch(error => { + if (cancelled) { + return + } + + console.error('shiki highlight failed; rendering plain code', error) + setHtml(plainTextHtml(code)) + }) + }, HIGHLIGHT_DELAY_MS) + + return () => { + cancelled = true + window.clearTimeout(timer) + } + }, [cacheKey, code, language, replacements, themeConfig]) + + if (html === null) { + // Nothing to paint yet (miss, debounce pending). Matches react-shiki's + // own empty render while the highlight is in flight. + return null + } + + return
+} diff --git a/apps/desktop/src/components/chat/shiki-config.ts b/apps/desktop/src/components/chat/shiki-config.ts new file mode 100644 index 0000000000..4ba81aaf06 --- /dev/null +++ b/apps/desktop/src/components/chat/shiki-config.ts @@ -0,0 +1,33 @@ +// Shiki theme/color constants shared by the chat code-block renderer +// (shiki-highlighter.tsx) and the lazy shiki chunk (shiki-block.tsx). Kept in +// their own dependency-free module so the lazy chunk can import them without +// pulling the main chat module (or react-shiki) into the shiki bundle. + +// `github-dark-dimmed` is GitHub's lower-contrast dark palette — the vivid +// `github-dark-default` tokens read harsh at our small code size. Shared by the +// inline diff renderer too (see diff-lines.tsx) so code + diffs match. +export const SHIKI_THEME = { dark: 'github-dark-dimmed', light: 'github-light-default' } as const + +/** + * `github-light-default` colors comments `#6e7781` (~4.2:1 against the code + * card background) — borderline unreadable at our 11px code size, and worst of + * all for shell snippets where a single `#` turns the rest of the line into one + * long comment span. Remap light-mode comments to GitHub's darker muted gray + * (`#57606a`, ~6.4:1). Dark mode (`#8b949e`, ~6.1:1) already reads fine, so we + * leave it untouched. Keyed per theme name so the bump only applies in light. + */ +export const SHIKI_COLOR_REPLACEMENTS: Record> = { + 'github-light-default': { '#6e7781': '#57606a' } +} + +/** + * Cache-key scope for the content-addressed highlight cache. Bumping this + * invalidates every cached highlight at once — bump it whenever the rendering + * options (themes, color replacements) change, because keys are NOT allowed to + * silently produce a different DOM than the one they were computed with. + */ +export const SHIKI_HIGHLIGHT_SCOPE = `hermes-shiki-v1:${JSON.stringify({ + dark: SHIKI_THEME.dark, + light: SHIKI_THEME.light, + colorReplacements: SHIKI_COLOR_REPLACEMENTS +})}` diff --git a/apps/desktop/src/components/chat/shiki-highlight-cache.test.ts b/apps/desktop/src/components/chat/shiki-highlight-cache.test.ts new file mode 100644 index 0000000000..77dde0d941 --- /dev/null +++ b/apps/desktop/src/components/chat/shiki-highlight-cache.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest' + +import { + HIGHLIGHT_CACHE_MAX_CHARS, + HIGHLIGHT_CACHE_MAX_ENTRIES, + HighlightCache, + highlightCacheKey +} from '@/components/chat/shiki-highlight-cache' + +describe('highlightCacheKey', () => { + it('separates scope, language and code so distinct blocks never collide', () => { + const a = highlightCacheKey('scope-1', 'ts', 'const x = 1') + const b = highlightCacheKey('scope-1', 'ts', 'const x = 2') + + expect(a).not.toBe(b) + expect(highlightCacheKey('scope-1', 'js', 'const x = 1')).not.toBe(a) + expect(highlightCacheKey('scope-2', 'ts', 'const x = 1')).not.toBe(a) + }) +}) + +describe('HighlightCache', () => { + it('round-trips an entry and refreshes recency on get', () => { + const cache = new HighlightCache(3, 1000) + + cache.set('a', 'A') + cache.set('b', 'B') + cache.set('c', 'C') + // Touch the oldest entry so it becomes the newest. + expect(cache.get('a')).toBe('A') + cache.set('d', 'D') + + // 'b' is now the least recently used and must be evicted first. + expect(cache.has('b')).toBe(false) + expect(cache.get('a')).toBe('A') + expect(cache.get('c')).toBe('C') + expect(cache.get('d')).toBe('D') + }) + + it('evicts oldest-first past the entry cap', () => { + const cache = new HighlightCache(2, 1_000_000) + + cache.set('a', 'A') + cache.set('b', 'B') + cache.set('c', 'C') + + expect(cache.size).toBe(2) + expect(cache.has('a')).toBe(false) + expect(cache.has('b')).toBe(true) + expect(cache.has('c')).toBe(true) + }) + + it('evicts oldest-first past the total-char cap', () => { + const cache = new HighlightCache(HIGHLIGHT_CACHE_MAX_ENTRIES, 10) + + cache.set('a', '12345') + cache.set('b', '123456') + + expect(cache.size).toBe(1) + expect(cache.has('a')).toBe(false) + expect(cache.has('b')).toBe(true) + expect(cache.totalChars).toBeLessThanOrEqual(10) + }) + + it('replaces an existing key in place without double-counting chars', () => { + const cache = new HighlightCache(2, 100) + + cache.set('a', '12345') + cache.set('a', '1234567890') + + expect(cache.size).toBe(1) + expect(cache.totalChars).toBe(10) + }) + + it('stays within both caps for a large burst of unique blocks', () => { + const cache = new HighlightCache(HIGHLIGHT_CACHE_MAX_ENTRIES, HIGHLIGHT_CACHE_MAX_CHARS) + + for (let i = 0; i < 2_000; i++) { + cache.set(`block-${i}`, `html-${i}`.repeat(100)) + } + + expect(cache.size).toBeLessThanOrEqual(HIGHLIGHT_CACHE_MAX_ENTRIES) + expect(cache.totalChars).toBeLessThanOrEqual(HIGHLIGHT_CACHE_MAX_CHARS) + }) + + it('clear drops everything', () => { + const cache = new HighlightCache() + + cache.set('a', 'A') + cache.clear() + + expect(cache.size).toBe(0) + expect(cache.totalChars).toBe(0) + expect(cache.get('a')).toBeUndefined() + }) +}) diff --git a/apps/desktop/src/components/chat/shiki-highlight-cache.ts b/apps/desktop/src/components/chat/shiki-highlight-cache.ts new file mode 100644 index 0000000000..3b0b7fe93b --- /dev/null +++ b/apps/desktop/src/components/chat/shiki-highlight-cache.ts @@ -0,0 +1,90 @@ +// ── Content-addressed syntax-highlight cache (#95595) ──────────────────────── +// Switching to a warm session remounts the incoming transcript, and every +// mounted code block used to be re-tokenized from scratch by shiki — N blocks +// × full tokenization on the renderer main thread, on every switch, even +// though the code had not changed. The fix is a module-level LRU cache keyed +// by (scope, language, code) holding the final highlighted HTML, so a remount +// of an unchanged block paints the cached markup synchronously and never +// touches the highlighter. +// +// Bounds: shiki's tokenized HTML is ~5-10x the source size, so an unbounded +// cache would leak renderer memory over a long session list. Cap both the +// entry count and the total cached characters; evict oldest-first. +// +// This module is intentionally dependency-free (no React, no shiki) so the +// cache logic can be unit-tested in isolation. + +export const HIGHLIGHT_CACHE_MAX_ENTRIES = 512 +export const HIGHLIGHT_CACHE_MAX_CHARS = 6 * 1024 * 1024 + +/** Unique key for one highlighted block: scope + language + exact code. */ +export function highlightCacheKey(scope: string, language: string, code: string): string { + return `${scope}\u0000${language}\u0000${code}` +} + +/** + * Bounded LRU map of highlight cache keys to rendered HTML. `get` refreshes + * recency (Map insertion order is used as the LRU clock); `set` evicts the + * oldest entries until both caps hold. + */ +export class HighlightCache { + private readonly entries = new Map() + private chars = 0 + + constructor( + private readonly maxEntries: number = HIGHLIGHT_CACHE_MAX_ENTRIES, + private readonly maxChars: number = HIGHLIGHT_CACHE_MAX_CHARS + ) {} + + get size(): number { + return this.entries.size + } + + get totalChars(): number { + return this.chars + } + + has(key: string): boolean { + return this.entries.has(key) + } + + get(key: string): string | undefined { + const value = this.entries.get(key) + + if (value !== undefined) { + // Refresh recency: re-inserting moves the entry to the newest position. + this.entries.delete(key) + this.entries.set(key, value) + } + + return value + } + + set(key: string, html: string): void { + if (this.entries.has(key)) { + this.chars -= this.entries.get(key)!.length + this.entries.delete(key) + } + + this.entries.set(key, html) + this.chars += html.length + this.evict() + } + + clear(): void { + this.entries.clear() + this.chars = 0 + } + + private evict(): void { + while ((this.entries.size > this.maxEntries || this.chars > this.maxChars) && this.entries.size > 0) { + const oldestKey = this.entries.keys().next().value as string + const oldest = this.entries.get(oldestKey)! + this.entries.delete(oldestKey) + this.chars -= oldest.length + } + } +} + +/** The renderer-wide highlight cache. Lives for the lifetime of the module. */ +export const highlightCache = new HighlightCache() diff --git a/apps/desktop/src/components/chat/shiki-highlighter.tsx b/apps/desktop/src/components/chat/shiki-highlighter.tsx index 6823bba7cf..dcb27bd9a1 100644 --- a/apps/desktop/src/components/chat/shiki-highlighter.tsx +++ b/apps/desktop/src/components/chat/shiki-highlighter.tsx @@ -1,15 +1,20 @@ 'use client' import type { SyntaxHighlighterProps } from '@assistant-ui/react-streamdown' -import { type ComponentProps, type FC, lazy, Suspense, useMemo } from 'react' -import type ShikiHighlighter from 'react-shiki' +import { type FC, lazy, Suspense, useMemo } from 'react' import { CodeCard, CodeCardBody } from '@/components/chat/code-card' import { ExpandableBlock } from '@/components/chat/expandable-block' +// Theme constants live in shiki-config (dependency-free) so the lazy shiki +// chunk can import them without pulling this module into the shiki bundle. +import { SHIKI_COLOR_REPLACEMENTS } from '@/components/chat/shiki-config' import { CopyButton } from '@/components/ui/copy-button' import { useI18n } from '@/i18n' import { isLikelyProseCodeBlock } from '@/lib/markdown-code' +import type { CachedShikiBlockProps } from './shiki-block' +export { SHIKI_COLOR_REPLACEMENTS, SHIKI_THEME } from '@/components/chat/shiki-config' + /** * Streamdown's code adapter renders header + body as inline siblings, so we * own the wrapping `` here and neutralize the upstream @@ -17,46 +22,34 @@ import { isLikelyProseCodeBlock } from '@/lib/markdown-code' * background-only — no header row, no language label — so a fence reads as a * tinted slab of the reply; copy is a hover-reveal control in the corner. * - * `react-shiki` full bundle so all `bundledLanguages` work; theme switches - * follow the document `color-scheme` via `defaultColor="light-dark()"`. + * The heavy lifting lives in the lazy `shiki-block` chunk (full bundle so all + * `bundledLanguages` work; theme switches follow the document `color-scheme` + * via `defaultColor="light-dark()"`), and its output is cached by content so + * warm-session switches never re-tokenize unchanged blocks (#95595). */ interface HermesSyntaxHighlighterProps extends SyntaxHighlighterProps { defer?: boolean } -// `github-dark-dimmed` is GitHub's lower-contrast dark palette — the vivid -// `github-dark-default` tokens read harsh at our small code size. Shared by the -// inline diff renderer too (see diff-lines.tsx) so code + diffs match. -export const SHIKI_THEME = { dark: 'github-dark-dimmed', light: 'github-light-default' } as const - -/** - * `github-light-default` colors comments `#6e7781` (~4.2:1 against the code - * card background) — borderline unreadable at our 11px code size, and worst of - * all for shell snippets where a single `#` turns the rest of the line into one - * long comment span. Remap light-mode comments to GitHub's darker muted gray - * (`#57606a`, ~6.4:1). Dark mode (`#8b949e`, ~6.1:1) already reads fine, so we - * leave it untouched. Keyed per theme name so the bump only applies in light. - */ -const SHIKI_COLOR_REPLACEMENTS: Record> = { - 'github-light-default': { '#6e7781': '#57606a' } -} - const MAX_HIGHLIGHT_CHARS = 150_000 const MAX_HIGHLIGHT_LINES = 3_000 const CHUNK_LINES = 200 const EST_LINE_PX = 16 -// react-shiki (and through it the multi-MB shiki grammar/theme bundle) is the +// shiki (and through it the multi-MB grammar/theme/wasm bundle) is the // heaviest dependency in the renderer. `shiki-block.tsx` is its only static // importer, so this lazy() is the single seam that keeps shiki out of the // entry chunk — it loads on the first highlighted code block, not at boot. +// The lazy module is cache-aware (#95595): unchanged blocks paint from a +// content-keyed cache instead of re-tokenizing on every mount. const ShikiBlock = lazy(() => import('./shiki-block')) -/** Drop-in ShikiHighlighter that suspends on first use and renders the code - * as plain preformatted text until the shiki chunk arrives. */ -export const LazyShiki: FC> = props => ( - }> - +/** Suspends on first use and renders the code as plain preformatted text + * until the shiki chunk arrives. Highlighted output is cached by + * (theme, language, code), so revisits never re-tokenize (#95595). */ +export const LazyShiki: FC = ({ language, code, theme, colorReplacements }) => ( + }> + ) @@ -160,18 +153,7 @@ export const SyntaxHighlighter: FC = ({ {plain ? ( ) : ( - - {trimmed} - + )} diff --git a/apps/desktop/src/components/model-picker.tsx b/apps/desktop/src/components/model-picker.tsx index 11c34daa9e..f2d7283f17 100644 --- a/apps/desktop/src/components/model-picker.tsx +++ b/apps/desktop/src/components/model-picker.tsx @@ -347,9 +347,7 @@ function ModelResults({ style={{ width: `${Math.max(2, loadProgress.percent)}%` }} /> - - {loadProgress.percent}% - + {loadProgress.percent}% )} {locked && ( @@ -393,17 +391,10 @@ function DownloadingModelRow({ jobId, target }: { jobId: string; target: string const { t } = useI18n() const copy = t.modelPicker - const percent = useStoreSelector( - $localRuntimeJobs, - jobs => jobs.find(job => job.job_id === jobId)?.percent ?? null - ) + const percent = useStoreSelector($localRuntimeJobs, jobs => jobs.find(job => job.job_id === jobId)?.percent ?? null) return ( - + {target} diff --git a/apps/desktop/src/components/notifications.tsx b/apps/desktop/src/components/notifications.tsx index 788873c531..e8a8dda8d6 100644 --- a/apps/desktop/src/components/notifications.tsx +++ b/apps/desktop/src/components/notifications.tsx @@ -236,9 +236,9 @@ function NotificationItem({ notification }: { notification: AppNotification }) { notification.action?.onClick() dismissNotification(notification.id) }} - size="xs" + size="sm" type="button" - variant="textStrong" + variant="default" > {notification.action.label} diff --git a/apps/desktop/src/components/onboarding/providers.tsx b/apps/desktop/src/components/onboarding/providers.tsx index d5ab346788..2612989b2e 100644 --- a/apps/desktop/src/components/onboarding/providers.tsx +++ b/apps/desktop/src/components/onboarding/providers.tsx @@ -100,7 +100,9 @@ export function FireworksProviderRow({ onClick }: { onClick: () => void }) { export function LocalModelsProviderRow({ onClick }: { onClick: () => void }) { const { t } = useI18n() - return + return ( + + ) } export function OpenRouterProviderRow({ onClick }: { onClick: () => void }) { diff --git a/apps/desktop/src/components/pane-shell/tree/store.ts b/apps/desktop/src/components/pane-shell/tree/store.ts index a451e570e3..07e9ed09f7 100644 --- a/apps/desktop/src/components/pane-shell/tree/store.ts +++ b/apps/desktop/src/components/pane-shell/tree/store.ts @@ -1040,6 +1040,13 @@ export function revealTreePane(paneId: string) { // Reveal beats a Close: un-dismiss and let adoption put the pane back. if ($dismissedPanes.get().has(paneId)) { setDismissed(paneId, false) + } + + // A layout replacement can omit a still-registered pane without dismissing + // it. Reconcile that saved contribution before claiming to reveal it. + const currentTree = $layoutTree.get() + + if (currentTree && !findGroupOfPane(currentTree, paneId)) { adoptContributedPanes() } @@ -1064,8 +1071,8 @@ export function revealTreePane(paneId: string) { if (hiddenNow.has(paneId)) { setTreePaneHidden(paneId, false) - - return + // Reactive unhide preserves a visible sibling. Explicit reveal must also + // front this pane and restore its group below. } const tree = $layoutTree.get() diff --git a/apps/desktop/src/components/pane-shell/workspace-scope.test.ts b/apps/desktop/src/components/pane-shell/workspace-scope.test.ts index e90d5cb02a..14e93c0703 100644 --- a/apps/desktop/src/components/pane-shell/workspace-scope.test.ts +++ b/apps/desktop/src/components/pane-shell/workspace-scope.test.ts @@ -9,7 +9,9 @@ import { rememberActivePane, resetRememberedActivePanes, resolveRememberedActivePane, - setWorkspaceScope + setWorkspaceOwnerLabel, + setWorkspaceScope, + workspaceOwnerTitle } from './workspace-scope' afterEach(() => { @@ -67,6 +69,21 @@ describe('workspace scope', () => { }) }) +describe('workspace owner title', () => { + it('captions a bot chat by its bot instead of the canonical stored title, and leaves everything else alone (#99152)', () => { + setWorkspaceOwnerLabel('bot:alpha', 'Alpha') + const botChat = { workspaceMode: 'bots' as const, workspaceOwnerKey: 'bot:alpha', workspaceTabTitle: 'Bot Chat' } + + expect(workspaceOwnerTitle('Bot Chat', botChat)).toBe('Alpha') + // A `+` side thread under the same bot keeps its own title. + expect(workspaceOwnerTitle('Plan the launch', botChat)).toBe('Plan the launch') + // A Sessions tab titled the same way is not a bot chat. + expect(workspaceOwnerTitle('Bot Chat', { workspaceMode: 'sessions' })).toBe('Bot Chat') + // No label yet (roster not loaded): the stored title stands. + expect(workspaceOwnerTitle('Bot Chat', { ...botChat, workspaceOwnerKey: 'bot:beta' })).toBe('Bot Chat') + }) +}) + describe('remembered active panes', () => { beforeEach(() => resetRememberedActivePanes()) diff --git a/apps/desktop/src/components/pane-shell/workspace-scope.ts b/apps/desktop/src/components/pane-shell/workspace-scope.ts index 7b62390908..4b0972265c 100644 --- a/apps/desktop/src/components/pane-shell/workspace-scope.ts +++ b/apps/desktop/src/components/pane-shell/workspace-scope.ts @@ -46,6 +46,33 @@ export type WorkspaceNewSessionTarget = /** Sessions uses its established ambient behavior (`null`). */ export const $workspaceNewSessionTarget = atom(null) +/** Display name per exact owner key, published by the workspace that owns the + * key (Bot Mode: the roster's display name). Presentation only — never a + * session title, which for a canonical Bot Chat is an identity the backend + * resolves by name and must stay exactly as stored. */ +export const $workspaceOwnerLabels = atom>>({}) + +export function setWorkspaceOwnerLabel(ownerKey: string, label: string): void { + if ($workspaceOwnerLabels.get()[ownerKey] !== label) { + $workspaceOwnerLabels.set({ ...$workspaceOwnerLabels.get(), [ownerKey]: label }) + } +} + +/** The caption a workspace-owned tab shows: its owner's label while the stored + * row still carries only the placeholder its opener registered — every bot's + * canonical chat is stored under the same name, so the tab reads the bot's + * (#99152). Any other title (a `+` side thread, a Sessions tab) is untouched. */ +export function workspaceOwnerTitle( + title: string, + scope: { workspaceMode?: WorkspaceMode; workspaceOwnerKey?: string; workspaceTabTitle?: string } | undefined +): string { + if (scope?.workspaceMode !== 'bots' || !scope.workspaceOwnerKey || title !== scope.workspaceTabTitle) { + return title + } + + return $workspaceOwnerLabels.get()[scope.workspaceOwnerKey] ?? title +} + /** One key for window-local active-pane memory. Owner keys stay opaque. */ export function workspaceScopeKey(mode: WorkspaceMode, ownerKey: string | null): string { return mode === 'sessions' ? 'sessions' : `bots:${ownerKey ?? ''}` diff --git a/apps/desktop/src/global.d.ts b/apps/desktop/src/global.d.ts index 8f29d581ff..b658780318 100644 --- a/apps/desktop/src/global.d.ts +++ b/apps/desktop/src/global.d.ts @@ -1,6 +1,8 @@ import type { GatewayWsUrlResult } from '@hermes/shared' import type { TranslucencyState } from '@hermes/shared/translucency' +import type { PoolLimits } from '../electron/pool-limits' + import type { WakeIndicatorState } from './lib/wake-indicator' import type { PetOverlayBounds, @@ -46,13 +48,24 @@ declare global { // Keepalive: mark a pool profile backend as recently used so the idle // reaper spares it while its chat is active. touchBackend: (profile?: string | null) => Promise<{ ok: boolean }> + // Pool sizing (Settings → Advanced): device-local, live-applied by the + // main process. get resolves the limits currently in force; set applies + // (and persists) new ones, evicting/reaping to converge immediately. + getPoolLimits: () => Promise + setPoolLimits: (limits: { maxBackends?: number; idleMs?: number }) => Promise<{ + ok: boolean + limits: PoolLimits + }> getGatewayWsUrl: (profile?: null | string) => Promise // Open (or focus) a standalone OS window for a single chat session so // the user can work with multiple chats side by side. Returns ok:false // with an error code when the sessionId is empty/invalid. `watch` opens // a spectator window (lazy resume — no agent build) for live-streaming // a running subagent's session. - openSessionWindow: (sessionId: string, opts?: { watch?: boolean }) => Promise<{ ok: boolean; error?: string }> + openSessionWindow: ( + sessionId: string, + opts?: { profile?: null | string; watch?: boolean } + ) => Promise<{ ok: boolean; error?: string }> // Resume this session in the user's own terminal emulator (`hermes --tui // --resume `) — the external terminal, not the in-app pane. openSessionInTerminal: ( @@ -508,6 +521,8 @@ declare global { * bisect disables, failed rebuilds, update-check results. null when * no venv operation has run yet. */ getSyncStatus: () => Promise + /** Restart the app in place — loads the swapped bundle when bundleSwapPending. */ + relaunchApp?: () => Promise getRemoteDisplayReason?: () => Promise updates: { check: () => Promise @@ -630,6 +645,9 @@ export interface DesktopVersionInfo { * external backend resolved from. Bundled artifacts run their payload; light * artifacts have no runtime and only reach remote backends. */ hermesRuntime?: { type: 'embedded' } | { type: 'light' } | { type: 'external'; source?: RuntimeSource } + /** True when the bundle on disk is newer than the running process — a plain + * app restart (no rebuild, no installer) is enough to load it. */ + bundleSwapPending?: boolean } /** Where an external build's backend came from. Mirrors the resolution ladder diff --git a/apps/desktop/src/hermes.test.ts b/apps/desktop/src/hermes.test.ts index 00611c6f68..ec9b94ada6 100644 --- a/apps/desktop/src/hermes.test.ts +++ b/apps/desktop/src/hermes.test.ts @@ -290,6 +290,41 @@ describe('Hermes REST helpers', () => { expect(paths).toContainEqual(expect.stringContaining('exclude_sources=cron%2Ctool')) }) + it('keeps per-slice errors on the legacy fallback so a cron failure does not taint recents', async () => { + resetSidebarBatchCapability() + const row = (id: string) => ({ id, title: id, profile: 'default' }) + + api.mockImplementation(({ path }: { path: string }) => { + if (path.startsWith('/api/profiles/sessions/sidebar')) { + return Promise.reject(new Error('404: {"detail":"No such API endpoint: /api/profiles/sessions/sidebar"}')) + } + + if (path.includes('source=cron')) { + return Promise.resolve({ + ...emptySessionsResponse, + sessions: [], + errors: [{ profile: 'default', error: 'disk I/O error' }] + }) + } + + return Promise.resolve({ ...emptySessionsResponse, sessions: [row('recent-1')] }) + }) + + const result = await listSidebarSessions({ + recentsProfile: 'default', + recentsLimit: 20, + recentsExclude: [], + cronLimit: 50, + messagingLimit: 100, + messagingExclude: [] + }) + + expect(result.recents.sessions.map(s => s.id)).toEqual(['recent-1']) + expect(result.recents.errors).toBeUndefined() + expect(result.cron.errors).toEqual([{ profile: 'default', error: 'disk I/O error' }]) + expect(result.errors).toBeUndefined() + }) + it('remembers endpoint-missing and skips re-probing the batched route on later refreshes', async () => { api.mockImplementation(({ path }: { path: string }) => path.startsWith('/api/profiles/sessions/sidebar') diff --git a/apps/desktop/src/hermes.ts b/apps/desktop/src/hermes.ts index 7bb69bc085..f5fb2ee7bd 100644 --- a/apps/desktop/src/hermes.ts +++ b/apps/desktop/src/hermes.ts @@ -40,6 +40,7 @@ export type { AnalyticsTotals, AudioSpeakResponse, AudioTranscriptionResponse, + AudioTtsLeaseResponse, AutomationBlueprint, AutomationBlueprintField, AuxiliaryModelsResponse, diff --git a/apps/desktop/src/i18n/ar.ts b/apps/desktop/src/i18n/ar.ts index b36a33ce31..899213afc5 100644 --- a/apps/desktop/src/i18n/ar.ts +++ b/apps/desktop/src/i18n/ar.ts @@ -493,6 +493,8 @@ export const ar = defineLocale({ embedsAlways: 'دائما', embedsOff: 'إيقاف', embedsReset: count => `إعادة تعيين ${count} ${count === 1 ? 'خدمة مسموح بها' : 'خدمة مسموح بها'}`, + resumeLastSessionTitle: 'إعادة فتح آخر محادثة عند التشغيل', + resumeLastSessionDesc: 'متابعة من حيث توقفت عند بدء التطبيق. أوقفه للبدء دائمًا بمحادثة جديدة.', product: 'مبسط', productDesc: 'عرض أنظف يركز على النتيجة.', technical: 'تقني', @@ -670,6 +672,43 @@ export const ar = defineLocale({ 'updates.nonInteractiveLocalChanges': 'عندما يحدّث Hermes نفسه من التطبيق دون موجه طرفية، احتفظ بتعديلات المصدر المحلية أو تجاهلها.' }, + about: { + heading: 'حول Hermes', + version: value => `الإصدار ${value}`, + versionUnavailable: 'الإصدار غير متاح', + bundleOutOfSync: 'إصدار التطبيق قديم', + bundleOutOfSyncDesc: + 'تم تحديث وقت تشغيل Hermes، لكن تطبيق سطح المكتب نفسه لا يزال إصدارًا قديمًا — لن تظهر ميزات الواجهة الجديدة (مثل Bot Mode) حتى يتم تحديث التطبيق. شغّل التحديث أدناه لإعادة بناء التطبيق. إذا لم يختفِ هذا التحذير، فأعد التثبيت من أحدث مثبّت لسطح المكتب.', + bundleOutOfSyncAction: 'الحصول على المثبّت', + bundleSwapPending: 'أعد التشغيل لإكمال التحديث', + bundleSwapPendingDesc: + 'تم تثبيت التطبيق المحدَّث بالفعل — يكفي إعادة تشغيل Hermes لتحميله. لن تتأثر المحادثات أو الإعدادات.', + bundleSwapPendingAction: 'إعادة تشغيل Hermes', + updates: 'التحديثات', + checkNow: 'التحقق الآن', + checking: 'جار التحقق...', + seeWhatsNew: 'عرض الجديد', + updateNow: 'تحديث الآن', + releaseNotes: 'ملاحظات الإصدار', + onLatest: 'أنت على أحدث إصدار', + installing: 'جار التثبيت...', + cantUpdate: 'تعذر التحديث', + cantReach: 'تعذر الوصول لخدمة التحديث', + tapCheck: 'اضغط للتحقق من التحديثات.', + updateReady: count => `${count} تحديث متاح`, + updateReadyUnknown: 'تحديث جديد جاهز.', + lastChecked: age => `آخر تحقق ${age}`, + justNowSuffix: 'الآن', + automaticUpdates: 'التحديثات التلقائية', + automaticUpdatesDesc: 'اسمح لـ Hermes بالتحقق من التحديثات وتثبيتها.', + branchCommit: (branch, commit) => `${branch} عند ${commit}`, + never: 'أبدا', + justNow: 'الآن', + minAgo: count => `قبل ${count} دقيقة`, + hoursAgo: count => `قبل ${count} ساعة`, + daysAgo: count => `قبل ${count} يوم` + }, + config: { none: 'لا شيء', noneParen: '(لا شيء)', @@ -2893,10 +2932,6 @@ export const ar = defineLocale({ title: 'المرفقات والأوامر', text: 'اكتب @ لإحضار ملف إلى المحادثة، و / لتشغيل أمر.' }, - 'model-switch': { - title: 'بدّل النموذج أثناء المحادثة', - text: 'اسم النموذج زر. غيّره كلما تغيّرت طبيعة العمل.' - }, 'local-setup': { title: 'هذا الجهاز يمكنه تشغيل النماذج محليًا', text: 'عتادك قادر على تشغيل نموذج محلي. تبقى محادثاتك على جهازك ولا تكلف شيئًا.', diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index a5d8dc2d9b..5bc867abce 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -619,6 +619,9 @@ export const en: Translations = { embedsAlways: 'Always', embedsOff: 'Off', embedsReset: (count: number) => `Reset ${count} allowed ${count === 1 ? 'service' : 'services'}`, + resumeLastSessionTitle: 'Reopen Last Chat on Launch', + resumeLastSessionDesc: + 'When enabled, the app reopens your most recent chat on cold start. Turn off to always start with a fresh new chat.', product: 'Product', productDesc: 'Human-friendly tool activity with concise summaries.', technical: 'Technical', @@ -686,6 +689,10 @@ export const en: Translations = { bundleOutOfSyncDesc: 'The Hermes runtime was updated, but the desktop app itself is still an older build — new interface features (like Bot Mode) will be missing until it updates. Run the update below to rebuild the app. If that doesn\u2019t clear this warning, reinstall from the latest desktop installer.', bundleOutOfSyncAction: 'Get the installer', + bundleSwapPending: 'Restart to finish the update', + bundleSwapPendingDesc: + 'The updated app is already installed — Hermes only needs to restart to load it. Chats and settings are untouched.', + bundleSwapPendingAction: 'Restart Hermes', updates: 'Updates', checkNow: 'Check now', checking: 'Checking…', @@ -1161,8 +1168,7 @@ export const en: Translations = { 'A higher-quality model fits this machine but would respond too slowly on its memory bandwidth — this is the best model that stays fast.', 'fastest-resident': 'No model reaches full speed on this hardware; this one comes closest while running entirely in GPU memory.', - 'least-painful-spilled': - 'No model fits entirely in GPU memory here — this one runs best from system RAM.' + 'least-painful-spilled': 'No model fits entirely in GPU memory here — this one runs best from system RAM.' } as Record, downloaded: 'Downloaded', downloadAction: size => `Download · ${size}`, @@ -1175,7 +1181,8 @@ export const en: Translations = { quickstartTitle: 'Run a model on this machine', quickstartDetail: (model, size) => `One click sets everything up: the local engine, ${model} (${size} download), and your default for new chats. Nothing leaves this computer.`, - quickstartDetailReady: model => `One click makes ${model} your default for new chats. Everything runs on this machine.`, + quickstartDetailReady: model => + `One click makes ${model} your default for new chats. Everything runs on this machine.`, quickstartAction: 'Set up for me', quickstartConfigure: 'Configure…', quickstartDoneToast: model => `${model} is set up — new chats run on this machine.`, @@ -1186,7 +1193,8 @@ export const en: Translations = { useAction: 'Use', activePill: 'Default', updateTitle: 'Engine update available', - updateDetail: (next, current) => `A newer llama.cpp build (${next}) is ready to install — you're on ${current}. Models keep working during the download.`, + updateDetail: (next, current) => + `A newer llama.cpp build (${next}) is ready to install — you're on ${current}. Models keep working during the download.`, updateAction: 'Update engine', updating: 'Updating engine…', upToDateTitle: 'Engine up to date', @@ -1206,7 +1214,8 @@ export const en: Translations = { ejectFailed: 'Could not unload the model', stopServer: 'Turn off', startServer: 'Turn on', - runtimeRunningDetail: 'The local server is running. Turning it off frees all GPU memory and stops new chats from using local models until you turn it back on.', + runtimeRunningDetail: + 'The local server is running. Turning it off frees all GPU memory and stops new chats from using local models until you turn it back on.', serverStopped: 'Local server stopped — GPU memory freed.', serverStarted: 'Local server running.', serverStopFailed: 'Could not stop the local server', @@ -1219,7 +1228,8 @@ export const en: Translations = { pillUsesRam: 'Uses system RAM', pillTooBig: 'Too big for this machine', browseTitle: 'Find more models', - browseHint: 'Search all of Hugging Face. Models you download here are sized to your machine automatically, but not tested by us.', + browseHint: + 'Search all of Hugging Face. Models you download here are sized to your machine automatically, but not tested by us.', browsePlaceholder: 'Search models by name or author…', browseSearching: 'Searching Hugging Face', browseListing: 'Reading model files', @@ -3134,13 +3144,17 @@ export const en: Translations = { resetStatusbar: 'Reset to defaults', toggleApprovalMode: 'Approvals', toggleBackendVersion: 'Backend version', + toggleCacheHitRate: 'Cache hit rate', toggleCommandCenter: 'Command Center', toggleContextUsage: 'Context meter', toggleRunningTimer: 'Turn timer', toggleSessionTimer: 'Session timer', toggleTerminal: 'Terminal', + toggleTokensPerSecond: 'Tokens per second', toggleVersion: 'Version & updates', toggleWorkspace: 'Workspace', + cacheHitRateTitle: 'Prompt cache hit rate this session — cached tokens cost less, so higher is cheaper', + tokensPerSecondTitle: 'Output tokens per second, averaged over the last 10 model calls', agents: 'Agents', closeAgents: 'Close agents', openAgents: 'Open agents', @@ -3746,10 +3760,6 @@ export const en: Translations = { title: 'Attach and command', text: 'Type @ to bring a file into the conversation, / to run a command.' }, - 'model-switch': { - title: 'Switch models mid-thread', - text: 'The model name is a button. Change it whenever the work changes shape.' - }, 'local-setup': { title: 'This machine can run models locally', text: 'Your hardware can serve a local model. Chats stay on your computer and cost nothing.', diff --git a/apps/desktop/src/i18n/ja.ts b/apps/desktop/src/i18n/ja.ts index 11d9326f8b..dc0f68c26a 100644 --- a/apps/desktop/src/i18n/ja.ts +++ b/apps/desktop/src/i18n/ja.ts @@ -438,6 +438,9 @@ export const ja = defineLocale({ embedsAlways: '常に', embedsOff: 'オフ', embedsReset: (count: number) => `許可した${count}件のサービスをリセット`, + resumeLastSessionTitle: '起動時に前回のチャットを再開', + resumeLastSessionDesc: + 'オンの場合、コールドスタート時に直近のチャットを再び開きます。オフにすると常に新しいチャットから始まります。', product: 'プロダクト', productDesc: '読みやすいツール活動と簡潔な要約を表示します。', technical: 'テクニカル', @@ -728,6 +731,10 @@ export const ja = defineLocale({ bundleOutOfSyncDesc: 'Hermes ランタイムは更新されましたが、デスクトップアプリ自体は古いビルドのままです。アプリを更新するまで、新しいインターフェース機能(Bot Mode など)は表示されません。下の更新を実行してアプリを再ビルドしてください。それでもこの警告が消えない場合は、最新のデスクトップインストーラーから再インストールしてください。', bundleOutOfSyncAction: 'インストーラーを入手', + bundleSwapPending: '再起動して更新を完了', + bundleSwapPendingDesc: + '更新されたアプリはすでにインストール済みです。Hermes を再起動するだけで新しいビルドが読み込まれます。チャットや設定はそのまま保持されます。', + bundleSwapPendingAction: 'Hermes を再起動', updates: '更新', checkNow: '今すぐ確認', checking: '確認中…', @@ -1072,26 +1079,30 @@ export const ja = defineLocale({ useAction: '使用する', activePill: 'デフォルト', updateTitle: 'エンジンの更新があります', - updateDetail: (next, current) => `新しい llama.cpp ビルド（${next}）をインストールできます——現在は ${current} です。ダウンロード中もモデルは引き続き使えます。`, + updateDetail: (next, current) => + `新しい llama.cpp ビルド（${next}）をインストールできます——現在は ${current} です。ダウンロード中もモデルは引き続き使えます。`, updateAction: 'エンジンを更新', updating: 'エンジンを更新中…', upToDateTitle: 'エンジンは最新です', upToDateDetail: (tag, backend) => `llama.cpp ${tag}（${backend}）で動作中——Hermes が提供する最新ビルドです。`, - updateToast: next => `ローカルエンジンの新しいビルド（${next}）があります。設定 → ローカルモデル から更新できます。`, + updateToast: next => + `ローカルエンジンの新しいビルド（${next}）があります。設定 → ローカルモデル から更新できます。`, activeDetail: '新しいチャットはこのモデルを使用——最初のメッセージ送信時に読み込みます', activeNotLoaded: '最初のメッセージで読み込みます', loadedPill: '読み込み済み', placementResident: 'すべて GPU 上', placementSpilled: '一部 RAM 上', placementResidentTip: 'このコンテキストウィンドウで GPU メモリ内で完全に動作しています — フルスピード。', - placementSpilledTip: 'モデルの一部がシステム RAM から動作しています — 動作しますが遅くなります。よりコンパクトなビルドか小さいコンテキストなら完全に収まります。', + placementSpilledTip: + 'モデルの一部がシステム RAM から動作しています — 動作しますが遅くなります。よりコンパクトなビルドか小さいコンテキストなら完全に収まります。', loadingPill: '読み込み中…', ejectTip: 'GPU メモリを解放（必要時に再読み込み）', ejected: 'モデルをアンロードしました——GPU メモリを解放しました。', ejectFailed: 'モデルをアンロードできませんでした', stopServer: 'オフにする', startServer: 'オンにする', - runtimeRunningDetail: 'ローカルサーバーが実行中です。オフにすると GPU メモリを全て解放し、再度オンにするまで新しいチャットはローカルモデルを使用しません。', + runtimeRunningDetail: + 'ローカルサーバーが実行中です。オフにすると GPU メモリを全て解放し、再度オンにするまで新しいチャットはローカルモデルを使用しません。', serverStopped: 'ローカルサーバーを停止しました——GPU メモリを解放しました。', serverStarted: 'ローカルサーバー実行中。', serverStopFailed: 'ローカルサーバーを停止できませんでした', @@ -1104,7 +1115,8 @@ export const ja = defineLocale({ pillUsesRam: 'システム RAM を使用', pillTooBig: 'このマシンには大きすぎます', browseTitle: 'さらにモデルを探す', - browseHint: 'Hugging Face 全体を検索できます。ここでダウンロードしたモデルは自動でマシンに合わせて動作しますが、当方でのテストは行われていません。', + browseHint: + 'Hugging Face 全体を検索できます。ここでダウンロードしたモデルは自動でマシンに合わせて動作しますが、当方でのテストは行われていません。', browsePlaceholder: 'モデル名または作者で検索…', browseSearching: 'Hugging Face を検索中', browseListing: 'モデルファイルを読み込み中', @@ -3279,10 +3291,6 @@ export const ja = defineLocale({ title: 'ファイルとコマンド', text: '@ でファイルを会話に取り込み、/ でコマンドを実行できます。' }, - 'model-switch': { - title: '会話の途中でモデルを変更', - text: 'モデル名はボタンです。作業の性質が変わったら切り替えてください。' - }, 'local-setup': { title: 'このマシンはローカルでモデルを実行できます', text: 'お使いのハードウェアでローカルモデルを動かせます。会話はこのコンピュータから出ず、料金もかかりません。', diff --git a/apps/desktop/src/i18n/ru.ts b/apps/desktop/src/i18n/ru.ts index 3c0916ed3b..3a0d3ed3fd 100644 --- a/apps/desktop/src/i18n/ru.ts +++ b/apps/desktop/src/i18n/ru.ts @@ -593,6 +593,9 @@ export const ru = defineLocale({ embedsOff: 'Выкл', embedsReset: count => `Сбросить ${count} ${RU_NOUN(count, 'разрешённый сервис', 'разрешённых сервиса', 'разрешённых сервисов')}`, + resumeLastSessionTitle: 'Открывать последний чат при запуске', + resumeLastSessionDesc: + 'Продолжайте с того места, где остановились. Выключите, чтобы всегда начинать с нового чата.', product: 'Продукт', productDesc: 'Дружелюбная активность инструментов с краткими сводками.', technical: 'Технический', @@ -3094,13 +3097,17 @@ export const ru = defineLocale({ resetStatusbar: 'Сбросить к значениям по умолчанию', toggleApprovalMode: 'Подтверждения', toggleBackendVersion: 'Версия бэкенда', + toggleCacheHitRate: 'Попадания в кэш', toggleCommandCenter: 'Командный центр', toggleContextUsage: 'Шкала контекста', toggleRunningTimer: 'Таймер хода', toggleSessionTimer: 'Таймер сеанса', toggleTerminal: 'Терминал', + toggleTokensPerSecond: 'Токенов в секунду', toggleVersion: 'Версия и обновления', toggleWorkspace: 'Рабочее пространство', + cacheHitRateTitle: 'Доля попаданий в кэш промпта за сеанс — кэшированные токены дешевле, чем выше, тем дешевле', + tokensPerSecondTitle: 'Выходных токенов в секунду, среднее за последние 10 вызовов модели', agents: 'Агенты', closeAgents: 'Закрыть агентов', openAgents: 'Открыть агентов', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 3056a9e84d..632b20b3a9 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -506,6 +506,8 @@ export interface Translations { embedsAlways: string embedsOff: string embedsReset: (count: number) => string + resumeLastSessionTitle: string + resumeLastSessionDesc: string product: string productDesc: string technical: string @@ -569,6 +571,9 @@ export interface Translations { bundleOutOfSync: string bundleOutOfSyncDesc: string bundleOutOfSyncAction: string + bundleSwapPending: string + bundleSwapPendingDesc: string + bundleSwapPendingAction: string updates: string checkNow: string checking: string @@ -2677,13 +2682,17 @@ export interface Translations { resetStatusbar: string toggleApprovalMode: string toggleBackendVersion: string + toggleCacheHitRate: string toggleCommandCenter: string toggleContextUsage: string toggleRunningTimer: string toggleSessionTimer: string toggleTerminal: string + toggleTokensPerSecond: string toggleVersion: string toggleWorkspace: string + cacheHitRateTitle: string + tokensPerSecondTitle: string agents: string closeAgents: string openAgents: string diff --git a/apps/desktop/src/i18n/zh-hant.ts b/apps/desktop/src/i18n/zh-hant.ts index 9fc59f86f7..780ce48033 100644 --- a/apps/desktop/src/i18n/zh-hant.ts +++ b/apps/desktop/src/i18n/zh-hant.ts @@ -424,6 +424,8 @@ export const zhHant = defineLocale({ embedsAlways: '一律', embedsOff: '關閉', embedsReset: (count: number) => `重設 ${count} 個已允許的服務`, + resumeLastSessionTitle: '啟動時恢復上次會話', + resumeLastSessionDesc: '開啟後，應用冷啟動時重新打開最近的聊天。關閉則始終從空白新會話開始。', product: '產品', productDesc: '易讀的工具活動與精簡摘要。', technical: '技術', @@ -710,6 +712,10 @@ export const zhHant = defineLocale({ bundleOutOfSyncDesc: 'Hermes 執行環境已更新,但桌面應用程式本身仍是舊建置——在應用程式更新之前,新的介面功能(如 Bot Mode)不會顯示。請執行下方的更新以重新建置應用程式。如果此警告仍未消除,請從最新的桌面安裝程式重新安裝。', bundleOutOfSyncAction: '取得安裝程式', + bundleSwapPending: '重新啟動以完成更新', + bundleSwapPendingDesc: + '更新後的應用程式已安裝完成，只需重新啟動 Hermes 即可載入新版本。聊天記錄和設定不會受到影響。', + bundleSwapPendingAction: '重新啟動 Hermes', updates: '更新', checkNow: '立即檢查', checking: '檢查中…', @@ -1001,8 +1007,7 @@ export const zhHant = defineLocale({ runtimeInstalled: '已安裝 llama.cpp 執行環境', runtimeInstalledDetail: (tag, backend) => `組建 ${tag}，${backend} 後端。Hermes 會為您啟動並管理伺服器。`, installTitle: '安裝本地執行環境', - installDetail: - '下載 llama.cpp 推理引擎（數百 MB）。下載的模型完全在本機執行——無需帳號，資料不會離開您的電腦。', + installDetail: '下載 llama.cpp 推理引擎（數百 MB）。下載的模型完全在本機執行——無需帳號，資料不會離開您的電腦。', installAction: '安裝執行環境', installing: '正在安裝執行環境…', installFailed: '執行環境安裝失敗', @@ -1015,7 +1020,8 @@ export const zhHant = defineLocale({ recommended: '推薦', recommendedReason: { 'best-quality-resident': '在完全駐留 GPU 且保持全速的模型中品質最高。推薦會在品質與該硬體的預計速度之間權衡。', - 'speed-gated-quality': '有更高品質的模型可以裝入這台機器，但受記憶體頻寬限制回應會太慢——這是保持流暢的最佳模型。', + 'speed-gated-quality': + '有更高品質的模型可以裝入這台機器，但受記憶體頻寬限制回應會太慢——這是保持流暢的最佳模型。', 'fastest-resident': '沒有模型能在該硬體上達到全速；這是完全駐留 GPU 記憶體中最快的一個。', 'least-painful-spilled': '沒有模型能完全裝入 GPU 記憶體——這是從系統記憶體執行表現最好的一個。' } as Record, @@ -1030,7 +1036,8 @@ export const zhHant = defineLocale({ useAction: '使用', activePill: '預設', updateTitle: '引擎有可用更新', - updateDetail: (next, current) => `新的 llama.cpp 組建（${next}）可以安裝——目前為 ${current}。下載期間模型仍可正常使用。`, + updateDetail: (next, current) => + `新的 llama.cpp 組建（${next}）可以安裝——目前為 ${current}。下載期間模型仍可正常使用。`, updateAction: '更新引擎', updating: '正在更新引擎…', upToDateTitle: '引擎已是最新', @@ -1042,7 +1049,8 @@ export const zhHant = defineLocale({ placementResident: '全部在 GPU', placementSpilled: '部分在記憶體', placementResidentTip: '完全在 GPU 記憶體中以此上下文視窗執行——全速。', - placementSpilledTip: '模型的一部分從系統記憶體執行——可用但較慢。更緊湊的版本或更小的上下文可以完全放入顯示記憶體。', + placementSpilledTip: + '模型的一部分從系統記憶體執行——可用但較慢。更緊湊的版本或更小的上下文可以完全放入顯示記憶體。', loadingPill: '載入中…', ejectTip: '釋放顯示記憶體（需要時重新載入）', ejected: '模型已卸載——顯示記憶體已釋放。', @@ -3152,10 +3160,6 @@ export const zhHant = defineLocale({ title: '附件與指令', text: '輸入 @ 把檔案帶入對話，輸入 / 執行指令。' }, - 'model-switch': { - title: '對話中隨時換模型', - text: '模型名稱就是按鈕。工作性質變了就換一個。' - }, 'local-setup': { title: '這台電腦可以本地執行模型', text: '你的硬體可以執行本地模型。對話不離開你的電腦，而且完全免費。', diff --git a/apps/desktop/src/i18n/zh.ts b/apps/desktop/src/i18n/zh.ts index 48bf174f6c..36b2fd788e 100644 --- a/apps/desktop/src/i18n/zh.ts +++ b/apps/desktop/src/i18n/zh.ts @@ -602,6 +602,8 @@ export const zh: Translations = { embedsAlways: '总是', embedsOff: '关闭', embedsReset: (count: number) => `重置 ${count} 个已允许的服务`, + resumeLastSessionTitle: '启动时恢复上次会话', + resumeLastSessionDesc: '开启后，应用冷启动时重新打开最近的聊天。关闭则始终从空白新会话开始。', product: '产品', productDesc: '易读的工具活动与简洁摘要。', technical: '技术', @@ -888,6 +890,9 @@ export const zh: Translations = { bundleOutOfSyncDesc: 'Hermes 运行时已更新,但桌面应用本身仍是旧构建——在应用更新之前,新的界面功能(如 Bot Mode)不会显示。请运行下方的更新以重新构建应用。如果此警告仍未消除,请从最新的桌面安装程序重新安装。', bundleOutOfSyncAction: '获取安装程序', + bundleSwapPending: '重启以完成更新', + bundleSwapPendingDesc: '更新后的应用已安装完成，只需重启 Hermes 即可加载新版本。聊天记录和设置不会受到影响。', + bundleSwapPendingAction: '重启 Hermes', updates: '更新', checkNow: '立即检查', checking: '检查中…', @@ -1370,7 +1375,8 @@ export const zh: Translations = { useAction: '使用', activePill: '默认', updateTitle: '引擎有可用更新', - updateDetail: (next, current) => `新的 llama.cpp 构建（${next}）可以安装——当前为 ${current}。下载期间模型仍可正常使用。`, + updateDetail: (next, current) => + `新的 llama.cpp 构建（${next}）可以安装——当前为 ${current}。下载期间模型仍可正常使用。`, updateAction: '更新引擎', updating: '正在更新引擎…', upToDateTitle: '引擎已是最新', @@ -3285,13 +3291,17 @@ export const zh: Translations = { resetStatusbar: '恢复默认设置', toggleApprovalMode: '审批', toggleBackendVersion: '后端版本', + toggleCacheHitRate: '缓存命中率', toggleCommandCenter: '命令中心', toggleContextUsage: '上下文用量', toggleRunningTimer: '回合计时', toggleSessionTimer: '会话计时', toggleTerminal: '终端', + toggleTokensPerSecond: '每秒 token 数', toggleVersion: '版本与更新', toggleWorkspace: '工作区', + cacheHitRateTitle: '本会话的提示缓存命中率 — 缓存 token 更便宜，越高越省', + tokensPerSecondTitle: '每秒输出 token 数，取最近 10 次模型调用的平均值', agents: '代理', closeAgents: '关闭代理', openAgents: '打开代理', @@ -3877,10 +3887,6 @@ export const zh: Translations = { title: '附件与命令', text: '输入 @ 把文件带入对话，输入 / 运行命令。' }, - 'model-switch': { - title: '对话中随时换模型', - text: '模型名称就是按钮。工作性质变了就换一个。' - }, 'local-setup': { title: '这台电脑可以本地运行模型', text: '你的硬件可以运行本地模型。对话不离开你的电脑，而且完全免费。', diff --git a/apps/desktop/src/lib/gateway-events.test.ts b/apps/desktop/src/lib/gateway-events.test.ts index 5ec527d0bc..b9af988f8c 100644 --- a/apps/desktop/src/lib/gateway-events.test.ts +++ b/apps/desktop/src/lib/gateway-events.test.ts @@ -9,6 +9,18 @@ describe('gateway event routing', () => { expect(approvalReplaySessionId('message.delta', 'active-1', 'routed-1')).toBeNull() }) + it('does not replay against an active runtime the gateway already reported gone', () => { + const isGone = (sid: string) => sid === 'dead-1' + + // Unscoped fan-out attributed to a dead active session: skip. + expect(approvalReplaySessionId('session.info', 'dead-1', 'dead-1', { explicit: false, isGone })).toBeNull() + expect(approvalReplaySessionId('gateway.ready', 'dead-1', null, { explicit: false, isGone })).toBeNull() + // A live active session still replays. + expect(approvalReplaySessionId('session.info', 'live-1', 'live-1', { explicit: false, isGone })).toBe('live-1') + // An explicitly scoped frame is the runtime speaking for itself — never skipped. + expect(approvalReplaySessionId('session.info', 'dead-1', 'dead-1', { explicit: true, isGone })).toBe('dead-1') + }) + it('drops only unscoped subagent events (genuinely background work)', () => { expect(gatewayEventRequiresSessionId('subagent.progress')).toBe(true) expect(gatewayEventRequiresSessionId('subagent.start')).toBe(true) diff --git a/apps/desktop/src/lib/gateway-events.ts b/apps/desktop/src/lib/gateway-events.ts index 6375955510..7c994a71ce 100644 --- a/apps/desktop/src/lib/gateway-events.ts +++ b/apps/desktop/src/lib/gateway-events.ts @@ -80,20 +80,34 @@ export interface GatewayEventSessionRoute { sessionId: null | string } +/** Which session (if any) to re-pull `approval.pending` for after `eventType`. + * + * `gateway.ready` and `session.info` are the two rehydration points. An + * UNSCOPED `session.info` (the approvals-loop / broadcast fan-out, no + * `session_id` on the frame) reaches here attributed to the active session by + * the routing fallback; when `isGone(activeSessionId)` — the gateway already + * answered 4001 for that runtime — replaying would only re-send the dead id + * on every fan-out tick (#100639), so return null. A frame that names the + * session explicitly is the runtime speaking for itself and is never gone. */ export function approvalReplaySessionId( eventType: string | undefined, activeSessionId: null | string, - routedSessionId: null | string + routedSessionId: null | string, + options?: { explicit?: boolean; isGone?: (sessionId: string) => boolean } ): null | string { + let target: null | string = null + if (eventType === 'gateway.ready') { - return activeSessionId + target = activeSessionId + } else if (eventType === 'session.info') { + target = routedSessionId } - if (eventType === 'session.info') { - return routedSessionId + if (target && !options?.explicit && options?.isGone?.(target)) { + return null } - return null + return target } /** diff --git a/apps/desktop/src/lib/model-status-label.test.ts b/apps/desktop/src/lib/model-status-label.test.ts index 326296dd27..f5dd3143ec 100644 --- a/apps/desktop/src/lib/model-status-label.test.ts +++ b/apps/desktop/src/lib/model-status-label.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from 'vitest' -import { currentPickerSelection, displayModelName, formatModelStatusLabel, modelDisplayParts } from './model-status-label' +import { + currentPickerSelection, + displayModelName, + formatModelStatusLabel, + modelDisplayParts +} from './model-status-label' import { reasoningEffortLabel } from './reasoning-effort' describe('model-status-label', () => { diff --git a/apps/desktop/src/lib/preview-annotate/group.test.ts b/apps/desktop/src/lib/preview-annotate/group.test.ts new file mode 100644 index 0000000000..fa3d9dd417 --- /dev/null +++ b/apps/desktop/src/lib/preview-annotate/group.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from 'vitest' + +import { annotateSplitDepth, groupAnnotations } from './group' +import type { ComposerReadyAnnotation } from './pack' + +function item(number: number, selector?: string): ComposerReadyAnnotation { + return { + imageDataUrl: '', + note: `note ${number}`, + number, + prompt: `Comment ${number}`, + identity: selector ? { css: {}, html: '', selector, tag: 'div', text: '' } : undefined + } +} + +describe('annotateSplitDepth', () => { + it('splits at the shallowest region where the comments disagree', () => { + expect(annotateSplitDepth(['body>main>div.header>h1', 'body>main>div.header>p', 'body>main>div.footer>a'])).toBe(3) + }) + + it('does not split siblings inside one component', () => { + // Differ only at the leaf, so the ancestor paths are identical: the depth + // runs to the end of the shared path and both land in the same group. + const selectors = ['body>div.card>h1', 'body>div.card>p'] + + expect(annotateSplitDepth(selectors)).toBe(2) + expect(groupAnnotations(selectors.map((selector, index) => item(index + 1, selector)))).toHaveLength(1) + }) + + it('does not split when every comment is on the same element', () => { + expect(annotateSplitDepth(['body>div.card', 'body>div.card'])).toBe(1) + }) + + it('separates a container comment from comments nested inside it', () => { + const depth = annotateSplitDepth(['body>main', 'body>main>div.a>span', 'body>main>div.b>span']) + + expect(depth).toBe(2) + }) + + it('handles a single selector and an empty batch', () => { + expect(annotateSplitDepth(['body>div.only'])).toBe(1) + expect(annotateSplitDepth([])).toBe(0) + }) +}) + +describe('groupAnnotations', () => { + it('gathers comments on the same region and separates different regions', () => { + const groups = groupAnnotations([ + item(1, 'body>main>section.hero>h1'), + item(2, 'body>main>section.pricing>button'), + item(3, 'body>main>section.hero>p'), + item(4, 'body>main>section.pricing>span') + ]) + + expect(groups).toHaveLength(2) + expect(groups[0]?.label).toBe('section.hero') + expect(groups[0]?.items.map(entry => entry.number)).toEqual([1, 3]) + expect(groups[1]?.label).toBe('section.pricing') + expect(groups[1]?.items.map(entry => entry.number)).toEqual([2, 4]) + }) + + it('produces groups whose subtrees do not overlap, so they can run in parallel', () => { + const groups = groupAnnotations([ + item(1, 'body>main>section.hero>h1'), + item(2, 'body>main>section.pricing>button'), + item(3, 'body>main>section.faq>li') + ]) + + const keys = groups.map(group => group.key) + const overlapping = keys.filter(key => keys.some(other => other !== key && other.startsWith(`${key}>`))) + + expect(keys).toHaveLength(3) + expect(overlapping).toEqual([]) + }) + + it('keeps area pins in their own trailing group rather than guessing a subtree', () => { + const groups = groupAnnotations([item(1, 'body>main>div.a>h1'), item(2), item(3, 'body>main>div.b>h1'), item(4)]) + + const loose = groups[groups.length - 1] + + expect(loose?.key).toBe('') + expect(loose?.label).toBe('') + expect(loose?.items.map(entry => entry.number)).toEqual([2, 4]) + }) + + it('returns one group when every comment lands in the same region', () => { + const groups = groupAnnotations([item(1, 'body>div.card>h1'), item(2, 'body>div.card>p')]) + + expect(groups).toHaveLength(1) + }) + + it('orders groups by first appearance so the numbering still reads in click order', () => { + const groups = groupAnnotations([ + item(1, 'body>main>div.b>h1'), + item(2, 'body>main>div.a>h1'), + item(3, 'body>main>div.b>p') + ]) + + expect(groups.map(group => group.label)).toEqual(['div.b', 'div.a']) + expect(groups[0]?.items.map(entry => entry.number)).toEqual([1, 3]) + }) + + it('survives a batch with no element comments at all', () => { + const groups = groupAnnotations([item(1), item(2)]) + + expect(groups).toHaveLength(1) + expect(groups[0]?.items).toHaveLength(2) + }) +}) + +describe('groupAnnotations refinement', () => { + // A normal page: header / main / footer part company at the top, so a single + // split buries every section under `main`. + const page = [ + ...['a.logo', 'ul.links>li', 'button.menu'].map((tail, index) => item(index + 1, `body>header.nav>${tail}`)), + ...['h1', 'p.sub', 'a.cta', 'img.art'].map((tail, index) => item(index + 4, `body>main>section.hero>${tail}`)), + ...['table', 'button.buy', 'span.note'].map((tail, index) => item(index + 8, `body>main>section.pricing>${tail}`)), + ...['details:nth-of-type(1)', 'details:nth-of-type(4)'].map((tail, index) => + item(index + 11, `body>main>section.faq>${tail}`) + ), + ...['div.cols>ul', 'small.copy'].map((tail, index) => item(index + 13, `body>footer.foot>${tail}`)) + ] + + it('breaks up the branch that would otherwise swallow most of the batch', () => { + const groups = groupAnnotations(page) + const labels = groups.map(group => group.label) + + expect(labels).toContain('section.hero') + expect(labels).toContain('section.pricing') + expect(labels).toContain('section.faq') + expect(labels).not.toContain('main') + }) + + it('leaves no group holding more than a third of the batch', () => { + const groups = groupAnnotations(page) + const ceiling = Math.max(2, Math.ceil(page.length / 3)) + + for (const group of groups) { + expect(group.items.length).toBeLessThanOrEqual(ceiling) + } + }) + + it('loses and duplicates nothing while refining', () => { + const numbers = groupAnnotations(page) + .flatMap(group => group.items.map(entry => entry.number)) + .sort((a, b) => a - b) + + expect(numbers).toEqual(page.map(entry => entry.number)) + }) + + it('keeps refined groups on non-overlapping subtrees', () => { + const keys = groupAnnotations(page).map(group => group.key) + const nested = keys.filter(key => keys.some(other => other !== key && other.startsWith(`${key}>`))) + + expect(nested).toEqual([]) + }) + + it('stops instead of looping when an oversized group cannot divide further', () => { + const identical = Array.from({ length: 9 }, (_, index) => item(index + 1, 'body>div.card>span')) + const groups = groupAnnotations(identical) + + expect(groups).toHaveLength(1) + expect(groups[0]?.items).toHaveLength(9) + }) +}) diff --git a/apps/desktop/src/lib/preview-annotate/group.ts b/apps/desktop/src/lib/preview-annotate/group.ts new file mode 100644 index 0000000000..04b9e3f802 --- /dev/null +++ b/apps/desktop/src/lib/preview-annotate/group.ts @@ -0,0 +1,170 @@ +/** + * Structural grouping for a comment batch. + * + * Twenty-three comments used to arrive as twenty-three flat blocks, so the + * agent made twenty-three todos and worked them one at a time. The fix is not + * a classifier in the renderer — "is this a UI nit or a functional bug" is a + * judgment only the model can make, and prose-matching it here would be wrong + * constantly. What the renderer CAN know is structure: which pins sit in the + * same part of the DOM, and therefore which ones are likely the same component + * and the same source file. + * + * So this splits the batch by shared ancestor path and hands the model groups + * that touch disjoint subtrees. Disjoint is the property that makes parallel + * work safe — grouping by theme instead ("all the UI ones") would put five + * agents in the same files. The model still owns the semantics and can regroup; + * these are labelled starting points, not orders. + * + * Two properties keep the split honest without a tuning knob: + * + * - It compares ANCESTOR paths, not full selectors. Two comments on the heading + * and the paragraph of one card differ at the leaf, and splitting there would + * hand out singletons — the thing this exists to prevent. Their parents are + * identical, so they group. + * - The depth is derived, then refined: descend the shared prefix until it + * stops being shared, and sub-split any group that ends up holding most of + * the batch. So the group count follows the page the user commented on rather + * than a constant someone picked. + */ + +import type { ComposerReadyAnnotation } from './pack' + +export interface AnnotateGroup { + /** Shared ancestor prefix, or '' for the group that has no element. */ + key: string + items: ComposerReadyAnnotation[] + /** Short human label for the shared region, e.g. `section.hero`. */ + label: string +} + +const SEP = '>' + +function segments(selector: string): string[] { + return selector.split(SEP).filter(Boolean) +} + +/** + * The element's container. A one-segment selector is its own container — + * dropping to nothing would collide with the unanchored group's empty key. + */ +function ancestorPath(selector: string): string[] { + const parts = segments(selector) + + return parts.length > 1 ? parts.slice(0, -1) : parts +} + +function prefixAt(parts: string[], depth: number): string { + return parts.slice(0, depth).join(SEP) +} + +/** + * First depth at which the ancestor paths stop agreeing. + * + * Grouping by a prefix of this depth yields the top-level regions the user + * touched. When every path is identical there is no boundary and everything + * belongs to one group. + */ +export function annotateSplitDepth(selectors: readonly string[]): number { + const parts = selectors.map(ancestorPath) + + if (parts.length < 2) { + return parts[0]?.length ? 1 : 0 + } + + const shortest = Math.min(...parts.map(list => list.length)) + + for (let depth = 1; depth <= shortest; depth++) { + const seen = new Set(parts.map(list => prefixAt(list, depth))) + + if (seen.size > 1) { + return depth + } + } + + // Every path shares the whole of the shortest one: the shorter paths are + // ancestors of the longer ones, so one segment deeper is where they part. + return parts.some(list => list.length > shortest) ? shortest + 1 : shortest +} + +function labelFor(key: string): string { + const parts = segments(key) + + return parts[parts.length - 1] || '' +} + +function bucket(items: readonly ComposerReadyAnnotation[], depth: number): AnnotateGroup[] { + const byKey = new Map() + + for (const item of items) { + const key = prefixAt(ancestorPath(item.identity?.selector || ''), depth) + const group = byKey.get(key) + + if (group) { + group.items.push(item) + + continue + } + + byKey.set(key, { items: [item], key, label: labelFor(key) }) + } + + return Array.from(byKey.values()) +} + +/** + * One pass of the split leaves the deepest branch lumped together: on a normal + * page `header`, `main`, and `footer` part company at the top, so every comment + * inside `main` — hero, pricing, faq — lands in one oversized group. That group + * is not foldable into a single change and not safely divisible among workers, + * which is the whole point of grouping. + * + * So refine: while some group holds more than a third of the batch and its + * members do diverge further down, replace it with its own sub-split. A group + * holding most of the batch has not separated anything. Each pass strictly + * shrinks the largest group or finds it indivisible, so this terminates. + */ +function refine(groups: AnnotateGroup[], total: number): AnnotateGroup[] { + const ceiling = Math.max(2, Math.ceil(total / 3)) + let current = groups + + for (let pass = 0; pass < total; pass++) { + const target = current.find(group => group.items.length > ceiling) + + if (!target) { + break + } + + const selectors = target.items.map(item => item.identity?.selector || '') + const deeper = annotateSplitDepth(selectors) + const split = bucket(target.items, deeper) + + if (split.length < 2) { + break + } + + current = current.flatMap(group => (group === target ? split : [group])) + } + + return current +} + +/** + * Split a packed batch into groups the model can hand out in parallel. + * + * Comments with no element (area pins) cannot be placed in the tree, so they + * collect in one trailing group rather than being guessed into someone else's + * subtree. Group order follows first appearance, so numbering still reads in + * the order the user clicked. + */ +export function groupAnnotations(items: readonly ComposerReadyAnnotation[]): AnnotateGroup[] { + const placed = items.filter(item => item.identity?.selector) + const loose = items.filter(item => !item.identity?.selector) + const depth = annotateSplitDepth(placed.map(item => item.identity?.selector || '')) + const groups = refine(bucket(placed, depth), placed.length) + + if (loose.length) { + groups.push({ items: [...loose], key: '', label: '' }) + } + + return groups +} diff --git a/apps/desktop/src/lib/preview-annotate/identity.ts b/apps/desktop/src/lib/preview-annotate/identity.ts index f8b94e4ea2..578dbb86fc 100644 --- a/apps/desktop/src/lib/preview-annotate/identity.ts +++ b/apps/desktop/src/lib/preview-annotate/identity.ts @@ -1,8 +1,9 @@ -import { ANNOTATE_CSS_KEYS } from './tokens' +import { ANNOTATE_CSS_KEYS, ANNOTATE_HTML_BUDGET } from './tokens' export interface ElementSnapshot { className?: string css: Record + html?: string id?: string role?: string selector: string @@ -12,6 +13,7 @@ export interface ElementSnapshot { export interface CompactIdentity { css: Record + html: string selector: string tag: string text: string @@ -47,6 +49,16 @@ function clip(value: string, max: number): string { return `${trimmed.slice(0, max - 1)}…` } +/** + * Markup keeps its own clip: it arrives already budgeted and redacted from the + * guest, and this is the backstop for a snapshot built anywhere else. Newlines + * collapse but the tag structure survives — `clip` alone would be fine, this + * just names the different budget. + */ +function clipHtml(value: string): string { + return clip(value, ANNOTATE_HTML_BUDGET) +} + /** Keep only the curated CSS snapshot, drop empties and the whole document. */ export function compactIdentity(snapshot: ElementSnapshot): CompactIdentity { const css: Record = {} @@ -64,8 +76,9 @@ export function compactIdentity(snapshot: ElementSnapshot): CompactIdentity { const tag = (snapshot.tag || 'div').toLowerCase() const selector = clip(snapshot.selector || tag, MAX_SELECTOR) const text = clip(snapshot.text || '', MAX_TEXT) + const html = clipHtml(snapshot.html || '') - return { css, selector, tag, text } + return { css, html, selector, tag, text } } export function formatIdentityLine(identity: CompactIdentity): string { diff --git a/apps/desktop/src/lib/preview-annotate/in-page.test.ts b/apps/desktop/src/lib/preview-annotate/in-page.test.ts index 06a9799ff9..15c31383e9 100644 --- a/apps/desktop/src/lib/preview-annotate/in-page.test.ts +++ b/apps/desktop/src/lib/preview-annotate/in-page.test.ts @@ -140,9 +140,138 @@ describe('annotateInPage overlay', () => { expect(event.identity.tag).toBe('button') expect(event.identity.selector).toContain('go') expect(event.identity.text).toBe('Go') + expect(event.identity.html).toContain(' { + const form = document.createElement('form') + form.setAttribute('data-api-key', 'sk-live-1234567890') + form.innerHTML = + '' + + '' + document.body.appendChild(form) + form.getBoundingClientRect = () => ({ + bottom: 60, + height: 50, + left: 0, + right: 200, + toJSON: () => ({}), + top: 10, + width: 200, + x: 0, + y: 10 + }) + document.elementFromPoint = () => form + + const api = annotateInPage(document) + api.install() + const pending = api.wait() + const host = document.querySelector('hermes-annotate') as HTMLElement + + host.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, button: 0, clientX: 20, clientY: 20 })) + host.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, button: 0, clientX: 21, clientY: 21 })) + + const event = await pending + + expect(event.type).toBe('pick-element') + + if (event.type === 'pick-element') { + expect(event.identity.html).not.toContain('hunter2') + expect(event.identity.html).not.toContain('tok_abc') + expect(event.identity.html).not.toContain('sk-live-1234567890') + expect(event.identity.html).toContain('[redacted]') + // A non-secret field keeps its value — redaction is targeted, not a blanket wipe. + expect(event.identity.html).toContain('me@example.com') + } + + api.teardown() + }) + + it('budgets the markup so one comment cannot paste a whole section', async () => { + const section = document.createElement('section') + section.innerHTML = '

filler filler filler

'.repeat(200) + document.body.appendChild(section) + section.getBoundingClientRect = () => ({ + bottom: 400, + height: 400, + left: 0, + right: 300, + toJSON: () => ({}), + top: 0, + width: 300, + x: 0, + y: 0 + }) + document.elementFromPoint = () => section + + const api = annotateInPage(document) + api.install() + const pending = api.wait() + const host = document.querySelector('hermes-annotate') as HTMLElement + + host.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, button: 0, clientX: 20, clientY: 20 })) + host.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, button: 0, clientX: 21, clientY: 21 })) + + const event = await pending + + if (event.type === 'pick-element') { + expect(event.identity.html.length).toBeLessThanOrEqual(600) + expect(event.identity.html.startsWith('
')).toBe(true) + } + + api.teardown() + }) + + it('shows the draft marker before a capture can photograph it', async () => { + const api = annotateInPage(document) + api.install() + api.showDraft({ height: 24, width: 80, x: 10, y: 10 }, 2) + + const ready = await api.beginCapture() + const shadow = document.querySelector('hermes-annotate')!.shadowRoot! + const draft = shadow.querySelector('[data-annotate-outline="draft"]') as HTMLElement + const marker = draft.querySelector('[data-annotate-marker]') + + expect(ready).toBe(true) + expect(draft.style.display).toBe('block') + expect(marker?.getAttribute('data-annotate-marker')).toBe('2') + api.teardown() + }) + + it('hides saved pins during a capture so a neighbour marker cannot land in the crop', async () => { + const api = annotateInPage(document) + api.install() + api.showPins([ + { kind: 'element', number: 1, rect: { height: 24, width: 80, x: 10, y: 10 } }, + { kind: 'element', number: 2, rect: { height: 24, width: 80, x: 10, y: 40 } } + ]) + api.showDraft({ height: 24, width: 80, x: 10, y: 70 }, 3) + + const shadow = document.querySelector('hermes-annotate')!.shadowRoot! + const pins = () => shadow.querySelector('[data-annotate-pin="1"]')!.parentElement as HTMLElement + + expect(pins().style.display).not.toBe('none') + + await api.beginCapture() + + expect(pins().style.display).toBe('none') + // The draft's own marker must survive — it is the point of the crop. + expect((shadow.querySelector('[data-annotate-outline="draft"]') as HTMLElement).style.display).toBe('block') + + api.endCapture() + + expect(pins().style.display).toBe('block') + api.teardown() + }) + + it('refuses to dress a capture when the overlay is gone', async () => { + const api = annotateInPage(document) + + expect(await api.beginCapture()).toBe(false) + }) + it('owns wheel scrolling instead of also allowing the native wheel action', () => { const scroller = document.createElement('div') scroller.style.overflowY = 'auto' diff --git a/apps/desktop/src/lib/preview-annotate/in-page.ts b/apps/desktop/src/lib/preview-annotate/in-page.ts index 48216ee47c..2de7bf5a0b 100644 --- a/apps/desktop/src/lib/preview-annotate/in-page.ts +++ b/apps/desktop/src/lib/preview-annotate/in-page.ts @@ -19,6 +19,7 @@ export interface AnnotatePageRect { export interface AnnotatePageIdentity { css: Record + html: string selector: string tag: string text: string @@ -39,6 +40,8 @@ export interface AnnotatePinChrome { } export interface AnnotateInPage { + beginCapture: () => Promise + endCapture: () => void getMarkerNumbers: () => number[] getOutlineColor: () => string hideDraft: () => void @@ -72,16 +75,25 @@ export function annotateInPage(doc: Document): AnnotateInPage { 'position', 'width', 'height', + 'max-width', 'padding', 'margin', + 'border', 'border-radius', + 'box-shadow', 'opacity', + 'overflow', + 'z-index', + 'transform', 'flex-direction', 'gap', + 'grid-template-columns', 'justify-content', 'align-items' ] + const htmlBudget = 600 + let host: HTMLElement | null = null let shadow: ShadowRoot | null = null let hoverBox: HTMLElement | null = null @@ -240,11 +252,70 @@ export function annotateInPage(doc: Document): AnnotateInPage { return out } + /** + * Markup for the picked element, with anything secret-shaped stripped first. + * + * A comment is user-authored context, but the element under the cursor is + * whatever the page put there: a filled password box, a token in a hidden + * input, an api-key data attribute. Redaction happens on a clone here, in + * the guest, so the secret never reaches the host, the composer, or the + * model — the same reason `browser_type` masks what it types. + */ + const markupOf = (el: Element): string => { + let clone: Element + + try { + clone = el.cloneNode(true) as Element + } catch { + return '' + } + + const nodes: Element[] = [clone] + const nested = clone.querySelectorAll('input, textarea, select, [data-secret]') + + for (let i = 0; i < nested.length; i++) { + nodes.push(nested[i]) + } + + for (const node of nodes) { + const tag = node.tagName.toLowerCase() + const type = (node.getAttribute('type') || '').toLowerCase() + const secretField = tag === 'input' && (type === 'password' || type === 'hidden') + const names = node.getAttributeNames() + + for (const name of names) { + const lower = name.toLowerCase() + + if (lower === 'value' && (secretField || node.getAttribute('value'))) { + node.setAttribute(name, secretField ? '[redacted]' : node.getAttribute(name) || '') + } + + if (/key|token|secret|password|auth|session|credential/.test(lower)) { + node.setAttribute(name, '[redacted]') + } + } + + if (secretField) { + node.setAttribute('value', '[redacted]') + } + } + + const html = clone.outerHTML || '' + + if (html.length <= htmlBudget) { + return html + } + + // Keep the opening tag — where the classes and props live — over the tail. + return `${html.slice(0, htmlBudget - 1)}…` + } + const identityOf = (el: Element): AnnotatePageIdentity => { const text = (el.textContent || '').replace(/\s+/g, ' ').trim() return { css: readCss(el), + html: markupOf(el), selector: cssPath(el), tag: el.tagName.toLowerCase(), text: text.length > 80 ? `${text.slice(0, 79)}…` : text @@ -663,6 +734,58 @@ export function annotateInPage(doc: Document): AnnotateInPage { } } + /** + * Two frames. `executeJavaScript` resolving only means the style property is + * set — the compositor has not drawn it yet, and `capturePage` grabs whatever + * is on screen. Capturing straight after `showDraft` therefore photographs + * the page one frame before the marker exists, which is why saved crops came + * back outlined but unnumbered. One rAF schedules us before the next paint; + * the second lands after it. + */ + const afterPaint = (): Promise => + new Promise(resolve => { + if (!win) { + resolve() + + return + } + + win.requestAnimationFrame(() => win.requestAnimationFrame(() => resolve())) + }) + + /** + * Dress the page for one crop: the draft's own marker, nothing else. + * + * Saved pins live in the page, so a neighbour's marker lands inside this + * crop whenever the two elements sit within the crop padding of each other — + * a comment on a heading came back carrying the marker of the comment on the + * paragraph below it, and "Image 2 marks the target in blue" then pointed at + * a 1. Hover chrome is transient but would be captured just the same. + */ + const beginCapture = async (): Promise => { + if (!host || !host.isConnected) { + return false + } + + if (pinsLayer) { + style(pinsLayer, { display: 'none' }) + } + + if (hoverBox) { + style(hoverBox, { display: 'none' }) + } + + await afterPaint() + + return true + } + + const endCapture = () => { + if (pinsLayer) { + style(pinsLayer, { display: 'block' }) + } + } + const teardown = () => { unbind() emit({ type: 'end' }) @@ -681,6 +804,8 @@ export function annotateInPage(doc: Document): AnnotateInPage { } return { + beginCapture, + endCapture, getMarkerNumbers: () => { if (!shadow) { return [] diff --git a/apps/desktop/src/lib/preview-annotate/index.ts b/apps/desktop/src/lib/preview-annotate/index.ts index ad45c02768..5ecd0bc9e5 100644 --- a/apps/desktop/src/lib/preview-annotate/index.ts +++ b/apps/desktop/src/lib/preview-annotate/index.ts @@ -1,4 +1,5 @@ export { type AnnotateFlushPorts, type AnnotateFlushResult, flushAnnotateStack } from './flush' +export { type AnnotateGroup, annotateSplitDepth, groupAnnotations } from './group' export { compactIdentity, type CompactIdentity, type ElementSnapshot, formatIdentityLine } from './identity' export { ANNOTATE_HOST_TAG, @@ -42,6 +43,7 @@ export { ANNOTATE_CARD_WIDTH, ANNOTATE_CROP_PAD, ANNOTATE_CSS_KEYS, + ANNOTATE_HTML_BUDGET, ANNOTATE_MARKER_SIZE, ANNOTATE_OUTLINE_WIDTH, ANNOTATE_PILL_BG, diff --git a/apps/desktop/src/lib/preview-annotate/pack.test.ts b/apps/desktop/src/lib/preview-annotate/pack.test.ts index b24b8c2da3..a49d562a12 100644 --- a/apps/desktop/src/lib/preview-annotate/pack.test.ts +++ b/apps/desktop/src/lib/preview-annotate/pack.test.ts @@ -4,6 +4,7 @@ import { flushAnnotateStack } from './flush' import { compactIdentity } from './identity' import { annotateFlushPrompt, packageAnnotatePin, packageAnnotateStack } from './pack' import { addAnnotatePin, type AnnotatePin, emptyAnnotateStack } from './stack' +import { ANNOTATE_HTML_BUDGET } from './tokens' const png = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==' @@ -20,6 +21,7 @@ function pin(partial: Partial = {}): AnnotatePin { rect: { height: 40, width: 120, x: 8, y: 8 }, identity: { css: { color: 'rgb(24, 24, 24)', 'font-size': '14px' }, + html: '', selector: 'button.plan', tag: 'button', text: 'Select plan' @@ -29,13 +31,23 @@ function pin(partial: Partial = {}): AnnotatePin { } describe('packageAnnotatePin', () => { - it('describes text in a generic container without exposing DOM and style internals', () => { + it('carries the selector, markup, and computed styles the agent needs to find the source', () => { + const packed = packageAnnotatePin(pin()) + + expect(packed.prompt).toContain('Selector: button.plan') + expect(packed.prompt).toContain('HTML: ') + expect(packed.prompt).toContain('color: rgb(24, 24, 24)') + expect(packed.prompt).toContain('font-size: 14px') + }) + + it('keeps the target line prose while the DOM detail rides its own labelled lines', () => { const text = 'גם בקיבוץ חולית הקטן יש ילד שעושה את הצעד הראשון במערכת החינוך' const packed = packageAnnotatePin( pin({ identity: { css: { color: 'rgb(0, 0, 0)', 'font-family': 'Moses, NarkisBlock', 'font-size': '18px' }, + html: '
…
', selector: 'div.DraftEditor-editorContainer>div.public-DraftEditor-content>div>div.text_editor_paragraph.rtl:nth-of-type(9)', tag: 'div', @@ -45,11 +57,13 @@ describe('packageAnnotatePin', () => { }) ) - expect(packed.prompt).toContain(`Target: "${text}"`) + const target = packed.prompt.split('\n').find(line => line.startsWith('Target:')) + + expect(target).toBe(`Target: "${text}"`) + expect(target).not.toContain('div') + expect(target).not.toContain('DraftEditor') expect(packed.prompt).toContain('Note: תסכם את זה') - expect(packed.prompt).not.toContain('div') - expect(packed.prompt).not.toContain('DraftEditor') - expect(packed.prompt).not.toContain('font-size') + expect(packed.prompt).toContain('Selector: div.DraftEditor-editorContainer') }) it('packs a numbered crop, compact identity, and the note', () => { @@ -62,8 +76,6 @@ describe('packageAnnotatePin', () => { expect(packed.prompt).toContain('Target: button "Select plan"') expect(packed.prompt).toContain('Image 1 marks the target in blue.') expect(packed.prompt).toContain('Note: This button overflows on mobile.') - expect(packed.prompt).not.toContain('button.plan') - expect(packed.prompt).not.toContain('font-size') expect(packed.prompt).not.toContain(' { pin({ identity: { css: { display: 'block' }, + html: '
', selector: '#sales-chart', tag: 'div', text: '' @@ -81,15 +94,17 @@ describe('packageAnnotatePin', () => { ) expect(packed.prompt).toContain('Target: #sales-chart') - expect(packed.prompt).not.toContain('display: block') }) - it('packages an area pin without pretending it has a selector', () => { + it('invents no element detail for an area pin', () => { const packed = packageAnnotatePin(pin({ identity: undefined, kind: 'area', note: 'too tight' })) expect(packed.prompt).toContain('area') expect(packed.prompt).toContain('120×40px') expect(packed.prompt).toContain('too tight') + expect(packed.prompt).not.toContain('Selector:') + expect(packed.prompt).not.toContain('HTML:') + expect(packed.prompt).not.toContain('Styles:') }) }) @@ -109,6 +124,24 @@ describe('compactIdentity', () => { expect(compact.css.margin).toBeUndefined() expect(compact.css.color).toBe('red') }) + + it('clips markup to the budget rather than pasting a whole section', () => { + const compact = compactIdentity({ + css: {}, + html: `
${'

filler

'.repeat(400)}
`, + selector: 'section', + tag: 'section', + text: '' + }) + + expect(compact.html.length).toBeLessThanOrEqual(ANNOTATE_HTML_BUDGET) + expect(compact.html.startsWith('
')).toBe(true) + expect(compact.html.endsWith('…')).toBe(true) + }) + + it('tolerates a snapshot with no markup', () => { + expect(compactIdentity({ css: {}, selector: 'div', tag: 'div', text: '' }).html).toBe('') + }) }) describe('flushAnnotateStack', () => { @@ -159,3 +192,79 @@ describe('flushAnnotateStack', () => { expect(annotateFlushPrompt(stacked)).toContain('2 comments') }) }) + +describe('annotateFlushPrompt batching', () => { + function at(number: number, selector: string): AnnotatePin { + return pin({ + id: `annotate-${number}`, + number, + identity: { css: {}, html: '', selector, tag: 'div', text: '' } + }) + } + + const batch = packageAnnotateStack([ + at(1, 'body>main>section.hero>h1'), + at(2, 'body>main>section.hero>p'), + at(3, 'body>main>section.pricing>button'), + at(4, 'body>main>section.pricing>span'), + at(5, 'body>main>section.faq>li') + ]) + + it('heads each region so a long batch is fewer pieces of work than comments', () => { + const prompt = annotateFlushPrompt(batch, 'http://localhost:5173/') + + expect(prompt).toContain('Group 1 — `section.hero` (2 comments)') + expect(prompt).toContain('Group 2 — `section.pricing` (2 comments)') + expect(prompt).toContain('Group 3 — `section.faq` (1 comment)') + expect(prompt).toContain('Work them as 3 pieces of work, not 5.') + }) + + it('warns against the theme split that would put workers in the same files', () => { + const prompt = annotateFlushPrompt(batch) + + expect(prompt).toContain('delegate whole groups') + expect(prompt).toContain('never form new groups by theme') + expect(prompt).toContain('Regroup if the code disagrees') + }) + + it('still lists every comment exactly once', () => { + const prompt = annotateFlushPrompt(batch) + + for (const item of batch) { + expect(prompt.split(`Comment ${item.number}\n`)).toHaveLength(2) + } + }) + + it('leaves a short batch flat — grouping two comments is noise', () => { + const prompt = annotateFlushPrompt(batch.slice(0, 2)) + + expect(prompt).not.toContain('Group 1') + expect(prompt).not.toContain('pieces of work') + }) + + it('leaves a batch flat when every comment is in one region', () => { + const prompt = annotateFlushPrompt( + packageAnnotateStack([ + at(1, 'body>div.card>h1'), + at(2, 'body>div.card>p'), + at(3, 'body>div.card>a'), + at(4, 'body>div.card>span') + ]) + ) + + expect(prompt).not.toContain('Group 1') + }) + + it('gives dragged areas their own section instead of a guessed region', () => { + const prompt = annotateFlushPrompt( + packageAnnotateStack([ + at(1, 'body>main>section.hero>h1'), + at(2, 'body>main>section.pricing>button'), + pin({ id: 'annotate-3', identity: undefined, kind: 'area', number: 3 }), + at(4, 'body>main>section.faq>li') + ]) + ) + + expect(prompt).toContain('Unanchored (dragged areas) (1 comment)') + }) +}) diff --git a/apps/desktop/src/lib/preview-annotate/pack.ts b/apps/desktop/src/lib/preview-annotate/pack.ts index c1827eeb9a..07d36a8203 100644 --- a/apps/desktop/src/lib/preview-annotate/pack.ts +++ b/apps/desktop/src/lib/preview-annotate/pack.ts @@ -1,3 +1,4 @@ +import { type AnnotateGroup, groupAnnotations } from './group' import { type CompactIdentity, formatIdentityLine } from './identity' import type { AnnotatePin } from './stack' @@ -17,13 +18,36 @@ function identityBlock(pin: AnnotatePin): string { return formatIdentityLine(pin.identity) } +function cssBlock(identity: CompactIdentity): string { + const entries = Object.entries(identity.css) + + if (!entries.length) { + return '' + } + + return `Styles: ${entries.map(([name, value]) => `${name}: ${value}`).join('; ')}` +} + +/** + * One comment, as much as the agent needs to find the element in source. + * + * The human-readable target line stays first and stays prose — it is what the + * user actually pointed at. Selector, markup, and computed styles follow as + * labelled lines, because the crop shows what is wrong and the DOM shows where + * it lives; an agent given only the picture greps for the wrong div. Area pins + * have no element, so they get the crop and the note and nothing invented. + */ export function packageAnnotatePin(pin: AnnotatePin): ComposerReadyAnnotation { const target = identityBlock(pin) const note = pin.note.trim() + const identity = pin.identity const prompt = [ `Comment ${pin.number}`, `Target: ${target}`, + identity?.selector ? `Selector: ${identity.selector}` : '', + identity?.html ? `HTML: ${identity.html}` : '', + identity ? cssBlock(identity) : '', note ? `Note: ${note}` : '', `Image ${pin.number} marks the target in blue.` ] @@ -31,7 +55,7 @@ export function packageAnnotatePin(pin: AnnotatePin): ComposerReadyAnnotation { .join('\n') return { - identity: pin.identity, + identity, imageDataUrl: pin.imageDataUrl, note, number: pin.number, @@ -43,16 +67,73 @@ export function packageAnnotateStack(pins: readonly AnnotatePin[]): ComposerRead return pins.map(packageAnnotatePin) } +/** Below this a flat list is easier to read than a set of headed sections. */ +const GROUP_THRESHOLD = 4 + +function groupHeading(group: AnnotateGroup, index: number): string { + const what = group.label ? `\`${group.label}\`` : 'Unanchored (dragged areas)' + + return `Group ${index + 1} — ${what} (${group.items.length} comment${group.items.length === 1 ? '' : 's'})` +} + +/** + * How to work a batch this size. + * + * Two things the model gets wrong when handed a long flat list: it makes one + * task per comment and grinds through them serially, and — told to parallelize + * — it splits by theme, which puts several workers in the same component. So + * say both. The groups below are structural (disjoint DOM subtrees, so usually + * disjoint files), which is what makes handing them out concurrently safe; + * "all the styling ones" is not. + * + * It stays advice, not instruction: the model can see whether these comments + * are really one refactor, and a grouping computed from selectors cannot. + */ +function batchGuidance(groupCount: number, total: number): string { + return [ + `These ${total} comments are pre-grouped by where they sit in the page — each group is a different part of the DOM, so the groups should touch mostly separate files.`, + `Work them as ${groupCount} pieces of work, not ${total}. Fold comments in the same group into one change.`, + 'If you delegate, delegate whole groups — never split one group across workers, and never form new groups by theme (all the spacing ones, all the copy ones): those cut across the same files and the workers will collide.', + 'Regroup if the code disagrees with this split — it is derived from the page structure, not from your source layout.' + ].join(' ') +} + export function annotateFlushPrompt(items: readonly ComposerReadyAnnotation[], pageUrl?: string): string { const where = pageUrl ? ` on ${pageUrl}` : '' const count = items.length - const header = - count === 1 - ? `I left a comment${where} in the in-app browser. Address it and keep the scope narrow.` - : `I left ${count} comments${where} in the in-app browser. Address them and keep the scope narrow.` + if (count === 1) { + return [ + `I left a comment${where} in the in-app browser. Address it and keep the scope narrow.`, + '', + ...items.map(item => item.prompt) + ].join('\n') + } - return [header, '', ...items.map(item => item.prompt)].join('\n') + const groups = groupAnnotations(items) + + if (count < GROUP_THRESHOLD || groups.length < 2) { + return [ + `I left ${count} comments${where} in the in-app browser. Address them and keep the scope narrow.`, + '', + ...items.map(item => item.prompt) + ].join('\n') + } + + const sections = groups.flatMap((group, index) => [ + groupHeading(group, index), + ...group.items.map(item => item.prompt), + '' + ]) + + return [ + `I left ${count} comments${where} in the in-app browser. Address them and keep the scope narrow.`, + batchGuidance(groups.length, count), + '', + ...sections + ] + .join('\n') + .trimEnd() } export function dataUrlToBlob(dataUrl: string): Blob { diff --git a/apps/desktop/src/lib/preview-annotate/stack.test.ts b/apps/desktop/src/lib/preview-annotate/stack.test.ts index e4edb80697..da1da95d3b 100644 --- a/apps/desktop/src/lib/preview-annotate/stack.test.ts +++ b/apps/desktop/src/lib/preview-annotate/stack.test.ts @@ -25,7 +25,13 @@ function draft(note: string, kind: 'area' | 'element' = 'element'): AnnotatePinD rect: { height: 24, width: 80, x: 10, y: 12 }, identity: kind === 'element' - ? { css: { 'font-size': '14px' }, selector: 'button.go', tag: 'button', text: 'Go' } + ? { + css: { 'font-size': '14px' }, + html: '', + selector: 'button.go', + tag: 'button', + text: 'Go' + } : undefined } } diff --git a/apps/desktop/src/lib/preview-annotate/stack.ts b/apps/desktop/src/lib/preview-annotate/stack.ts index 2c0a2ea152..f25c1faefd 100644 --- a/apps/desktop/src/lib/preview-annotate/stack.ts +++ b/apps/desktop/src/lib/preview-annotate/stack.ts @@ -1,3 +1,5 @@ +import type { CompactIdentity } from './identity' + /** * Numbered pin stack for comment mode. Saving a pin only appends — it never * sends a turn. Numbers are assigned 1..N in add order and stay put if a pin @@ -13,12 +15,8 @@ export interface AnnotateRect { y: number } -export interface AnnotateIdentity { - css: Record - selector: string - tag: string - text: string -} +/** What a pin knows about its element. One shape, owned by `identity`. */ +export type AnnotateIdentity = CompactIdentity export interface AnnotatePin { id: string diff --git a/apps/desktop/src/lib/preview-annotate/tokens.ts b/apps/desktop/src/lib/preview-annotate/tokens.ts index cafbbbb427..73f255deb1 100644 --- a/apps/desktop/src/lib/preview-annotate/tokens.ts +++ b/apps/desktop/src/lib/preview-annotate/tokens.ts @@ -34,14 +34,28 @@ export const ANNOTATE_CSS_KEYS = [ 'position', 'width', 'height', + 'max-width', 'padding', 'margin', + 'border', 'border-radius', + 'box-shadow', 'opacity', + 'overflow', + 'z-index', + 'transform', 'flex-direction', 'gap', + 'grid-template-columns', 'justify-content', 'align-items' ] as const export type AnnotateCssKey = (typeof ANNOTATE_CSS_KEYS)[number] + +/** + * Markup budget for one comment. Enough for the opening tag plus a couple of + * levels of children — the part an agent greps a component out of — without + * pasting a whole section into the composer. + */ +export const ANNOTATE_HTML_BUDGET = 600 diff --git a/apps/desktop/src/lib/statusbar.test.ts b/apps/desktop/src/lib/statusbar.test.ts new file mode 100644 index 0000000000..f4bbb5d995 --- /dev/null +++ b/apps/desktop/src/lib/statusbar.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' + +import { cacheHitLabel, tokensPerSecondLabel } from '@/lib/statusbar' + +const base = { calls: 0, input: 0, output: 0, total: 0 } + +describe('statusbar usage readouts', () => { + it('paints the backend cache-hit and throughput fields, and stays blank when they are absent', () => { + // The backend omits both fields (rather than sending 0) when it has no data + // — a provider with no cache reads, or a session before its first call. + expect(cacheHitLabel(base)).toBe('') + expect(tokensPerSecondLabel(base)).toBe('') + + expect(cacheHitLabel({ ...base, cache_hit_pct: 87 })).toBe('87%') + expect(tokensPerSecondLabel({ ...base, avg_tps: 41.6 })).toBe('42 t/s') + }) +}) diff --git a/apps/desktop/src/lib/statusbar.tsx b/apps/desktop/src/lib/statusbar.tsx index 01ca3b645a..5b9670eef7 100644 --- a/apps/desktop/src/lib/statusbar.tsx +++ b/apps/desktop/src/lib/statusbar.tsx @@ -59,6 +59,22 @@ export function contextBarLabel(usage: UsageStats): string { return `[${contextBar(usage.context_percent)}] ${pct}%` } +/** `87%` for a reported hit rate; '' when the backend omitted it (no cache + * reads yet, or a provider that doesn't report them). The backend already + * clamps and rounds, so this only guards a malformed/absent field. */ +export function cacheHitLabel(usage: UsageStats): string { + const pct = usage.cache_hit_pct + + return typeof pct === 'number' && Number.isFinite(pct) ? `${Math.round(pct)}%` : '' +} + +/** `42 t/s` for the rolling throughput; '' before the first completed call. */ +export function tokensPerSecondLabel(usage: UsageStats): string { + const tps = usage.avg_tps + + return typeof tps === 'number' && Number.isFinite(tps) && tps > 0 ? `${Math.round(tps)} t/s` : '' +} + export function LiveDuration({ since }: { since: number | null | undefined }) { const [now, setNow] = useState(() => Date.now()) diff --git a/apps/desktop/src/lib/tips/catalog.ts b/apps/desktop/src/lib/tips/catalog.ts index 69d3f434ec..157968c85e 100644 --- a/apps/desktop/src/lib/tips/catalog.ts +++ b/apps/desktop/src/lib/tips/catalog.ts @@ -35,7 +35,6 @@ export type TipId = | 'composer-mentions' | 'cron' | 'messaging' - | 'model-switch' | 'new-session' | 'profiles' | 'right-pane' @@ -53,6 +52,5 @@ export const TIP_CATALOG: readonly TipDef[] = [ { id: 'command-palette', keybind: 'nav.commandPalette', side: 'right', targets: ['[data-tour="sessions-sidebar"]'] }, { id: 'profiles', keybind: 'profile.next', side: 'right', targets: ['[data-tour="profile-rail"]'] }, { id: 'composer-mentions', side: 'top', targets: ['[data-tour="composer"]'] }, - { id: 'model-switch', keybind: 'composer.modelPicker', side: 'top', targets: ['[data-tour="model-pill"]'] }, { id: 'right-pane', keybind: 'view.toggleRightSidebar', side: 'bottom', targets: ['[data-tour="right-pane-toggle"]'] } ] diff --git a/apps/desktop/src/lib/tts-lease.test.ts b/apps/desktop/src/lib/tts-lease.test.ts new file mode 100644 index 0000000000..166c54ac9e --- /dev/null +++ b/apps/desktop/src/lib/tts-lease.test.ts @@ -0,0 +1,99 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const setTtsLease = vi.fn(async (_lease: string, _active: boolean) => ({ ok: true })) + +vi.mock('@/hermes', () => ({ + setTtsLease: (lease: string, active: boolean) => setTtsLease(lease, active) +})) + +import { CONVERSATION_LEASE, READ_ALOUD_LEASE, resetTtsLeasesForTests, syncTtsLease } from './tts-lease' + +describe('syncTtsLease', () => { + beforeEach(() => { + resetTtsLeasesForTests() + setTtsLease.mockReset() + setTtsLease.mockImplementation(async () => ({ ok: true })) + }) + + afterEach(() => { + resetTtsLeasesForTests() + }) + + it('acquires on the first on and releases on off', async () => { + await syncTtsLease(READ_ALOUD_LEASE, true) + await syncTtsLease(READ_ALOUD_LEASE, false) + + expect(setTtsLease.mock.calls).toEqual([ + [READ_ALOUD_LEASE, true], + [READ_ALOUD_LEASE, false] + ]) + }) + + it('skips an initial off — never releases a lease it did not hold', async () => { + await syncTtsLease(CONVERSATION_LEASE, false) + + expect(setTtsLease).not.toHaveBeenCalled() + }) + + it('dedupes a repeat of the last sent state', async () => { + await syncTtsLease(READ_ALOUD_LEASE, true) + await syncTtsLease(READ_ALOUD_LEASE, true) + await syncTtsLease(READ_ALOUD_LEASE, true) + + expect(setTtsLease).toHaveBeenCalledTimes(1) + }) + + it('queues an off behind an in-flight on so the wire never sees them reordered', async () => { + let finishAcquire: () => void = () => undefined + setTtsLease.mockImplementationOnce( + () => + new Promise(resolve => { + finishAcquire = () => resolve({ ok: true }) + }) + ) + + const on = syncTtsLease(CONVERSATION_LEASE, true) + // Let the acquire actually go out (it runs on a microtask). + await Promise.resolve() + expect(setTtsLease.mock.calls).toEqual([[CONVERSATION_LEASE, true]]) + + const off = syncTtsLease(CONVERSATION_LEASE, false) + await Promise.resolve() + // Still only the acquire — the release waits for it to finish. + expect(setTtsLease).toHaveBeenCalledTimes(1) + + finishAcquire() + await Promise.all([on, off]) + + expect(setTtsLease.mock.calls).toEqual([ + [CONVERSATION_LEASE, true], + [CONVERSATION_LEASE, false] + ]) + }) + + it('coalesces a flip that reverses before its call went out — latest intent wins', async () => { + const on = syncTtsLease(CONVERSATION_LEASE, true) + const off = syncTtsLease(CONVERSATION_LEASE, false) + await Promise.all([on, off]) + + // The acquire never had a chance to go out; only the terminal state is sent + // (a release of a never-held lease is a backend no-op). + expect(setTtsLease.mock.calls).toEqual([[CONVERSATION_LEASE, false]]) + }) + + it('forgets the sent state on failure so the next flip retries', async () => { + setTtsLease.mockImplementationOnce(async () => { + throw new Error('backend not ready') + }) + + await expect(syncTtsLease(READ_ALOUD_LEASE, true)).resolves.toBeUndefined() + await syncTtsLease(READ_ALOUD_LEASE, true) + + expect(setTtsLease).toHaveBeenCalledTimes(2) + }) + + it('conversation lease is per renderer, read-aloud lease is shared', () => { + expect(CONVERSATION_LEASE).toMatch(/^desktop:conversation:[a-z0-9]+$/) + expect(READ_ALOUD_LEASE).toBe('desktop:read-aloud') + }) +}) diff --git a/apps/desktop/src/lib/tts-lease.ts b/apps/desktop/src/lib/tts-lease.ts new file mode 100644 index 0000000000..987d2765e1 --- /dev/null +++ b/apps/desktop/src/lib/tts-lease.ts @@ -0,0 +1,76 @@ +import { setTtsLease } from '@/hermes' + +// The desktop's speech-output toggles — "Read replies aloud" and voice +// conversation mode — are the user telling us TTS is about to be needed (or no +// longer is). The backend turns that into engine lifecycle: acquiring a lease +// pre-loads the configured provider (a local piper/kittentts model, a lazily +// installed SDK) so the first spoken reply starts hot instead of paying the load +// as dead air; releasing the last lease unloads resident local models. +// +// This module is the renderer's single choke point for that signal. It dedupes +// (several composers/tiles observe the same toggle), serializes per lease so a +// fast on→off→on can't be reordered on the wire, and never surfaces failures — +// warm-up is an optimization; the toggle itself must not depend on it. + +// Per-renderer id so two windows in conversation mode hold DISTINCT leases — +// window A ending its conversation must not release the engine window B is +// still speaking through. Read-aloud mirrors one config key shared by every +// window, so it deliberately uses one shared lease name. +const RENDERER_ID = Math.random().toString(36).slice(2, 10) + +export const READ_ALOUD_LEASE = 'desktop:read-aloud' +export const CONVERSATION_LEASE = `desktop:conversation:${RENDERER_ID}` + +const sent = new Map() +const inFlight = new Map>() + +/** + * Bring the backend's view of `lease` in line with `active`. Idempotent: a + * repeat of the last sent state is a no-op. The initial `false` (nothing was + * ever acquired) is also skipped — releasing a lease we never held would only + * churn the backend on app start. + */ +export function syncTtsLease(lease: string, active: boolean): Promise { + const last = sent.get(lease) + + if (last === active || (last === undefined && !active)) { + return inFlight.get(lease) ?? Promise.resolve() + } + + sent.set(lease, active) + + const previous = inFlight.get(lease) ?? Promise.resolve() + + const next = previous + .then(async () => { + // Latest intent wins: if the toggle flipped again while we were queued, + // the newer call sends its own state and this one has nothing to say. + if (sent.get(lease) !== active) { + return + } + + await setTtsLease(lease, active) + }) + .catch(() => { + // Backend not up yet / older backend without the endpoint / warm-up + // failure: forget what we "sent" so the next flip retries honestly. + if (sent.get(lease) === active) { + sent.delete(lease) + } + }) + .finally(() => { + if (inFlight.get(lease) === next) { + inFlight.delete(lease) + } + }) + + inFlight.set(lease, next) + + return next +} + +/** Test seam — forget every sent state. */ +export function resetTtsLeasesForTests() { + sent.clear() + inFlight.clear() +} diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row-keeps-closed-chat.test.ts b/apps/desktop/src/plugins/hermes-bots/bot-row-keeps-closed-chat.test.ts deleted file mode 100644 index a9928cb04a..0000000000 --- a/apps/desktop/src/plugins/hermes-bots/bot-row-keeps-closed-chat.test.ts +++ /dev/null @@ -1,291 +0,0 @@ -/** - * A bot row click is "go to this bot", not "open its Bot Chat". Before this, - * every click resolved the canonical chat by name and opened it as a tab — and - * with no record of a close anywhere (this plugin keeps no closed set; core's - * tile bucket only forgets), a Bot Chat the user closed came back beside every - * newer thread on every bot switch. Now a bot whose workspace already holds - * tabs comes back to the one the user left; the forever-chat is opened only - * when nothing is open, or on the explicit ask (the row menu's "Open Bot Chat"). - * - * Ported from tests/bot-row-keeps-closed-chat.test.mjs, which drove a `vm` - * copy of plugin.js. Its two source-reading cases are dropped for real - * assertions: the menu's call site is now a render in bot-row.test.tsx, and - * the reclaim guard's text is asserted here as the claim-shape invariant the - * guard actually reads. - */ - -import { beforeEach, describe, expect, it, vi } from 'vitest' - -import type { RosterRow } from './types' - -const { openBotCanonicalChat, prepareBotSource } = vi.hoisted(() => ({ - openBotCanonicalChat: vi.fn(), - prepareBotSource: vi.fn() -})) - -vi.mock('./canonical-chat', () => ({ - CANONICAL_CHAT_TITLE: 'Bot Chat', - ensureBotMetadata: vi.fn(async () => ({})), - notifyBotOpenFailure: vi.fn(), - openBotCanonicalChat, - prepareBotSource, - PROFILE_SESSION_LIST_LIMIT: 200 -})) - -const { host } = await import('@hermes/plugin-sdk') -const { $openBotChat, $selectedBot } = await import('./bot-state') -const { openRosterBot } = await import('./roster-actions') - -const bot = { connectionId: 'local', name: 'alpha' } as RosterRow - -/** Swap in a focus API for one test, restoring whatever the SDK really has — - * including its absence, which is the older-shell case. */ -function withFocusApi(impl: null | (() => null | string)) { - const had = Object.hasOwn(host, 'focusOpenWorkspaceSession') - const original = host.focusOpenWorkspaceSession - - if (impl) { - host.focusOpenWorkspaceSession = impl - } else { - // @ts-expect-error — modelling a Desktop old enough to lack the verb. - delete host.focusOpenWorkspaceSession - } - - return () => { - if (had) { - host.focusOpenWorkspaceSession = original - } else { - // @ts-expect-error — same. - delete host.focusOpenWorkspaceSession - } - } -} - -beforeEach(() => { - vi.clearAllMocks() - prepareBotSource.mockResolvedValue(undefined) - openBotCanonicalChat.mockResolvedValue({ openedId: 'bot-chat', registryId: 'bot-chat' }) - $openBotChat.set(null) - $selectedBot.set('') -}) - -describe('a row click returns to the tabs the bot already has open', () => { - it('fronts the remembered tab and resolves no canonical chat', async () => { - const focus = vi.fn(() => 'thread-2') - const restore = withFocusApi(focus) - - try { - await expect(openRosterBot(bot)).resolves.toBe(true) - - expect(focus).toHaveBeenCalledWith('bot:alpha', expect.any(Function)) - expect(openBotCanonicalChat).not.toHaveBeenCalled() - // Open tabs need no source activation either — the bot is already live. - expect(prepareBotSource).not.toHaveBeenCalled() - } finally { - restore() - } - }) - - it('claims only the fronted tab, with no registry id', async () => { - const restore = withFocusApi(() => 'thread-2') - - try { - await openRosterBot(bot) - - expect($openBotChat.get()).toEqual({ - key: 'local::alpha', - openedRegistryId: '', - openedSessionId: 'thread-2' - }) - } finally { - restore() - } - }) -}) - -describe('the canonical chat still opens when it is what was asked for', () => { - it('opens it when the bot has nothing open', async () => { - const restore = withFocusApi(() => null) - - try { - await expect(openRosterBot(bot)).resolves.toBe(true) - - expect(openBotCanonicalChat).toHaveBeenCalled() - expect($openBotChat.get()?.openedRegistryId).toBe('bot-chat') - } finally { - restore() - } - }) - - it('skips the open-tab shortcut on the explicit ask', async () => { - const focus = vi.fn(() => 'thread-2') - const restore = withFocusApi(focus) - - try { - await expect(openRosterBot(bot, { canonical: true })).resolves.toBe(true) - - expect(focus).not.toHaveBeenCalled() - expect($openBotChat.get()?.openedRegistryId).toBe('bot-chat') - } finally { - restore() - } - }) -}) - -describe('the fronted-tab shortcut reconciles with the canonical registry (#90102)', () => { - // The stuck shape: a persisted "Bot Chat" tile names a session the - // registry no longer resolves to (superseded pointer-era row, re-minted - // canonical chat, stale finished session). The roster click must judge - // that tile against the server-resolved canonical_session and fall - // through to the authoritative registry open instead of fronting it. - const staleBot = { - connectionId: 'local', - name: 'alpha', - canonical_session: { id: 'bot-chat', resolved_id: 'bot-chat-tip' } - } as RosterRow - - /** The probe openRosterBot hands the focus verb, captured. */ - function captureProbe() { - let probe: ((tile: { storedSessionId: string; workspaceTabTitle?: string }) => boolean) | undefined - - const focus = vi.fn((_key: string, isStaleTile?: typeof probe) => { - probe = isStaleTile - - return null - }) - - return { focus, probe: () => probe } - } - - it('classifies a canonical-titled tile at a foreign id as stale', async () => { - const { focus, probe } = captureProbe() - const restore = withFocusApi(focus as unknown as () => null | string) - - try { - await openRosterBot(staleBot) - - const isStale = probe()! - expect(isStale({ storedSessionId: 'old-finished-session', workspaceTabTitle: 'Bot Chat' })).toBe(true) - } finally { - restore() - } - }) - - it('keeps the tile that matches the registry row or its lineage tip', async () => { - const { focus, probe } = captureProbe() - const restore = withFocusApi(focus as unknown as () => null | string) - - try { - await openRosterBot(staleBot) - - const isStale = probe()! - expect(isStale({ storedSessionId: 'bot-chat', workspaceTabTitle: 'Bot Chat' })).toBe(false) - expect(isStale({ storedSessionId: 'bot-chat-tip', workspaceTabTitle: 'Bot Chat' })).toBe(false) - } finally { - restore() - } - }) - - it('never judges side-chat tabs — only canonical-titled tiles carry registry identity', async () => { - const { focus, probe } = captureProbe() - const restore = withFocusApi(focus as unknown as () => null | string) - - try { - await openRosterBot(staleBot) - - const isStale = probe()! - expect(isStale({ storedSessionId: 'scratch-thread', workspaceTabTitle: 'Group: writers' })).toBe(false) - expect(isStale({ storedSessionId: 'scratch-thread' })).toBe(false) - } finally { - restore() - } - }) - - it('an older gateway without canonical_session cannot judge — every tile survives', async () => { - const { focus, probe } = captureProbe() - const restore = withFocusApi(focus as unknown as () => null | string) - - try { - await openRosterBot(bot) // no canonical_session on this row - - const isStale = probe()! - expect(isStale({ storedSessionId: 'anything', workspaceTabTitle: 'Bot Chat' })).toBe(false) - } finally { - restore() - } - }) - - it('falls through to the authoritative canonical open when the stale tile was the only tab', async () => { - // The store discards the stale tile and reports null; the click must - // then resolve the registry — the backend-truth path — not give up. - const restore = withFocusApi(() => null) - - try { - await expect(openRosterBot(staleBot)).resolves.toBe(true) - - expect(openBotCanonicalChat).toHaveBeenCalled() - expect($openBotChat.get()?.openedRegistryId).toBe('bot-chat') - } finally { - restore() - } - }) -}) - -describe('a shell that cannot report open tabs behaves as it did before', () => { - it('opens the canonical chat when the verb is missing', async () => { - const restore = withFocusApi(null) - - try { - await expect(openRosterBot(bot)).resolves.toBe(true) - - expect(openBotCanonicalChat).toHaveBeenCalled() - } finally { - restore() - } - }) - - it('opens the canonical chat when the verb throws', async () => { - const restore = withFocusApi(() => { - throw new Error('no tree yet') - }) - - try { - await expect(openRosterBot(bot)).resolves.toBe(true) - - expect(openBotCanonicalChat).toHaveBeenCalled() - } finally { - restore() - } - }) -}) - -describe('the claim a fronted tab records cannot resurrect the closed chat', () => { - // The reclaim listener re-resolves the canonical chat for a claim it owns, - // and guards on the registry id to avoid doing so for a fronted tab. That - // guard is only correct because a fronted-tab claim leaves the id empty - // while a real canonical open always fills it — the invariant asserted here. - it('leaves the registry id empty for a fronted tab', async () => { - const restore = withFocusApi(() => 'thread-2') - - try { - await openRosterBot(bot) - - expect($openBotChat.get()?.openedRegistryId).toBe('') - expect($openBotChat.get()?.openedSessionId).toBeTruthy() - } finally { - restore() - } - }) - - it('fills the registry id for a real canonical open', async () => { - const restore = withFocusApi(() => null) - - try { - await openRosterBot(bot) - - expect($openBotChat.get()?.openedRegistryId).toBeTruthy() - } finally { - restore() - } - }) -}) diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row-opens-canonical-chat.test.ts b/apps/desktop/src/plugins/hermes-bots/bot-row-opens-canonical-chat.test.ts new file mode 100644 index 0000000000..a50c880280 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/bot-row-opens-canonical-chat.test.ts @@ -0,0 +1,146 @@ +/** + * A bot row click lands on the bot's canonical Bot Chat — the session the row + * previews (`canonical_session`, resolved by name on every roster poll). + * + * The regression this pins: a plain click used to front whatever + * bots-workspace tab the user last had open for that bot. A `+` side thread + * persisted in Local Storage across restarts and won every click forever while + * the row kept previewing the Bot Chat — sidebar and center described two + * different conversations ("[Bots] - Sessions is not in sync again"). + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { RosterRow } from './types' + +const { openBotCanonicalChat, prepareBotSource } = vi.hoisted(() => ({ + openBotCanonicalChat: vi.fn(), + prepareBotSource: vi.fn() +})) + +vi.mock('./canonical-chat', () => ({ + CANONICAL_CHAT_TITLE: 'Bot Chat', + ensureBotMetadata: vi.fn(async () => ({})), + notifyBotOpenFailure: vi.fn(), + openBotCanonicalChat, + prepareBotSource, + PROFILE_SESSION_LIST_LIMIT: 200 +})) + +const { host } = await import('@hermes/plugin-sdk') +const { $openBotChat, $selectedBot } = await import('./bot-state') +const { openRosterBot, trackInboundActivity } = await import('./roster-actions') +const { $selectedStoredSessionId } = await import('@/store/session') + +const bot = { connectionId: 'local', name: 'alpha' } as RosterRow + +beforeEach(() => { + vi.clearAllMocks() + prepareBotSource.mockResolvedValue(undefined) + openBotCanonicalChat.mockResolvedValue({ openedId: 'bot-chat-tip', registryId: 'bot-chat' }) + $openBotChat.set(null) + $selectedBot.set('') +}) + +describe('a row click lands on the canonical chat, never a remembered side tab', () => { + const canonicalBot = { + ...bot, + canonical_session: { id: 'bot-chat', resolved_id: 'bot-chat-tip' } + } as RosterRow + + afterEach(() => { + // @ts-expect-error — restore the harness default (no focus verb). + delete host.focusOpenWorkspaceSession + }) + + it('fronts an open Bot Chat tab without a registry round-trip, side tabs excluded', async () => { + const focus = vi.fn((_key: string, _probe: unknown, only?: readonly string[]) => + only?.includes('bot-chat-tip') ? 'bot-chat-tip' : null + ) + + host.focusOpenWorkspaceSession = focus as never + + await expect(openRosterBot(canonicalBot)).resolves.toBe(true) + + expect(focus).toHaveBeenCalledWith('bot:alpha', expect.any(Function), ['bot-chat', 'bot-chat-tip']) + expect(openBotCanonicalChat).not.toHaveBeenCalled() + expect($openBotChat.get()).toEqual({ + key: 'local::alpha', + openedRegistryId: 'bot-chat', + openedSessionId: 'bot-chat-tip' + }) + }) + + it('fronting an already-open Bot Chat refreshes its transcript in place', async () => { + // The front is presentation-only: the pane keeps whatever transcript it + // last painted, which can predate rows the bot wrote while the user was + // elsewhere (a cron delivery, a teammate's message_agent, another bot's + // turn). Fronting must force a registry open so forceResume re-pulls the + // latest rows instead of leaving a stale snapshot until the next turn + // (#99393 class; #95600 only covered the not-yet-open path). + host.focusOpenWorkspaceSession = vi.fn((_key: string, _probe: unknown, only?: readonly string[]) => + only?.includes('bot-chat-tip') ? 'bot-chat-tip' : null + ) as never + $selectedStoredSessionId.set('bot-chat-tip') + + await expect(openRosterBot(canonicalBot)).resolves.toBe(true) + + expect(openBotCanonicalChat).toHaveBeenCalledWith(canonicalBot, expect.any(Function)) + $selectedStoredSessionId.set(null) + }) + + it('resolves the registry when only a side thread is open', async () => { + // The shell would happily front 'side-thread' — the allowlist excludes it. + host.focusOpenWorkspaceSession = vi.fn((_key: string, _probe: unknown, only?: readonly string[]) => + only?.includes('side-thread') ? 'side-thread' : null + ) as never + + await expect(openRosterBot(canonicalBot)).resolves.toBe(true) + + expect(openBotCanonicalChat).toHaveBeenCalledWith(canonicalBot, expect.any(Function)) + expect($openBotChat.get()?.openedSessionId).toBe('bot-chat-tip') + }) + + it('a failed open records no claim', async () => { + openBotCanonicalChat.mockRejectedValueOnce(new Error('gateway away')) + + await expect(openRosterBot(bot)).resolves.toBe(false) + + expect($openBotChat.get()).toBeNull() + }) +}) + +describe('the open Bot Chat follows its session on the gateway', () => { + // The roster poll is the only signal for turns that never reach this + // window's stream (cron bot-chat deliveries, message_agent, group rounds). + // When the FOCUSED chat's canonical session moves, it re-resolves so the pane + // repaints from the gateway instead of waiting for a restart (#99393). + const activeBot = (lastActive: number) => + ({ + connectionId: 'local', + name: 'alpha', + canonical_session: { id: 'bot-chat', resolved_id: 'bot-chat-tip', last_active: lastActive } + }) as RosterRow + + it('re-opens the focused Bot Chat when its canonical session advances', () => { + $selectedBot.set('alpha') + $selectedStoredSessionId.set('bot-chat-tip') + trackInboundActivity([activeBot(100)]) // seeds the watermark + + trackInboundActivity([activeBot(200)]) + + expect(openBotCanonicalChat).toHaveBeenCalledTimes(1) + $selectedStoredSessionId.set(null) + }) + + it('leaves the center alone when the Bot Chat is not what is focused', () => { + $selectedBot.set('alpha') + $selectedStoredSessionId.set('some-group-room') + trackInboundActivity([activeBot(300)]) + + trackInboundActivity([activeBot(400)]) + + expect(openBotCanonicalChat).not.toHaveBeenCalled() + $selectedStoredSessionId.set(null) + }) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row.test.tsx b/apps/desktop/src/plugins/hermes-bots/bot-row.test.tsx index ce51a8fe97..55dcc8e2df 100644 --- a/apps/desktop/src/plugins/hermes-bots/bot-row.test.tsx +++ b/apps/desktop/src/plugins/hermes-bots/bot-row.test.tsx @@ -59,7 +59,7 @@ vi.mock('./roster-actions', () => ({ openRosterBot })) const noop = () => undefined function renderRow(bot: RosterRow) { - render() + render() return screen.getByRole('button') } @@ -132,14 +132,14 @@ describe('the row delegates the open and claims no activation authority', () => }) }) -describe('the menu carries the explicit ask for the forever-chat', () => { - it('opens the canonical chat, which a plain row click deliberately does not', async () => { +describe('the menu opens the same forever-chat a row click does', () => { + it('opens the canonical chat', async () => { const bot = { name: 'alpha' } as RosterRow fireEvent.contextMenu(renderRow(bot)) fireEvent.click(await screen.findByText('Open Bot Chat')) - expect(openRosterBot.mock.calls).toEqual([[bot, { canonical: true }]]) + expect(openRosterBot.mock.calls).toEqual([[bot]]) }) }) diff --git a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx index dad5305d9b..48f60acc4d 100644 --- a/apps/desktop/src/plugins/hermes-bots/bot-row.tsx +++ b/apps/desktop/src/plugins/hermes-bots/bot-row.tsx @@ -14,6 +14,9 @@ import { ContextMenuContent, ContextMenuItem, ContextMenuSeparator, + ContextMenuSub, + ContextMenuSubContent, + ContextMenuSubTrigger, ContextMenuTrigger, haptic, host, @@ -62,6 +65,7 @@ import { openRosterBot } from './roster-actions' import { botRosterMeta, botWorkspaceOwnerKey, setBotsWorkspaceOwner } from './routing' import { A2A_PREFIX_RE, botCanonicalSessionId, botRowOwnsWorkspace, previewKind, workerActiveAt } from './row-helpers' import type { GroupMember, RosterRow, SidebarRowLabels } from './types' +import { $botSections, $draggingBot, BOT_DRAG_MIME, botSectionId, moveBotsToSection } from './user-sections' // ── bot row ────────────────────────────────────────────────────────────────── @@ -81,10 +85,12 @@ interface BotRowProps { onDelete: (bot: RosterRow) => void onEdit: (bot: RosterRow) => void onGroup: (bot: RosterRow) => void + /** Opens the New section dialog; the bot is filed into it on create. */ + onNewSection: (bot: RosterRow) => void showHandle?: boolean } -export function BotRow({ bot, onDelete, onEdit, onGroup, showHandle }: BotRowProps) { +export function BotRow({ bot, onDelete, onEdit, onGroup, onNewSection, showHandle }: BotRowProps) { const { t } = useI18n() const b = useBots() const activeProfile = useValue(host.state.profile) @@ -207,15 +213,35 @@ export function BotRow({ bot, onDelete, onEdit, onGroup, showHandle }: BotRowPro // activate a source and resolve the canonical Bot Chat. const open = () => void openRosterBot(bot) + // DRAG lives on the row button itself: it already takes pointer events, so + // the click that opens the bot and the drag that files it are one element's + // gestures. The drag carries the roster key under a private MIME type, so + // only a section block can accept it. + const rosterKey = botRosterKey(bot) + const sections = useValue($botSections) + const dragging = useValue($draggingBot) === rosterKey + const currentSectionId = botSectionId(bot, allMeta) + const row = ( $draggingBot.set(null)} + onDragStart={event => { + event.dataTransfer.setData(BOT_DRAG_MIME, rosterKey) + event.dataTransfer.effectAllowed = 'move' + $draggingBot.set(rosterKey) + }} onPointerEnter={warm} >
@@ -284,9 +310,7 @@ export function BotRow({ bot, onDelete, onEdit, onGroup, showHandle }: BotRowPro {row} - void openRosterBot(bot, { canonical: true })}> - {b.bot.openBotChat} - + void openRosterBot(bot)}>{b.bot.openBotChat} { @@ -381,6 +405,36 @@ export function BotRow({ bot, onDelete, onEdit, onGroup, showHandle }: BotRowPro > {b.bot.newChatWith} + + {/* Filing. Membership is one field on the bot's meta (`sectionId`), so + this is a one-field write and no list anywhere has to be kept in + sync with it. */} + + {b.sections.moveTo} + + {sections.map(section => ( + void moveBotsToSection([bot], section.id)} + > + + {section.name} + + ))} + {sections.length ? : null} + onNewSection(bot)}> + + {b.sections.newSectionEllipsis} + + {currentSectionId ? ( + void moveBotsToSection([bot], null)}> + + {b.sections.removeFromSection} + + ) : null} + + {isDefaultBot(bot) ? null : } {isDefaultBot(bot) ? null : ( onDelete(bot)} variant="destructive"> diff --git a/apps/desktop/src/plugins/hermes-bots/canonical-chat-registry.test.ts b/apps/desktop/src/plugins/hermes-bots/canonical-chat-registry.test.ts index 03e34f8e6c..791b710dd3 100644 --- a/apps/desktop/src/plugins/hermes-bots/canonical-chat-registry.test.ts +++ b/apps/desktop/src/plugins/hermes-bots/canonical-chat-registry.test.ts @@ -280,4 +280,29 @@ describe('a failed lookup fails CLOSED — never "no chat exists"', () => { await expect(createCanonicalChat('ops')).rejects.toThrow(/Bot Chat registry/) expect(calls.some(call => call.method === 'session.create')).toBe(false) }) + + // #98383: a profile backend mid-restart can answer `session.list` + // SUCCESSFULLY with an empty list instead of throwing. `rows.find(...) || + // null` used to read that identically to "this bot never had a chat", + // which minted a replacement and re-fired the kickoff on every click. + it('refuses to mint on an empty lookup when the roster already confirmed a canonical chat', async () => { + const calls = respondWith(method => { + if (method === 'session.list') { + return { sessions: [] } + } + + if (method === 'session.create') { + throw new Error('must not create: an empty result is not confirmed absence') + } + + return {} + }) + + const bot = { canonical_session: { id: 'forever-chat' }, name: 'ops' } as RosterRow + const { openBotCanonicalChat } = await loadModule() + + await expect(openBotCanonicalChat(bot)).rejects.toThrow(/Bot Chat registry/) + expect(calls.some(call => call.method === 'session.create')).toBe(false) + expect(hostMock.openSession).not.toHaveBeenCalled() + }) }) diff --git a/apps/desktop/src/plugins/hermes-bots/canonical-chat.ts b/apps/desktop/src/plugins/hermes-bots/canonical-chat.ts index abac5a0c0e..f25c72963b 100644 --- a/apps/desktop/src/plugins/hermes-bots/canonical-chat.ts +++ b/apps/desktop/src/plugins/hermes-bots/canonical-chat.ts @@ -213,8 +213,25 @@ async function findExistingCanonicalChat(owner: RosterRow | string): Promise isCanonicalBotChatHistory(row)) - return rows.find(row => isCanonicalBotChatHistory(row)) || null + if (match) { + return match + } + + // A zero-row result is NOT the same as a thrown error, but it is just as + // capable of forking the forever chat: a profile backend mid-restart can + // answer `session.list` successfully with an empty list rather than + // failing it, and `|| null` used to read that identically to "this bot + // never had a chat" (#98383). The roster's own `canonical_session` is the + // last positive confirmation this profile HAD one — when that exists, + // an empty lookup is unconfirmed absence, not confirmed absence, so fail + // closed the same way a thrown RPC error already does instead of minting. + if (bot?.canonical_session?.id) { + throw new Error(`Could not confirm ${name}'s Bot Chat registry — not starting a new chat`) + } + + return null } interface CreateCanonicalChatOptions { diff --git a/apps/desktop/src/plugins/hermes-bots/create-dialog.tsx b/apps/desktop/src/plugins/hermes-bots/create-dialog.tsx index 79c339d749..27a23a1a8c 100644 --- a/apps/desktop/src/plugins/hermes-bots/create-dialog.tsx +++ b/apps/desktop/src/plugins/hermes-bots/create-dialog.tsx @@ -1271,7 +1271,7 @@ export function CreateGroupChatDialog({ open, roster, onClose, onCreated }: Crea return (
) @@ -641,6 +764,11 @@ export function BotsPane() { {b.group.newTitle} + + setSectionDialog({ mode: 'create' })}> + + {b.sections.newSection} +
@@ -808,14 +936,14 @@ export function BotsPane() { />
) : ( -
+
{showGatewaySections ? [ sortedGroupRows.length ? renderGroupChatSection() : null, ...gatewaySections.sections.map(renderGatewaySection) ].filter(Boolean) - : rosterRows.map(row => (row.kind === 'group' ? renderGroupRow(row) : renderBotRow(row.bot)))} + : renderUserSections(rosterRows)} {showHiddenSection ? (
+ { + if (!open) { + setSectionDialog(null) + } + }} + onSubmit={name => { + if (sectionDialog?.mode === 'rename') { + renameBotSection(sectionDialog.id, name) + } else { + createBotSection(name, sectionDialog?.bot ? [sectionDialog.bot] : []) + } + }} + open={Boolean(sectionDialog)} + /> { diff --git a/apps/desktop/src/plugins/hermes-bots/roster-sections.tsx b/apps/desktop/src/plugins/hermes-bots/roster-sections.tsx index dd4cd01378..fb1d1e3a6d 100644 --- a/apps/desktop/src/plugins/hermes-bots/roster-sections.tsx +++ b/apps/desktop/src/plugins/hermes-bots/roster-sections.tsx @@ -7,7 +7,8 @@ * without either half knowing about a bot row. */ -import { Codicon, ConnectionGlyph, DisclosureCaret, RowButton, Tip } from '@hermes/plugin-sdk' +import { cn, Codicon, ConnectionGlyph, DisclosureCaret, RowButton, Tip } from '@hermes/plugin-sdk' +import type { ReactNode } from 'react' import { botHandle, botRosterKey, botSourceStatus, filterBots } from './data' import { displayName } from './labels' @@ -222,22 +223,28 @@ export function GatewayKindGlyph({ className, kind }: GatewayKindGlyphProps) { /** Foldable roster heading. It organizes rows visually but never supplies or * reconstructs ownership; every action still receives the full bot row. */ interface RosterSectionHeaderProps { + /** Trailing control drawn beside the heading (outside its button — a + * button cannot nest a button). User sections put their ⋯ menu here. */ + action?: ReactNode collapsed: boolean count: number gatewayKind?: string icon?: string label: string + onDoubleClick?: () => void onToggle: () => void status?: { available: boolean; label: string } tip?: string } export function RosterSectionHeader({ + action, collapsed, count, gatewayKind, icon, label, + onDoubleClick, onToggle, status, tip @@ -245,8 +252,12 @@ export function RosterSectionHeader({ const button = ( {gatewayKind ? ( @@ -269,7 +280,18 @@ export function RosterSectionHeader({ ) - return tip ? {button} : button + const heading = tip ? {button} : button + + // With a trailing action, heading and action share one hover group so the + // action can reveal on hover of the whole row. + return action ? ( +
+ {heading} + {action} +
+ ) : ( + heading + ) } interface GatewaySectionHeadingProps { diff --git a/apps/desktop/src/plugins/hermes-bots/types.ts b/apps/desktop/src/plugins/hermes-bots/types.ts index e21298f544..7770c1b5d9 100644 --- a/apps/desktop/src/plugins/hermes-bots/types.ts +++ b/apps/desktop/src/plugins/hermes-bots/types.ts @@ -53,6 +53,12 @@ export interface SessionPreview { /** Per-bot presentation state, persisted in the profile's `ui_meta`. */ export interface BotMeta { + /** Which user-made section this bot is filed under (`user-sections.ts`). + * Membership lives on the BOT, not as a member list on the section: a bot + * can only be in one place, deleting a section cannot orphan anybody, and + * the assignment rides the same profile.yaml sync every other bot setting + * already uses — so sections follow the profile to another machine. */ + sectionId?: null | string color?: string /** Set when the user has customized the avatar, so defaults stop applying. */ custom?: boolean diff --git a/apps/desktop/src/plugins/hermes-bots/user-sections-ui.tsx b/apps/desktop/src/plugins/hermes-bots/user-sections-ui.tsx new file mode 100644 index 0000000000..609f1a7a97 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/user-sections-ui.tsx @@ -0,0 +1,344 @@ +/** + * The chrome for user sections: the foldable heading (the roster's own + * `RosterSectionHeader`, with a ⋯ menu and a right-click menu that drive the + * same actions), the name dialog used for both New section and Rename (the + * same shape the app's session rename uses), and the drop zone a section + * block sits in. The model is in `user-sections.ts`; nothing here holds state + * that outlives a dialog. + */ + +import { + Button, + cn, + Codicon, + ContextMenu, + ContextMenuContent, + ContextMenuItem, + ContextMenuSeparator, + ContextMenuTrigger, + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, + Input, + useI18n, + useValue +} from '@hermes/plugin-sdk' +import { type DragEvent, type ReactNode, useEffect, useRef, useState } from 'react' + +import { useBots } from './i18n' +import { RosterSectionHeader } from './roster-sections' +import { $draggingBot, BOT_DRAG_MIME } from './user-sections' + +// ── name dialog ────────────────────────────────────────────────────────────── + +interface SectionNameDialogProps { + /** Blank for New section, the current name for Rename. */ + initialName: string + mode: 'create' | 'rename' + onOpenChange: (open: boolean) => void + onSubmit: (name: string) => void + open: boolean +} + +/** One small dialog for both creating and renaming a section — the app renames + * sessions through the same Dialog + Input + Cancel/Save shape, so a section + * rename feels like every other rename. */ +export function SectionNameDialog({ initialName, mode, onOpenChange, onSubmit, open }: SectionNameDialogProps) { + const { t } = useI18n() + const b = useBots() + const [value, setValue] = useState(initialName) + const inputRef = useRef(null) + + useEffect(() => { + if (open) { + setValue(initialName) + window.setTimeout(() => inputRef.current?.select(), 0) + } + }, [initialName, open]) + + const submit = () => { + const next = value.trim() + + if (!next) { + return + } + + onOpenChange(false) + + if (mode === 'create' || next !== initialName.trim()) { + onSubmit(next) + } + } + + return ( + + + + {mode === 'create' ? b.sections.newTitle : b.sections.renameTitle} + + setValue(event.target.value)} + onKeyDown={event => { + if (event.key === 'Enter' && !event.nativeEvent.isComposing) { + event.preventDefault() + submit() + } + }} + placeholder={b.sections.namePlaceholder} + ref={inputRef} + value={value} + /> + + + + + + + ) +} + +// ── heading ────────────────────────────────────────────────────────────────── + +interface UserSectionHeaderProps { + canMoveDown: boolean + canMoveUp: boolean + collapsed: boolean + count: number + /** null for Unassigned, which has no record and therefore no menu. */ + id: null | string + name: string + onDelete: () => void + onMove: (delta: number) => void + onRename: () => void + onToggle: () => void +} + +export function UserSectionHeader({ + canMoveDown, + canMoveUp, + collapsed, + count, + id, + name, + onDelete, + onMove, + onRename, + onToggle +}: UserSectionHeaderProps) { + const b = useBots() + const { t } = useI18n() + + // Unassigned has no record to rename, reorder or delete — it is whatever is + // left over — so it gets the plain heading rather than a menu of disabled + // items. + if (!id) { + return ( + + ) + } + + // RIGHT-CLICK IS THE SAME MENU. The ⋯ button only appears on hover and is a + // small target; right-clicking the heading is what people actually try + // first. Both drive the identical actions, so neither can drift. + const items = [ + { icon: 'edit', label: b.sections.rename, onSelect: onRename }, + { disabled: !canMoveUp, icon: 'arrow-up', label: b.sections.moveUp, onSelect: () => onMove(-1) }, + { disabled: !canMoveDown, icon: 'arrow-down', label: b.sections.moveDown, onSelect: () => onMove(1) } + ] + + const action = ( + + + + + + {items.map(item => ( + + + {item.label} + + ))} + + + + {t.common.delete} + + + + ) + + return ( + + +
+ +
+ + + {items.map(item => ( + + {item.label} + + ))} + + + {t.common.delete} + + + + ) +} + +// ── drop zone ──────────────────────────────────────────────────────────────── + +/** While a bot is in flight, Escape cancels the gesture. Mount once in the + * roster pane. */ +export function useEscapeCancelsBotDrag(): void { + const dragging = useValue($draggingBot) + + useEffect(() => { + if (!dragging) { + return + } + + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') { + $draggingBot.set(null) + } + } + + window.addEventListener('keydown', onKeyDown, true) + + return () => window.removeEventListener('keydown', onKeyDown, true) + }, [dragging]) +} + +interface SectionDropZoneProps { + children: ReactNode + /** Whether the dragged bot is already filed here — then the zone is not a + * target, and the OS shows the no-drop cursor instead of a highlight that + * promises a move that would change nothing. */ + isSource: boolean + /** Drawn inside a gateway bucket: indented under a hairline rail so the + * two heading levels read as parent and child. */ + nested?: boolean + onDropBot: (rosterKey: string) => void +} + +/** A section block as a drop target: the whole block (heading + rows, or the + * empty placeholder) lights up while a bot is over it. */ +export function SectionDropZone({ children, isSource, nested, onDropBot }: SectionDropZoneProps) { + const dragging = useValue($draggingBot) + const [over, setOver] = useState(false) + const armed = Boolean(dragging) && !isSource + const lit = armed && over + + // Escape cancels the gesture: the in-flight key is cleared (see the + // keydown hook in the roster pane), so every zone disarms at once and a + // drop that still lands is refused below. Reset the hover so the next drag + // starts clean. + useEffect(() => { + if (!dragging) { + setOver(false) + } + }, [dragging]) + + const accepts = (event: DragEvent) => armed && event.dataTransfer.types.includes(BOT_DRAG_MIME) + + return ( +
{ + if (accepts(event)) { + event.preventDefault() + setOver(true) + } + }} + onDragLeave={event => { + // Only clear when the pointer leaves the BLOCK, not when it crosses + // between the rows inside it — dragleave fires on every child + // boundary, which otherwise strobes the highlight. + if (!event.currentTarget.contains(event.relatedTarget as Node | null)) { + setOver(false) + } + }} + onDragOver={event => { + if (!accepts(event)) { + return + } + + // preventDefault is what MAKES this a drop target — without it the + // browser refuses the drop and the cursor stays "no entry". + event.preventDefault() + event.dataTransfer.dropEffect = 'move' + + if (!over) { + setOver(true) + } + }} + onDrop={event => { + setOver(false) + // The dropped row remounts under its new section, so its own dragend + // never reaches the new node — clear the in-flight state here or the + // row stays faded after a successful drop. + $draggingBot.set(null) + + const key = event.dataTransfer.getData(BOT_DRAG_MIME) + + // No in-flight key means the user pressed Escape mid-drag: refuse. + if (!key || !dragging || isSource) { + return + } + + event.preventDefault() + onDropBot(key) + }} + > + {children} +
+ ) +} diff --git a/apps/desktop/src/plugins/hermes-bots/user-sections.test.ts b/apps/desktop/src/plugins/hermes-bots/user-sections.test.ts new file mode 100644 index 0000000000..a16ca4fccc --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/user-sections.test.ts @@ -0,0 +1,123 @@ +/** + * User sections — the three invariants that make membership-on-the-bot safe: + * filing persists through `saveBotMeta` (so it rides profile sync), every row + * lands in exactly one block with the remainder as Unassigned, and deleting a + * section returns its bots to Unassigned rather than losing them. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { saveBotMeta, storage } = vi.hoisted(() => ({ + saveBotMeta: vi.fn<(bot: { name: string }, patch: Record) => Promise>(), + storage: new Map() +})) + +vi.mock('./data', async () => { + const { atom } = await import('nanostores') + const $botMeta = atom>({}) + + saveBotMeta.mockImplementation(async (bot: { name: string }, patch: Record) => { + $botMeta.set({ ...$botMeta.get(), [bot.name]: { ...$botMeta.get()[bot.name], ...patch } }) + + return { serverOutcome: 'persisted', serverPersisted: true } + }) + + return { $botMeta, saveBotMeta } +}) + +vi.mock('./routing', () => ({ + botRosterMeta: (bot: { name: string }, meta: Record) => meta[bot.name] +})) + +vi.mock('./shared', () => ({ + getPluginCtx: () => ({ + storage: { + get: (key: string, fallback: unknown) => (storage.has(key) ? storage.get(key) : fallback), + set: (key: string, value: unknown) => storage.set(key, value) + } + }) +})) + +import { $botMeta } from './data' +import type { RosterRow } from './types' +import { + $botSections, + createBotSection, + deleteBotSection, + groupRowsBySection, + loadBotSections, + moveBotsToSection, + UNASSIGNED_SECTION_KEY +} from './user-sections' + +const bot = (name: string) => ({ name }) as RosterRow +const row = (name: string) => ({ bot: bot(name), kind: 'bot' as const }) + +beforeEach(() => { + storage.clear() + $botMeta.set({}) + $botSections.set([]) + saveBotMeta.mockClear() +}) + +describe('user sections', () => { + it('filing writes one sectionId per bot through saveBotMeta and survives a reload', async () => { + const section = createBotSection('Clients', [bot('nanox'), bot('scout')])! + + // Membership rides the bot's own meta write (profile ui_meta), one per bot. + await vi.waitFor(() => expect(saveBotMeta).toHaveBeenCalledTimes(2)) + expect(saveBotMeta).toHaveBeenCalledWith(bot('nanox'), { sectionId: section.id }) + + // A no-op move (already there) writes nothing. + await moveBotsToSection([bot('nanox')], section.id) + expect(saveBotMeta).toHaveBeenCalledTimes(2) + + // The section record itself persists in plugin storage. + $botSections.set([]) + loadBotSections() + expect($botSections.get()).toEqual([{ id: section.id, name: 'Clients' }]) + }) + + it('groups every row exactly once; unknown or missing sections fall to Unassigned, drawn last', () => { + const rows = [row('nanox'), row('scout'), row('ghost'), { kind: 'group' as const, name: 'Room' }] + + const meta = { + nanox: { sectionId: 'sec-clients' }, + scout: { sectionId: 'sec-workforce' }, + ghost: { sectionId: 'sec-deleted' } + } + + const blocks = groupRowsBySection( + rows, + [ + { id: 'sec-clients', name: 'Clients' }, + { id: 'sec-workforce', name: 'Workforce' } + ], + meta + ) + + expect(blocks.map(b => [b.key, b.rows.length])).toEqual([ + ['section:sec-clients', 1], + ['section:sec-workforce', 1], + [UNASSIGNED_SECTION_KEY, 2] + ]) + expect(blocks.flatMap(b => b.rows)).toHaveLength(rows.length) + expect(groupRowsBySection(rows, [], meta)).toEqual([{ id: null, key: UNASSIGNED_SECTION_KEY, name: '', rows }]) + }) + + it('deleting a section returns its bots to Unassigned, and undo refiles them', async () => { + const section = createBotSection('Clients', [bot('nanox')])! + createBotSection('Team') + await vi.waitFor(() => expect($botMeta.get().nanox?.sectionId).toBe(section.id)) + + const { members, undo } = deleteBotSection(section.id, [bot('nanox'), bot('scout')]) + + expect(members).toEqual([bot('nanox')]) + expect($botSections.get().map(s => s.name)).toEqual(['Team']) + await vi.waitFor(() => expect($botMeta.get().nanox?.sectionId).toBeNull()) + + undo() + expect($botSections.get().map(s => s.name)).toEqual(['Clients', 'Team']) + await vi.waitFor(() => expect($botMeta.get().nanox?.sectionId).toBe(section.id)) + }) +}) diff --git a/apps/desktop/src/plugins/hermes-bots/user-sections.ts b/apps/desktop/src/plugins/hermes-bots/user-sections.ts new file mode 100644 index 0000000000..d653475535 --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/user-sections.ts @@ -0,0 +1,247 @@ +/** + * USER SECTIONS — folders the user makes, not folders the topology makes. + * + * The roster already had sections (`roster-sections.tsx`), but only AUTOMATIC + * ones: one per gateway connection, plus the group-chat bucket. Those answer + * "where does this bot run", which is not the question you are asking when you + * want two client bots filed together under "Clients". + * + * So this is a SECOND axis, and it composes with the first rather than + * replacing it. Two deliberate choices: + * + * * The membership lives on the BOT (`sectionId` in its ui_meta), not as a + * member list on the section. A bot can only be in one place, deleting a + * section cannot orphan anybody, and the assignment rides the same + * profile.yaml sync every other bot setting already uses — so sections + * follow the profile to another machine. + * * "Unassigned" is not a section. It is whatever is left, always drawn + * last, and it is where members of a deleted section land. With no + * sections at all the roster renders exactly as it did before. + * + * Pure model + session atoms. No JSX — the pane composes it. + */ + +import { atom } from 'nanostores' + +import { $botMeta, saveBotMeta } from './data' +import { botRosterMeta } from './routing' +import { getPluginCtx } from './shared' +import type { BotMeta, RosterRow } from './types' + +export const UNASSIGNED_SECTION_KEY = 'section:unassigned' +export const BOT_SECTIONS_KEY = 'bot-sections-v1' + +export interface BotSection { + id: string + name: string +} + +/** `[{ id, name }]`, in display order. */ +export const $botSections = atom([]) + +/** Roster key of the bot in flight during a drag. Session-only, and cleared + * on dragend even when the drop lands outside any target — a stuck + * "dragging" state outlives the gesture and reads as a broken pane. */ +export const $draggingBot = atom(null) + +export function normalizeBotSections(value: unknown): BotSection[] { + if (!Array.isArray(value)) { + return [] + } + + const seen = new Set() + const out: BotSection[] = [] + + for (const entry of value) { + const id = String((entry as BotSection)?.id || '').trim() + const name = String((entry as BotSection)?.name || '').trim() + + if (!id || !name || seen.has(id)) { + continue + } + + seen.add(id) + out.push({ id, name }) + } + + return out +} + +function persistBotSections(next: BotSection[]): void { + $botSections.set(next) + + try { + getPluginCtx()?.storage?.set?.(BOT_SECTIONS_KEY, next) + } catch { + // No storage — sections live for this window only, which is strictly + // better than the pane throwing while the user drags a bot into a folder. + } +} + +/** Read the persisted list back at plugin start. */ +export function loadBotSections(): void { + try { + $botSections.set(normalizeBotSections(getPluginCtx()?.storage?.get?.(BOT_SECTIONS_KEY, []))) + } catch { + $botSections.set([]) + } +} + +function newSectionId(): string { + return `sec-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}` +} + +/** Create a section and file `bots` into it. Returns the new section, or + * null when the name is blank. */ +export function createBotSection(name: string, bots: RosterRow[] = []): BotSection | null { + const clean = String(name || '').trim() + + if (!clean) { + return null + } + + const section: BotSection = { id: newSectionId(), name: clean } + + persistBotSections([...$botSections.get(), section]) + void moveBotsToSection(bots, section.id) + + return section +} + +export function renameBotSection(id: string, name: string): void { + const clean = String(name || '').trim() + + if (!clean) { + return + } + + persistBotSections($botSections.get().map(s => (s.id === id ? { ...s, name: clean } : s))) +} + +/** + * Delete the section only. Its members are not deleted and not hidden — they + * fall back to Unassigned, which is the whole reason membership lives on the + * bot rather than on the section. Returns an undo that puts the section back + * in its slot and refiles the same bots, so the delete needs no confirmation. + */ +export function deleteBotSection(id: string, roster: RosterRow[] = []): { members: RosterRow[]; undo: () => void } { + const list = $botSections.get() + const index = list.findIndex(s => s.id === id) + const section = list[index] + const members = (roster || []).filter(bot => botSectionId(bot, $botMeta.get()) === id) + + persistBotSections(list.filter(s => s.id !== id)) + void moveBotsToSection(members, null) + + return { + members, + undo: () => { + if (!section) { + return + } + + const current = $botSections.get().filter(s => s.id !== id) + + current.splice(Math.min(index, current.length), 0, section) + persistBotSections(current) + void moveBotsToSection(members, id) + } + } +} + +export function moveBotSection(id: string, delta: number): void { + const list = $botSections.get() + const from = list.findIndex(s => s.id === id) + const to = from + delta + + if (from < 0 || to < 0 || to >= list.length) { + return + } + + const next = list.slice() + const [moved] = next.splice(from, 1) + + next.splice(to, 0, moved!) + persistBotSections(next) +} + +/** + * `null` clears the assignment (back to Unassigned). One `saveBotMeta` per + * bot — membership is a field on each bot's own profile, so that IS one write + * per profile — and the writes run in sequence rather than fanned out, so the + * shared local snapshot is never committed by two saves at once. + */ +export async function moveBotsToSection(bots: RosterRow[], sectionId: null | string): Promise { + for (const bot of bots || []) { + if (bot && botSectionId(bot, $botMeta.get()) !== (sectionId || null)) { + await saveBotMeta(bot, { sectionId: sectionId || null }) + } + } +} + +export function botSectionId(bot: RosterRow, metaByName: Record): null | string { + const id = botRosterMeta(bot, metaByName)?.sectionId + + return id ? String(id) : null +} + +export interface SectionBlock { + id: null | string + key: string + name: string + rows: TRow[] +} + +/** + * Split roster rows into section blocks, in section order, with Unassigned + * last. Pure, and returns EVERY row exactly once: a row whose `sectionId` + * names a section that no longer exists lands in Unassigned rather than + * vanishing, which is what makes deleting a section safe. + */ +export function groupRowsBySection( + rows: TRow[], + sections: unknown, + metaByName: Record +): SectionBlock[] { + const list = normalizeBotSections(sections) + const known = new Set(list.map(s => s.id)) + const byId = new Map(list.map(s => [s.id, [] as TRow[]])) + const loose: TRow[] = [] + + for (const row of rows || []) { + const bot = ((row as { bot?: RosterRow })?.bot || row) as RosterRow + const id = bot ? botSectionId(bot, metaByName) : null + + if (id && known.has(id)) { + byId.get(id)!.push(row) + } else { + loose.push(row) + } + } + + const blocks: SectionBlock[] = list.map(section => ({ + id: section.id, + key: `section:${section.id}`, + name: section.name, + rows: byId.get(section.id) || [] + })) + + blocks.push({ id: null, key: UNASSIGNED_SECTION_KEY, name: '', rows: loose }) + + return blocks +} + +// ── drag and drop ──────────────────────────────────────────────────────────── +// +// Filing a bot by dragging it onto a section, which is the gesture people +// reach for first; the row's "Move to section" submenu is the same action +// for anyone who does not. +// +// A CUSTOM MIME TYPE, not `text/plain`: the roster shares a window with the +// composer, the transcript and the tab strip, all of which accept dropped +// text. A private type means a bot dragged onto any of them is simply not a +// valid payload there, instead of pasting its roster key into someone's +// message. `dataTransfer.types` is readable during dragover (the DATA itself +// is not, by design), so a drop target can still light up correctly. + +export const BOT_DRAG_MIME = 'application/x-hermes-bot-key' diff --git a/apps/desktop/src/sdk/index.ts b/apps/desktop/src/sdk/index.ts index 4ae9303e40..bf28c8fb16 100644 --- a/apps/desktop/src/sdk/index.ts +++ b/apps/desktop/src/sdk/index.ts @@ -36,6 +36,7 @@ import { $workspaceMode, $workspaceOwnerKey, setWorkspaceScope as publishWorkspaceScope, + setWorkspaceOwnerLabel, type WorkspaceNewSessionTarget } from '@/components/pane-shell/workspace-scope' import { onGatewayEvent } from '@/contrib/events' @@ -1160,6 +1161,11 @@ export const host = { return close }, + /** Name a workspace owner on its tabs (a bot's display name). A canonical + * chat's STORED title is an identity the backend resolves by name; this is + * the caption shown for it. Feature-detect on older desktops. */ + setWorkspaceOwnerLabel, + /** Switch the visible main-pane workspace without unregistering retained panes. */ setWorkspaceScope: ( mode: WorkspaceMode, @@ -1217,8 +1223,9 @@ export const host = { * caller falls through to its authoritative open path. */ focusOpenWorkspaceSession: ( workspaceOwnerKey: string, - isStaleTile?: (tile: { storedSessionId: string; workspaceTabTitle?: string }) => boolean - ): null | string => focusWorkspaceOwnerSessionTile(workspaceOwnerKey, isStaleTile), + isStaleTile?: (tile: { storedSessionId: string; workspaceTabTitle?: string }) => boolean, + onlyStoredIds?: readonly string[] + ): null | string => focusWorkspaceOwnerSessionTile(workspaceOwnerKey, isStaleTile, onlyStoredIds), /** Reactive on-screen visibility of a contributed pane: true while it is in * the layout tree, not dismissed/hidden, its zone un-minimized, AND holding @@ -1510,6 +1517,11 @@ export { ContextMenuContent, ContextMenuItem, ContextMenuSeparator, + // Submenus: Bot Mode files a bot into a user section from its row menu, and + // a flat list of every folder would swamp the items already there. + ContextMenuSub, + ContextMenuSubContent, + ContextMenuSubTrigger, ContextMenuTrigger } from '@/components/ui/context-menu' export { CopyButton } from '@/components/ui/copy-button' diff --git a/apps/desktop/src/store/composer-status.test.ts b/apps/desktop/src/store/composer-status.test.ts index d47103b0cb..547d269c26 100644 --- a/apps/desktop/src/store/composer-status.test.ts +++ b/apps/desktop/src/store/composer-status.test.ts @@ -7,9 +7,14 @@ import { isSessionGoneForBackgroundPolling, reconcileBackgroundProcesses, refreshBackgroundProcesses, - resetBackgroundPollingGuard + resetBackgroundPollingGuard, + stopBackgroundProcess } from './composer-status' import { $gateway } from './gateway' +import { markSessionGone } from './runtime-gone' + +vi.mock('./notifications', () => ({ notifyError: vi.fn() })) +import { notifyError } from './notifications' const SID = 'sess-1' @@ -263,6 +268,52 @@ describe('refreshBackgroundProcesses dead-session guard', () => { expect(request).toHaveBeenCalledTimes(2) }) + + it('dismisses a stale process row when Stop is clicked after the runtime is gone', async () => { + reconcileBackgroundProcesses(SID, [running('stale')]) + markSessionGone(SID) + $gateway.set({ request: vi.fn() } as never) + + await stopBackgroundProcess(SID, 'stale') + + expect(items()).toEqual([]) + }) + + it('dismisses a stale process row while the gateway is disconnected', async () => { + reconcileBackgroundProcesses(SID, [running('disconnected')]) + markSessionGone(SID) + $gateway.set(null as never) + + await stopBackgroundProcess(SID, 'disconnected') + + expect(items()).toEqual([]) + }) + + it('keeps the row and reports failure when the gateway is disconnected', async () => { + reconcileBackgroundProcesses(SID, [running('unreachable')]) + $gateway.set(null as never) + vi.mocked(notifyError).mockClear() + + await stopBackgroundProcess(SID, 'unreachable') + + expect(items()).toEqual([expect.objectContaining({ id: 'unreachable', state: 'running' })]) + expect(notifyError).toHaveBeenCalledWith(expect.any(Error), 'Could not stop the process') + }) + + it('dismisses and latches when Stop discovers the runtime is gone', async () => { + const request = vi.fn(async () => { + throw new Error('session not found') + }) + + reconcileBackgroundProcesses(SID, [running('rejected')]) + $gateway.set({ request } as never) + + await stopBackgroundProcess(SID, 'rejected') + await stopBackgroundProcess(SID, 'rejected') + + expect(items()).toEqual([]) + expect(request).toHaveBeenCalledTimes(1) + }) }) // ── Review-thread hardenings on the guard (#94950) ─────────────────────────── diff --git a/apps/desktop/src/store/composer-status.ts b/apps/desktop/src/store/composer-status.ts index ce891688e5..f6970880b6 100644 --- a/apps/desktop/src/store/composer-status.ts +++ b/apps/desktop/src/store/composer-status.ts @@ -10,7 +10,8 @@ import { dispatchNativeNotification } from './native-notifications' import { notifyError } from './notifications' import { isSessionGone, isSessionGoneForBackgroundPolling, markSessionGone, noteRuntimeAlive } from './runtime-gone' import { $sessions, lineageAliases } from './session' -import { $sessionStates } from './session-states' +import { ambientRequestFor } from './session-gone-latch' +import { $sessionStates, requestForOwnedSession } from './session-states' import { $subagentsBySession, type SubagentProgress } from './subagents' import { $todosBySession } from './todos' @@ -401,7 +402,12 @@ export async function refreshBackgroundProcesses(sid: string): Promise { } try { - const result = await gateway.request<{ processes?: GatewayProcessEntry[] }>('process.list', { session_id: sid }) + const result = await requestForOwnedSession<{ processes?: GatewayProcessEntry[] }>( + sid, + ambientRequestFor(gateway), + 'process.list', + { session_id: sid } + ) reconcileBackgroundProcesses(sid, result?.processes ?? []) // The binding answered, so it is healthy: refund the stored session's @@ -441,10 +447,34 @@ export function dismissBackgroundProcess(sid: string, id: string) { * row while the process lived on, stranding rogue tasks. On failure the row * stays so the user can retry / see it didn't die. */ export async function stopBackgroundProcess(sid: string, id: string): Promise { + const gateway = $gateway.get() + + if (isSessionGone(sid)) { + // The backend has already declared this runtime gone, so there is no + // authoritative process left to kill through this session. Remove the + // stale local row instead of leaving the Stop button permanently inert. + dismissBackgroundProcess(sid, id) + + return + } + + if (!gateway) { + notifyError(new Error('Gateway is not connected'), 'Could not stop the process') + + return + } + try { - await $gateway.get()?.request('process.kill', { process_id: id, session_id: sid }) + await requestForOwnedSession(sid, ambientRequestFor(gateway), 'process.kill', { process_id: id, session_id: sid }) dismissBackgroundProcess(sid, id) } catch (err) { + if (isSessionGoneForBackgroundPolling(err)) { + dismissBackgroundProcess(sid, id) + markSessionGone(sid) + + return + } + notifyError(err, 'Could not stop the process') } } @@ -471,7 +501,16 @@ export function resetSessionBackground(sid: string) { dismissed.add(item.id) if (item.state === 'running') { - void gateway?.request('process.kill', { process_id: item.id, session_id: sid }).catch(() => undefined) + if (gateway && !isSessionGone(sid)) { + void requestForOwnedSession(sid, ambientRequestFor(gateway), 'process.kill', { + process_id: item.id, + session_id: sid + }).catch(error => { + if (isSessionGoneForBackgroundPolling(error)) { + markSessionGone(sid) + } + }) + } } } diff --git a/apps/desktop/src/store/gateway.ts b/apps/desktop/src/store/gateway.ts index 7601ea258e..5bf36abaea 100644 --- a/apps/desktop/src/store/gateway.ts +++ b/apps/desktop/src/store/gateway.ts @@ -530,19 +530,6 @@ async function openSecondary(entry: Secondary): Promise { // Best effort for partial test/HMR graphs. Production always loads the // real store; a failed import must not make the transport unrecoverable. } - - // Runtime re-mint also invalidates the status-stack gone-latch: ids - // the dead runtime 4001'd may be live again once tiles re-resume. - // Fire-and-forget: composer-status imports from this module, so the - // import must stay dynamic (cycle), and it must NOT sit on the timed - // redial path — awaiting the module load here pushed cold-start - // redials past test/waitFor budgets. The reset needs no ordering - // guarantee relative to the dial. - void import('@/store/composer-status') - .then(({ resetBackgroundPollingGuard }) => resetBackgroundPollingGuard()) - .catch(() => { - // Best effort for partial test/HMR graphs, same as above. - }) } // Registry-scoped entries dial through getConnectionFor when the bridge has @@ -1597,6 +1584,24 @@ export function reconnectSecondaryGateways({ forceOpenSockets = false }: { force } } +// How many non-primary backends currently hold an open socket. Hover-intent +// prewarming consults this before spawning: a speculative spawn that pushes +// the pool past its cap causes the Electron main to LRU-evict a warm backend +// — often one the user is about to click — turning the prewarm into churn +// (the #91545 evict/respawn cascade). The active gateway's backend is +// primary-routed and never counts toward the pool cap. +export function openSecondaryCount(): number { + let count = 0 + + for (const entry of g.secondaries.values()) { + if (isOpen(entry.gateway)) { + count += 1 + } + } + + return count +} + // Keep the idle reaper from killing a backend we still need: ping every live // secondary. The active one is pinged separately (touchActiveGatewayBackend). export function touchSecondaryGateways(): void { diff --git a/apps/desktop/src/store/goals.test.ts b/apps/desktop/src/store/goals.test.ts index e355776f36..c69ab23b2d 100644 --- a/apps/desktop/src/store/goals.test.ts +++ b/apps/desktop/src/store/goals.test.ts @@ -1,3 +1,4 @@ +import { JsonRpcGatewayError } from '@hermes/shared' import { afterEach, describe, expect, it, vi } from 'vitest' import { $gateway } from './gateway' @@ -8,6 +9,8 @@ describe('goal store', () => { afterEach(() => { vi.useRealTimers() $goalsBySession.set({}) + $gateway.set(null as never) + resetBackgroundPollingGuard() }) it('stores active goals from /goal output', () => { @@ -109,6 +112,19 @@ describe('goal store', () => { expect($goalsBySession.get().s2).toMatchObject({ status: 'paused', title: 'other work' }) }) + + it('does not retry goal hydration for a runtime rejected as session-not-found', async () => { + const request = vi.fn(async () => { + throw new JsonRpcGatewayError('session not found', { code: 4001 }) + }) + + $gateway.set({ request } as never) + + await refreshSessionGoal('dead-runtime') + await refreshSessionGoal('dead-runtime') + + expect(request).toHaveBeenCalledTimes(1) + }) }) describe('refreshSessionGoal dead-session guard', () => { diff --git a/apps/desktop/src/store/goals.ts b/apps/desktop/src/store/goals.ts index aa51be46ce..8bd1ba4a0b 100644 --- a/apps/desktop/src/store/goals.ts +++ b/apps/desktop/src/store/goals.ts @@ -4,6 +4,8 @@ import { keyedTimeouts } from '@/lib/keyed-timeouts' import { $gateway } from './gateway' import { isSessionGone, isSessionGoneForBackgroundPolling, markSessionGone } from './runtime-gone' +import { ambientRequestFor } from './session-gone-latch' +import { requestForOwnedSession } from './session-states' export type GoalStatus = 'active' | 'done' | 'paused' | 'waiting' @@ -169,7 +171,10 @@ export async function refreshSessionGoal(sid: string): Promise { } try { - const result = await gateway.request<{ output?: string }>('slash.exec', { command: 'goal status', session_id: sid }) + const result = await requestForOwnedSession<{ output?: string }>(sid, ambientRequestFor(gateway), 'slash.exec', { + command: 'goal status', + session_id: sid + }) applyGoalStatusText(sid, result?.output ?? '', { hydrate: true }) } catch (error) { diff --git a/apps/desktop/src/store/native-notifications.test.ts b/apps/desktop/src/store/native-notifications.test.ts index c1fd51d30f..69adfb0df0 100644 --- a/apps/desktop/src/store/native-notifications.test.ts +++ b/apps/desktop/src/store/native-notifications.test.ts @@ -15,6 +15,7 @@ import { } from './native-notifications' import { __resetNativeNotifyBaselineForTests, markNativeNotifyBaseline } from './notify-baseline' import { $approvalRequest, setApprovalRequest } from './prompts' +import { markSessionGone, resetBackgroundPollingGuard } from './runtime-gone' import { $activeSessionId, setActiveSessionId } from './session' const desktopWindow = window as unknown as { hermesDesktop?: Window['hermesDesktop'] } @@ -47,6 +48,7 @@ beforeEach(() => { } setActiveSessionId(null) + resetBackgroundPollingGuard() setWindowState({ focused: false, hidden: true }) __resetNativeNotifyBaselineForTests() }) @@ -59,6 +61,8 @@ afterEach(() => { } else { delete desktopWindow.hermesDesktop } + + resetBackgroundPollingGuard() }) describe('dispatchNativeNotification focus gating', () => { @@ -339,4 +343,12 @@ describe('respondToApprovalAction', () => { await respondToApprovalAction('bg', 'approve') expect(request).not.toHaveBeenCalled() }) + + it('does not retry an approval action for a runtime already marked gone', async () => { + markSessionGone('bg') + + await respondToApprovalAction('bg', 'approve') + + expect(request).not.toHaveBeenCalled() + }) }) diff --git a/apps/desktop/src/store/native-notifications.ts b/apps/desktop/src/store/native-notifications.ts index 09edafc2bb..4a76acb8a0 100644 --- a/apps/desktop/src/store/native-notifications.ts +++ b/apps/desktop/src/store/native-notifications.ts @@ -6,6 +6,7 @@ import { persistString, storedString } from '@/lib/storage' import { $gateway } from './gateway' import { withinNativeNotifyBaseline } from './notify-baseline' import { clearApprovalRequest } from './prompts' +import { isSessionGone, isSessionGoneForBackgroundPolling, markSessionGone } from './runtime-gone' import { $activeSessionId } from './session' import { requestForOwnedSession } from './session-states' @@ -353,6 +354,10 @@ export async function respondToApprovalAction(sessionId: null | string, actionId return } + if (sessionId && isSessionGone(sessionId)) { + return + } + const gateway = $gateway.get() if (!gateway) { @@ -373,7 +378,11 @@ export async function respondToApprovalAction(sessionId: null | string, actionId { choice, session_id: sessionId ?? undefined } ) clearApprovalRequest(sessionId) - } catch { + } catch (error) { + if (sessionId && isSessionGoneForBackgroundPolling(error)) { + markSessionGone(sessionId) + } + // Leave the prompt parked so the user can still resolve it in-app. } } diff --git a/apps/desktop/src/store/pool-limits.ts b/apps/desktop/src/store/pool-limits.ts new file mode 100644 index 0000000000..71bd9e4ade --- /dev/null +++ b/apps/desktop/src/store/pool-limits.ts @@ -0,0 +1,65 @@ +/** + * Pool limits — how many bot backends may stay spawned, and how long an + * unused one survives before it is shut down. + * + * A device-local preference (each machine trades RAM against switching + * speed for itself). The MAIN process is authoritative: it owns the pool + * and the persisted copy, and applies a new max immediately by evicting + * least-recently-used idle backends — no restart. This store mirrors the + * live values for the Settings rows and feeds prewarmProfileBackend's + * saturation guard. + */ + +import { atom } from 'nanostores' + +export interface PoolLimits { + /** Max concurrently spawned non-primary profile backends. */ + maxBackends: number + /** Idle lifetime of an unused pool backend, in milliseconds. */ + idleMs: number +} + +export const POOL_LIMITS_DEFAULTS: PoolLimits = { + maxBackends: 3, + idleMs: 10 * 60_000 +} + +export const $poolLimits = atom({ ...POOL_LIMITS_DEFAULTS }) + +/** Seed from main's authoritative state once at startup; no-op without the + * bridge (web/older builds just keep the defaults for the UI). */ +export async function loadPoolLimits(): Promise { + try { + const limits = await window.hermesDesktop?.getPoolLimits?.() + + if (limits) { + $poolLimits.set(limits) + } + } catch { + // Keep defaults — Settings rows still render and can retry on save. + } +} + +/** Push new limits to main; adopt the post-clamp values it reports. */ +export async function savePoolLimits(next: { maxBackends?: number; idleMs?: number }): Promise { + const current = $poolLimits.get() + + const optimistic: PoolLimits = { + maxBackends: next.maxBackends ?? current.maxBackends, + idleMs: next.idleMs ?? current.idleMs + } + + // Optimistic paint, then honest reconciliation with the clamped result. + $poolLimits.set(optimistic) + + try { + const result = await window.hermesDesktop?.setPoolLimits?.(next) + + if (result?.limits) { + $poolLimits.set(result.limits) + } + } catch { + $poolLimits.set(current) + throw new Error('Applying pool limits failed') + } +} diff --git a/apps/desktop/src/store/profile.test.ts b/apps/desktop/src/store/profile.test.ts index a4c6d75e3d..33c9578427 100644 --- a/apps/desktop/src/store/profile.test.ts +++ b/apps/desktop/src/store/profile.test.ts @@ -9,10 +9,24 @@ import type { ProfileInfo } from '@/types/hermes' const ensureGatewayForProfile = vi.fn(async () => undefined) const ensureGatewayForAgent = vi.fn(async () => undefined) const openGatewayForProfile = vi.fn(async (_profile: string) => undefined) -const $gateway = atom({ id: 'live-socket' }) +const openSecondaryCount = vi.fn(() => 0) +const $gateway = atom({ id: 'live-socket', connectionState: 'open' }) const resetStarmapGraph = vi.fn() -vi.mock('@/store/gateway', () => ({ $gateway, ensureGatewayForAgent, ensureGatewayForProfile, openGatewayForProfile })) +vi.mock('@/store/gateway', () => ({ + $gateway, + ensureGatewayForAgent, + ensureGatewayForProfile, + openGatewayForProfile, + openSecondaryCount +})) +// The pool-limits atom is profile.ts's live saturation signal — keep the real +// one so tests can move the cap via the store, but stub its IPC bridge. +vi.mock('@/store/pool-limits', async () => { + const { atom } = await import('nanostores') + + return { $poolLimits: atom({ idleMs: 600_000, maxBackends: 3 }) } +}) vi.mock('@/hermes', () => ({ getProfiles: vi.fn(async () => ({ profiles: [] })), setApiRequestProfile: vi.fn() @@ -29,6 +43,8 @@ const { refreshProfiles } = await import('./profile') +const { $poolLimits } = await import('@/store/pool-limits') + const { $connection } = await import('./session') const { invalidateProfileScopedQueries } = await import('@/lib/query-client') const { getProfiles } = await import('@/hermes') @@ -55,7 +71,8 @@ beforeEach(() => { getConnection.mockReset() ensureGatewayForProfile.mockClear() openGatewayForProfile.mockClear() - $gateway.set({ id: 'live-socket' }) + openSecondaryCount.mockReturnValue(0) + $gateway.set({ id: 'live-socket', connectionState: 'open' }) $activeGatewayProfile.set('default') $connection.set(localConn()) $profiles.set([]) @@ -115,6 +132,17 @@ describe('ensureGatewayProfile → $connection sync (#46651)', () => { expect(ensureGatewayForProfile).not.toHaveBeenCalled() expect($connection.get()?.mode).toBe('remote') }) + + it('reconnects when the target profile is active but its gateway socket is closed', async () => { + $activeGatewayProfile.set('vps-remote') + $connection.set(remoteConn()) + $gateway.set({ connectionState: 'closed' }) + getConnection.mockResolvedValue(remoteConn()) + + await ensureGatewayProfile('vps-remote') + + expect(ensureGatewayForProfile).toHaveBeenCalledWith('vps-remote') + }) }) describe('profile-scoped cache invalidation', () => { @@ -158,6 +186,43 @@ describe('prewarmProfileBackend (hover-intent pool spawn)', () => { expect(() => prewarmProfileBackend('warm-failing')).not.toThrow() }) + + it('skips pre-warm when the pool is saturated (#91545 evict/respawn cascade)', () => { + // Every pool slot occupied: a speculative spawn would LRU-evict a warm + // backend — often the one the user is about to click. Default limit 3, + // 3 open secondaries → the next spawn would exceed the cap. + openSecondaryCount.mockReturnValue(3) + + prewarmProfileBackend('warm-saturated') + + expect(openGatewayForProfile).not.toHaveBeenCalled() + }) + + it('pre-warms while pool slots are free', () => { + openSecondaryCount.mockReturnValue(1) + + prewarmProfileBackend('warm-slot-free') + + expect(openGatewayForProfile).toHaveBeenCalledWith('warm-slot-free') + }) + + it('follows the live pool-limit atom, not a hard-coded cap', () => { + // User raises Warm Bot Backends to 8 in Settings: prewarming must keep + // working well past the old default of 3. + openSecondaryCount.mockReturnValue(5) + $poolLimits.set({ idleMs: 600_000, maxBackends: 8 }) + + prewarmProfileBackend('warm-raised-cap') + + expect(openGatewayForProfile).toHaveBeenCalledWith('warm-raised-cap') + + // And lowering the cap re-engages the guard at the new boundary. + $poolLimits.set({ idleMs: 600_000, maxBackends: 2 }) + + prewarmProfileBackend('warm-lowered-cap') + + expect(openGatewayForProfile).not.toHaveBeenCalledWith('warm-lowered-cap') + }) }) describe('refreshProfiles shared rail list (#49289)', () => { diff --git a/apps/desktop/src/store/profile.ts b/apps/desktop/src/store/profile.ts index 09ab31619f..cf903e120e 100644 --- a/apps/desktop/src/store/profile.ts +++ b/apps/desktop/src/store/profile.ts @@ -21,9 +21,11 @@ import { ensureGatewayForAgent, ensureGatewayForProfile, openGatewayForAgent, - openGatewayForProfile + openGatewayForProfile, + openSecondaryCount } from '@/store/gateway' import { notifyError } from '@/store/notifications' +import { $poolLimits } from '@/store/pool-limits' import { notifyRemoteOverrideAuthFailure } from '@/store/profile-remote-override' import { clearComposerSelectionOwner, setComposerSelectionOwner, setConnection } from '@/store/session' import type { SessionOwnerRoute } from '@/store/session-request-router' @@ -423,6 +425,17 @@ export function prewarmProfileBackend(name: string): void { return } + // Prewarm/cap harmony (#91545): the pool caps spawned backends at the + // configured max, and a spawn over the cap LRU-evicts the warmest idle + // backend. A hover sweep across the rail therefore evicted backends for + // profiles the user was about to click — prewarming caused the exact churn + // it exists to prevent. Skip speculative spawns once every pool slot is + // occupied by an open socket; the real click still spawns on demand, it + // just doesn't get a head start. + if (openSecondaryCount() + 1 > $poolLimits.get().maxBackends) { + return + } + prewarmedAt.set(key, now) openGatewayForProfile(key).catch(() => undefined) } @@ -491,7 +504,7 @@ export async function ensureGatewayProfile(profile: string | null | undefined): const target = normalizeProfileKey(profile) - if (normalizeProfileKey($activeGatewayProfile.get()) === target && $gateway.get()) { + if (normalizeProfileKey($activeGatewayProfile.get()) === target && $gateway.get()?.connectionState === 'open') { return } @@ -503,7 +516,7 @@ export async function ensureGatewayProfile(profile: string | null | undefined): await gatewaySwitch.catch(() => undefined) } - if (normalizeProfileKey($activeGatewayProfile.get()) === target && $gateway.get()) { + if (normalizeProfileKey($activeGatewayProfile.get()) === target && $gateway.get()?.connectionState === 'open') { return } @@ -857,6 +870,18 @@ function activateOnCurrentSource(target: string): Promise { return connectionId ? ensureGatewayAgent(connectionId, target) : ensureGatewayProfile(target) } +// Pin the next new chat to `name` (legacy profile-only door) so session.create +// reads the profile the user clicked "+" under, not whatever +// $activeGatewayProfile holds once an in-flight profile swap settles (#79005). +export function pinNewChatProfile(name: string): string { + const target = normalizeProfileKey(name) + $newChatProfile.set(target) + $newChatRoute.set(null) + captureNewChatSource(profilePickConnectionId(target)) + + return target +} + // Start a fresh session in `name` WITHOUT collapsing the "All profiles" browse // view. Unlike selectProfile, it leaves $showAllProfiles untouched, so the // unified sidebar stays put — used by the per-profile "+" in the all-profiles @@ -864,10 +889,7 @@ function activateOnCurrentSource(target: string): Promise { // is in. Points new chats at the profile and opens its backend so the next // message lands in the right place. export function newSessionInProfile(name: string): void { - const target = normalizeProfileKey(name) - $newChatProfile.set(target) - $newChatRoute.set(null) - captureNewChatSource(profilePickConnectionId(target)) + const target = pinNewChatProfile(name) requestFreshSession() // #81094: surface the failed dial instead of failing silently. void activateOnCurrentSource(target).catch((error: unknown) => { diff --git a/apps/desktop/src/store/projects.test.ts b/apps/desktop/src/store/projects.test.ts index 7c6f62c117..a334d98dd5 100644 --- a/apps/desktop/src/store/projects.test.ts +++ b/apps/desktop/src/store/projects.test.ts @@ -12,13 +12,9 @@ import { $projectScope, $projectsRpcAvailable, $projectTree, - $removedSessionIds, - $sessionMutationsInFlight, $worktreeRefreshToken, ALL_PROJECTS, - beginSessionMutation, createProject, - endSessionMutation, enterProject, exitProjectScope, fetchProjectSessions, @@ -31,9 +27,15 @@ import { refreshWorktrees, resolveNewSessionCwd, scanAndRecordRepos, - startWorkInRepo, - tombstoneSessions + startWorkInRepo } from './projects' +import { + $removedSessionIds, + $sessionMutationsInFlight, + beginSessionMutation, + endSessionMutation, + tombstoneSessions +} from './session-removal' vi.mock('@/i18n', () => ({ translateNow: (key: string) => key diff --git a/apps/desktop/src/store/projects.ts b/apps/desktop/src/store/projects.ts index 22a42cf38f..9730ae6b08 100644 --- a/apps/desktop/src/store/projects.ts +++ b/apps/desktop/src/store/projects.ts @@ -30,6 +30,7 @@ import { setSessions, workspaceCwdForNewSession } from '@/store/session' +import { $removedSessionIds, $sessionMutationsInFlight } from '@/store/session-removal' import type { ProjectInfo, ProjectsPayload } from '@/types/hermes' // First-class, per-profile Projects (named, multi-folder workspaces). State is @@ -65,78 +66,6 @@ function projectsStaleBackendError(): Error { return new Error(translateNow('sidebar.projects.staleBackend')) } -// Client-side cache eviction (Apollo-style optimistic layer): ids the user just -// deleted/archived. The backend tree is a snapshot that still lists them until -// its next refresh, so the render-time overlay strips these so the tree matches -// the live `$sessions` cache exactly — same as the flat Recents list. Pruned on -// refresh once the server snapshot has caught up. -export const $removedSessionIds = atom>(new Set()) - -export function tombstoneSessions(ids: Array): void { - const next = new Set($removedSessionIds.get()) - const before = next.size - - for (const id of ids) { - const trimmed = id?.trim() - - if (trimmed) { - next.add(trimmed) - } - } - - if (next.size !== before) { - $removedSessionIds.set(next) - } -} - -export function untombstoneSessions(ids: Array): void { - const current = $removedSessionIds.get() - - if (!current.size) { - return - } - - const next = new Set(current) - - for (const id of ids) { - const trimmed = id?.trim() - - if (trimmed) { - next.delete(trimmed) - } - } - - if (next.size !== current.size) { - $removedSessionIds.set(next) - } -} - -// Ids whose delete/archive RPC is still in flight. Their tombstones are pinned -// against the projects.tree prune below: a refresh whose snapshot predates the -// mutation completing must NOT drop the tombstone, or the row flashes back until -// the backend catches up. Keyed by id, so concurrent deletes stay independent. -export const $sessionMutationsInFlight = atom>(new Set()) - -function mutateInFlight(ids: Array, add: boolean): void { - const current = $sessionMutationsInFlight.get() - const next = new Set(current) - - for (const id of ids) { - const trimmed = id?.trim() - - if (trimmed) { - add ? next.add(trimmed) : next.delete(trimmed) - } - } - - if (next.size !== current.size) { - $sessionMutationsInFlight.set(next) - } -} - -export const beginSessionMutation = (ids: Array): void => mutateInFlight(ids, true) -export const endSessionMutation = (ids: Array): void => mutateInFlight(ids, false) - // True while the disk scan is in flight (drives the "finding repos" hint). export const $reposScanning = atom(false) @@ -349,7 +278,7 @@ async function gatewayRequest(method: string, params: Record return gateway.request(method, params) } -function projectProfile(): null | string { +export function projectProfile(): null | string { const profile = normalizeProfileKey($activeGatewayProfile.get()) return $profileScope.get() === ALL_PROFILES || profile === ALL_PROFILES ? null : profile diff --git a/apps/desktop/src/store/prompts.test.ts b/apps/desktop/src/store/prompts.test.ts index 9f0e761f71..3b9e4ac825 100644 --- a/apps/desktop/src/store/prompts.test.ts +++ b/apps/desktop/src/store/prompts.test.ts @@ -1,4 +1,5 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { JsonRpcGatewayError } from '@hermes/shared' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { clearClarifyRequest, setClarifyRequest } from './clarify' import { @@ -16,8 +17,8 @@ import { setSecretRequest, setSudoRequest } from './prompts' -import { resetBackgroundPollingGuard } from './runtime-gone' -import { $activeSessionId } from './session' +import { isSessionGone, resetBackgroundPollingGuard } from './runtime-gone' +import { $activeSessionId, setActiveSessionId } from './session' // Prompts are parked per-session; the exported $*Request views are scoped to the // active session, so each test focuses the session it's asserting on. @@ -29,6 +30,7 @@ afterEach(() => { clearAllPrompts() clearClarifyRequest() $activeSessionId.set(null) + resetBackgroundPollingGuard() }) describe('approval prompt store', () => { @@ -130,6 +132,64 @@ describe('approval prompt store', () => { ['approval.received', { request_id: 'r1', session_id: 's1' }] ]) }) + + it('does not replay a pending approval after the runtime is rejected as gone', async () => { + const request = vi.fn(async () => { + throw new JsonRpcGatewayError('session not found', { code: 4001 }) + }) + + await replayPendingApproval({ request }, 'dead-runtime') + await replayPendingApproval({ request }, 'dead-runtime') + + expect(request).toHaveBeenCalledTimes(1) + expect(isSessionGone('dead-runtime')).toBe(true) + expect($approvalRequest.get()).toBeNull() + }) + + it('propagates transient approval replay failures without latching the runtime', async () => { + const request = vi.fn(async () => { + throw new Error('gateway timed out') + }) + + await expect(replayPendingApproval({ request }, 'transient-runtime')).rejects.toThrow('gateway timed out') + expect(isSessionGone('transient-runtime')).toBe(false) + }) + + it('keeps approval receipt failures contained and marks the runtime gone', async () => { + const request = vi.fn(async () => { + throw new JsonRpcGatewayError('session not found', { code: 4001 }) + }) + + $activeSessionId.set('dead-runtime') + + await expect( + receiveApprovalRequest( + { request }, + { command: 'x', description: 'd', requestId: 'r1', sessionId: 'dead-runtime' } + ) + ).resolves.toBeUndefined() + + expect(isSessionGone('dead-runtime')).toBe(true) + expect($approvalRequest.get()?.requestId).toBe('r1') + }) + + it('propagates transient approval receipt failures without latching the runtime', async () => { + const request = vi.fn(async () => { + throw new Error('gateway timed out') + }) + + setActiveSessionId('transient-runtime') + + await expect( + receiveApprovalRequest( + { request }, + { command: 'x', description: 'd', requestId: 'r2', sessionId: 'transient-runtime' } + ) + ).rejects.toThrow('gateway timed out') + + expect(isSessionGone('transient-runtime')).toBe(false) + expect($approvalRequest.get()?.requestId).toBe('r2') + }) }) describe('sudo prompt store', () => { diff --git a/apps/desktop/src/store/prompts.ts b/apps/desktop/src/store/prompts.ts index 4aca295ede..13637fc797 100644 --- a/apps/desktop/src/store/prompts.ts +++ b/apps/desktop/src/store/prompts.ts @@ -3,6 +3,8 @@ import { atom, computed, type ReadableAtom } from 'nanostores' import { $clarifyRequest, $clarifyRequests } from './clarify' import { isSessionGone, isSessionGoneForBackgroundPolling, markSessionGone } from './runtime-gone' import { $activeSessionId } from './session' +import { ambientRequestFor } from './session-gone-latch' +import { requestForOwnedSession } from './session-states' // Blocking interactive prompts the gateway raises mid-turn. Each maps to a // `*.request` event the Python side emits while it blocks the agent thread @@ -120,10 +122,20 @@ export async function receiveApprovalRequest(gateway: ApprovalGateway | null, re setApprovalRequest(request) if (gateway && request.requestId && request.sessionId) { - await gateway.request('approval.received', { - request_id: request.requestId, - session_id: request.sessionId - }) + try { + await requestForOwnedSession(request.sessionId, ambientRequestFor(gateway), 'approval.received', { + request_id: request.requestId, + session_id: request.sessionId + }) + } catch (error) { + if (isSessionGoneForBackgroundPolling(error)) { + markSessionGone(request.sessionId) + + return + } + + throw error + } } } @@ -135,7 +147,7 @@ export async function replayPendingApproval(gateway: ApprovalGateway | null, ses let rawResult: unknown try { - rawResult = await gateway.request('approval.pending', { + rawResult = await requestForOwnedSession(sessionId, ambientRequestFor(gateway), 'approval.pending', { session_id: sessionId }) } catch (error) { diff --git a/apps/desktop/src/store/provider-wait.test.ts b/apps/desktop/src/store/provider-wait.test.ts index f5422fe4a8..9c5e6bc143 100644 --- a/apps/desktop/src/store/provider-wait.test.ts +++ b/apps/desktop/src/store/provider-wait.test.ts @@ -22,7 +22,9 @@ describe('providerWaitText', () => { describe('parseModelLoadWait', () => { it('extracts model and percent from a load frame', () => { expect( - parseModelLoadWait('⏳ loading Qwen3.6-35B-A3B-UD-Q4_K_M into memory — 42% (responses start once the model is loaded)') + parseModelLoadWait( + '⏳ loading Qwen3.6-35B-A3B-UD-Q4_K_M into memory — 42% (responses start once the model is loaded)' + ) ).toEqual({ kind: 'load', model: 'Qwen3.6-35B-A3B-UD-Q4_K_M', percent: 42 }) }) diff --git a/apps/desktop/src/store/runtime-gone.test.ts b/apps/desktop/src/store/runtime-gone.test.ts index c7e5280616..7c0198825b 100644 --- a/apps/desktop/src/store/runtime-gone.test.ts +++ b/apps/desktop/src/store/runtime-gone.test.ts @@ -1,9 +1,19 @@ +import { JsonRpcGatewayError } from '@hermes/shared' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { refreshBackgroundProcesses, resetBackgroundPollingGuard } from './composer-status' import { $gateway } from './gateway' -import { markRuntimeGone, noteRuntimeAlive, resetRuntimeGoneHealing } from './runtime-gone' +import { + isSessionGone, + isSessionGoneForBackgroundPolling, + markRuntimeGone, + markSessionGone, + noteRuntimeAlive, + resetBackgroundPollingGuardAfterRebind, + resetRuntimeGoneHealing +} from './runtime-gone' import { $activeSessionId, $sessionResumeRequest } from './session' +import { $removedSessionIds, tombstoneSessions } from './session-removal' import { $sessionStates, $sessionTiles } from './session-states' const STORED = 'stored-1' @@ -20,6 +30,7 @@ beforeEach(() => { $sessionTiles.set([]) $activeSessionId.set(null) $sessionResumeRequest.set(null) + $removedSessionIds.set(new Set()) }) afterEach(() => { @@ -30,6 +41,7 @@ afterEach(() => { $sessionTiles.set([]) $activeSessionId.set(null) $sessionResumeRequest.set(null) + $removedSessionIds.set(new Set()) }) describe('markRuntimeGone', () => { @@ -66,6 +78,19 @@ describe('markRuntimeGone', () => { expect($sessionResumeRequest.get()).toBeNull() }) + it('does not resurrect a chat the user just deleted', () => { + // An idle reap can land its 4001 in the same tick as the delete. The stored + // id is already tombstoned, so requestSessionResume drops the request + // instead of re-selecting a doomed chat and toasting "Resume failed". + $sessionStates.set({ [RUNTIME]: cachedState(STORED) }) + $activeSessionId.set(RUNTIME) + tombstoneSessions([STORED]) + + markRuntimeGone(RUNTIME) + + expect($sessionResumeRequest.get()).toBeNull() + }) + it('heals a runtime id exactly once, however many pollers report it', () => { $sessionStates.set({ [RUNTIME]: cachedState(STORED) }) $activeSessionId.set(RUNTIME) @@ -147,3 +172,64 @@ describe('refreshBackgroundProcesses recovery', () => { expect($sessionResumeRequest.get()).toBeNull() }) }) + +describe('gone-latch classifier and rebind seam', () => { + it('recognizes structured 4001 and bare legacy text without misclassifying coded errors', () => { + expect(isSessionGoneForBackgroundPolling(new JsonRpcGatewayError('gone', { code: 4001 }))).toBe(true) + expect(isSessionGoneForBackgroundPolling(new JsonRpcGatewayError('session not found', { code: 5007 }))).toBe(false) + expect(isSessionGoneForBackgroundPolling(new JsonRpcGatewayError('session not found'))).toBe(true) + expect( + isSessionGoneForBackgroundPolling(new Error("Error invoking remote method 'x': Error: session not found")) + ).toBe(true) + expect(isSessionGoneForBackgroundPolling(new Error('tool failed: upstream said session not found'))).toBe(false) + }) + + it('clears the latch only for ids a successful resume/activate rebound', () => { + markSessionGone('rt-dead') + markSessionGone('rt-other') + + resetBackgroundPollingGuardAfterRebind('process.list', { session_id: 'rt-dead' }, { session_id: 'rt-dead' }) + expect(isSessionGone('rt-dead')).toBe(true) + + resetBackgroundPollingGuardAfterRebind('session.resume', { session_id: 'stored-1' }, { session_id: 'rt-dead' }) + expect(isSessionGone('rt-dead')).toBe(false) + expect(isSessionGone('rt-other')).toBe(true) + + resetBackgroundPollingGuardAfterRebind('session.activate', { session_id: 'rt-other' }, undefined) + expect(isSessionGone('rt-other')).toBe(false) + }) + + it('a respawned backend (global clear) also resets every heal budget', () => { + for (const rt of ['rt-1', 'rt-2', 'rt-3']) { + $sessionStates.set({ [rt]: cachedState(STORED) }) + $sessionTiles.set([tile(STORED, rt)]) + expect(markRuntimeGone(rt)).toBe(true) + } + + resetBackgroundPollingGuard() + + $sessionStates.set({ 'rt-4': cachedState(STORED) }) + $sessionTiles.set([tile(STORED, 'rt-4')]) + expect(markRuntimeGone('rt-4')).toBe(true) + }) + + it('refunds the stored session heal budget on a successful rebind', () => { + // Three reaps exhaust MAX_CONSECUTIVE_HEALS for STORED... + for (const rt of ['rt-1', 'rt-2', 'rt-3']) { + $sessionStates.set({ [rt]: cachedState(STORED) }) + $sessionTiles.set([tile(STORED, rt)]) + expect(markRuntimeGone(rt)).toBe(true) + } + + $sessionStates.set({ 'rt-4': cachedState(STORED) }) + $sessionTiles.set([tile(STORED, 'rt-4')]) + expect(markRuntimeGone('rt-4')).toBe(false) + + // ...but a rebind of STORED proves it alive, so the next reap heals again. + resetBackgroundPollingGuardAfterRebind('session.resume', { session_id: STORED }, { session_id: 'rt-5' }) + + $sessionStates.set({ 'rt-5': cachedState(STORED) }) + $sessionTiles.set([tile(STORED, 'rt-5')]) + expect(markRuntimeGone('rt-5')).toBe(true) + }) +}) diff --git a/apps/desktop/src/store/runtime-gone.ts b/apps/desktop/src/store/runtime-gone.ts index 8d65c7d693..c529cbb8d9 100644 --- a/apps/desktop/src/store/runtime-gone.ts +++ b/apps/desktop/src/store/runtime-gone.ts @@ -1,41 +1,19 @@ import { $activeSessionId, requestSessionResume } from './session' +import { + healsByStoredId, + isSessionGone, + isSessionGoneForBackgroundPolling, + latchSessionGone, + resetBackgroundPollingGuard, + resetBackgroundPollingGuardAfterRebind +} from './session-gone-latch' import { $sessionStates, $sessionTiles, unbindTileRuntime } from './session-states' -/** Session ids the gateway has told us are gone. A session-scoped RPC against a - * runtime the gateway no longer holds fails 4001 "session not found" — terminal - * for THIS runtime id, not a transient socket loss. - * - * Shared by every background poller (process.list, approval.pending, goal - * status). One set, one clear path: a fresh-runtime rebind calls - * {@link resetBackgroundPollingGuard} and every poller resumes. */ -const goneSessions = new Set() - -/** Gateway JSON-RPC code for "session not found" (tui_gateway `_sess_nowait`). */ -const GATEWAY_SESSION_NOT_FOUND_CODE = 4001 - -/** A gone session is unrecoverable for THIS runtime id; a timeout or transport - * blip is not. Only the former may stop a poll — misclassifying a transient - * failure would silently freeze a healthy session. - * - * Match the gateway's 4001 code when the error carries one. The message - * fallback survives only for errors with no numeric code at all. */ -export function isSessionGoneForBackgroundPolling(error: unknown): boolean { - const code = - error && typeof error === 'object' && typeof (error as { code?: unknown }).code === 'number' - ? (error as { code: number }).code - : undefined - - if (code !== undefined) { - return code === GATEWAY_SESSION_NOT_FOUND_CODE - } - - const message = error instanceof Error ? error.message : String(error ?? '') - - return /session not found/i.test(message) -} - -export function isSessionGone(sid: string): boolean { - return goneSessions.has(sid) +export { + isSessionGone, + isSessionGoneForBackgroundPolling, + resetBackgroundPollingGuard, + resetBackgroundPollingGuardAfterRebind } /** Latch `sid` off and heal the bound view. Safe to call on every 4001. */ @@ -44,23 +22,10 @@ export function markSessionGone(sid: string): void { return } - goneSessions.add(sid) + latchSessionGone(sid) markRuntimeGone(sid) } -/** Clear the gone-latch. Called with a session id when a fresh runtime binds to - * it (so polling resumes), or with no argument to reset everything (tests / - * gateway reconnect). */ -export function resetBackgroundPollingGuard(sid?: string): void { - if (sid) { - goneSessions.delete(sid) - - return - } - - goneSessions.clear() -} - /** Heal a session view whose bound runtime id the gateway no longer holds. * * The desktop learns a runtime is gone through two channels: @@ -96,11 +61,12 @@ export function resetBackgroundPollingGuard(sid?: string): void { * for the same id could only come from a duplicate report of the same death. */ const healedRuntimes = new Set() -/** Consecutive heals per stored session id, reset by {@link noteRuntimeAlive}. - * A backend that reaps as fast as we resume would otherwise turn this into the - * very storm it exists to stop — one resume per poll tick, forever. Cap it and - * let the user's next action (which carries its own recovery) take over. */ -const healsByStoredId = new Map() +/** Consecutive heals per stored session id live in `session-gone-latch` + * (`healsByStoredId`), reset by {@link noteRuntimeAlive} and by a successful + * rebind. A backend that reaps as fast as we resume would otherwise turn this + * into the very storm it exists to stop — one resume per poll tick, forever. + * Cap it and let the user's next action (which carries its own recovery) + * take over. */ /** Enough to ride out a reap that races a resume, low enough that a backend * reaping on sight cannot be turned into a resume loop. */ diff --git a/apps/desktop/src/store/session-gone-latch.ts b/apps/desktop/src/store/session-gone-latch.ts new file mode 100644 index 0000000000..5e22ebf550 --- /dev/null +++ b/apps/desktop/src/store/session-gone-latch.ts @@ -0,0 +1,130 @@ +import { JsonRpcGatewayError } from '@hermes/shared' + +/** Session ids the gateway has told us are gone. A session-scoped RPC against a + * runtime the gateway no longer holds fails 4001 "session not found" — terminal + * for THIS runtime id, not a transient socket loss. + * + * Shared by every background poller (process.list, approval.pending, goal + * status) and by the owner-routed RPC seam that clears it. This module is a + * dependency-free leaf on purpose: `session-request-router` (which every + * store imports) must be able to clear the latch after a successful rebind + * without pulling the session/tile stores into its import graph. Stores that + * also need the heal levers import through `runtime-gone.ts` (which re-exports + * this module); cycle-sensitive callers (the router, the gateway event loop) + * import the leaf directly. */ +const goneSessions = new Set() + +/** Gateway JSON-RPC code for "session not found" (tui_gateway `_sess_nowait`). */ +const GATEWAY_SESSION_NOT_FOUND_CODE = 4001 + +/** Consecutive heals per stored session id (see `runtime-gone.ts` + * `markRuntimeGone`). Lives here so the rebind seam below can refund it + * without importing the heal module. */ +export const healsByStoredId = new Map() + +/** A gone session is unrecoverable for THIS runtime id; a timeout or transport + * blip is not. Only the former may stop a poll — misclassifying a transient + * failure would silently freeze a healthy session. + * + * Match the gateway's 4001 code when the error carries one. Codeless errors + * (the frame's structure was lost across the IPC bridge or a wrapped rethrow) + * are accepted only with a bare "session not found" body — a tool or report + * string that merely mentions the phrase must not latch a live runtime. */ +export function isSessionGoneForBackgroundPolling(error: unknown): boolean { + if (error instanceof JsonRpcGatewayError && typeof error.code === 'number') { + return error.code === GATEWAY_SESSION_NOT_FOUND_CODE + } + + const code = + error && typeof error === 'object' && typeof (error as { code?: unknown }).code === 'number' + ? (error as { code: number }).code + : undefined + + if (code !== undefined) { + return code === GATEWAY_SESSION_NOT_FOUND_CODE + } + + const message = (error instanceof Error ? error.message : String(error ?? '')) + .trim() + .replace(/^Error invoking remote method '[^']+':\s*Error:\s*/i, '') + .replace(/^Error:\s*/i, '') + + return /^(?:4001\s*[:,-]?\s*)?session not found[.!]?$/i.test(message) +} + +export function isSessionGone(sid: null | string | undefined): boolean { + return Boolean(sid && goneSessions.has(sid)) +} + +/** Latch `sid` off. Idempotent. */ +export function latchSessionGone(sid: string): void { + if (sid) { + goneSessions.add(sid) + } +} + +/** Clear the gone-latch. Called with a session id when a fresh runtime binds to + * it (so polling resumes), or with no argument to reset everything (tests / + * a respawned backend that re-mints every runtime id). */ +export function resetBackgroundPollingGuard(sid?: string): void { + if (sid) { + goneSessions.delete(sid) + + return + } + + goneSessions.clear() + // Same lifetime as the latch: a respawned backend re-mints every runtime + // id, so every stored session's heal budget starts over too. + healsByStoredId.clear() +} + +/** Ids a successful `session.resume` / `session.activate` just rebound — the + * stored id it was asked for and the runtime id it answered with. Empty for + * any other method: a socket reconnect is NOT a rebind (the backend may have + * reaped the old runtime, and reopening a WebSocket does not make that id + * valid again). Only a successful resume/activate response is proof. */ +function reboundSessionIds(method: string, params: Record, result: unknown): string[] { + if (method !== 'session.activate' && method !== 'session.resume') { + return [] + } + + const ids: string[] = [] + + for (const value of [params.session_id, (result as { session_id?: unknown } | null)?.session_id]) { + if (typeof value === 'string' && value.trim()) { + ids.push(value.trim()) + } + } + + return ids +} + +/** Un-latch the ids a successful `session.resume` / `session.activate` just + * rebound and refund the stored session's heal budget: a rebind is proof of + * life, so the NEXT reap can still be healed. Without the refund a backend + * that reaps a detached runtime a few times (per-request lease sockets + * closing between polls) exhausts the heal cap and the view is stuck on a + * phantom id forever (#100639: 1,230 approval.pending 4001s on one runtime + * id in 42 minutes, zero recovery). Called by the session request router on + * every routed RPC result; a no-op for every method but resume/activate. */ +export function resetBackgroundPollingGuardAfterRebind( + method: string, + params: Record, + result: unknown +): void { + for (const id of reboundSessionIds(method, params, result)) { + goneSessions.delete(id) + healsByStoredId.delete(id) + } +} + +/** Adapt a store-level gateway handle (`$gateway.get()` or the narrower + * `ApprovalGateway` shape) to the ambient-request callback + * `requestForOwnedSession` expects. The pollers never pass a deadline, so the + * 2-arg call shape is kept exactly (gateway.request callers assert on it). */ +export function ambientRequestFor(gateway: { + request: (method: string, params: Record) => Promise +}): (method: string, params?: Record) => Promise { + return (method: string, params?: Record) => gateway.request(method, params ?? {}) as Promise +} diff --git a/apps/desktop/src/store/session-pane-focus.test.ts b/apps/desktop/src/store/session-pane-focus.test.ts new file mode 100644 index 0000000000..0c0b2688b1 --- /dev/null +++ b/apps/desktop/src/store/session-pane-focus.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +async function setup() { + const tree = await import('@/components/pane-shell/tree/store') + const model = await import('@/components/pane-shell/tree/model') + const { registry } = await import('@/contrib/registry') + const session = await import('@/store/session') + const states = await import('@/store/session-states') + const { paneMirror } = await import('@/app/chat/pane-mirror') + + registry.register({ + area: 'panes', + data: { placement: 'main', uncloseable: true }, + id: 'workspace', + render: () => null, + title: 'Chat' + }) + tree.declareDefaultTree(model.group(['workspace'], { active: 'workspace', id: 'main' })) + tree.watchContributedPanes() + paneMirror({ + source: states.$sessionTiles, + key: tile => tile.storedSessionId, + prefix: 'session-tile', + dir: () => 'center', + minWidth: '20rem', + title: id => id, + render: () => null, + close: states.closeSessionTile + })() + session.$selectedStoredSessionId.set('previous-chat') + + const scope = { + ownerRoute: { connectionId: 'remote-a', mode: 'remote' as const, profile: 'writer' }, + workspaceMode: 'bots' as const, + workspaceOwnerKey: 'remote-a::writer', + workspaceTabTitle: 'Bot Chat' + } + + states.openSessionTile('canonical-chat', 'center', 'workspace', undefined, scope) + + return { model, scope, session, states, tree } +} + +describe('focusing a saved Bot Chat requires a visible pane', () => { + let ctx: Awaited> + const paneId = 'session-tile:canonical-chat' + + beforeEach(async () => { + window.localStorage.clear() + vi.resetModules() + ctx = await setup() + }) + + it('re-adopts a saved tab after a profile overlay replaces the layout', async () => { + const { applyDesktopOverlay } = await import('@/store/profile-share') + const { model, scope, states, tree } = ctx + const saved = states.$sessionTiles.get() + applyDesktopOverlay('imported-profile', { + version: 1, + layoutTree: model.group(['workspace'], { active: 'workspace', id: 'imported-main' }) + }) + expect(model.findGroupOfPane(tree.$layoutTree.get()!, paneId)).toBeNull() + + expect(states.focusWorkspaceOwnerSessionTile(scope.workspaceOwnerKey, undefined, ['canonical-chat'])).toBe( + 'canonical-chat' + ) + expect(tree.isPaneVisible(paneId)).toBe(true) + expect(tree.$activeTreeGroup.get()).toBe('imported-main') + expect(states.$sessionTiles.get()).toEqual(saved) + expect(states.sessionTileOwnerRoute('canonical-chat')).toEqual(scope.ownerRoute) + }) + + it('reports a miss through both helpers if the layout cannot place the saved tab', () => { + const { scope, session, states, tree } = ctx + tree.$layoutTree.set(null) + + expect(states.focusOpenSession('canonical-chat', scope)).toBeNull() + expect(states.focusWorkspaceOwnerSessionTile(scope.workspaceOwnerKey, undefined, ['canonical-chat'])).toBeNull() + expect(session.$selectedStoredSessionId.get()).toBe('previous-chat') + expect(states.$sessionTiles.get().map(tile => tile.storedSessionId)).toEqual(['canonical-chat']) + }) +}) diff --git a/apps/desktop/src/store/session-pin-sync.test.ts b/apps/desktop/src/store/session-pin-sync.test.ts index f9c39e76b3..9abfad3c91 100644 --- a/apps/desktop/src/store/session-pin-sync.test.ts +++ b/apps/desktop/src/store/session-pin-sync.test.ts @@ -15,7 +15,7 @@ vi.mock('@/hermes', () => ({ import { $pinnedSessionIds } from '@/store/layout' import { $activeGatewayProfile } from '@/store/profile' -import { $sessions } from '@/store/session' +import { $cronSessions, $messagingSessions, $sessions } from '@/store/session' import { $unconfirmedPinWrites, resetSessionPinMirror, watchSessionPins } from './session-pin-sync' @@ -33,6 +33,8 @@ beforeAll(() => { beforeEach(() => { $sessions.set([]) + $cronSessions.set([]) + $messagingSessions.set([]) $pinnedSessionIds.set([]) // The mirror/pending/unconfirmed maps are module-global, so one test's // bookkeeping would otherwise suppress the next test's PATCH (or fence out @@ -43,6 +45,8 @@ beforeEach(() => { afterEach(() => { $sessions.set([]) + $cronSessions.set([]) + $messagingSessions.set([]) $pinnedSessionIds.set([]) }) @@ -103,6 +107,51 @@ describe('watchSessionPins', () => { }) describe('watchSessionPins remote pull', () => { + it('adopts and durably unpins a backend-only messaging pin', async () => { + $messagingSessions.set([row('photon-pin', { pinned: true, profile: 'messages', source: 'photon' })]) + await flush() + + expect($pinnedSessionIds.get()).toEqual(['photon-pin']) + patch.mockClear() + + $pinnedSessionIds.set([]) + await flush() + + expect(patch).toHaveBeenCalledWith('photon-pin', false, 'messages') + }) + + it('adopts and durably unpins a backend-only cron pin', async () => { + $cronSessions.set([row('cron-pin', { pinned: true, profile: 'jobs', source: 'cron' })]) + await flush() + + expect($pinnedSessionIds.get()).toEqual(['cron-pin']) + patch.mockClear() + + $pinnedSessionIds.set([]) + await flush() + + expect(patch).toHaveBeenCalledWith('cron-pin', false, 'jobs') + }) + + it('routes a cross-slice unpin to the active profile', async () => { + $activeGatewayProfile.set('work') + + try { + $sessions.set([row('shared', { pinned: true, profile: 'default' })]) + $messagingSessions.set([row('shared', { pinned: true, profile: 'work', source: 'photon' })]) + await flush() + expect($pinnedSessionIds.get()).toEqual(['shared']) + patch.mockClear() + + $pinnedSessionIds.set([]) + await flush() + + expect(patch).toHaveBeenCalledWith('shared', false, 'work') + } finally { + $activeGatewayProfile.set('default') + } + }) + it('adopts a pin another app made', async () => { $sessions.set([row('remote', { pinned: true })]) await flush() diff --git a/apps/desktop/src/store/session-pin-sync.ts b/apps/desktop/src/store/session-pin-sync.ts index 8d162a7e83..5ca8c694b5 100644 --- a/apps/desktop/src/store/session-pin-sync.ts +++ b/apps/desktop/src/store/session-pin-sync.ts @@ -27,7 +27,7 @@ import { setSessionPinnedRemote } from '@/hermes' import { onConnectionScopeChange } from '@/lib/connection-scoped' import { $pinnedSessionIds, pinSession, unpinSession } from '@/store/layout' import { $activeGatewayProfile, normalizeProfileKey } from '@/store/profile' -import { $sessions, sessionMatchesStoredId, sessionPinId } from '@/store/session' +import { $cronSessions, $messagingSessions, $sessions, sessionMatchesStoredId, sessionPinId } from '@/store/session' import type { SessionInfo } from '@/types/hermes' // pin ids we've successfully PATCHed pinned=true this session. @@ -71,7 +71,24 @@ function publishUnconfirmed(): void { } function profileFor(pinId: string): null | string | undefined { - return $sessions.get().find(row => sessionMatchesStoredId(row, pinId))?.profile + return loadedRowFor(pinId)?.profile +} + +function loadedSessionRows(): SessionInfo[] { + return [...$sessions.get(), ...$cronSessions.get(), ...$messagingSessions.get()] +} + +/** + * The row a stored pin id resolves to, across every slice. Same tie-break as + * `rowsByPinId`: when two profiles share the id, the write must target the + * row the pull adopted — the active gateway's — or an unpin PATCHes the other + * profile and the next page re-adopts the pin. + */ +function loadedRowFor(pinId: string): SessionInfo | undefined { + const rows = loadedSessionRows().filter(row => sessionMatchesStoredId(row, pinId)) + const gateway = normalizeProfileKey($activeGatewayProfile.get()) + + return rows.find(row => normalizeProfileKey(row.profile) === gateway) ?? rows[0] } /** @@ -140,7 +157,7 @@ function writePin(id: string, pinned: boolean, profile?: null | string): Promise function pullRemotePins(): void { const local = new Set($pinnedSessionIds.get()) - for (const row of rowsByPinId($sessions.get()).values()) { + for (const row of rowsByPinId(loadedSessionRows()).values()) { // A backend without the flag has no opinion; never act on `undefined`. if (typeof row.pinned !== 'boolean') { continue @@ -190,8 +207,8 @@ function pullRemotePins(): void { } } -// Re-entrancy guard: reconcile() is subscribed to BOTH $sessions and -// $pinnedSessionIds, and pullRemotePins() mutates $pinnedSessionIds (via +// Re-entrancy guard: reconcile() is subscribed to every loaded-session slice +// and $pinnedSessionIds, and pullRemotePins() mutates $pinnedSessionIds (via // pinSession/unpinSession), which fires reconcile() again synchronously. // Without this guard, a session whose pin state oscillates — two rows with the // same durable id but conflicting `pinned` flags, possible when profile @@ -247,9 +264,9 @@ function reconcileInner(): void { } // Flush whatever we can resolve now; unresolved ids (row not loaded yet) - // retry on the next $sessions change. + // retry on the next loaded-session slice change. for (const id of [...pending]) { - const row = $sessions.get().find(entry => sessionMatchesStoredId(entry, id)) + const row = loadedRowFor(id) if (!row) { continue @@ -276,6 +293,8 @@ export function watchSessionPins(): void { reconcile() $pinnedSessionIds.listen(reconcile) $sessions.listen(reconcile) + $cronSessions.listen(reconcile) + $messagingSessions.listen(reconcile) } /** diff --git a/apps/desktop/src/store/session-removal.test.ts b/apps/desktop/src/store/session-removal.test.ts new file mode 100644 index 0000000000..2757663a6c --- /dev/null +++ b/apps/desktop/src/store/session-removal.test.ts @@ -0,0 +1,80 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { $sessionResumeRequest, requestSessionResume } from './session' +import { + $removedSessionIds, + $sessionMutationsInFlight, + beginSessionMutation, + endSessionMutation, + isSessionRemovalPending, + tombstoneSessions, + untombstoneSessions +} from './session-removal' + +afterEach(() => { + $removedSessionIds.set(new Set()) + $sessionMutationsInFlight.set(new Set()) + $sessionResumeRequest.set(null) +}) + +describe('isSessionRemovalPending', () => { + it('is true for a tombstoned id and for one whose delete RPC is still in flight', () => { + tombstoneSessions(['gone']) + beginSessionMutation(['deleting']) + + expect(isSessionRemovalPending('gone')).toBe(true) + expect(isSessionRemovalPending('deleting')).toBe(true) + expect(isSessionRemovalPending('alive')).toBe(false) + }) + + it('goes false again when a failed delete rolls the row back', () => { + tombstoneSessions(['rolled-back']) + beginSessionMutation(['rolled-back']) + expect(isSessionRemovalPending('rolled-back')).toBe(true) + + untombstoneSessions(['rolled-back']) + endSessionMutation(['rolled-back']) + + expect(isSessionRemovalPending('rolled-back')).toBe(false) + }) + + it('ignores blank ids rather than treating them as pending', () => { + expect(isSessionRemovalPending('')).toBe(false) + expect(isSessionRemovalPending(' ')).toBe(false) + expect(isSessionRemovalPending(null)).toBe(false) + }) +}) + +describe('requestSessionResume refuses a doomed session', () => { + it('queues a resume for a live session', () => { + requestSessionResume('live-1') + + expect($sessionResumeRequest.get()?.sessionId).toBe('live-1') + }) + + it('drops the request once the id is tombstoned', () => { + tombstoneSessions(['deleted-1']) + + requestSessionResume('deleted-1') + + expect($sessionResumeRequest.get()).toBeNull() + }) + + it('drops the request while the delete RPC is still in flight', () => { + beginSessionMutation(['deleting-1']) + + requestSessionResume('deleting-1') + + expect($sessionResumeRequest.get()).toBeNull() + }) + + it('leaves an earlier live request intact instead of clobbering it', () => { + requestSessionResume('live-1') + const queued = $sessionResumeRequest.get() + + tombstoneSessions(['deleted-1']) + requestSessionResume('deleted-1') + + expect($sessionResumeRequest.get()).toBe(queued) + }) +}) diff --git a/apps/desktop/src/store/session-removal.ts b/apps/desktop/src/store/session-removal.ts new file mode 100644 index 0000000000..ae67b64e09 --- /dev/null +++ b/apps/desktop/src/store/session-removal.ts @@ -0,0 +1,95 @@ +import { atom } from 'nanostores' + +// Client-side cache eviction (Apollo-style optimistic layer): ids the user just +// deleted/archived. The backend tree is a snapshot that still lists them until +// its next refresh, so the render-time overlay strips these so the tree matches +// the live `$sessions` cache exactly — same as the flat Recents list. Pruned on +// refresh once the server snapshot has caught up. +// +// This lives beside the session stores rather than in `projects.ts` so the +// resume path can consult it without importing the project tree (which itself +// reads `store/session`). +export const $removedSessionIds = atom>(new Set()) + +export function tombstoneSessions(ids: Array): void { + const next = new Set($removedSessionIds.get()) + const before = next.size + + for (const id of ids) { + const trimmed = id?.trim() + + if (trimmed) { + next.add(trimmed) + } + } + + if (next.size !== before) { + $removedSessionIds.set(next) + } +} + +export function untombstoneSessions(ids: Array): void { + const current = $removedSessionIds.get() + + if (!current.size) { + return + } + + const next = new Set(current) + + for (const id of ids) { + const trimmed = id?.trim() + + if (trimmed) { + next.delete(trimmed) + } + } + + if (next.size !== current.size) { + $removedSessionIds.set(next) + } +} + +// Ids whose delete/archive RPC is still in flight. Their tombstones are pinned +// against the projects.tree prune: a refresh whose snapshot predates the +// mutation completing must NOT drop the tombstone, or the row flashes back until +// the backend catches up. Keyed by id, so concurrent deletes stay independent. +export const $sessionMutationsInFlight = atom>(new Set()) + +function mutateInFlight(ids: Array, add: boolean): void { + const current = $sessionMutationsInFlight.get() + const next = new Set(current) + + for (const id of ids) { + const trimmed = id?.trim() + + if (trimmed) { + add ? next.add(trimmed) : next.delete(trimmed) + } + } + + if (next.size !== current.size) { + $sessionMutationsInFlight.set(next) + } +} + +export const beginSessionMutation = (ids: Array): void => mutateInFlight(ids, true) +export const endSessionMutation = (ids: Array): void => mutateInFlight(ids, false) + +/** The session is on its way out: already tombstoned, or its delete/archive RPC + * is still in flight. Either way the durable row is doomed, so nothing may + * resume it — a resume would 404 and toast "Resume failed / Session not found" + * for a chat the user deliberately removed. + * + * Deletion tombstones synchronously and only untombstones if the RPC fails + * (which restores the row and the route), so this predicate is the single + * answer every resume actuator asks. */ +export function isSessionRemovalPending(sessionId: null | string | undefined): boolean { + const id = sessionId?.trim() + + if (!id) { + return false + } + + return $removedSessionIds.get().has(id) || $sessionMutationsInFlight.get().has(id) +} diff --git a/apps/desktop/src/store/session-request-router.test.ts b/apps/desktop/src/store/session-request-router.test.ts index 3ec852e284..8c3261b402 100644 --- a/apps/desktop/src/store/session-request-router.test.ts +++ b/apps/desktop/src/store/session-request-router.test.ts @@ -1,5 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { isSessionGone, latchSessionGone, resetBackgroundPollingGuard } from './session-gone-latch' + // Regression coverage for the #89206 wake-failure class: session-scoped RPCs // routed to a backend that does not own the session's profile. Three layers: // 1. The registry publishes the ACTIVE route's profile ($activeGatewayRoute) @@ -105,6 +107,7 @@ beforeEach(() => { secondaryGateways.length = 0 promptAckStatus = null $connectionsRegistry.set(null) + resetBackgroundPollingGuard() configureGatewayRegistry({ onEvent: vi.fn() }) closeSecondaryGateways() }) @@ -112,6 +115,7 @@ beforeEach(() => { afterEach(() => { closeSecondaryGateways() vi.clearAllMocks() + resetBackgroundPollingGuard() delete (window as unknown as { hermesDesktop?: unknown }).hermesDesktop }) @@ -179,6 +183,62 @@ describe('sessionRpcNeedsProfileRoute', () => { }) describe('requestForSessionProfile', () => { + it('clears a dead-runtime latch only after a successful resume or activate', async () => { + const ambient = vi.fn(async () => ({ session_id: 'rt-rebound' })) + + latchSessionGone('rt-rebound') + expect(isSessionGone('rt-rebound')).toBe(true) + + await requestForSessionProfile(null, ambient as never, 'session.activate', { session_id: 'rt-rebound' }) + expect(isSessionGone('rt-rebound')).toBe(false) + + latchSessionGone('rt-rebound') + await expect( + requestForSessionProfile( + null, + vi.fn(async () => { + throw new Error('resume failed') + }) as never, + 'session.resume', + { session_id: 'rt-rebound' } + ) + ).rejects.toThrow('resume failed') + expect(isSessionGone('rt-rebound')).toBe(true) + }) + + it('clears a dead-runtime latch through the bare profile owner route', async () => { + const primary = makePrimary() + setPrimaryGateway(primary as never, 'default') + installDesktop() + const ambient = vi.fn(async () => ({ ambient: true })) + + latchSessionGone('profile-rebound') + + await requestForSessionProfile('loki', ambient as never, 'session.resume', { + session_id: 'profile-rebound' + }) + + expect(isSessionGone('profile-rebound')).toBe(false) + }) + + it('clears a dead-runtime latch through an explicit connection owner route', async () => { + const primary = makePrimary() + setPrimaryGateway(primary as never, 'default') + installDesktop() + const ambient = vi.fn(async () => ({ ambient: true })) + + latchSessionGone('connection-rebound') + + await requestForSessionProfile( + { connectionId: 'source-a', profile: 'default' }, + ambient as never, + 'session.activate', + { session_id: 'connection-rebound' } + ) + + expect(isSessionGone('connection-rebound')).toBe(false) + }) + it('keeps routing a bare profile owner through its legacy profile pool when a connection registry exists', async () => { // A profile pick on the primary or the explicit `local` source takes the // legacy profile-only door (store/profile activateOnCurrentSource), so a diff --git a/apps/desktop/src/store/session-request-router.ts b/apps/desktop/src/store/session-request-router.ts index 3cb1993c33..943ed192e7 100644 --- a/apps/desktop/src/store/session-request-router.ts +++ b/apps/desktop/src/store/session-request-router.ts @@ -1,5 +1,7 @@ import { requestGatewayForAgent, requestGatewayForProfile, retainGatewayForSessionTurn } from '@/store/gateway' +import { resetBackgroundPollingGuardAfterRebind } from './session-gone-latch' + /** * The ONE authoritative exact owner of a session: the registry connection whose * socket minted (or resumed) the runtime, plus the Desktop profile that selects @@ -107,13 +109,14 @@ async function withRoutedTurnLease( const sessionId = promptSessionId(method, params) if (!sessionId) { - return request() + return requestWithRebindGuard(method, params, request) } const release = await retainGatewayForSessionTurn(connectionId, profile, sessionId) try { const result = await request() + resetBackgroundPollingGuardAfterRebind(method, params, result) if (!turnKeepsRunning(result)) { release() @@ -126,6 +129,17 @@ async function withRoutedTurnLease( } } +async function requestWithRebindGuard( + method: string, + params: Record, + request: () => Promise +): Promise { + const result = await request() + resetBackgroundPollingGuardAfterRebind(method, params, result) + + return result +} + /** * True when a session-scoped RPC must be pinned to `ownerProfile`'s own socket. * @@ -193,14 +207,14 @@ export function requestForSessionProfile( // for a deadline (the plugin host bridge in contrib/wiring is the only one // that does). if (signal !== undefined) { - return ambientRequest(method, params, timeoutMs, signal) + return requestWithRebindGuard(method, params, () => ambientRequest(method, params, timeoutMs, signal)) } if (timeoutMs !== undefined) { - return ambientRequest(method, params, timeoutMs) + return requestWithRebindGuard(method, params, () => ambientRequest(method, params, timeoutMs)) } - return ambientRequest(method, params) + return requestWithRebindGuard(method, params, () => ambientRequest(method, params)) } const profile = normKey(ownerProfile) diff --git a/apps/desktop/src/store/session-states-foreground-scopes.test.ts b/apps/desktop/src/store/session-states-foreground-scopes.test.ts index 97ce973a14..2518a9fce4 100644 --- a/apps/desktop/src/store/session-states-foreground-scopes.test.ts +++ b/apps/desktop/src/store/session-states-foreground-scopes.test.ts @@ -69,6 +69,25 @@ describe('foregroundSessionScopes: owner hold across the create → foreground g expect(foregroundSessionScopes()).toEqual(new Set()) }) + it('does not retire a hold for a tile that carries neither the route nor a scoped runtime', () => { + const pandora = { connectionId: '100-125-133-71-9119', mode: 'remote' as const, profile: 'default' } + + holdSessionOwnerUntilForeground('stored-branch', pandora) + // The pre-fix branch path: openSessionTile with no workspaceScope mints a + // route-less tile. It pins nothing, so its mere existence must not retire + // the hold — that reopened the create→foreground gap and the pruner closed + // the owner socket under the draft runtime (resume→reclaim loop, #93892). + $sessionTiles.set([{ storedSessionId: 'stored-branch' }]) + expect(foregroundSessionScopes()).toEqual(new Set(['conn:100-125-133-71-9119::default'])) + + // Once the tile actually names the owner (persisted route), it covers the + // hold and the hold retires for good. + $sessionTiles.set([{ ownerRoute: pandora, storedSessionId: 'stored-branch' }]) + expect(foregroundSessionScopes()).toEqual(new Set(['conn:100-125-133-71-9119::default'])) + $sessionTiles.set([]) + expect(foregroundSessionScopes()).toEqual(new Set()) + }) + it('is released explicitly by the caller (failed create / drift close) and expires on its own', () => { vi.useFakeTimers() diff --git a/apps/desktop/src/store/session-states-runtime-map.test.ts b/apps/desktop/src/store/session-states-runtime-map.test.ts index 1b12f91a9a..8d8f5df9ed 100644 --- a/apps/desktop/src/store/session-states-runtime-map.test.ts +++ b/apps/desktop/src/store/session-states-runtime-map.test.ts @@ -1,14 +1,18 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { createClientSessionState } from '@/lib/chat-runtime' +import { $connectionsRegistry } from '@/store/connection-registry-state' +import { setPrimaryGateway, setPrimaryGatewayConnection } from '@/store/gateway' import { $profiles } from '@/store/profile' import { _resetSessionOwnerHintsForTests, setSessionOwnerHint, setSessions } from '@/store/session' import { isSessionOwnerResolutionError } from '@/store/session-owner-resolution' import { $sessionTiles, clearAllSessionStates, + dropSessionState, knownOwnerForSession, publishSessionState, + recordSessionEventScope, requestForOwnedSession, storedSessionIdForRuntimeId } from '@/store/session-states' @@ -126,4 +130,85 @@ describe('knownOwnerForSession / requestForOwnedSession', () => { ).resolves.toEqual({ ok: true }) expect(ambient).toHaveBeenCalledWith('approval.respond', { session_id: 'rt-orphan' }) }) + + it('routes a connection-tagged orphan runtime through the owner its inbound event recorded (#97511)', () => { + // Registry topology, multiple profiles, no tile/hint/row binding for the + // runtime — the approval.request event itself proved the exact owner. + $profiles.set([{ name: 'default' }, { name: 'omar' }] as never) + recordSessionEventScope({ connectionId: 'homelab', profile: 'omar', session_id: 'rt-unbound' }) + + expect(knownOwnerForSession('rt-unbound')).toEqual({ connectionId: 'homelab', profile: 'omar' }) + + // An event without a profile tag still records the 'default' convention + // every other owner source uses. + recordSessionEventScope({ connectionId: 'homelab', session_id: 'rt-unprofiled' }) + expect(knownOwnerForSession('rt-unprofiled')).toEqual({ connectionId: 'homelab', profile: 'default' }) + }) + + it('still prefers the durable stored owner when a stale runtime ledger entry collides with a stored id (#97511)', () => { + // Pathological collision: some dead runtime's id equals a live stored id. + // The persisted hint (durable identity) must outrank the ledger entry. + setSessionOwnerHint('stored-live', { connectionId: 'local', profile: 'omar' }) + recordSessionEventScope({ connectionId: 'spark', profile: 'default', session_id: 'stored-live' }) + + expect(knownOwnerForSession('stored-live')).toEqual({ connectionId: 'local', profile: 'omar' }) + }) + + it('keeps failing closed for untagged or unknown runtimes in multi-profile topology (#97511)', () => { + $profiles.set([{ name: 'default' }, { name: 'omar' }] as never) + // Untagged events carry no connectionId and record nothing. + recordSessionEventScope({ profile: 'omar', session_id: 'rt-untagged' }) + + expect(knownOwnerForSession('rt-untagged')).toBeUndefined() + expect(knownOwnerForSession('rt-never-seen')).toBeUndefined() + }) + + it('drops the recorded event owner together with the runtime state (#97511)', () => { + recordSessionEventScope({ connectionId: 'homelab', profile: 'omar', session_id: 'rt-dropped' }) + expect(knownOwnerForSession('rt-dropped')).toEqual({ connectionId: 'homelab', profile: 'omar' }) + + dropSessionState('rt-dropped') + expect(knownOwnerForSession('rt-dropped')).toBeUndefined() + }) + + it('answers an approval on a sole-local registry install through the primary socket (#96394)', async () => { + // The reported topology: a modern Desktop (connections bridge present, + // registry loaded with exactly one `local` connection), one profile, and + // an approval.request whose runtime id has no tile / hint / row binding. + // hasRegistryTopology() is true here, so the ambient escape hatch is + // closed by design — the exact owner must come from the event itself. + ;(window as unknown as { hermesDesktop: unknown }).hermesDesktop = { connections: { list: async () => null } } + $connectionsRegistry.set({ + activeConnectionId: 'local', + connections: [{ id: 'local', kind: 'local', label: 'Local' }] + } as never) + $profiles.set([{ name: 'default' }] as never) + + const primaryRequest = vi.fn(async (method: string, params: unknown) => ({ method, params, via: 'primary' })) + + setPrimaryGateway({ onEvent: () => () => undefined, request: primaryRequest, state: 'open' } as never, 'default') + setPrimaryGatewayConnection({ connectionId: 'local' }) + + const ambient = vi.fn(async () => ({ via: 'ambient' })) + + try { + // Before the event lands the owner is unknown and routing still fails closed. + await expect( + requestForOwnedSession('rt-approval', ambient as never, 'approval.respond', { session_id: 'rt-approval' }) + ).rejects.toSatisfy(isSessionOwnerResolutionError) + + // use-gateway-boot stamps every primary event with the active connection + // id (Electron resolves the sole local connection to `local`). + recordSessionEventScope({ connectionId: 'local', profile: 'default', session_id: 'rt-approval' }) + + await expect( + requestForOwnedSession('rt-approval', ambient as never, 'approval.respond', { session_id: 'rt-approval' }) + ).resolves.toEqual({ method: 'approval.respond', params: { session_id: 'rt-approval' }, via: 'primary' }) + expect(ambient).not.toHaveBeenCalled() + } finally { + setPrimaryGateway(null) + $connectionsRegistry.set(null) + delete (window as unknown as { hermesDesktop?: unknown }).hermesDesktop + } + }) }) diff --git a/apps/desktop/src/store/session-states.test.ts b/apps/desktop/src/store/session-states.test.ts index 4e2152a108..1926948f27 100644 --- a/apps/desktop/src/store/session-states.test.ts +++ b/apps/desktop/src/store/session-states.test.ts @@ -200,6 +200,32 @@ describe('resetTileRuntimeBindings', () => { expect(invalidateRuntimeBindings).toHaveBeenCalledWith(new Set(['stored-barry-sibling-bot', 'stored-work-bot'])) }) + it('keeps an owner-routed SESSIONS tile (branch child) bound across an unrelated reconnect', () => { + const invalidateRuntimeBindings = vi.fn() + setSessionTileDelegate({ invalidateRuntimeBindings } as unknown as SessionTileDelegate) + $sessionTiles.set([ + { + ownerRoute: { connectionId: '100-125-133-71-9119', mode: 'remote', profile: 'default' }, + runtimeId: 'runtime-branch-live', + storedSessionId: 'stored-branch-child', + workspaceMode: 'sessions' + } + ]) + + // A flapping sibling connection reconnects; the branch child's runtime + // lives on its parent's backend and must keep its binding — dropping it + // re-arms the tile's resume, and repeated sibling flaps latch the + // resume-storm error card over a healthy session. + resetTileRuntimeBindings({ connectionId: 'other-ssh-source', profile: 'default' }) + + expect($sessionTiles.get()[0]?.runtimeId).toBe('runtime-branch-live') + expect(invalidateRuntimeBindings).toHaveBeenCalledWith(new Set(['stored-branch-child'])) + + // Its OWN connection reconnecting still drops the binding for re-resume. + resetTileRuntimeBindings({ connectionId: '100-125-133-71-9119', profile: 'default' }) + expect($sessionTiles.get()[0]?.runtimeId).toBeUndefined() + }) + it('unknown restarted identity preserves only Bot runtimes owned by provably-live connections', () => { // Legacy remote primary: no registry connectionId to scope by. The dead // owner can't be named, so keep only owners we know are alive elsewhere — @@ -244,6 +270,27 @@ describe('SessionTile workspace scope', () => { $sessionTiles.set([]) }) + it('persists a sessions-mode owner route so a branch child tile pins its owning socket', () => { + const ownerRoute = { connectionId: '100-125-133-71-9119', mode: 'remote' as const, profile: 'default' } + + openSessionTile('branch-child', 'center', undefined, null, { ownerRoute, workspaceMode: 'sessions' }) + + expect($sessionTiles.get()).toEqual([ + expect.objectContaining({ ownerRoute, storedSessionId: 'branch-child', workspaceMode: 'sessions' }) + ]) + }) + + it('keeps an existing sessions-mode owner route on a route-less re-scope', () => { + const ownerRoute = { connectionId: '100-125-133-71-9119', mode: 'remote' as const, profile: 'default' } + + openSessionTile('branch-child', 'center', undefined, null, { ownerRoute, workspaceMode: 'sessions' }) + // A plain sidebar re-open routes through setSessionTileWorkspaceScope with + // no route — absence of information, not a revocation. + setSessionTileWorkspaceScope('branch-child', { workspaceMode: 'sessions' }) + + expect($sessionTiles.get()).toEqual([expect.objectContaining({ ownerRoute, storedSessionId: 'branch-child' })]) + }) + it('stores an exact Bot owner and keeps it through placement patches', () => { const ownerRoute = { connectionId: 'connection-a', @@ -273,6 +320,7 @@ describe('SessionTile workspace scope', () => { $selectedStoredSessionId.set('bot-chat') openSessionTile('bot-chat', 'center', undefined, undefined, scope) + $layoutTree.set(group(['workspace', tilePane('bot-chat')], { active: 'workspace', id: 'main' })) expect($sessionTiles.get()).toEqual([ expect.objectContaining({ @@ -284,6 +332,23 @@ describe('SessionTile workspace scope', () => { expect(focusOpenSession('bot-chat', scope)).toBe('tile') }) + it('fronts the existing tab when compaction rotated the tip id — never a duplicate', () => { + // The tile was opened when seg-2 was the tip; the conversation has since + // rotated to seg-3 (projected row carries the full chain). Opening the + // new tip must front that tile, not open the same chat twice. + setSessions([{ _lineage_ids: ['seg-1', 'seg-2', 'seg-3'], _lineage_root_id: 'seg-1', id: 'seg-3' } as never]) + openSessionTile('seg-2') + $layoutTree.set(group(['workspace', tilePane('seg-2')], { active: 'workspace', id: 'main' })) + + expect(focusOpenSession('seg-3')).toBe('tile') + expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['seg-2']) + + // The open path dedupes through the same lineage test. + openSessionTile('seg-3') + expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['seg-2']) + setSessions([]) + }) + it('keeps Bot tabs while a profile publication swaps the Sessions bucket', () => { const scope = { workspaceMode: 'bots' as const, workspaceOwnerKey: 'connection-a::writer' } @@ -317,6 +382,45 @@ describe('SessionTile workspace scope', () => { }) }) + it('preserves an existing Bot tile scope when moving it without an explicit scope', () => { + const scope = { + ownerRoute: { + connectionId: 'connection-a', + mode: 'remote' as const, + profile: 'default', + targetProfile: 'default' + }, + workspaceMode: 'bots' as const, + workspaceOwnerKey: 'bot:connection-a::default', + workspaceTabTitle: 'Bot chat' + } + + openSessionTile('bot-chat', 'right', undefined, undefined, scope) + $layoutTree.set(group(['workspace', 'session-tile:bot-chat'], { id: 'workspace-group' })) + // A split drag re-docks the tab with no scope (session-drag onCommit). + openSessionTile('bot-chat', 'left', 'workspace') + + expect($sessionTiles.get()).toEqual([ + expect.objectContaining({ + anchor: 'workspace', + dir: 'left', + ownerRoute: scope.ownerRoute, + storedSessionId: 'bot-chat', + workspaceMode: 'bots', + workspaceOwnerKey: scope.workspaceOwnerKey, + workspaceTabTitle: 'Bot chat' + }) + ]) + + // An explicit scope from the caller still wins over the tile's current one. + openSessionTile('bot-chat', 'right', 'workspace', undefined, { workspaceMode: 'sessions' }) + + expect($sessionTiles.get()).toEqual([ + expect.objectContaining({ dir: 'right', storedSessionId: 'bot-chat', workspaceMode: 'sessions' }) + ]) + expect($sessionTiles.get()[0]).not.toHaveProperty('workspaceOwnerKey', scope.workspaceOwnerKey) + }) + it('preserves workspace scope while dropping a stale runtime binding', () => { $sessionTiles.set([ { @@ -385,6 +489,7 @@ describe('focusWorkspaceOwnerSessionTile', () => { openSessionTile('thread', 'center', 'workspace', undefined, botA) rememberActivePane(workspaceScopeKey('bots', 'bot:a'), tilePane('closed-bot-chat')) $sessionTiles.set($sessionTiles.get().filter(t => t.storedSessionId !== 'closed-bot-chat')) + $layoutTree.set(group(['workspace', tilePane('thread')], { active: 'workspace', id: 'main' })) expect(focusWorkspaceOwnerSessionTile('bot:a')).toBe('thread') }) @@ -431,6 +536,7 @@ describe('focusWorkspaceOwnerSessionTile', () => { it('a throwing probe keeps the tile — reconciliation must not break the click', () => { openSessionTile('bot-chat', 'center', 'workspace', undefined, botA) + $layoutTree.set(group(['workspace', tilePane('bot-chat')], { active: 'workspace', id: 'main' })) expect( focusWorkspaceOwnerSessionTile('bot:a', () => { @@ -440,8 +546,9 @@ describe('focusWorkspaceOwnerSessionTile', () => { expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['bot-chat']) }) - it('no probe keeps the old behavior byte for byte', () => { + it('fronts a visible tile without a probe', () => { openSessionTile('bot-chat', 'center', 'workspace', undefined, botA) + $layoutTree.set(group(['workspace', tilePane('bot-chat')], { active: 'workspace', id: 'main' })) expect(focusWorkspaceOwnerSessionTile('bot:a')).toBe('bot-chat') expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['bot-chat']) diff --git a/apps/desktop/src/store/session-states.ts b/apps/desktop/src/store/session-states.ts index d3402dc369..afd217b79b 100644 --- a/apps/desktop/src/store/session-states.ts +++ b/apps/desktop/src/store/session-states.ts @@ -25,6 +25,7 @@ import { $activeTreeGroup, $layoutTree, focusedSessionTabAnchor, + isPaneVisible, moveTreePane, noteActiveTreeGroup, revealTreePane @@ -83,9 +84,22 @@ export const $sessionStates = atom>({}) const sessionScopeByRuntimeId = new Map() +// Structured twin of the scope ledger: the same inbound events also carry the +// exact (connectionId, profile) owner, which the composite scope string +// cannot give back. Consumed as the LAST rung of knownOwnerForSession so a +// runtime whose event source already proved its owner can still route +// session-scoped RPCs (approval.respond) when every durable binding +// (tile / hint / row) is absent — while durable stored identity keeps +// outranking it (#97511). +const sessionOwnerByRuntimeId = new Map() + export function recordSessionEventScope(event: { connectionId?: string; profile?: string; session_id?: string }): void { if (event.session_id && event.connectionId) { sessionScopeByRuntimeId.set(event.session_id, registryBackendScopeKey(event.connectionId, event.profile)) + sessionOwnerByRuntimeId.set(event.session_id, { + connectionId: event.connectionId, + profile: String(event.profile ?? '').trim() || 'default' + }) } } @@ -506,6 +520,7 @@ export function dropSessionState(runtimeId: string) { clearWatchdog(runtimeId) clearSessionProviderWait(runtimeId) sessionScopeByRuntimeId.delete(runtimeId) + sessionOwnerByRuntimeId.delete(runtimeId) const current = $sessionStates.get() setSessionStalled(current[runtimeId]?.storedSessionId, false) @@ -532,6 +547,7 @@ export function clearAllSessionStates() { settledExpiry.clear() clearAllProviderWaits() sessionScopeByRuntimeId.clear() + sessionOwnerByRuntimeId.clear() $stalledSessionIds.set([]) $sessionStates.set({}) } @@ -970,6 +986,13 @@ export function openTileGatewayScopes(): Set { * `profile` stamp) was already loaded for the sidebar's cron section. The * hint outranks the row for the same reason as contrib/wiring's ladder: a * row can be stamped from the ambient profile and carries no connection. + * Last rung: the owner recorded from the inbound runtime event itself + * (sessionOwnerByRuntimeId, #97511) — an orphan runtime whose tile/hint/row + * binding is absent or stale still routes through the exact + * (connectionId, profile) its events proved, while every durable rung above + * keeps outranking it, so a stored-id collision never inherits a stale + * runtime ledger entry. Untagged events record nothing, so unknown owners in + * multi-profile topology still fail closed. * Returns undefined when no owner is known — the caller fails closed * (assertSessionOwnerResolved), never falls to "active". */ @@ -983,7 +1006,8 @@ export function knownOwnerForSession(sessionId: null | string | undefined): Sess return ( sessionTileOwnerRoute(storedSessionId) ?? getSessionOwnerHint(storedSessionId) ?? - knownSessionOwner(ownerLookupSessionRows(), storedSessionId) + knownSessionOwner(ownerLookupSessionRows(), storedSessionId) ?? + sessionOwnerByRuntimeId.get(sessionId) ) } @@ -1087,8 +1111,23 @@ export const $botChatSessionIds = atom>( new Set((readJson(BOT_CHAT_SCOPE_KEY) as unknown[] | null)?.filter(id => typeof id === 'string') ?? []) ) -function rememberBotChatScope(storedSessionId: string, isBotChat: boolean): void { +/** The bot-mode scope each stored id was last opened under, for the main tab + * (which has no tile to carry one). Window-local: the caption falls back to + * the stored title until the chat is opened again. */ +export const $botChatScopes = atom>>({}) + +function rememberBotChatScope(storedSessionId: string, scope: SessionTileWorkspaceScope): void { + const isBotChat = scope.workspaceMode === 'bots' const current = $botChatSessionIds.get() + const { [storedSessionId]: previous, ...rest } = $botChatScopes.get() + + const changed = isBotChat + ? previous?.workspaceOwnerKey !== scope.workspaceOwnerKey || previous?.workspaceTabTitle !== scope.workspaceTabTitle + : Boolean(previous) + + if (changed) { + $botChatScopes.set(isBotChat ? { ...rest, [storedSessionId]: scope } : rest) + } if (current.has(storedSessionId) === isBotChat) { return @@ -1118,11 +1157,16 @@ export function isBotChatSession(sessionId: null | string | undefined): boolean export function setSessionTileWorkspaceScope(storedSessionId: string, scope: SessionTileWorkspaceScope): boolean { // Before the tile lookup: openSession routes every open through here, and a // bot chat usually has no tile to record the scope on. - rememberBotChatScope(storedSessionId, scope.workspaceMode === 'bots') + rememberBotChatScope(storedSessionId, scope) const tile = $sessionTiles.get().find(candidate => candidate.storedSessionId === storedSessionId) const workspaceOwnerKey = scope.workspaceMode === 'bots' ? scope.workspaceOwnerKey : undefined - const ownerRoute = scope.workspaceMode === 'bots' ? scope.ownerRoute : undefined + // Sessions-mode re-opens (sidebar click on an already-tiled session) pass no + // route; that is absence of information, not a revocation — keep the exact + // owner the tile was opened with (a branch child's parent connection) so a + // plain re-open can't unpin the owning socket. Bot scopes stay authoritative + // both ways: they always name their route explicitly. + const ownerRoute = scope.workspaceMode === 'bots' ? scope.ownerRoute : (scope.ownerRoute ?? tile?.ownerRoute) const workspaceTabTitle = scope.workspaceMode === 'bots' ? scope.workspaceTabTitle : undefined if ( @@ -1206,8 +1250,15 @@ export function resetTileRuntimeBindings( const preservedStoredIds = new Set( tiles .filter( + // Any tile with an EXACT owner route — bot tabs always, and a + // sessions tile whose opener stamped one (a branch child on its + // parent's connection). Its runtime lives on that owner's socket, + // not the ambient gateway, so an unrelated connection's reconnect + // must not drop the binding: each drop re-arms the tile's resume, + // and a flapping sibling connection turns that into 4+ re-resumes + // inside the storm window — latching the "keeps losing its backend + // runtime" card over a session that is actually healthy. tile => - tile.workspaceMode === 'bots' && Boolean(tile.ownerRoute?.connectionId) && (!(reconnected || liveConnectionIds) || !belongsToReconnectedRuntime(tile)) ) @@ -1359,9 +1410,17 @@ export function openSessionTile( dir: TileDock = 'right', anchor?: string, before?: null | string, - workspaceScope: SessionTileWorkspaceScope = { workspaceMode: 'sessions' } + explicitScope?: SessionTileWorkspaceScope ) { const tiles = $sessionTiles.get() + const existing = tiles.find(t => t.storedSessionId === storedSessionId) + + // No scope on an already-open tile is a MOVE (a split drag re-docking a tab), + // not a re-scope: keep the workspace it lives in instead of re-bucketing it + // into Sessions — a Bot tab used to vanish from the Bot workspace on drop. + const workspaceScope: SessionTileWorkspaceScope = explicitScope ?? { + workspaceMode: existing?.workspaceMode ?? 'sessions' + } // Opening a session in a tab/tile is "reading" it — clear its unread dot // exactly like main-thread resume does. Previously only @@ -1371,7 +1430,9 @@ export function openSessionTile( markSessionRead(storedSessionId) ackStoredSessionId(storedSessionId) - if (workspaceScope.workspaceMode === 'sessions' && storedSessionId === $selectedStoredSessionId.get()) { + const aliases = lineageAliases(storedSessionId, $sessions.get()) + + if (workspaceScope.workspaceMode === 'sessions' && aliases.includes($selectedStoredSessionId.get() ?? '')) { return } @@ -1379,14 +1440,22 @@ export function openSessionTile( const workspaceOwnerKey = workspaceScope.workspaceMode === 'bots' ? workspaceScope.workspaceOwnerKey : undefined - if (!tiles.some(t => t.storedSessionId === storedSessionId)) { + if (!tiles.some(t => aliases.includes(t.storedSessionId))) { saveTiles([ ...tiles, { anchor: dock, before, dir, - ownerRoute: workspaceScope.workspaceMode === 'bots' ? workspaceScope.ownerRoute : undefined, + // The owner route pins the owning backend's socket in the gateway + // keep-set (openTileGatewayScopes / foregroundSessionScopes) for as + // long as the tile is open. Bot tabs always carry one; a sessions-mode + // tile carries one when its opener knows the exact owner — e.g. a + // branch child created on its parent's owning connection, whose + // draft runtime is otherwise orphan-reaped the moment the pruner + // closes the unpinned socket (the resume/reclaim flicker loop, + // #93892 shape). + ownerRoute: workspaceScope.ownerRoute, storedSessionId, workspaceMode: workspaceScope.workspaceMode, workspaceOwnerKey, @@ -1399,7 +1468,9 @@ export function openSessionTile( return } - setSessionTileWorkspaceScope(storedSessionId, workspaceScope) + if (explicitScope) { + setSessionTileWorkspaceScope(storedSessionId, explicitScope) + } // Already open: relocate the existing pane to the drop target (pane-mirror // only docks on first adoption, so a re-drag must move the tree pane itself). @@ -1467,22 +1538,32 @@ export function focusOpenSession( storedSessionId: string, workspaceScope: SessionTileWorkspaceScope = { workspaceMode: 'sessions' } ): 'main' | 'tile' | null { - if ($sessionTiles.get().some(t => t.storedSessionId === storedSessionId)) { - const paneId = `${TILE_PANE_PREFIX}${storedSessionId}` + // Compression rotates a conversation's tip id while tiles stay keyed by + // whichever segment id they were opened with. An exact-id test right after + // a rotation said "not open" for a conversation that IS on screen, and + // callers opened the same chat in a second tab. Match any id of the + // lineage instead, and front the tile under ITS key. + const aliases = lineageAliases(storedSessionId, $sessions.get()) + const tile = $sessionTiles.get().find(t => aliases.includes(t.storedSessionId)) + + if (tile) { + const paneId = `${TILE_PANE_PREFIX}${tile.storedSessionId}` revealTreePane(paneId) // un-dismiss + adopt + front in its group const tree = $layoutTree.get() const group = tree ? findGroupOfPane(tree, paneId) : null - if (group) { - noteActiveTreeGroup(group.id) + if (!group || !isPaneVisible(paneId)) { + return null } + noteActiveTreeGroup(group.id) + return 'tile' } // Already the main session: front the workspace tab and drop tile focus so // the readouts + sidebar highlight come home (a no-op when main is focused). - if (workspaceScope.workspaceMode === 'sessions' && storedSessionId === $selectedStoredSessionId.get()) { + if (workspaceScope.workspaceMode === 'sessions' && aliases.includes($selectedStoredSessionId.get() ?? '')) { revealTreePane('workspace') noteActiveTreeGroup(null) @@ -1514,7 +1595,8 @@ export function focusOpenSession( * falls through to its authoritative open. No probe = the old behavior. */ export function focusWorkspaceOwnerSessionTile( workspaceOwnerKey: string, - isStaleTile?: (tile: SessionTile) => boolean + isStaleTile?: (tile: SessionTile) => boolean, + onlyStoredIds?: readonly string[] ): null | string { const allOwned = $sessionTiles .get() @@ -1539,6 +1621,13 @@ export function focusWorkspaceOwnerSessionTile( owned = allOwned.filter(tile => !stale.includes(tile)) } + // `onlyStoredIds`: the sessions this call may front (Bot Mode passes the + // canonical chat's registry id + lineage tip). Other tabs in the owner's + // zone stay open; they are simply not what the caller asked for. + if (onlyStoredIds) { + owned = owned.filter(tile => onlyStoredIds.includes(tile.storedSessionId)) + } + if (owned.length === 0) { return null } @@ -1548,9 +1637,9 @@ export function focusWorkspaceOwnerSessionTile( const paneId = resolveRememberedActivePane(workspaceScopeKey('bots', workspaceOwnerKey), paneIds) ?? paneIds[0] const storedSessionId = paneId.slice(TILE_PANE_PREFIX.length) - focusOpenSession(storedSessionId, { workspaceMode: 'bots', workspaceOwnerKey }) - - return storedSessionId + return focusOpenSession(storedSessionId, { workspaceMode: 'bots', workspaceOwnerKey }) === 'tile' + ? storedSessionId + : null } /** Does a sidebar click still need to navigate after `focusOpenSession`? A miss diff --git a/apps/desktop/src/store/session.test.ts b/apps/desktop/src/store/session.test.ts index 036a52aab7..95e436c480 100644 --- a/apps/desktop/src/store/session.test.ts +++ b/apps/desktop/src/store/session.test.ts @@ -30,6 +30,7 @@ import { _resetLegacyDiscardForTests, _resetSessionOwnerHintsForTests, applyConfiguredDefaultProjectDir, + carryForwardFailedProfileSessions, commitWorkspaceCwdForSelectedSession, ensureDefaultWorkspaceCwd, forgetSessionOwnerHintsForConnection, @@ -41,12 +42,15 @@ import { getSessionOwnerHint, getSessionOwnerHints, hydrateSessionOwnerHints, + keepFailedProfileMeta, knownSessionOwner, knownSessionProfile, + lineageAliases, mergeSessionPage, rememberedSessionProfile, resolveComposerSessionKey, sessionBelongsToProfile, + sessionMatchesStoredId, sessionOwnerRouteFromRow, sessionPinId, setComposerSelectionOwner, @@ -364,6 +368,18 @@ describe('sessionPinId', () => { }) }) +describe('lineageAliases across a deep compression chain', () => { + it('aliases every segment, intermediates included', () => { + // The projected row carries the full chain: a tile or route can hold a + // MIDDLE segment's id from when IT was the tip. + const rows = [session({ _lineage_ids: ['root', 'mid', 'tip'], _lineage_root_id: 'root', id: 'tip' })] + + expect(lineageAliases('mid', rows).sort()).toEqual(['mid', 'root', 'tip']) + expect(lineageAliases('tip', rows).sort()).toEqual(['mid', 'root', 'tip']) + expect(sessionMatchesStoredId(rows[0], 'mid')).toBe(true) + }) +}) + describe('resolveComposerSessionKey', () => { it('keeps the lineage root across compression tip rotation', () => { const tipBefore = '20260720_062637_ad96b3' @@ -673,6 +689,87 @@ describe('mergeSessionPage', () => { }) }) +describe('carryForwardFailedProfileSessions', () => { + it('is a no-op when the backend reported no profile errors', () => { + const previous = [session({ id: 'yesterday', profile: 'default' })] + const incoming = [session({ id: 'today', profile: 'default' })] + + expect(carryForwardFailedProfileSessions(previous, incoming, undefined)).toBe(incoming) + expect(carryForwardFailedProfileSessions(previous, incoming, [])).toBe(incoming) + }) + + it('re-attaches idle rows for a profile whose slice failed (empty 200 + errors)', () => { + // Repro: current session is running, sidebar scan hits disk I/O, backend + // returns recents=[] with errors=[{profile:default}]. mergeSessionPage then + // keeps only working/pinned/selected and Yesterday/This-week vanish. + const previous = [ + session({ id: 'running', last_active: 300, profile: 'default', title: 'Now' }), + session({ id: 'yesterday', last_active: 200, profile: 'default', title: 'Yesterday' }), + session({ id: 'week', last_active: 100, profile: 'default', title: 'This week' }) + ] + + const carried = carryForwardFailedProfileSessions(previous, [], [{ profile: 'default', error: 'disk I/O error' }]) + + expect(carried.map(s => s.id)).toEqual(['running', 'yesterday', 'week']) + expect(carried[1]).toBe(previous[1]) + }) + + it('does not resurrect a successful profile’s omitted rows, and does not duplicate', () => { + const previous = [ + session({ id: 'work-old', profile: 'work' }), + session({ id: 'home-idle', profile: 'default' }), + session({ id: 'home-fresh', profile: 'default' }) + ] + + const incoming = [session({ id: 'home-fresh', message_count: 4, profile: 'default' })] + + const carried = carryForwardFailedProfileSessions(previous, incoming, [{ profile: 'work' }]) + + expect(carried.map(s => `${s.profile}:${s.id}`)).toEqual(['default:home-fresh', 'work:work-old']) + }) + + it('re-ranks carried rows by recency instead of parking them at the tail', () => { + const previous = [ + session({ id: 'idle-newer', last_active: 500, profile: 'work' }), + session({ id: 'idle-older', last_active: 50, profile: 'work' }) + ] + + const incoming = [session({ id: 'home', last_active: 100, profile: 'default' })] + + expect(carryForwardFailedProfileSessions(previous, incoming, [{ profile: 'work' }]).map(s => s.id)).toEqual([ + 'idle-newer', + 'home', + 'idle-older' + ]) + }) + + it('treats a missing profile tag on the error as default', () => { + const previous = [session({ id: 'idle', profile: 'default' })] + + expect(carryForwardFailedProfileSessions(previous, [], [{ error: 'disk I/O error' }]).map(s => s.id)).toEqual([ + 'idle' + ]) + }) +}) + +describe('keepFailedProfileMeta', () => { + it('is a no-op when the backend reported no profile errors', () => { + const incoming = { default: { cost_usd: 1, tokens: 2 } } + + expect(keepFailedProfileMeta({ default: { cost_usd: 9, tokens: 9 } }, incoming, [])).toBe(incoming) + }) + + it('restores previous usage/truncated flags for profiles whose slice failed', () => { + const previous = { default: { cost_usd: 4, tokens: 40 }, work: { cost_usd: 1, tokens: 10 } } + const incoming = { work: { cost_usd: 2, tokens: 20 } } + + expect(keepFailedProfileMeta(previous, incoming, [{ profile: 'default' }])).toEqual({ + default: { cost_usd: 4, tokens: 40 }, + work: { cost_usd: 2, tokens: 20 } + }) + }) +}) + describe('touchSessionActivity', () => { afterEach(() => { setSessions([]) diff --git a/apps/desktop/src/store/session.ts b/apps/desktop/src/store/session.ts index 023d8d2205..5ce4c9e620 100644 --- a/apps/desktop/src/store/session.ts +++ b/apps/desktop/src/store/session.ts @@ -14,6 +14,7 @@ import { persistBoolean, persistString, readJson, storedBoolean, storedString, w import { syncCronModelImpactConnection } from '@/store/cron-model-impact-scope' import type { SessionInfo, UsageStats } from '@/types/hermes' +import { isSessionRemovalPending } from './session-removal' import type { SessionOwnerRoute, SessionOwnerScope } from './session-request-router' import { clearUnreadOnOpen } from './session-unread-remote' @@ -359,16 +360,19 @@ export const sessionPinId = (session: Pick, + session: Pick, storedSessionId: string -): boolean => session.id === storedSessionId || session._lineage_root_id === storedSessionId +): boolean => + session.id === storedSessionId || + session._lineage_root_id === storedSessionId || + Boolean(session._lineage_ids?.includes(storedSessionId)) // Alias lookup, memoized per sessions-list reference. `lineageAliases` runs // per cached session state per status projection per message delta — an // O(sessions) scan there multiplies out to states × sessions × ~30Hz per busy // session, which is what made a populated recents list drag every stream. The // list is replaced wholesale (never mutated), so its reference is the cache key. -type LineageRow = Pick +type LineageRow = Pick const lineageIndexBySessions = new WeakMap>() function lineageIndex(sessions: readonly LineageRow[]): Map { @@ -398,6 +402,21 @@ function lineageIndex(sessions: readonly LineageRow[]): Map { add(session._lineage_root_id, session.id) add(session._lineage_root_id, session._lineage_root_id) } + + // Chains three+ segments deep: the projected row carries every id the + // conversation has answered to, so a surface keyed to a MIDDLE segment + // (it was the tip when the surface opened) still aliases to the rest. + // Without this, only tip↔root connect and such a surface reads as a + // different conversation — one chat open twice after a compaction. + const ids = session._lineage_ids + + if (ids && ids.length > 1) { + for (const a of ids) { + for (const b of ids) { + add(a, b) + } + } + } } lineageIndexBySessions.set(sessions, index) @@ -629,6 +648,85 @@ export function mergeSessionPage( return interleaved } +function sidebarProfileKey(session: Pick): string { + return (session.profile ?? '').trim() || 'default' +} + +function sessionListIdentity(session: Pick): string { + return `${sidebarProfileKey(session)}::${session.id}` +} + +/** + * Re-attach previous rows for profiles whose sidebar slice failed this refresh. + * + * The batched sidebar endpoint reports a disk I/O / lock failure as HTTP 200 + * with `recents: []` and `errors: [{ profile }]`. `mergeSessionPage` only keeps + * working / pinned / selected ids, so idle Yesterday / This-week rows would + * otherwise vanish until a later successful scan (#73847, #88528). + * + * Successful profiles are left alone: their incoming page is still authoritative. + */ +export function carryForwardFailedProfileSessions( + previous: SessionInfo[], + incoming: SessionInfo[], + errors: Array<{ profile?: string; error?: string }> | undefined | null +): SessionInfo[] { + if (!errors?.length || previous.length === 0) { + return incoming + } + + const failed = new Set(errors.map(error => (error.profile ?? '').trim() || 'default')) + const incomingIds = new Set(incoming.map(sessionListIdentity)) + const carried: SessionInfo[] = [] + + for (const session of previous) { + if (!failed.has(sidebarProfileKey(session)) || incomingIds.has(sessionListIdentity(session))) { + continue + } + + carried.push(session) + } + + if (carried.length === 0) { + return incoming + } + + // Incoming-first concat parks the failed profile at the tail of an + // all-profiles list. Re-rank by the same recency key the backend uses. + const recency = (session: SessionInfo): number => Math.max(session.last_active || 0, session.started_at || 0) + + return [...incoming, ...carried].sort((a, b) => recency(b) - recency(a)) +} + +/** Keep previous per-profile sidebar meta for profiles whose slice failed. + * + * A failed scan returns `{}` / falsey truncated flags. Applying those + * would zero usage and hide Load more under a list we just carried forward. + */ +export function keepFailedProfileMeta( + previous: Record, + incoming: Record, + errors: Array<{ profile?: string; error?: string }> | undefined | null +): Record { + if (!errors?.length) { + return incoming + } + + const next = { ...incoming } + + for (const error of errors) { + const key = (error.profile ?? '').trim() || 'default' + + if (Object.prototype.hasOwnProperty.call(previous, key)) { + next[key] = previous[key] + } else { + delete next[key] + } + } + + return next +} + /** Raise a session in recents on user send (before stream / turn resolve). */ export function touchSessionActivity( sessionId: string | null | undefined, @@ -1186,6 +1284,16 @@ export const requestSessionResume = (sessionId: string, ownerRoute?: SessionOwne return } + // A chat on its way out must never be re-selected. The push path + // (markRuntimeGone) and the RPC seam both queue a resume off a 4001, and an + // idle reap can land one in the same tick as a delete — that queued request + // then resumes a tombstoned id, 404s, and toasts "Resume failed / Session + // not found" for a chat the user deliberately removed. Filtering at the + // producer means no consumer has to re-derive "is this id doomed". + if (isSessionRemovalPending(id)) { + return + } + if (ownerRoute) { setSessionOwnerHint(id, ownerRoute) } diff --git a/apps/desktop/src/store/statusbar-prefs.ts b/apps/desktop/src/store/statusbar-prefs.ts index 9be85c5ce4..a82e91617d 100644 --- a/apps/desktop/src/store/statusbar-prefs.ts +++ b/apps/desktop/src/store/statusbar-prefs.ts @@ -16,17 +16,20 @@ export function toggleStatusbarVisible() { // bar's job is to answer "is the backend healthy, where am I, what's it doing" — // route shortcuts (cron/webhooks/agents), the terminal toggle, and the approval // pill are navigation, not status, so they start out of the way. The per-turn -// session readouts (running/session timers, context meter) are diagnostics most -// users don't watch, so they start hidden too and the bar stays quiet mid-turn. +// session readouts (running/session timers, context meter, cache hit rate, +// tokens/sec) are diagnostics most users don't watch, so they start hidden too +// and the bar stays quiet mid-turn. export const STATUSBAR_HIDDEN_BY_DEFAULT: readonly string[] = [ 'agents', 'approval-mode', + 'cache-hit-rate', 'context-usage', 'cron', 'running-timer', 'session-timer', 'system-resources', 'terminal', + 'tokens-per-second', 'webhooks' ] diff --git a/apps/desktop/src/store/subagents.ts b/apps/desktop/src/store/subagents.ts index 7196f14a47..ae923aae8d 100644 --- a/apps/desktop/src/store/subagents.ts +++ b/apps/desktop/src/store/subagents.ts @@ -18,6 +18,9 @@ export interface SubagentProgress { goal: string /** The child's own stored session id — lets UIs open its session window. */ sessionId?: string + /** Batch (delegation) id — exact grouping key for one fan-out's workers, + * so concurrent/nested batches never merge into one group. */ + delegationId?: string model?: string status: SubagentStatus taskCount: number @@ -189,6 +192,7 @@ function toProgress(payload: SubagentPayload, prev: SubagentProgress | undefined parentId: str(payload.parent_id) || prev?.parentId || null, goal: str(payload.goal) || prev?.goal || 'Subagent', sessionId: str(payload.child_session_id) || prev?.sessionId, + delegationId: str(payload.delegation_id) || prev?.delegationId, model: str(payload.model) || prev?.model, status, taskCount: num(payload.task_count) ?? prev?.taskCount ?? 1, diff --git a/apps/desktop/src/store/updates.test.ts b/apps/desktop/src/store/updates.test.ts index cea46b004f..a3a7c9cc81 100644 --- a/apps/desktop/src/store/updates.test.ts +++ b/apps/desktop/src/store/updates.test.ts @@ -89,6 +89,8 @@ const { applyUpdates, applyEverythingUpdate, hasMultipleUpdateTargets, + openUpdatesWindow, + startActiveUpdate, $updateApply, $updateEverything, $updateOverlayOpen, @@ -117,7 +119,7 @@ const status = (over: Partial = {}): DesktopUpdateStatus => ...over }) -const lastToast = () => notifySpy.mock.calls.at(-1)?.[0] as { onDismiss: () => void } +const lastToast = () => notifySpy.mock.calls.at(-1)?.[0] as { action: { onClick: () => void }; onDismiss: () => void } const setRemote = (on: boolean) => setConnection({ @@ -410,6 +412,102 @@ describe('requestActiveUpdate', () => { }) }) +// Surface-bound update entry points. A surface that displays ONE target's +// status must act on that target: the overlay has no target switcher, so +// inheriting the connection-mode default silently pointed the user at the +// other machine. This is what left a Mac desktop on a months-old build while +// its remote Linux backend updated fine, with no error anywhere (#70266). +describe('explicit update targets', () => { + const applyClientMock = vi.fn() + const checkClientMock = vi.fn() + + beforeEach(() => { + storage.clear() + notifySpy.mockClear() + dismissSpy.mockClear() + applyClientMock.mockReset().mockResolvedValue({ ok: true, handedOff: true }) + checkClientMock.mockReset().mockResolvedValue(status({ behind: 4, updateAvailable: true })) + updateHermesSpy.mockReset().mockResolvedValue({ ok: true, name: 'update' }) + checkHermesUpdateSpy.mockReset().mockResolvedValue({ + install_method: 'git', + current_version: '0.4.2', + behind: 0, + update_available: false, + can_apply: true, + update_command: null, + message: null + }) + getActionStatusSpy.mockReset().mockResolvedValue({ lines: [], running: false, exit_code: 0 }) + resetUpdateApplyState() + $updateStatus.set(null) + $backendUpdateStatus.set(null) + $updateOverlayOpen.set(false) + $updateOverlayTarget.set('backend') + $mockConnectionsRegistry.set(null) + setRemote(true) + ;(globalThis as unknown as { window: unknown }).window = { + hermesDesktop: { updates: { apply: applyClientMock, check: checkClientMock } } + } + vi.useRealTimers() + }) + + afterEach(async () => { + await vi.waitFor(() => expect($updateEverything.get().running).toBe(false), { timeout: 5000 }) + await vi.waitFor(() => expect($backendUpdateApply.get().applying).toBe(false), { timeout: 5000 }) + setRemote(false) + delete (globalThis as unknown as { window?: unknown }).window + }) + + // The macOS "Check for Updates…" app-menu item — the OS-standard affordance + // for updating THIS app — routes here via `hermes:open-updates`. + it('opens the client overlay on an explicit client target, even in remote mode', async () => { + openUpdatesWindow('client') + + expect($updateOverlayTarget.get()).toBe('client') + await vi.waitFor(() => expect(checkClientMock).toHaveBeenCalledTimes(1)) + expect(checkHermesUpdateSpy).not.toHaveBeenCalled() + }) + + it('still defaults to the connected machine when no target is named', async () => { + openUpdatesWindow() + + expect($updateOverlayTarget.get()).toBe('backend') + await vi.waitFor(() => expect(checkHermesUpdateSpy).toHaveBeenCalled()) + expect(checkClientMock).not.toHaveBeenCalled() + }) + + it('applies the client update on an explicit client target, without fanning out', async () => { + startActiveUpdate('client') + + expect($updateOverlayTarget.get()).toBe('client') + await vi.waitFor(() => expect(applyClientMock).toHaveBeenCalledTimes(1)) + expect(updateHermesSpy).not.toHaveBeenCalled() + expect($updateEverything.get().running).toBe(false) + }) + + it('keeps the everything-flow for the generic, target-less apply', async () => { + $backendUpdateStatus.set(status({ behind: 3 })) + + startActiveUpdate() + + await vi.waitFor(() => expect(updateHermesSpy).toHaveBeenCalled(), { timeout: 5000 }) + }) + + // A toast raised by the CLIENT check must open the client overlay: the user + // was told the app is behind, so landing them on the backend's (current) + // status reads as the update having vanished. + it('opens the overlay for the target whose status raised the toast', () => { + maybeNotifyUpdateAvailable(status(), 'client') + lastToast().action.onClick() + expect($updateOverlayTarget.get()).toBe('client') + + storage.clear() // clear the snooze the click just set + maybeNotifyUpdateAvailable(status({ targetSha: 'sha-b' }), 'backend') + lastToast().action.onClick() + expect($updateOverlayTarget.get()).toBe('backend') + }) +}) + // The everything-flow: on multi-target installs (remote mode / multi-connection // registry) "update" must mean every machine — active backend, other registered // sources via the Electron fan-out, and the client LAST. Before this flow, @@ -564,6 +662,35 @@ describe('applyEverythingUpdate', () => { expect(updateAllMock).toHaveBeenCalledTimes(1) }) + it('re-checks the client instead of trusting a stale cached status', async () => { + setRemote(true) + $backendUpdateStatus.set(status({ behind: 3 })) + // FAIL-BEFORE: `$updateStatus.get() ?? (await checkUpdates())` short-circuits + // on this cached row — captured up to a poll interval (30 min) ago, and + // before the backend leg ran — so the client apply was skipped and the app + // stayed stale. The live check says otherwise and must win. + $updateStatus.set(status({ behind: 0, updateAvailable: false })) + checkClientMock.mockResolvedValue(status({ behind: 7, updateAvailable: true })) + + await applyEverythingUpdate() + + expect(applyClientMock).toHaveBeenCalledTimes(1) + }) + + it('falls back to the cached client status when the live re-check fails', async () => { + setRemote(true) + $backendUpdateStatus.set(status({ behind: 3 })) + $updateStatus.set(status({ behind: 7, updateAvailable: true })) + // `checkUpdates()` never rejects — it resolves with an error-status and + // overwrites the atom with it, so an unreachable bridge must not read as + // "client is current" and skip the leg. + checkClientMock.mockRejectedValue(new Error('bridge gone')) + + await applyEverythingUpdate() + + expect(applyClientMock).toHaveBeenCalledTimes(1) + }) + it('requestActiveUpdate routes through the everything-flow when EITHER target is behind', async () => { setRemote(true) // Backend current, client behind — the exact case the old remote-only diff --git a/apps/desktop/src/store/updates.ts b/apps/desktop/src/store/updates.ts index e317c234e3..746d8e8564 100644 --- a/apps/desktop/src/store/updates.ts +++ b/apps/desktop/src/store/updates.ts @@ -205,11 +205,15 @@ export function reportInstallMethodWarning(message: string | undefined): void { * Closing the toast — dismissing it or opening the updates window from it — * (re)starts the cooldown, so a busy upstream branch doesn't re-spam the user * on every new commit. The snooze is persisted, so it survives relaunches too. + * + * `target` is the target whose status produced this toast. The overlay has no + * target switcher, so a client-status toast that opened the backend overlay + * showed the user a machine they weren't told about, with no way back. */ -export function maybeNotifyUpdateAvailable(status: DesktopUpdateStatus | null) { +export function maybeNotifyUpdateAvailable(status: DesktopUpdateStatus | null, target: UpdateTarget = 'client') { // Either signal means "update ready": behind > 0 (git checkout) or // updateAvailable (app-installer, shallow clone). - if (!status || status.supported === false || status.error) { + if (!status || status.supported === false || status.error || !status.targetSha) { return } @@ -232,7 +236,7 @@ export function maybeNotifyUpdateAvailable(status: DesktopUpdateStatus | null) { label: translateNow('notifications.seeWhatsNew'), onClick: () => { snoozeUpdateToast() - openUpdatesWindow() + openUpdateOverlayFor(target) } }, durationMs: 0, @@ -250,8 +254,24 @@ export function maybeNotifyUpdateAvailable(status: DesktopUpdateStatus | null) { }) } -export function openUpdatesWindow(): void { - openUpdateOverlayFor(isRemoteMode() ? 'backend' : 'client') +/** The target a generic, surface-less update command acts on: the machine the + * user is connected to. Surfaces that display one target's status must pass + * that target explicitly instead of inheriting this. */ +function activeUpdateTarget(): UpdateTarget { + return isRemoteMode() ? 'backend' : 'client' +} + +/** + * Open the updates overlay and kick off its check. + * + * Callers tied to a specific status surface pass its target; only genuinely + * generic entry points take the connection-mode default. The macOS "Check for + * Updates…" menu item is the former — it is the OS-standard affordance for + * updating *this app*, so in remote mode it checked the wrong machine and the + * Mac client silently drifted behind (#70266). + */ +export function openUpdatesWindow(target: UpdateTarget = activeUpdateTarget()): void { + openUpdateOverlayFor(target) } /** @@ -265,19 +285,22 @@ export function openUpdatesWindow(): void { * through the everything-flow so "update" means every machine, not just the * active target — the single-target ternary is what left remote-mode users * updating the backend forever while the GUI itself went stale. + * + * An explicit `target` opts out of both: the caller is acting on one named + * machine's status and must not fan out to the others. */ -export function startActiveUpdate(): void { - if (hasMultipleUpdateTargets()) { +export function startActiveUpdate(target?: UpdateTarget): void { + if (!target && hasMultipleUpdateTargets()) { $updateOverlayOpen.set(true) void applyEverythingUpdate() return } - const target: UpdateTarget = isRemoteMode() ? 'backend' : 'client' - $updateOverlayTarget.set(target) + const effective = target ?? activeUpdateTarget() + $updateOverlayTarget.set(effective) $updateOverlayOpen.set(true) - void (target === 'backend' ? applyBackendUpdate() : applyUpdates()) + void (effective === 'backend' ? applyBackendUpdate() : applyUpdates()) } /** @@ -306,11 +329,11 @@ export function requestActiveUpdate(): void { } } - const target: UpdateTarget = isRemoteMode() ? 'backend' : 'client' + const target = activeUpdateTarget() const status = target === 'backend' ? $backendUpdateStatus.get() : $updateStatus.get() if ((status?.behind ?? 0) > 0 || status?.updateAvailable) { - startActiveUpdate() + startActiveUpdate(target) return } @@ -375,7 +398,7 @@ export async function checkBackendUpdates(): Promise try { const status = mapBackendCheck(await checkHermesUpdate(true)) $backendUpdateStatus.set(status) - maybeNotifyUpdateAvailable(status) + maybeNotifyUpdateAvailable(status, 'backend') return status } catch (error) { @@ -406,7 +429,7 @@ export async function checkUpdates(): Promise { try { const status = await bridge.check() $updateStatus.set(status) - maybeNotifyUpdateAvailable(status) + maybeNotifyUpdateAvailable(status, 'client') void refreshDesktopVersion() return status @@ -857,6 +880,12 @@ export function applyEverythingUpdate(): Promise { async function runEverythingUpdate(): Promise { $updateEverything.set({ running: true }) + // Snapshot the client status before any leg runs: the backend leg's own + // post-update nudge re-checks the client and overwrites `$updateStatus`, + // including with an error row when the bridge is unreachable. Step 3 needs a + // pre-flow value to fall back on when its own live check can't answer. + const cachedClientStatus = $updateStatus.get() + try { // 1. Active backend first (remote mode), with the detailed overlay flow. // Its own finish path re-checks and nudges, but the everything-flow @@ -913,7 +942,14 @@ async function runEverythingUpdate(): Promise { // 3. The client last — its apply relaunches or hands off the app, so it // must come after every dispatch above. Skipped when already current. - const clientStatus = $updateStatus.get() ?? (await checkUpdates()) + // Re-check rather than trusting `$updateStatus`: the cached value can be + // up to a poll interval (30 min) old and was captured BEFORE the backend + // update above, so a cached `behind: 0` would skip the client leg and + // leave the app stale — the exact failure this flow exists to prevent. + // `checkUpdates()` resolves with an error-status rather than rejecting, + // so fall back to the pre-flow snapshot when the live check can't answer. + const freshClientStatus = await checkUpdates().catch(() => null) + const clientStatus = freshClientStatus?.error ? cachedClientStatus : (freshClientStatus ?? cachedClientStatus) if ((clientStatus?.behind ?? 0) > 0 || clientStatus?.updateAvailable) { $updateOverlayTarget.set('client') diff --git a/apps/desktop/src/store/windows.test.ts b/apps/desktop/src/store/windows.test.ts index 7c032a07f0..69402ae14b 100644 --- a/apps/desktop/src/store/windows.test.ts +++ b/apps/desktop/src/store/windows.test.ts @@ -1,5 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { $activeGatewayProfile } from './profile' +import { $sessions } from './session' import { canOpenBrowserWindow, canOpenNewWindow, @@ -88,23 +90,17 @@ describe('openSessionInNewWindow', () => { expect(notifyError).not.toHaveBeenCalled() }) - it('invokes the bridge with the session id', async () => { + it('carries the owning profile: stamped row wins, an unstamped child inherits the viewed profile (#82768)', async () => { const open = vi.fn().mockResolvedValue({ ok: true }) installBridge(open) + $activeGatewayProfile.set('work') + $sessions.set([{ id: 's1', profile: 'research' } as never]) await openSessionInNewWindow('s1') + await openSessionInNewWindow('child-not-listed-yet', { watch: true }) - expect(open).toHaveBeenCalledWith('s1', undefined) - expect(notifyError).not.toHaveBeenCalled() - }) - - it('forwards the watch flag for spectator (subagent) windows', async () => { - const open = vi.fn().mockResolvedValue({ ok: true }) - installBridge(open) - - await openSessionInNewWindow('s1', { watch: true }) - - expect(open).toHaveBeenCalledWith('s1', { watch: true }) + expect(open).toHaveBeenCalledWith('s1', { profile: 'research' }) + expect(open).toHaveBeenCalledWith('child-not-listed-yet', { profile: 'work', watch: true }) expect(notifyError).not.toHaveBeenCalled() }) diff --git a/apps/desktop/src/store/windows.ts b/apps/desktop/src/store/windows.ts index 8015893d88..bf24e6f0fb 100644 --- a/apps/desktop/src/store/windows.ts +++ b/apps/desktop/src/store/windows.ts @@ -189,13 +189,27 @@ async function runWindowOpen(call: () => Promise, failMessage: // Open (or focus) a standalone OS window for a single chat session. No-ops // gracefully outside Electron so callers can wire it unconditionally. // `watch: true` opens a spectator window (lazy resume, live-mirror stream). +// The window is a full renderer that adopts the PRIMARY profile unless told +// otherwise, so the owning profile rides along (same ladder as openHud, +// #82285): the session's stamped owner wins, and an unstamped/uncached id — +// a brand-new subagent child — inherits the profile the user is looking at +// (#82768, #61286). export async function openSessionInNewWindow(sessionId: string, opts?: { watch?: boolean }): Promise { if (!sessionId || !canOpenSessionWindow()) { return } + // Lazy imports: `./profile` subscribes to the API client on load, so a + // static import here would drag it into every page that opens windows. + const [{ $activeGatewayProfile, normalizeProfileKey }, { $sessions, rememberedSessionProfile }] = await Promise.all([ + import('./profile'), + import('./session') + ]) + + const profile = normalizeProfileKey(rememberedSessionProfile($sessions.get(), sessionId, $activeGatewayProfile.get())) + await runWindowOpen( - () => window.hermesDesktop.openSessionWindow(sessionId, opts), + () => window.hermesDesktop.openSessionWindow(sessionId, { ...opts, profile }), 'Could not open chat in a new window' ) } diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 0f7529eb00..5402baa746 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -29,6 +29,21 @@ export interface AudioSpeakResponse { provider?: string } +/** `POST /api/audio/tts-lease` — TTS engine warm-up / release driven by speech toggles. */ +export interface AudioTtsLeaseResponse { + ok: boolean + lease: string + active: boolean + /** Live lease holders after this call (null when the backend call itself failed). */ + leases: null | number + /** Warm-up outcome: `loaded` | `cached` | `installed` | `noop` | `error`. */ + action?: string + provider?: string + /** Resident local models dropped (release path). */ + released?: number + error?: string +} + export interface ElevenLabsVoice { label: string name: string @@ -493,6 +508,10 @@ export interface SessionInfo { * continuation tip. Stable across compressions — used as the durable id for * pins so a pinned conversation survives auto-compression. */ _lineage_root_id?: null | string + /** Every id on the compression chain (root, intermediates, tip) when this + * entry is a projected continuation tip. Intermediates matter: a persisted + * tile or route can hold a middle segment's id from when IT was the tip. */ + _lineage_ids?: null | string[] input_tokens: number /** Spend for the session, straight off the `sessions` row. `actual` is set * when the provider reported a price; `estimated` is our own pricing-table @@ -716,6 +735,10 @@ export interface SessionRuntimeInfo { } export interface UsageStats { + /** Rolling tokens-per-second over the last ~10 API calls (tui_gateway `_get_usage`). */ + avg_tps?: number + /** Session prompt-cache hit rate, 0–100. Omitted (not 0) when the provider reports no cache reads. */ + cache_hit_pct?: number calls: number context_max?: number context_percent?: number diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 50f927c368..687badff7d 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -82,6 +82,17 @@ model: # api_key: "your-key-here" # Uncomment to set here instead of .env base_url: "https://openrouter.ai/api/v1" + # Stream API responses from the provider (default: true). The agent core + # prefers streaming for every turn — subagents included — for liveness + # health-checking. Set false to force non-streaming requests for the whole + # session (persists across mid-session model switches). Escape hatch for + # self-hosted OpenAI-compatible servers whose streaming tool-call path is + # broken (e.g. vLLM with --tool-call-parser qwen3_xml + a reasoning parser + # can leak tool calls into plain text instead of returning tool_calls — + # #72901). Orthogonal to display.streaming, which controls token rendering + # only. + # streaming: true + # Azure Foundry keyless auth example: # provider: "azure-foundry" # base_url: "https://.openai.azure.com/openai/v1" @@ -522,12 +533,14 @@ browser: # Tool Loop Guardrails # ============================================================================= # Soft warnings are enabled by default. They append guidance to repeated failed -# or non-progressing tool results but still let the tool execute. Hard stops are -# opt-in circuit breakers for autonomous/cron sessions where stopping a loop is -# preferable to spending the full iteration budget. +# or non-progressing tool results but still let the tool execute. Hard stops stay +# opt-in for interactive CLI/TUI/Desktop/ACP sessions, but default on for unattended +# gateway/cron sessions where nobody is present to interrupt a model that +# ignores loop warnings. tool_loop_guardrails: warnings_enabled: true hard_stop_enabled: false + non_interactive_hard_stop_enabled: true warn_after: exact_failure: 2 same_tool_failure: 3 @@ -688,6 +701,19 @@ compression: # summarization on a short idle thread. Example: 1800 = compact after 30 min idle. idle_compact_after_seconds: 0 + # Gateway session-hygiene turn-hold budget (default: 10). Max seconds an + # arriving user turn is held while a still-streaming hygiene summary + # finishes. Distinct from hygiene_timeout_seconds (compressor inactivity + # budget): this bounds user-visible latency so chat transports (Telegram + # ~30s) do not drop a silent connection. On expiry the turn proceeds + # uncompressed; the detached worker keeps its commit admission (when the + # commit is watermark-fenced) and the summary is adopted at the next safe + # boundary. Thinking-model summarizers often need longer than 10s to emit + # the first content token — raise to 300 (or >= your summarizer's real + # time-to-first-content) only if you want THIS turn to wait for the + # compression instead of adopting it one turn late. + hygiene_max_turn_hold_seconds: 10 + # Proactive tool-result prune (default: 0 = disabled). Opt-in token trigger # for a deterministic, no-LLM prune of OLD tool-result payloads, run # independently of `threshold` above. On large-window models (512K/1M) the @@ -922,6 +948,32 @@ max_concurrent_sessions: null # explicitly want one shared "room brain" per group/channel. group_sessions_per_user: true +# ============================================================================= +# Session Storage Retention (state.db) +# ============================================================================= +# ~/.hermes/state.db keeps every session, message, and tool call, plus the +# FTS5 search indexes. Since #54189, auto-pruning is ON by default so the file +# stays bounded: at CLI/gateway/cron startup (at most once per +# min_interval_hours) Hermes deletes ENDED sessions whose last activity is +# older than retention_days. Open, pinned, and in-progress sessions are never +# deleted. Stale automation sessions (cron/kanban/subagent/one-shot CLI) whose +# process died without closing them are first *closed*, then aged through a +# further full retention window before removal. +# +# After a prune that removed rows, VACUUM reclaims disk space only when both +# the time throttle (min_vacuum_interval_days) has elapsed AND more than 25% +# of the file's pages are reclaimable — a dense database never pays for a full +# rewrite to reclaim a few MB. +# +# Uncomment to change the defaults shown; set auto_prune: false to keep every +# ended session forever (the pre-#54189 behavior). +# sessions: +# auto_prune: true +# retention_days: 90 +# vacuum_after_prune: true +# min_vacuum_interval_days: 30 +# min_interval_hours: 24 + # Optional direct endpoint for autonomous Bot Mode rooms spanning gateways. # Leave unset for the safe default: Desktop coordinates cross-gateway rooms and # same-gateway rooms can still continue on their own. Set this only to the @@ -1155,7 +1207,17 @@ agent: # "claude-opus-4.6": "high" # bare model name also works # "deepseek/deepseek-v4-pro": "xhigh" # dots and dashes are interchangeable reasoning_overrides: {} - + + # Fast mode (OpenAI Priority Processing / xAI Grok 4.6 / Anthropic Fast Mode + # on Opus 4.8+). Premium pricing; only sent to first-party endpoints. + # "" / "normal" - off (default) + # "fast" - every request + # "auto" - only the first fast_auto_seconds of every turn + # "cold" - that window on the first turn of a session only + # Also: /fast normal|fast|auto|cold [--global] + service_tier: "" + fast_auto_seconds: 60 + # Custom personalities (use with /personality command). # Built-ins (helpful, concise, technical, creative, teacher, kawaii, catgirl, # pirate, shakespeare, surfer, noir, uwu, philosopher, hype) are always @@ -1179,6 +1241,13 @@ gateway: # if an agent has not unwound. Keep it below the service-manager stop budget. # signal_interrupt_grace_timeout: 1 + # Let platform adapters honor HTTP_PROXY / HTTPS_PROXY / NO_PROXY (and + # SSL_CERT_FILE) from the process environment, plus macOS system-proxy + # auto-detection. Set to false when the gateway inherits a proxy it must not + # use (e.g. a Windows Scheduled Task picking up a local Clash/V2Ray proxy that + # isn't running). Explicit per-platform vars like DISCORD_PROXY still apply. + # trust_env: true + # ============================================================================= # Toolsets # ============================================================================= @@ -1277,6 +1346,10 @@ platform_toolsets: # # Render live tool calls as Slack-native plan/task cards. This explicit # # opt-in works even though Slack text tool_progress defaults to off. # native_task_cards: false +# # Slack user IDs whose Web-API posts (user token, e.g. your own +# # dashboard/mobile front-end) count as human instead of being dropped +# # as app traffic. Narrower than allow_bots: all. Users only — never apps. +# api_human_users: ["U0AAAAAAA", "U0BBBBBBB"] # # Suppress automatic link-preview cards without removing clickable links. # # Omit either key to preserve Slack's default for that preview type. # unfurl_links: false @@ -1721,6 +1794,13 @@ display: # false: Silent (default) bell_on_complete: false + # Play terminal bell when a blocking prompt opens and waits on you: + # clarify questions, dangerous-command approvals, sudo password, secret + # capture. Same mechanism as bell_on_complete (\a) — works over SSH. + # true: Ring whenever the agent is waiting for your input + # false: Silent (default) + bell_on_prompt: false + # Show model reasoning/thinking before each response. # When enabled, a dim box shows the model's thought process above the response. # Toggle at runtime with /reasoning show or /reasoning hide. diff --git a/cli.py b/cli.py index 0174d7f3b0..e874afc737 100644 --- a/cli.py +++ b/cli.py @@ -314,6 +314,15 @@ def _strip_reasoning_tags(text: str) -> str: cleaned, flags=re.IGNORECASE, ) + # Unterminated opener / stray / markup = stream cut + # mid tool-call serialization (#101899); strip to end of text. + cleaned = re.sub( + r'(?:^|\n)[ \t]*<(?:tool_call|tool_calls|tool_result|function_call|function_calls)\b[^>]*>.*$' + r'|(?:^|\n)[^\n<]* dict | None: def _parse_service_tier_config(raw: str) -> str | None: - """Parse a persisted service-tier preference into a Responses API value.""" + """Parse a persisted fast-mode preference: None, "priority", "auto", or "cold".""" value = str(raw or "").strip().lower() if not value or value in {"normal", "default", "standard", "off", "none"}: return None if value in {"fast", "priority", "on"}: return "priority" + if value in {"auto", "cold"}: + return value logger.warning("Unknown service_tier '%s', ignoring", raw) return None @@ -1781,6 +1792,10 @@ def _setup_worktree(repo_root: str = None, sync_base: bool = True, """ import subprocess + from hermes_cli._subprocess_compat import ( + noninteractive_git_env as _noninteractive_git_env, + ) + repo_root = repo_root or _git_repo_root() if not repo_root: _cprint("\033[31m✗ --worktree requires being inside a git repository.\033[0m") @@ -1854,6 +1869,7 @@ def _setup_worktree(repo_root: str = None, sync_base: bool = True, result = subprocess.run( ["git", *_wt_add_cfg, "worktree", "add", str(wt_path), "-b", branch_name, base_ref], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, cwd=repo_root, + stdin=subprocess.DEVNULL, env=_noninteractive_git_env(), ) if result.returncode != 0: # If branching from the resolved remote ref failed for any reason @@ -1869,6 +1885,7 @@ def _setup_worktree(repo_root: str = None, sync_base: bool = True, result = subprocess.run( ["git", "worktree", "add", str(wt_path), "-b", branch_name, base_ref], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, cwd=repo_root, + stdin=subprocess.DEVNULL, env=_noninteractive_git_env(), ) if result.returncode != 0: _cleanup_failed_worktree_add(repo_root, wt_path, branch_name) @@ -5178,6 +5195,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): self.resume_display = CLI_CONFIG["display"].get("resume_display", "full") # bell_on_complete: play terminal bell (\a) when agent finishes a response self.bell_on_complete = CLI_CONFIG["display"].get("bell_on_complete", False) + # bell_on_prompt: play terminal bell (\a) whenever a blocking prompt + # modal opens (clarify, approval, sudo password, secret capture) + self.bell_on_prompt = CLI_CONFIG["display"].get("bell_on_prompt", False) # show_reasoning: display model thinking/reasoning before the response self.show_reasoning = CLI_CONFIG["display"].get("show_reasoning", True) # reasoning_full: when reasoning display is on, print the post-response @@ -11997,7 +12017,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): if not providers: _cprint(" No authenticated providers found.") _cprint("") - _cprint(" /model switch model (persists)") + _cprint(" /model switch model (this session)") + _cprint(" /model --global switch model and persist as default") _cprint(" /model --once switch for the next turn only") _cprint(" /model --session switch for this session only") _cprint(" /model --provider switch provider") @@ -14669,7 +14690,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): sees the updated tools on the next turn. """ try: - from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock + from tools.mcp_tool import ( + shutdown_mcp_servers, discover_mcp_tools, reprobe_tool_availability, _servers, _lock, + ) # Capture old server names with _lock: @@ -14681,6 +14704,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # Shutdown existing connections shutdown_mcp_servers() + # Explicit reload also re-probes tool availability (check_fn). + reprobe_tool_availability() # Reconnect (reads config.yaml fresh) new_tools = discover_mcp_tools() @@ -15656,6 +15681,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # _voice_message_prefix property and its usage in _process_message(). tts_status = " (TTS enabled)" if self._voice_tts else "" + if self._voice_tts: + # Speech output is on from the start — warm the engine now so the + # first spoken reply doesn't pay the model load as dead air. + self._tts_lease_async(True) # Use the startup-pinned cache so the advertised shortcut always # matches the live prompt_toolkit binding — reading live config # here would drift after a mid-session config edit (Copilot @@ -15714,6 +15743,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): self._voice_tts = False self._voice_continuous = False + # Speech output is off with the mode — release the TTS engine lease so + # a resident local model (piper/kittentts) is freed once nothing else + # in this process still needs it. + self._tts_lease_async(False) + # Shut down the persistent audio stream in background if recorder is not None: def _bg_shutdown(rec=recorder): @@ -15961,6 +15995,29 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): if not owned: _cprint(f" {_DIM}Enable with /wake on{_RST}") + def _tts_lease_async(self, active: bool) -> None: + """Acquire/release this CLI's TTS engine lease in the background. + + The /voice tts toggle (and voice-mode on/off with speech output set) + is the "TTS is about to be needed / no longer needed" signal: + acquiring pre-loads the configured provider so the first reply starts + hot; releasing lets the last-holder path unload resident local models. + Never blocks the toggle and never fails it. + """ + + def _run(): + try: + from tools.tts_tool import acquire_tts_lease, release_tts_lease + + if active: + acquire_tts_lease("cli:voice-tts") + else: + release_tts_lease("cli:voice-tts") + except Exception as e: + logger.debug("voice: tts lease active=%s failed: %s", active, e) + + threading.Thread(target=_run, name="tts-lease-cli", daemon=True).start() + def _toggle_voice_tts(self): """Toggle TTS output for voice mode.""" if not self._voice_mode: @@ -15976,6 +16033,10 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): if not check_tts_requirements(): _cprint(f"{_DIM}Warning: No TTS provider available. Install edge-tts or set API keys.{_RST}") + # Toggle = warm-up / release signal for the TTS engine (see + # tools.tts_tool.acquire_tts_lease). + self._tts_lease_async(self._voice_tts) + _cprint(f"{_ACCENT}Voice TTS {status}.{_RST}") def _show_voice_status(self): @@ -16016,6 +16077,39 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): outcome = outcome[:119] + "…" _cprint(f"\n{_DIM}{icon} {label}: {detail} → {outcome}{_RST}") + def _ring_bell(self, prompt: bool = False, context: str = "", detail: str = "") -> None: + """Write a terminal bell (\\a) if the matching display.bell_* flag is on. + + ``prompt=True`` is the blocking-modal variant (clarify / approval / + sudo / secret capture) gated by ``display.bell_on_prompt``; the default + is the end-of-turn bell gated by ``display.bell_on_complete``. Works + over SSH — the BEL propagates to the user's terminal. + + The same flag also emits an OSC 9 desktop notification (Ghostty, + iTerm2, Kitty, WezTerm) and, inside a supporting Warp build, a + ``warp://cli-agent`` OSC 777 event — see ``hermes_cli.terminal_notify``. + ``context`` is the short notification body (e.g. "approval"). + """ + flag = "bell_on_prompt" if prompt else "bell_on_complete" + if not getattr(self, flag, False): + return + try: + sys.stdout.write("\a") + sys.stdout.flush() + except Exception: + pass + try: + from hermes_cli.terminal_notify import notify as _terminal_notify + + _terminal_notify( + context or ("input needed" if prompt else "turn complete"), + prompt=prompt, + session_id=getattr(self, "session_id", "") or "", + detail=detail, + ) + except Exception: + pass + def _clarify_callback(self, question, choices, multi_select=False, questions=None): """ Platform callback for the clarify tool. Called from the agent thread. @@ -16063,6 +16157,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): self._clarify_freetext = is_open_ended self._clarify_multi_base = None + self._ring_bell(prompt=True, context="clarify") # Trigger an immediate prompt_toolkit repaint from this (non-main) # thread. Modal prompts must paint at once and must not be gated by the # _invalidate throttle / resize guard — see _paint_now / _invalidate (#41098). @@ -16254,6 +16349,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): self._clarify_state = state self._clarify_batch_set_active(state, 0) self._clarify_deadline = None if timeout <= 0 else _time.monotonic() + timeout + self._ring_bell(prompt=True, context="clarify") self._paint_now() _last_countdown_refresh = _time.monotonic() @@ -16304,6 +16400,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): "response_queue": response_queue, } self._sudo_deadline = _time.monotonic() + timeout + self._ring_bell(prompt=True, context="sudo password") # Modal prompt — paint immediately, bypassing the throttle/resize guard # so the prompt can't be dropped and time out unseen (#41098). @@ -16373,6 +16470,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): } self._approval_deadline = _time.monotonic() + timeout + self._ring_bell(prompt=True, context="approval", detail=command) # Modal prompt — paint immediately, bypassing the throttle/resize # guard. A throttled paint here can be silently dropped (250ms # window collision or in-flight resize), leaving the panel unseen so @@ -17516,9 +17614,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # Play terminal bell when agent finishes (if enabled). # Works over SSH — the bell propagates to the user's terminal. - if self.bell_on_complete: - sys.stdout.write("\a") - sys.stdout.flush() + self._ring_bell(context="turn complete") # Notify when iteration budget was hit if result and not result.get("completed") and not result.get("interrupted"): diff --git a/contributors/emails/16833782+Fatmylin@users.noreply.github.com b/contributors/emails/16833782+Fatmylin@users.noreply.github.com new file mode 100644 index 0000000000..f608e6e066 --- /dev/null +++ b/contributors/emails/16833782+Fatmylin@users.noreply.github.com @@ -0,0 +1,2 @@ +Fatmylin +# PR #95964 salvage diff --git a/contributors/emails/210261288+Christopher-Schulze@users.noreply.github.com b/contributors/emails/210261288+Christopher-Schulze@users.noreply.github.com new file mode 100644 index 0000000000..e40f1c1a02 --- /dev/null +++ b/contributors/emails/210261288+Christopher-Schulze@users.noreply.github.com @@ -0,0 +1,2 @@ +Christopher-Schulze +# PR #85806 salvage diff --git a/contributors/emails/296402666+ciabata-git@users.noreply.github.com b/contributors/emails/296402666+ciabata-git@users.noreply.github.com new file mode 100644 index 0000000000..317632f3c5 --- /dev/null +++ b/contributors/emails/296402666+ciabata-git@users.noreply.github.com @@ -0,0 +1,2 @@ +ciabata-git +# PR #96011 author email preserved by PR #97330 diff --git a/contributors/emails/298902573+pierrenode@users.noreply.github.com b/contributors/emails/298902573+pierrenode@users.noreply.github.com new file mode 100644 index 0000000000..ef6b9888a4 --- /dev/null +++ b/contributors/emails/298902573+pierrenode@users.noreply.github.com @@ -0,0 +1,2 @@ +pierrenode +# PR #84168 salvage diff --git a/contributors/emails/387700378@qq.com b/contributors/emails/387700378@qq.com new file mode 100644 index 0000000000..4b6c348909 --- /dev/null +++ b/contributors/emails/387700378@qq.com @@ -0,0 +1 @@ +OmniaZ1 diff --git a/contributors/emails/JLHunzicker@gmail.com b/contributors/emails/JLHunzicker@gmail.com new file mode 100644 index 0000000000..865bba7f71 --- /dev/null +++ b/contributors/emails/JLHunzicker@gmail.com @@ -0,0 +1,2 @@ +JackHunzicker +# PR #98371 salvage diff --git a/contributors/emails/TEDLANHAM@GMAIL.COM b/contributors/emails/TEDLANHAM@GMAIL.COM new file mode 100644 index 0000000000..6de0474e18 --- /dev/null +++ b/contributors/emails/TEDLANHAM@GMAIL.COM @@ -0,0 +1 @@ +Backroads4Me diff --git a/contributors/emails/agent@dynamicagency.com b/contributors/emails/agent@dynamicagency.com new file mode 100644 index 0000000000..3696dc87bf --- /dev/null +++ b/contributors/emails/agent@dynamicagency.com @@ -0,0 +1,2 @@ +agentdynamic +# PR #92413 salvage diff --git a/contributors/emails/andrewwikel@gmail.com b/contributors/emails/andrewwikel@gmail.com new file mode 100644 index 0000000000..dd9535b42e --- /dev/null +++ b/contributors/emails/andrewwikel@gmail.com @@ -0,0 +1,2 @@ +slash1andy +# PR #88217 salvage diff --git a/contributors/emails/benjaminperry6@yahoo.fr b/contributors/emails/benjaminperry6@yahoo.fr new file mode 100644 index 0000000000..16b94399dd --- /dev/null +++ b/contributors/emails/benjaminperry6@yahoo.fr @@ -0,0 +1,2 @@ +benperry6 +# PR #97330 author email diff --git a/contributors/emails/contact@danteschrauwen.be b/contributors/emails/contact@danteschrauwen.be new file mode 100644 index 0000000000..905026861f --- /dev/null +++ b/contributors/emails/contact@danteschrauwen.be @@ -0,0 +1,2 @@ +deinte +# PR #101090 salvage (cron timezone-migration catch-up) diff --git a/contributors/emails/crdesign8@hotmail.com b/contributors/emails/crdesign8@hotmail.com new file mode 100644 index 0000000000..fffa9b3583 --- /dev/null +++ b/contributors/emails/crdesign8@hotmail.com @@ -0,0 +1 @@ +crdesign8 diff --git a/contributors/emails/csreyes92@gmail.com b/contributors/emails/csreyes92@gmail.com new file mode 100644 index 0000000000..6d1d2cf38e --- /dev/null +++ b/contributors/emails/csreyes92@gmail.com @@ -0,0 +1 @@ +csreyes diff --git a/contributors/emails/deathxdefeat@users.noreply.github.com b/contributors/emails/deathxdefeat@users.noreply.github.com new file mode 100644 index 0000000000..3b4c4ee0dd --- /dev/null +++ b/contributors/emails/deathxdefeat@users.noreply.github.com @@ -0,0 +1 @@ +deathxdefeat diff --git a/contributors/emails/e2e@ikbi.test b/contributors/emails/e2e@ikbi.test new file mode 100644 index 0000000000..361be65ed9 --- /dev/null +++ b/contributors/emails/e2e@ikbi.test @@ -0,0 +1 @@ +RootZ3n diff --git a/contributors/emails/emanuele.cornaggia@gmail.com b/contributors/emails/emanuele.cornaggia@gmail.com new file mode 100644 index 0000000000..830fe71ba2 --- /dev/null +++ b/contributors/emails/emanuele.cornaggia@gmail.com @@ -0,0 +1,2 @@ +EmanueleCornaggia +# PR #101778 salvage diff --git a/contributors/emails/glitchbunny0@proton.me b/contributors/emails/glitchbunny0@proton.me new file mode 100644 index 0000000000..5f679947b3 --- /dev/null +++ b/contributors/emails/glitchbunny0@proton.me @@ -0,0 +1,2 @@ +glitchbunny0 +# PR #58957 salvage diff --git a/contributors/emails/globalveterinary@gmail.com b/contributors/emails/globalveterinary@gmail.com new file mode 100644 index 0000000000..4069ed192a --- /dev/null +++ b/contributors/emails/globalveterinary@gmail.com @@ -0,0 +1 @@ +globalvet2025 diff --git a/contributors/emails/gobeumsu@gmail.com b/contributors/emails/gobeumsu@gmail.com new file mode 100644 index 0000000000..f217b336bf --- /dev/null +++ b/contributors/emails/gobeumsu@gmail.com @@ -0,0 +1 @@ +GoBeromsu diff --git a/contributors/emails/harsha@usethread.io b/contributors/emails/harsha@usethread.io new file mode 100644 index 0000000000..78f9244a8c --- /dev/null +++ b/contributors/emails/harsha@usethread.io @@ -0,0 +1,2 @@ +harshmoney123 +# PR #100805 salvage diff --git a/contributors/emails/itsflownium@users.noreply.github.com b/contributors/emails/itsflownium@users.noreply.github.com new file mode 100644 index 0000000000..2c08cb46fe --- /dev/null +++ b/contributors/emails/itsflownium@users.noreply.github.com @@ -0,0 +1 @@ +itsflownium diff --git a/contributors/emails/jackal991@users.noreply.github.com b/contributors/emails/jackal991@users.noreply.github.com new file mode 100644 index 0000000000..8eda80766a --- /dev/null +++ b/contributors/emails/jackal991@users.noreply.github.com @@ -0,0 +1 @@ +Jackal991 diff --git a/contributors/emails/joel.taylor@ccmschools.edu.au b/contributors/emails/joel.taylor@ccmschools.edu.au new file mode 100644 index 0000000000..07f5a05228 --- /dev/null +++ b/contributors/emails/joel.taylor@ccmschools.edu.au @@ -0,0 +1 @@ +JoelMTaylor diff --git a/contributors/emails/kaneko@pxls.co.jp b/contributors/emails/kaneko@pxls.co.jp new file mode 100644 index 0000000000..9a1afa1685 --- /dev/null +++ b/contributors/emails/kaneko@pxls.co.jp @@ -0,0 +1,2 @@ +pxls-kaneko +# PR #95647 salvage diff --git a/contributors/emails/kbaicai@qq.com b/contributors/emails/kbaicai@qq.com new file mode 100644 index 0000000000..c8052461d0 --- /dev/null +++ b/contributors/emails/kbaicai@qq.com @@ -0,0 +1,2 @@ +ifastcc +# PR #34308 co-author diff --git a/contributors/emails/leocamilo@me.com b/contributors/emails/leocamilo@me.com new file mode 100644 index 0000000000..6adc1d27f3 --- /dev/null +++ b/contributors/emails/leocamilo@me.com @@ -0,0 +1 @@ +leomcamilo diff --git a/contributors/emails/michaelalexanderknaap@gmail.com b/contributors/emails/michaelalexanderknaap@gmail.com new file mode 100644 index 0000000000..47b5623e70 --- /dev/null +++ b/contributors/emails/michaelalexanderknaap@gmail.com @@ -0,0 +1 @@ +fortun8te diff --git a/contributors/emails/michel.alexander@gmail.com b/contributors/emails/michel.alexander@gmail.com new file mode 100644 index 0000000000..d990d4f5e5 --- /dev/null +++ b/contributors/emails/michel.alexander@gmail.com @@ -0,0 +1,2 @@ +SelfParody +# PR #19000 salvage diff --git a/contributors/emails/muhammad.gcs@gmail.com b/contributors/emails/muhammad.gcs@gmail.com new file mode 100644 index 0000000000..d4d782ab1b --- /dev/null +++ b/contributors/emails/muhammad.gcs@gmail.com @@ -0,0 +1 @@ +muhifni diff --git a/contributors/emails/nate@atxlakescapes.com b/contributors/emails/nate@atxlakescapes.com new file mode 100644 index 0000000000..66471fd497 --- /dev/null +++ b/contributors/emails/nate@atxlakescapes.com @@ -0,0 +1 @@ +Lakescape diff --git a/contributors/emails/nguyenngoctinh011258@gmail.com b/contributors/emails/nguyenngoctinh011258@gmail.com new file mode 100644 index 0000000000..aee57ffe94 --- /dev/null +++ b/contributors/emails/nguyenngoctinh011258@gmail.com @@ -0,0 +1 @@ +twotnguyen diff --git a/contributors/emails/omarlopz1985@gmail.com b/contributors/emails/omarlopz1985@gmail.com new file mode 100644 index 0000000000..4186351e1c --- /dev/null +++ b/contributors/emails/omarlopz1985@gmail.com @@ -0,0 +1 @@ +olopez25 diff --git a/contributors/emails/patryk.kopycinski@elastic.co b/contributors/emails/patryk.kopycinski@elastic.co new file mode 100644 index 0000000000..c42cf72661 --- /dev/null +++ b/contributors/emails/patryk.kopycinski@elastic.co @@ -0,0 +1 @@ +patrykkopycinski diff --git a/contributors/emails/praxis1244@gmail.com b/contributors/emails/praxis1244@gmail.com new file mode 100644 index 0000000000..9613e9a467 --- /dev/null +++ b/contributors/emails/praxis1244@gmail.com @@ -0,0 +1 @@ +praxis1244-consulting diff --git a/contributors/emails/rcarratalasanchez@gmail.com b/contributors/emails/rcarratalasanchez@gmail.com new file mode 100644 index 0000000000..6fffcb4a82 --- /dev/null +++ b/contributors/emails/rcarratalasanchez@gmail.com @@ -0,0 +1 @@ +rcarrata diff --git a/contributors/emails/robbert_camps@hotmail.com b/contributors/emails/robbert_camps@hotmail.com new file mode 100644 index 0000000000..85a92dac9e --- /dev/null +++ b/contributors/emails/robbert_camps@hotmail.com @@ -0,0 +1 @@ +RobbertC5 diff --git a/contributors/emails/sosxradar@gmail.com b/contributors/emails/sosxradar@gmail.com new file mode 100644 index 0000000000..02d4ffb5d0 --- /dev/null +++ b/contributors/emails/sosxradar@gmail.com @@ -0,0 +1 @@ +GTHell diff --git a/contributors/emails/talmoredder@gmail.com b/contributors/emails/talmoredder@gmail.com new file mode 100644 index 0000000000..0e8b281e2c --- /dev/null +++ b/contributors/emails/talmoredder@gmail.com @@ -0,0 +1,2 @@ +EdderTalmor +# PR #10110 salvage diff --git a/contributors/emails/tky.juani@gmail.com b/contributors/emails/tky.juani@gmail.com new file mode 100644 index 0000000000..b921c7e4ea --- /dev/null +++ b/contributors/emails/tky.juani@gmail.com @@ -0,0 +1 @@ +JuaniLezcano diff --git a/contributors/emails/umit.ediz@hotmail.com b/contributors/emails/umit.ediz@hotmail.com new file mode 100644 index 0000000000..35cc601087 --- /dev/null +++ b/contributors/emails/umit.ediz@hotmail.com @@ -0,0 +1 @@ +Edizzier diff --git a/contributors/emails/vibecoder@example.com b/contributors/emails/vibecoder@example.com new file mode 100644 index 0000000000..c2d1000c3e --- /dev/null +++ b/contributors/emails/vibecoder@example.com @@ -0,0 +1 @@ +zoser69 diff --git a/contributors/emails/yj2761@nyu.edu b/contributors/emails/yj2761@nyu.edu new file mode 100644 index 0000000000..a73c993b32 --- /dev/null +++ b/contributors/emails/yj2761@nyu.edu @@ -0,0 +1,2 @@ +yaojiejia +# PR #95160 salvage diff --git a/cron/delivery_queue.py b/cron/delivery_queue.py new file mode 100644 index 0000000000..635a295b2f --- /dev/null +++ b/cron/delivery_queue.py @@ -0,0 +1,379 @@ +"""Profile-local durable handoff for cron delivery through live gateway adapters. + +A restart-safe cron worker executes outside the gateway cgroup. It cannot own +relay/E2EE adapter objects, so it queues the final send here. A gateway claims +each row at most once. If that gateway dies after claiming, the outcome is +marked unknown and never retried: losing a delivery is safer than duplicating a +possibly-completed send. +""" + +from __future__ import annotations + +import json +import logging +import os +import sqlite3 +import threading +import time +import uuid +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Callable, Iterator, Optional + +from agent.redact import redact_sensitive_text +from cron.executions import _owner_is_live, _process_start_time +from hermes_cli.sqlite_util import add_column_if_missing +from hermes_constants import get_hermes_home +from hermes_time import now as _hermes_now + +logger = logging.getLogger(__name__) + +DELIVERY_DB: Optional[Path] = None +_PROCESS_ID = uuid.uuid4().hex +_lock = threading.RLock() +_ACTIVE_DELIVERIES: set[str] = set() +_TERMINAL = ("delivered", "failed", "unknown") +MAX_TERMINAL_DELIVERIES = 1000 +DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS = 300.0 + + +def _prune_terminal_unlocked(conn: sqlite3.Connection) -> None: + """Redact terminal payloads and retain only bounded outcome metadata.""" + conn.execute( + """UPDATE deliveries SET job_json='{}', content='' + WHERE status IN ('delivered','failed','unknown') + AND (job_json != '{}' OR content != '')""" + ) + keep = max(0, int(MAX_TERMINAL_DELIVERIES)) + terminal_count = int( + conn.execute( + "SELECT COUNT(*) FROM deliveries " + "WHERE status IN ('delivered','failed','unknown')" + ).fetchone()[0] + ) + excess = terminal_count - keep + if excess > 0: + conn.execute( + """INSERT OR IGNORE INTO delivery_tombstones + (execution_id, terminal_status, finished_at) + SELECT execution_id, status, finished_at FROM deliveries + WHERE status IN ('delivered','failed','unknown') + ORDER BY finished_at, created_at, execution_id + LIMIT ?""", + (excess,), + ) + conn.execute( + """DELETE FROM deliveries WHERE execution_id IN ( + SELECT execution_id FROM deliveries + WHERE status IN ('delivered','failed','unknown') + ORDER BY finished_at, created_at, execution_id + LIMIT ? + )""", + (excess,), + ) + + +def _path() -> Path: + return DELIVERY_DB or (get_hermes_home().resolve() / "cron" / "deliveries.db") + + +@contextmanager +def _transaction() -> Iterator[sqlite3.Connection]: + with _lock: + path = _path() + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path, timeout=5) + try: + path.chmod(0o600) + except OSError: + pass + conn.row_factory = sqlite3.Row + try: + from hermes_state import apply_wal_with_fallback + + conn.execute("PRAGMA busy_timeout=5000") + apply_wal_with_fallback(conn, db_label="cron/deliveries.db") + conn.execute("PRAGMA synchronous=FULL") + conn.execute( + """CREATE TABLE IF NOT EXISTS deliveries ( + execution_id TEXT PRIMARY KEY, + job_json TEXT NOT NULL, + content TEXT NOT NULL, + for_failure INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL CHECK(status IN + ('pending','delivering','delivered','failed','unknown')), + owner_process_id TEXT, + owner_pid INTEGER, + owner_started_at INTEGER, + created_at TEXT NOT NULL, + finished_at TEXT, + error TEXT + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS delivery_tombstones ( + execution_id TEXT PRIMARY KEY, + terminal_status TEXT NOT NULL CHECK(terminal_status IN + ('delivered','failed','unknown')), + finished_at TEXT + )""" + ) + add_column_if_missing( + conn, "deliveries", "for_failure", + "for_failure INTEGER NOT NULL DEFAULT 0", + ) + # Pruning is done explicitly by the paths that create terminal + # rows (_finish / recover_abandoned / _terminalize_wait_timeout); + # read-only polls must not pay for a full-table UPDATE + COUNT. + with conn: + yield conn + finally: + conn.close() + + +def enqueue( + execution_id: str, + job: dict, + content: str, + *, + for_failure: bool = False, +) -> dict: + """Persist one idempotent delivery request before the worker waits.""" + with _transaction() as conn: + tombstone = conn.execute( + "SELECT terminal_status, finished_at FROM delivery_tombstones " + "WHERE execution_id=?", + (str(execution_id),), + ).fetchone() + if tombstone is not None: + return { + "execution_id": str(execution_id), + "status": tombstone["terminal_status"], + "finished_at": tombstone["finished_at"], + } + conn.execute( + """INSERT OR IGNORE INTO deliveries + (execution_id, job_json, content, for_failure, status, created_at) + VALUES (?, ?, ?, ?, 'pending', ?)""", + ( + str(execution_id), + json.dumps(job, ensure_ascii=False, sort_keys=True), + str(content), + int(bool(for_failure)), + _hermes_now().isoformat(), + ), + ) + row = conn.execute( + "SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),) + ).fetchone() + return dict(row) + + +def get_status(execution_id: str) -> Optional[dict]: + with _transaction() as conn: + row = conn.execute( + "SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),) + ).fetchone() + if row is not None: + return dict(row) + tombstone = conn.execute( + "SELECT execution_id, terminal_status, finished_at " + "FROM delivery_tombstones WHERE execution_id=?", + (str(execution_id),), + ).fetchone() + if tombstone is None: + return None + return { + "execution_id": tombstone["execution_id"], + "status": tombstone["terminal_status"], + "finished_at": tombstone["finished_at"], + "error": None, + } + + +def claim_next() -> Optional[dict]: + """Atomically claim one pending send before touching the transport.""" + pid = os.getpid() + started = _process_start_time(pid) + with _transaction() as conn: + row = conn.execute( + "SELECT execution_id FROM deliveries WHERE status='pending' " + "ORDER BY created_at, execution_id LIMIT 1" + ).fetchone() + if row is None: + return None + cur = conn.execute( + """UPDATE deliveries SET status='delivering', owner_process_id=?, + owner_pid=?, owner_started_at=? + WHERE execution_id=? AND status='pending'""", + (_PROCESS_ID, pid, started, row["execution_id"]), + ) + if cur.rowcount != 1: + return None + claimed = conn.execute( + "SELECT * FROM deliveries WHERE execution_id=?", (row["execution_id"],) + ).fetchone() + _ACTIVE_DELIVERIES.add(row["execution_id"]) + result = dict(claimed) + result["job"] = json.loads(result.pop("job_json")) + return result + + +def _finish(execution_id: str, *, error: Optional[str]) -> bool: + status = "failed" if error else "delivered" + safe_error = ( + redact_sensitive_text(str(error), force=True, redact_url_credentials=True) + if error + else None + ) + with _transaction() as conn: + cur = conn.execute( + """UPDATE deliveries SET status=?, finished_at=?, error=? + WHERE execution_id=? AND status='delivering' + AND owner_process_id=? AND owner_pid=?""", + ( + status, + _hermes_now().isoformat(), + safe_error, + execution_id, + _PROCESS_ID, + os.getpid(), + ), + ) + _prune_terminal_unlocked(conn) + return cur.rowcount == 1 + + +def recover_abandoned() -> int: + """Fence dead delivery owners as unknown; never replay uncertain sends.""" + changed = 0 + with _transaction() as conn: + rows = conn.execute( + "SELECT execution_id, owner_process_id, owner_pid, owner_started_at " + "FROM deliveries WHERE status='delivering'" + ).fetchall() + for row in rows: + same_process = row["owner_process_id"] == _PROCESS_ID + if same_process: + with _lock: + if row["execution_id"] in _ACTIVE_DELIVERIES: + continue + elif _owner_is_live(int(row["owner_pid"]), row["owner_started_at"]): + continue + error = ( + "Gateway finished delivery but could not persist its outcome; " + "send was not retried." + if same_process + else "Gateway exited during delivery; send outcome is unknown and was not retried." + ) + cur = conn.execute( + """UPDATE deliveries SET status='unknown', finished_at=?, error=? + WHERE execution_id=? AND status='delivering'""", + ( + _hermes_now().isoformat(), + error, + row["execution_id"], + ), + ) + changed += cur.rowcount + _prune_terminal_unlocked(conn) + return changed + + +def drain( + send: Callable[[dict, str, bool], Optional[str]], *, limit: int = 20 +) -> int: + """Deliver pending rows through *send*, terminalizing every claimed row.""" + recover_abandoned() + processed = 0 + for _ in range(max(0, limit)): + row = claim_next() + if row is None: + break + with _lock: + _ACTIVE_DELIVERIES.add(row["execution_id"]) + try: + try: + error = send( + row["job"], row["content"], bool(row["for_failure"]) + ) + except BaseException as exc: + error = f"{type(exc).__name__}: {exc}" + _finish(row["execution_id"], error=error) + finally: + with _lock: + _ACTIVE_DELIVERIES.discard(row["execution_id"]) + processed += 1 + return processed + + +def _terminalize_wait_timeout(execution_id: str) -> str: + """Fence a delivery whose worker can no longer wait for confirmation. + + A row still ``pending`` was provably never attempted, so it is left queued + for whichever gateway comes up next (a restart that includes an update can + easily exceed the worker's wait budget). That is a deferral, not a + failure: report success so the job is not recorded ``delivery_failed`` for + a message the drain will still send. Only a row caught mid-send is + uncertain and gets fenced ``unknown``. + """ + now = _hermes_now().isoformat() + uncertain_error = ( + "timed out while gateway delivery was in progress; outcome is unknown and " + "was not retried" + ) + with _transaction() as conn: + row = conn.execute( + "SELECT status FROM deliveries WHERE execution_id=?", + (str(execution_id),), + ).fetchone() + if row is not None and row["status"] == "pending": + logger.warning( + "Cron delivery %s: no live gateway within the wait budget; " + "left queued for the next gateway", + execution_id, + ) + return "" + conn.execute( + """UPDATE deliveries SET status='unknown', finished_at=?, error=? + WHERE execution_id=? AND status='delivering'""", + (now, uncertain_error, str(execution_id)), + ) + row = conn.execute( + "SELECT status, error FROM deliveries WHERE execution_id=?", + (str(execution_id),), + ).fetchone() + _prune_terminal_unlocked(conn) + if row is None: + return "timed out waiting for live gateway delivery" + if row["status"] == "delivered": + return "" + return str(row["error"] or f"delivery {row['status']}") + + +def enqueue_and_wait( + execution_id: str, + job: dict, + content: str, + *, + for_failure: bool = False, + timeout: Optional[float] = None, +) -> Optional[str]: + """Queue delivery and wait for a gateway's terminal at-most-once outcome.""" + queued = enqueue(execution_id, job, content, for_failure=for_failure) + if queued["status"] in _TERMINAL: + return None if queued["status"] == "delivered" else str( + queued.get("error") or f"delivery {queued['status']}" + ) + wait_timeout = ( + DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS if timeout is None else max(0.0, timeout) + ) + deadline = time.monotonic() + wait_timeout + while time.monotonic() < deadline: + row = get_status(execution_id) + if row and row["status"] in _TERMINAL: + return None if row["status"] == "delivered" else str( + row.get("error") or f"delivery {row['status']}" + ) + time.sleep(1.0) + return _terminalize_wait_timeout(execution_id) or None diff --git a/cron/executions.py b/cron/executions.py index efadab464a..023ed75a4d 100644 --- a/cron/executions.py +++ b/cron/executions.py @@ -10,6 +10,7 @@ from __future__ import annotations import os import sqlite3 import threading +import time import uuid from contextlib import contextmanager from typing import Any, Dict, Iterator, List, Optional @@ -22,6 +23,7 @@ from hermes_time import now as _hermes_now # profile's execution records into the import-time home. EXECUTIONS_FILE: Optional[Path] = None MAX_TERMINAL_EXECUTIONS = 1000 +HANDOFF_ADOPTION_GRACE_SECONDS = 30.0 _TERMINAL_STATES = ("completed", "failed", "unknown") _lock = threading.RLock() _PROCESS_ID = uuid.uuid4().hex @@ -52,12 +54,23 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: process_started_at INTEGER, status TEXT NOT NULL CHECK(status IN ('claimed','running','completed','failed','unknown')), + handoff_pending INTEGER NOT NULL DEFAULT 0, + handoff_started_at REAL, claimed_at TEXT NOT NULL, started_at TEXT, finished_at TEXT, error TEXT )""" ) + from hermes_cli.sqlite_util import add_column_if_missing + + add_column_if_missing( + conn, "executions", "handoff_pending", + "handoff_pending INTEGER NOT NULL DEFAULT 0", + ) + add_column_if_missing( + conn, "executions", "handoff_started_at", "handoff_started_at REAL" + ) conn.execute( "CREATE INDEX IF NOT EXISTS idx_executions_job_claimed " "ON executions(job_id, claimed_at DESC, id DESC)" @@ -132,7 +145,7 @@ def _prune_unlocked(conn: sqlite3.Connection) -> None: """DELETE FROM executions WHERE id IN ( SELECT id FROM executions WHERE status IN ('completed','failed','unknown') - ORDER BY claimed_at DESC, id DESC LIMIT -1 OFFSET ? + ORDER BY finished_at DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ? )""", (limit,), ) @@ -160,14 +173,64 @@ def create_execution(job_id: str, *, source: str) -> Dict[str, Any]: return record # type: ignore[return-value] +def mark_execution_handoff_pending(execution_id: str) -> Optional[Dict[str, Any]]: + """Fence restart recovery while an external worker is adopting a claim.""" + with _transaction() as conn: + cur = conn.execute( + """UPDATE executions + SET handoff_pending=1, handoff_started_at=? + WHERE id=? AND status='claimed' + AND process_id=? AND pid=?""", + (time.time(), execution_id, _PROCESS_ID, os.getpid()), + ) + if cur.rowcount != 1: + return None + record = _record(conn.execute( + "SELECT * FROM executions WHERE id=?", (execution_id,) + ).fetchone()) + _emit_execution_state(record) + return record + + +def adopt_claimed_execution(execution_id: str) -> Optional[Dict[str, Any]]: + """Atomically transfer and start an attempt in its worker process. + + The dispatching gateway creates the row before spawning a restart-safe + worker. Adoption is the single ``claimed`` → ``running`` gate: only the + winner may acknowledge ownership or run side effects. + """ + pid = os.getpid() + process_started_at = _process_start_time(pid) + now = _hermes_now().isoformat() + with _transaction() as conn: + cur = conn.execute( + """UPDATE executions + SET process_id=?, pid=?, process_started_at=?, + status='running', started_at=?, handoff_pending=0, + handoff_started_at=NULL + WHERE id=? AND status='claimed' AND handoff_pending=1""", + (_PROCESS_ID, pid, process_started_at, now, execution_id), + ) + if cur.rowcount != 1: + return None + record = _record(conn.execute( + "SELECT * FROM executions WHERE id=?", (execution_id,) + ).fetchone()) + _emit_execution_state(record) + return record + + def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]: """Transition one claimed attempt to running exactly once.""" now = _hermes_now().isoformat() with _transaction() as conn: cur = conn.execute( - """UPDATE executions SET status='running', started_at=? - WHERE id=? AND status='claimed'""", - (now, execution_id), + """UPDATE executions + SET status='running', started_at=?, handoff_pending=0, + handoff_started_at=NULL + WHERE id=? AND status='claimed' AND handoff_pending=0 + AND process_id=? AND pid=?""", + (now, execution_id, _PROCESS_ID, os.getpid()), ) if cur.rowcount != 1: return None @@ -188,9 +251,12 @@ def finish_execution( detail = None if success else (str(error) if error else "unknown failure") with _transaction() as conn: cur = conn.execute( - """UPDATE executions SET status=?, finished_at=?, error=? - WHERE id=? AND status IN ('claimed','running')""", - (status, now, detail, execution_id), + """UPDATE executions + SET status=?, finished_at=?, error=?, handoff_pending=0, + handoff_started_at=NULL + WHERE id=? AND status IN ('claimed','running') + AND process_id=? AND pid=?""", + (status, now, detail, execution_id, _PROCESS_ID, os.getpid()), ) if cur.rowcount != 1: return None @@ -209,7 +275,9 @@ def recover_interrupted_executions() -> int: recovered: List[Dict[str, Any]] = [] with _transaction() as conn: rows = conn.execute( - """SELECT id, process_id, pid, process_started_at FROM executions + """SELECT id, status, process_id, pid, process_started_at, + handoff_pending, handoff_started_at + FROM executions WHERE status IN ('claimed','running')""" ).fetchall() for row in rows: @@ -217,13 +285,26 @@ def recover_interrupted_executions() -> int: continue if _owner_is_live(int(row["pid"]), row["process_started_at"]): continue + handoff_started_at = row["handoff_started_at"] + if ( + row["handoff_pending"] + and handoff_started_at is not None + and time.time() - float(handoff_started_at) + < HANDOFF_ADOPTION_GRACE_SECONDS + ): + continue cur = conn.execute( - """UPDATE executions SET status='unknown', finished_at=?, error=? - WHERE id=? AND status IN ('claimed','running')""", + """UPDATE executions + SET status='unknown', finished_at=?, error=?, + handoff_pending=0, handoff_started_at=NULL + WHERE id=? AND status=? AND process_id=? AND pid=? + AND handoff_pending=? + AND handoff_started_at IS ?""", (now, "Scheduler restarted after this execution's owner exited before a durable " "terminal state; whether side effects ran is unknown.", - row["id"]), + row["id"], row["status"], row["process_id"], row["pid"], + row["handoff_pending"], row["handoff_started_at"]), ) changed += cur.rowcount if cur.rowcount: @@ -263,6 +344,16 @@ def list_executions( return [dict(row) for row in rows] +def get_execution(execution_id: str) -> Optional[Dict[str, Any]]: + """Return one exact execution attempt, or ``None`` when it is absent.""" + with _transaction() as conn: + row = conn.execute( + "SELECT * FROM executions WHERE id=?", + (str(execution_id),), + ).fetchone() + return dict(row) if row is not None else None + + def latest_execution(job_id: str) -> Optional[Dict[str, Any]]: rows = list_executions(job_id=job_id, limit=1) return rows[0] if rows else None diff --git a/cron/jobs.py b/cron/jobs.py index 468fbffbfa..ed2153c60e 100644 --- a/cron/jobs.py +++ b/cron/jobs.py @@ -1248,7 +1248,8 @@ def _classify_dispatch_lateness(lateness_seconds: float, grace_seconds: int) -> # ``next_run_at`` to now so the next tick re-dispatches it, exactly like the # operator's force-run / mech_red_guard's ``cron resume`` but built-in. _persisted_error_recoveries: int = 0 -_PERSISTED_ERROR_RECOVERY_HISTORY = 20 +# Bounded in-memory history kept by every probe-visible fire-path counter. +_TELEMETRY_RECENT_HISTORY = 20 _persisted_error_recoveries_recent: list = [] @@ -1351,26 +1352,38 @@ def _schedule_cadence_seconds(schedule: Dict[str, Any]) -> Optional[float]: _cron_cadence_cache: Dict[str, Optional[float]] = {} +def _append_telemetry_record(filename: str, entry: Dict[str, Any], recent: list) -> None: + """Keep ``entry`` in the bounded in-memory ``recent`` list and append it to + ``/`` (best effort — telemetry must never break a tick). + + Shared by the probe-visible fire-path counters (persisted-error recovery, + timezone-migration catch-up); each keeps its own module-level int counter + because tests reset those by name. + """ + recent.append(entry) + del recent[:-_TELEMETRY_RECENT_HISTORY] + try: + path = _current_cron_store().cron_dir / filename + _ensure_cron_dir(path.parent) + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(entry) + "\n") + except Exception as exc: + logger.debug("Could not append %s record: %s", filename, exc) + + def _record_persisted_error_recovery(job: Dict[str, Any], previous_next_run: str) -> None: """Persist a countable, probe-visible signal for one stale-error re-arm.""" global _persisted_error_recoveries - now = _hermes_now() entry = { "job_id": job.get("id"), "name": job.get("name") or job.get("id"), "previous_next_run_at": previous_next_run, - "rearmed_at": now.isoformat(), + "rearmed_at": _hermes_now().isoformat(), } _persisted_error_recoveries += 1 - _persisted_error_recoveries_recent.append(entry) - del _persisted_error_recoveries_recent[:-_PERSISTED_ERROR_RECOVERY_HISTORY] - try: - path = _current_cron_store().cron_dir / "persisted_error_recoveries.jsonl" - _ensure_cron_dir(path.parent) - with open(path, "a", encoding="utf-8") as fh: - fh.write(json.dumps(entry) + "\n") - except Exception as exc: # never let telemetry break a tick - logger.debug("Could not append persisted-error-recovery record: %s", exc) + _append_telemetry_record( + "persisted_error_recoveries.jsonl", entry, _persisted_error_recoveries_recent + ) def get_persisted_error_recovery_stats() -> Dict[str, Any]: @@ -1410,6 +1423,95 @@ def _cron_next_run_matches_expr( return True +# Classification results for a due cron instant that is NOT an occurrence of +# the job's current expression (see _classify_stale_cron_next_run). +STALE_CRON_MATCH = "match" +STALE_CRON_TIMEZONE_MIGRATION = "timezone_migration" +STALE_CRON_EXPR_EDIT = "expr_edit" + + +def _classify_stale_cron_next_run( + schedule: Dict[str, Any], + raw_next_run_dt: datetime, + next_run_dt: datetime, +) -> str: + """Explain WHY a stored ``next_run_at`` misses the current cron lattice. + + ``_cron_next_run_matches_expr`` answers "does the stored instant occur in + the current expression?" but not "why not?", and the two answers call for + opposite actions: + + * ``expr_edit`` — a direct ``jobs.json`` edit changed ``schedule.expr`` + while leaving ``next_run_at`` computed under the old one (#93049). The + stored instant is a time the current expression *excludes*, so it must + be re-anchored WITHOUT firing. + * ``timezone_migration`` — the expression never changed; only the stored + value's *offset representation* did. Upgrading from a UTC-scheduling + build to one that honours the profile timezone leaves legacy rows like + ``2026-09-02T04:00:00+00:00`` for ``0 4 * * *``; normalizing to + Europe/Brussels turns that into ``06:00+02``, which the expression + excludes. Treating it as a stale edit re-anchored to tomorrow and + silently skipped a due occurrence that had never fired. + + The discriminator is whether *normalization itself* moved the wall clock. + Cron expressions describe local wall-clock intent, so a stored instant + whose OWN wall clock is a legal occurrence, and which only left the + lattice because ``_ensure_aware`` converted it into a different offset, is + a representation migration — not a schedule edit. When the offsets agree + (the common case, including every value this build wrote) the wall clock + is unchanged, so a genuine ``expr`` edit can never be misread as a + migration. + """ + if _cron_next_run_matches_expr(schedule, next_run_dt): + return STALE_CRON_MATCH + wall_clock_shifted = ( + raw_next_run_dt.replace(tzinfo=None) != next_run_dt.replace(tzinfo=None) + ) + if wall_clock_shifted and _cron_next_run_matches_expr(schedule, raw_next_run_dt): + return STALE_CRON_TIMEZONE_MIGRATION + return STALE_CRON_EXPR_EDIT + + +# Durable, probe-visible counter for offset-representation migrations caught +# on the fire path. Kept separate from `catch_up_occurrences` (runs skipped +# past their grace window — a migrated row that is ALSO past grace increments +# both) because this one means "an upgrade rewrote how next_run_at is +# represented" — an operator seeing it climb after a deploy is seeing the +# migration drain, and seeing it climb steadily afterwards is seeing a +# timezone that keeps changing under the store. +_timezone_migration_catchups: int = 0 +_timezone_migration_catchups_recent: list = [] + + +def _record_timezone_migration_catchup( + job: Dict[str, Any], + raw_next_run_dt: datetime, + next_run_dt: datetime, +) -> None: + """Persist a countable signal for one offset-migration catch-up fire.""" + global _timezone_migration_catchups + entry = { + "job_id": job.get("id"), + "name": job.get("name") or job.get("id"), + "expr": (job.get("schedule") or {}).get("expr"), + "stored_next_run_at": raw_next_run_dt.isoformat(), + "normalized_next_run_at": next_run_dt.isoformat(), + "fired_at": _hermes_now().isoformat(), + } + _timezone_migration_catchups += 1 + _append_telemetry_record( + "timezone_migration_catchups.jsonl", entry, _timezone_migration_catchups_recent + ) + + +def get_timezone_migration_catchup_stats() -> Dict[str, Any]: + """Probe-visible snapshot of offset-migration catch-up fires.""" + return { + "timezone_migration_catchups": _timezone_migration_catchups, + "recent": list(_timezone_migration_catchups_recent), + } + + def compute_next_run(schedule: Dict[str, Any], last_run_at: Optional[str] = None) -> Optional[str]: """ Compute the next run time for a schedule. @@ -2248,6 +2350,7 @@ def create_job( monitor_script: Optional[str] = None, monitor_url: Optional[str] = None, reasoning_effort: Optional[str] = None, + failure_deliver: Optional[str] = None, ) -> Dict[str, Any]: """ Create a new cron job. @@ -2350,6 +2453,18 @@ def create_job( normalized_no_agent = bool(no_agent) normalized_attach = attach_to_session if isinstance(attach_to_session, bool) else None normalized_reasoning_effort = _normalize_reasoning_effort(reasoning_effort) + # failure_deliver shares deliver's value grammar; the str/list + # flatten below mirrors the tool layer's _normalize_deliver_param for + # direct create_job callers (the tool pre-normalizes). Semantic + # validation happens at resolution time via the shared deliver path. + normalized_failure_deliver = ( + str(failure_deliver).strip() if isinstance(failure_deliver, str) else None + ) + if isinstance(failure_deliver, (list, tuple)): + normalized_failure_deliver = ",".join( + str(p).strip() for p in failure_deliver if str(p).strip() + ) + normalized_failure_deliver = normalized_failure_deliver or None normalized_monitor_script = str(monitor_script).strip() if isinstance(monitor_script, str) else None normalized_monitor_script = normalized_monitor_script or None normalized_monitor_url = str(monitor_url).strip() if isinstance(monitor_url, str) else None @@ -2450,6 +2565,9 @@ def create_job( "last_status": None, "last_error": None, "last_delivery_error": None, + # Live-adapter targets whose last send was acked with no message_id / + # raw_response (accepted, but UNVERIFIED — surfaced by cron list/doctor). + "last_delivery_unverified": None, "failure_streak": 0, # Delivery configuration "deliver": deliver, @@ -2466,6 +2584,10 @@ def create_job( # absent key = job follows config resolution (pre-feature behavior). if normalized_reasoning_effort is not None: job["reasoning_effort"] = normalized_reasoning_effort + # Conditional-persist for failure_deliver too: absent key = failures + # follow deliver, byte-identical to pre-feature jobs (NS-788). + if normalized_failure_deliver is not None: + job["failure_deliver"] = normalized_failure_deliver with _jobs_lock(): jobs = load_jobs() @@ -3033,7 +3155,13 @@ def _mark_job_run_locked( ``delivery_error`` is tracked separately from the agent error — a job can succeed (agent produced output) but fail delivery (platform down). - ``status`` overrides the derived ``last_status`` ("ok"/"error") with a + A run that succeeded but failed delivery records + ``last_status = "delivery_failed"`` (never "ok") so the failure is + visible to every reader, while ``failure_streak`` stays untouched — + the agent did its job. + + ``status`` overrides the derived ``last_status`` ("ok"/"error"/ + "delivery_failed") with a specific terminal status for this run — e.g. ``"blocked_config"`` when the pre-dispatch configuration validation refused to run the agent (T1-26), so `cronjob list` distinguishes "your config is broken" from @@ -3058,7 +3186,21 @@ def _mark_job_run_locked( # The transient manual-run context is single-fire: whatever # run just completed consumed it (or superseded it). job.pop("manual_run_prompt", None) - job["last_status"] = status or ("ok" if success else "error") + # A run whose agent succeeded but whose delivery failed is NOT + # "ok": recording it as such hid last_delivery_error behind a + # green status in `cron list`/the UI and made a job that never + # reached the user look like a quiet success (#83993). It gets + # its own status so every reader that keys off "ok" (CLI list, + # doctor, cronjob_tools) sees the failure. An explicit + # ``status`` override (e.g. "blocked_config") still wins. + if status: + job["last_status"] = status + elif not success: + job["last_status"] = "error" + elif isinstance(delivery_error, str) and delivery_error.strip(): + job["last_status"] = "delivery_failed" + else: + job["last_status"] = "ok" job["last_error"] = error if not success else None # A healthy run means the configuration validates again — drop # the preflight alert-dedup marker so a FUTURE config break @@ -4023,9 +4165,18 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: # so re-anchor before either can fire. Recomputation uses the # current expression, so this converges — it cannot defer # forever. - if not manual_run and kind == "cron" and not _cron_next_run_matches_expr( - schedule, next_run_dt - ): + # + # Not every mismatch is an edit, though: an offset-representation + # migration (UTC-scheduling build -> profile-timezone build) + # moves a legacy instant off the lattice without the expression + # ever changing, and re-anchoring THAT silently swallowed a due + # occurrence. Classify first, and only the edit case skips. + stale_class = ( + _classify_stale_cron_next_run(schedule, raw_next_run_dt, next_run_dt) + if not manual_run and kind == "cron" + else STALE_CRON_MATCH + ) + if stale_class == STALE_CRON_EXPR_EDIT: new_next = compute_next_run(schedule, now.isoformat()) logger.info( "Job '%s' next_run_at %s does not match its current " @@ -4043,6 +4194,27 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]: needs_save = True break continue + if stale_class == STALE_CRON_TIMEZONE_MIGRATION: + # Fall through to the normal due path: the occurrence is + # real and overdue, so it fires ONCE here and the usual + # advance/mark_job_run re-anchor writes the value back in + # the current offset. At-most-once is preserved because + # nothing re-reads the legacy instant after that. + logger.warning( + "cron.timezone_migration.catch_up job='%s' id=%s expr=%r " + "stored=%s normalized=%s — stored next_run_at carries a " + "pre-migration UTC offset (%s, now %s) and is a legal " + "occurrence at its own wall clock; firing the due run " + "instead of re-anchoring past it.", + job.get("name", job.get("id", "?")), + job.get("id"), + schedule.get("expr"), + next_run, + next_run_dt.isoformat(), + raw_next_run_dt.utcoffset(), + now.utcoffset(), + ) + _record_timezone_migration_catchup(job, raw_next_run_dt, next_run_dt) # For recurring jobs, check if the scheduled time is stale # (gateway was down and missed the window). Fast-forward to diff --git a/cron/lifecycle_guard.py b/cron/lifecycle_guard.py index fb4c18e91c..ca05319f73 100644 --- a/cron/lifecycle_guard.py +++ b/cron/lifecycle_guard.py @@ -291,9 +291,107 @@ _SHELL_EXECUTABLES = frozenset({"sh", "bash", "dash", "ksh", "zsh"}) _SHELL_OPTIONS_WITH_VALUES = frozenset({"-O", "+O", "-o", "+o"}) _MAX_REFERENCED_SCRIPT_BYTES = 1024 * 1024 _MAX_REFERENCED_SCRIPT_DEPTH = 8 +# Whole-walk work limits (#78398). The per-file cap and depth bound above +# limit one read, not the walk: a command can reference arbitrarily many +# scripts, and the pure-Python shlex pass (one lexer per line, quadratic on a +# giant token) once held the GIL for minutes on a broad command. These caps +# bound one whole walk and are charged BEFORE any text reaches shlex. +# Exhaustion fails closed (an unscanned script could hide a lifecycle command) +# and is logged at WARNING so an operator can tell it from a real block. Sizes +# sit well above any legitimate wrapper graph (a 200-script wrapper is ~5 KB) +# while keeping the worst text still admitted to a few seconds of lexing; +# remote reads are a backend roundtrip each, so they get a far tighter cap. +_MAX_LIFECYCLE_SCAN_BYTES = _MAX_REFERENCED_SCRIPT_BYTES # 1 MiB across the walk +_MAX_LIFECYCLE_SCAN_LINES = 16384 +_MAX_LIFECYCLE_SCAN_LINE_BYTES = 64 * 1024 +_MAX_LIFECYCLE_SCAN_PATHS = 1024 +_MAX_LIFECYCLE_SCAN_REMOTE_READS = 64 _CONTROL_CHARS = frozenset(";&|()") +class _LifecycleScanBudget: + """Shared work budget for one complete referenced-script walk.""" + + __slots__ = ( + "bytes_remaining", + "lines_remaining", + "paths_remaining", + "remote_reads_remaining", + ) + + def __init__(self) -> None: + # Read the module constants at construction so tests (and operators) + # can lower them without defaults capturing stale values at import. + self.bytes_remaining = _MAX_LIFECYCLE_SCAN_BYTES + self.lines_remaining = _MAX_LIFECYCLE_SCAN_LINES + self.paths_remaining = _MAX_LIFECYCLE_SCAN_PATHS + self.remote_reads_remaining = _MAX_LIFECYCLE_SCAN_REMOTE_READS + + def charge_text(self, text: str) -> bool: + """Charge *text* before tokenization; False when it does not fit.""" + # UTF-8 is at least one byte per code point, so the character count + # is a free lower bound — skip the encode for obviously-oversized input. + if len(text) > self.bytes_remaining: + return False + encoded = len(text.encode("utf-8", errors="replace")) + if encoded > self.bytes_remaining: + return False + lines = text.count("\n") + 1 + if lines > self.lines_remaining: + return False + # One huge token is the quadratic shlex case; bound the longest + # physical line. Measured in characters (a lower bound on bytes) — + # tight enough for a DoS bound without a per-line encode. + longest = max((len(line) for line in text.split("\n")), default=0) + if longest > _MAX_LIFECYCLE_SCAN_LINE_BYTES: + return False + self.bytes_remaining -= encoded + self.lines_remaining -= lines + return True + + def charge_path(self) -> bool: + """Charge one unique referenced path before any local/remote read.""" + if self.paths_remaining <= 0: + return False + self.paths_remaining -= 1 + return True + + def charge_remote_read(self) -> bool: + """Charge one remote-backend read (a network roundtrip each).""" + if self.remote_reads_remaining <= 0: + return False + self.remote_reads_remaining -= 1 + return True + + +def _capped_read_limit(max_bytes: Optional[int]) -> int: + """Per-read byte cap: never above the per-file cap, never negative. + + One definition so local and remote reads cannot diverge again (#76762, + #77703 were exactly that class of bug). + """ + if max_bytes is None: + return _MAX_REFERENCED_SCRIPT_BYTES + return min(_MAX_REFERENCED_SCRIPT_BYTES, max(0, int(max_bytes))) + + +def lifecycle_scan_root_within_budget(text: str) -> bool: + """Whether *text* may safely enter an optional tokenizer pass. + + Used by ``tools/terminal_tool.py`` to gate its launchctl-specific pre-scan + (which tokenizes with shlex). This is a FRESH budget, independent of the + one the full guard builds for its own walk: the pre-scan may pass while + the guard's walk later exhausts, and the outcome is still fail-closed — + only the friendlier launchctl diagnostic is lost. ``False`` is not a + verdict: callers must still run the full guard, which fails closed for + an over-budget root. + """ + try: + return _LifecycleScanBudget().charge_text(text) + except Exception: + return False + + # Directory names that sit directly under a `Library` path component and # mark a FileProvider-backed subtree: `Mobile Documents` is iCloud Drive; # `CloudStorage` hosts every third-party FileProvider domain (Dropbox, @@ -909,9 +1007,14 @@ def _has_binary_magic(data: bytes) -> bool: return data.startswith(_BINARY_MAGICS) -def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]: +def _read_referenced_script( + path: Path, *, max_bytes: Optional[int] = None +) -> tuple[Optional[str], bool]: """Return ``(text, unsafe)`` using bounded, regular-file-only reads. + ``max_bytes`` lowers the per-file cap to what the calling walk can still + afford (never raises it above ``_MAX_REFERENCED_SCRIPT_BYTES``). + This is the shared choke point for every local script read the guard performs (the terminal walk in ``_contains_unsafe_gateway_action`` AND the cron-script scan in ``_read_script_for_scanning``), so the @@ -921,6 +1024,7 @@ def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]: (#88052). The lexical check covers direct cloud paths; the resolved check covers local launchers that are symlinks into a cloud subtree. """ + byte_limit = _capped_read_limit(max_bytes) if _is_cloud_placeholder_path(path): return None, True try: @@ -964,12 +1068,14 @@ def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]: data = os.read(descriptor, _BINARY_SNIFF_BYTES) if data.startswith(_BINARY_MAGIC_PREFIXES): return None, False + # A regular file whose size already exceeds the cap fails closed + # without reading it (the walk budget can be far below 1 MiB). + if metadata.st_size > byte_limit: + return None, True # Read the remainder (bounded). Loop because os.read may return # short for non-regular-file-backed descriptors. - while len(data) <= _MAX_REFERENCED_SCRIPT_BYTES: - chunk = os.read( - descriptor, _MAX_REFERENCED_SCRIPT_BYTES + 1 - len(data) - ) + while len(data) <= byte_limit: + chunk = os.read(descriptor, byte_limit + 1 - len(data)) if not chunk: break data += chunk @@ -992,14 +1098,16 @@ def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]: # Check the size BEFORE stripping: stripping shrinks the buffer, so doing it # first would let an oversized file slip under the threshold and skip this # fail-closed branch. - if len(data) > _MAX_REFERENCED_SCRIPT_BYTES: + if len(data) > byte_limit: return None, True if b"\x00" in data: data = data.replace(b"\x00", b"") return data.decode("utf-8", errors="replace"), False -def _sanitize_remote_script_text(text: Optional[str]) -> tuple[Optional[str], bool]: +def _sanitize_remote_script_text( + text: Optional[str], *, max_bytes: Optional[int] = None +) -> tuple[Optional[str], bool]: """Apply the local-read contract to text from a ``read_remote_script`` callback. The recursion boundary must not trust its callbacks: any backend (SSH, @@ -1020,19 +1128,37 @@ def _sanitize_remote_script_text(text: Optional[str]) -> tuple[Optional[str], bo return None, False if "\x00" in text: return None, False - if len(text.encode("utf-8", errors="replace")) > _MAX_REFERENCED_SCRIPT_BYTES: + byte_limit = _capped_read_limit(max_bytes) + if len(text) > byte_limit: + return None, True # chars <= bytes: over the cap without encoding + if len(text.encode("utf-8", errors="replace")) > byte_limit: return None, True return text, False +def _budget_exhausted(what: str, depth: int) -> bool: + logger.warning( + "lifecycle guard scan budget exhausted (%s at depth %d); " + "failing closed — see _MAX_LIFECYCLE_SCAN_* in cron/lifecycle_guard.py", + what, depth, + ) + return True + + def _contains_unsafe_gateway_action( command: str, *, cwd: Optional[str], depth: int, visited: set[Path], + budget: _LifecycleScanBudget, read_remote_script: Optional[_ReadRemoteScriptFn] = None, ) -> bool: + # Charge BEFORE _direct_lifecycle_scan: every scan in it (including the + # wrapper-prefix lifecycle detector) tokenizes with shlex, so checking + # afterwards would keep the CPU spike. + if not budget.charge_text(command): + return _budget_exhausted("text", depth) if _direct_lifecycle_scan(command): return True if depth >= _MAX_REFERENCED_SCRIPT_DEPTH: @@ -1044,6 +1170,7 @@ def _contains_unsafe_gateway_action( cwd=cwd, depth=depth + 1, visited=visited, + budget=budget, read_remote_script=read_remote_script, ): return True @@ -1068,17 +1195,26 @@ def _contains_unsafe_gateway_action( return True if resolved in visited: continue + if not budget.charge_path(): + return _budget_exhausted("paths", depth) visited.add(resolved) - script_text, unsafe = _read_referenced_script(script_path) + # Never read more than the walk can still afford to tokenize; a file + # larger than the remainder fails closed exactly like an oversized one. + script_text, unsafe = _read_referenced_script( + script_path, max_bytes=budget.bytes_remaining + ) if unsafe: return True if script_text is None and read_remote_script is not None: # Local path missing; try the remote backend if one is available. + if not budget.charge_remote_read(): + return _budget_exhausted("remote reads", depth) # The callback's output crosses the same trust boundary as a # local read — sanitize it identically before it enters the # recursion (binary skip + size fail-closed). script_text, unsafe = _sanitize_remote_script_text( - read_remote_script(str(script_path)) + read_remote_script(str(script_path)), + max_bytes=budget.bytes_remaining, ) if unsafe: return True @@ -1092,6 +1228,7 @@ def _contains_unsafe_gateway_action( cwd=script_dir, depth=depth + 1, visited=visited, + budget=budget, read_remote_script=read_remote_script, ): return True @@ -1126,6 +1263,7 @@ def contains_gateway_lifecycle_command_or_referenced_script( cwd=cwd, depth=0, visited=set(), + budget=_LifecycleScanBudget(), read_remote_script=read_remote_script, ) except Exception: @@ -1254,7 +1392,12 @@ def check_gateway_lifecycle( # `hermes gateway restart` embedded in a .py script is still # blocked. Non-regular/oversized script files still fail closed # via the lifecycle-shaped sentinel in _read_script_for_scanning. - unsafe = _lifecycle_command_scan_with_data_exemption(combined) + # The data-exemption masker tokenizes the text with shlex, so it is + # charged against the same walk budget as the shell path (#78398). + if not _LifecycleScanBudget().charge_text(combined): + unsafe = _budget_exhausted("text", 0) + else: + unsafe = _lifecycle_command_scan_with_data_exemption(combined) else: script_dir = _resolve_script_directory(script) if script else None unsafe = contains_gateway_lifecycle_command_or_referenced_script( diff --git a/cron/notepad.py b/cron/notepad.py index 339e9a8823..450b951661 100644 --- a/cron/notepad.py +++ b/cron/notepad.py @@ -26,23 +26,34 @@ from __future__ import annotations import sqlite3 import threading from contextlib import contextmanager +from pathlib import Path from typing import Any, Dict, Iterator, List, Optional from hermes_constants import get_hermes_home from hermes_time import now as _hermes_now -NOTEPAD_FILE = get_hermes_home().resolve() / "cron" / "notepad.db" +# Optional test override. Production resolves the path at transaction time so +# multiplexed profile ticks (set_hermes_home_override) cannot leak one +# profile's notepad rows into the import-time home — and remove_job's +# clear_notepad cannot wipe the wrong profile's DB (#86519). Same pattern as +# cron/executions.py. +NOTEPAD_FILE: Optional[Path] = None MAX_VALUE_BYTES = 16 * 1024 MAX_KEY_CHARS = 128 MAX_JOB_TOTAL_BYTES = 64 * 1024 _lock = threading.RLock() +def _current_notepad_file() -> Path: + return NOTEPAD_FILE or (get_hermes_home().resolve() / "cron" / "notepad.db") + + def _connect() -> sqlite3.Connection: from cron.jobs import _ensure_cron_dir - _ensure_cron_dir(NOTEPAD_FILE.parent) - return sqlite3.connect(NOTEPAD_FILE, timeout=5) + path = _current_notepad_file() + _ensure_cron_dir(path.parent) + return sqlite3.connect(path, timeout=5) def _initialize_schema(conn: sqlite3.Connection) -> None: @@ -157,7 +168,7 @@ def clear_notepad(job_id: str) -> int: Called from ``cron.jobs.remove_job`` so deleted jobs don't orphan their rows. No-ops without creating the DB when no notepad file exists yet. """ - if not NOTEPAD_FILE.exists(): + if not _current_notepad_file().exists(): return 0 with _transaction() as conn: cur = conn.execute( diff --git a/cron/scheduler.py b/cron/scheduler.py index 2abf92032a..3c4da065f5 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -725,7 +725,15 @@ from cron.jobs import ( save_job_output, use_cron_store, ) -from cron.executions import create_execution, finish_execution, mark_execution_running +from cron.executions import ( + _TERMINAL_STATES, + create_execution, + finish_execution, + get_execution, + mark_execution_handoff_pending, + mark_execution_running, + recover_interrupted_executions, +) # Sentinel: when a cron agent has nothing new to report, it can start its # response with this marker to suppress delivery. Output is still saved @@ -769,6 +777,10 @@ _parallel_pool: Optional[concurrent.futures.ThreadPoolExecutor] = None _parallel_pool_max_workers: Optional[int] = None _running_job_ids: set = set() _running_fire_owners: dict[str, dict[object, tuple[Optional[str], Path]]] = {} +# Parent gateway threads synchronously waiting on restart-safe scope workers. +# Shutdown must not misclassify these as ownerless in-process runs: the tool +# process sweep cannot reach the worker's transient scope. +_restart_safe_waiter_job_ids: set[str] = set() _running_lock = threading.Lock() # Wall-clock (time.time()) instant each in-flight job id was claimed by @@ -1287,9 +1299,11 @@ def mark_running_jobs_interrupted( Returns the list of job IDs marked, for the caller to log. """ with _running_lock: + restart_safe_waiters = set(_restart_safe_waiter_job_ids) active_fires = [ (token, job_id, owner, profile_home) for job_id, executions in _running_fire_owners.items() + if job_id not in restart_safe_waiters for token, (owner, profile_home) in executions.items() ] if only_owners is not None: @@ -1301,7 +1315,9 @@ def mark_running_jobs_interrupted( if only_owners is None: active_fires.extend( (None, job_id, None, _get_hermes_home()) - for job_id in _running_job_ids - registered_ids + for job_id in ( + _running_job_ids - registered_ids - restart_safe_waiters + ) ) _interrupted_job_ids.update( token if token is not None else job_id @@ -2838,7 +2854,20 @@ def _expand_routing_tokens(part: str) -> List[str]: return expanded -def _resolve_delivery_targets(job: dict) -> List[dict]: +def _delivery_lane_value(job: dict, *, for_failure: bool = False): + """Raw deliver-lane value for a run outcome: the failure lane when + ``for_failure`` and the job overrides it, else ``deliver``. Keeps + delivery bookkeeping (outcome classification, unresolved-origin, + incident 'alerted' marking) reading the SAME lane the notice was + actually routed through (NS-788 review finding B1).""" + if for_failure: + failure_deliver = job.get("failure_deliver") + if failure_deliver is not None and str(failure_deliver).strip(): + return failure_deliver + return job.get("deliver", "local") + + +def _resolve_delivery_targets(job: dict, *, for_failure: bool = False) -> List[dict]: """Resolve all concrete auto-delivery targets for a cron job. Accepts the legacy comma-separated ``deliver`` string plus the @@ -2847,8 +2876,17 @@ def _resolve_delivery_targets(job: dict) -> List[dict]: targets: ``origin,all`` and ``all,telegram:-100:17`` both work. Duplicate (platform, chat_id, thread_id) tuples are collapsed by the existing dedup pass. + + ``for_failure=True`` resolves failure-category engine notices + (failure summaries, interrupted-run notices, drift/preflight + alerts): when the job carries a ``failure_deliver`` value, targets + resolve from it INSTEAD of ``deliver`` — ``failure_deliver: local`` + is the structural opt-out for shared channels (NS-788, Coatue). + Absent ``failure_deliver``, failure delivery follows ``deliver`` + exactly as before. """ - deliver = _normalize_deliver_value(job.get("deliver", "local")) + deliver_raw = _delivery_lane_value(job, for_failure=for_failure) + deliver = _normalize_deliver_value(deliver_raw) if deliver == "local": return [] @@ -2984,7 +3022,7 @@ def _send_media_via_adapter( return errors -def _confirm_adapter_delivery(send_result) -> bool: +def _confirm_adapter_delivery(send_result, job_id: str = "?", unverified: Optional[list] = None) -> bool: """Return True only if ``send_result`` unambiguously confirms delivery. A live adapter that returns ``None`` (e.g. a swallowed exception, a busy @@ -2993,16 +3031,54 @@ def _confirm_adapter_delivery(send_result) -> bool: scheduler to log ``"delivered to via live adapter"`` while the gateway never actually sees the message (#47056). - Likewise, an object missing a ``success`` attribute (e.g. a bare ``dict`` - or a partial mock) is a contract violation: it does not actually tell us - whether the send succeeded. Require an explicit, truthy ``success`` - attribute to count as confirmed. + Likewise, a result carrying no ``success`` at all (a partial mock, or a + ``dict`` from a code path that never reached the adapter) is a contract + violation: it does not actually tell us whether the send succeeded. + Require an explicit, truthy ``success`` to count as confirmed. + + Both shapes are inspected the same way, because ``_deliver_to_platform`` + returns either a ``SendResult`` object or a plain ``dict``: + + * ``delivered is False`` is a REJECTION even when ``success`` is truthy. + The silence-narration filter returns + ``{"success": True, "delivered": False}`` — a successfully *dropped* + message, not a delivered one. Reading only ``success`` there is how a + cron brief was logged as delivered while the user got nothing (#77763). + * No ``message_id`` and no ``raw_response`` means we have no positive + evidence of a send. That is not proof of failure either (some adapters + legitimately return a bare success), so it is still accepted — but + logged at WARNING so an UNVERIFIED delivery is visible in the log + instead of masquerading as a confirmed one. Telegram ``SendResult`` + objects carry ``message_id``; the dict-filter shape does not. """ if send_result is None: return False - if not hasattr(send_result, "success"): + if isinstance(send_result, dict): + if "success" not in send_result: + return False + success = bool(send_result.get("success")) + delivered = send_result.get("delivered") + message_id = send_result.get("message_id") + raw_response = send_result.get("raw_response") + else: + if not hasattr(send_result, "success"): + return False + success = bool(getattr(send_result, "success")) + delivered = getattr(send_result, "delivered", None) + message_id = getattr(send_result, "message_id", None) + raw_response = getattr(send_result, "raw_response", None) + if not success or delivered is False: return False - return bool(getattr(send_result, "success")) + if message_id is None and not raw_response: + logger.warning( + "Job '%s': live adapter reported success with no delivery evidence " + "(no message_id, no raw_response) — treating as delivered but " + "UNVERIFIED", + job_id, + ) + if unverified is not None: + unverified.append(True) + return True def _is_channel_dm_topic( @@ -3061,7 +3137,51 @@ def _is_channel_dm_topic( return is_channel -def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Optional[str]: +def _cron_delivery_notify_enabled(cfg: Optional[dict]) -> bool: + """Resolve ``cron.delivery.notify`` (config.yaml). Default True. + + Only an explicit boolean ``False`` (or a YAML ``false``/``off`` that parses + to it) disables the push notification; a missing/malformed section keeps + the default so a typo can never silently make cron briefs silent. + """ + try: + cron_cfg = (cfg or {}).get("cron") + if not isinstance(cron_cfg, dict): + return True + delivery_cfg = cron_cfg.get("delivery") + if not isinstance(delivery_cfg, dict): + return True + return delivery_cfg.get("notify", True) is not False + except Exception: + return True + + +def _record_delivery_verification(job: dict, unverified_targets: list) -> None: + """Persist the UNVERIFIED-delivery marker on the job record. + + ``last_delivery_unverified`` is a list of ``platform:chat_id`` targets + whose live adapter acked the send with no message_id/raw_response, or + ``None`` once a run delivered with positive evidence (or to no live + target). Skips the write when nothing changed so the common verified + path costs no jobs.json save. Never raises — status bookkeeping must not + fail a delivery. + """ + new_value = list(unverified_targets) or None + if (job.get("last_delivery_unverified") or None) == new_value: + return + try: + from cron.jobs import update_job + + update_job(job["id"], {"last_delivery_unverified": new_value}) + except Exception as exc: # pragma: no cover - defensive + logger.debug( + "Job '%s': could not record delivery verification: %s", job.get("id"), exc, + ) + + +def _deliver_result( + job: dict, content: str, adapters=None, loop=None, *, for_failure: bool = False +) -> Optional[str]: """ Deliver job output to the configured target(s) (origin chat, specific platform, etc.). @@ -3070,11 +3190,16 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option the standalone HTTP path cannot encrypt. Falls back to standalone send if the adapter path fails or is unavailable. + ``for_failure=True`` routes failure-category engine notices through the + job's ``failure_deliver`` override when present (NS-788). + Returns None on success, or an error string on failure. """ - targets = _resolve_delivery_targets(job) + targets = _resolve_delivery_targets(job, for_failure=for_failure) if not targets: - deliver_value = _normalize_deliver_value(job.get("deliver", "local")) + deliver_value = _normalize_deliver_value( + _delivery_lane_value(job, for_failure=for_failure) + ) if deliver_value == "local": return None # local-only jobs don't deliver — not a failure # deliver=origin with no resolvable origin and no configured home @@ -3094,6 +3219,28 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option logger.warning("Job '%s': %s", job["id"], msg) return msg + # Restart-safe workers intentionally have no live gateway adapter objects. + # Hand the send back through a durable queue so the current or replacement + # gateway performs it with relay/E2EE parity. The execution id is the + # idempotency key; the queue never retries an uncertain claimed send. + # Match on this job's own attempt: a worker's script may itself dispatch + # another job in-process (``hermes cron run``), and that nested delivery + # must not be keyed under the outer execution id. + external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER", "") + if ( + external_execution + and adapters is None + and external_execution == str(job.get("execution_id") or "") + ): + from cron.delivery_queue import enqueue_and_wait + + return enqueue_and_wait( + external_execution, + job, + content, + for_failure=for_failure, + ) + from tools.send_message_tool import _send_to_platform from gateway.config import load_gateway_config, Platform @@ -3108,6 +3255,18 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option except Exception: pass + # cron.delivery.notify (default True): mark live-adapter cron sends as + # FINAL notifications so the platform pushes them (Telegram's "important" + # mode otherwise sends with disable_notification=True). Configurable so + # operators who prefer silent briefs can opt back out. + notify_delivery = _cron_delivery_notify_enabled(user_cfg) + # Set when a live adapter acked a send with NO delivery evidence (no + # message_id / raw_response — the Slack/Matrix/Mattermost bare + # SendResult(success=True) shape). Persisted on the job as + # ``last_delivery_unverified`` so `hermes cron list` shows the state + # instead of it living only in a WARNING log line. + unverified_targets: list = [] + if wrap_response: task_name = job.get("name", job["id"]) job_id = job.get("id", "") @@ -3261,7 +3420,14 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option from gateway.delivery import resolve_delivery_transport - transport = resolve_delivery_transport(platform, config, adapters) + target_adapters = adapters + if isinstance(adapters, SharedRouteAdapters): + # Credentialless satellite: the primary adapter is a valid + # transport for THIS target only when an exact primary route maps + # it to this profile (#101113). Miss → fail closed below. + shared = adapters.get(platform, target) + target_adapters = {platform: shared} if shared is not None else {} + transport = resolve_delivery_transport(platform, config, target_adapters) if transport is not None: pconfig = transport.config runtime_adapter = transport.adapter @@ -3483,10 +3649,14 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option route_metadata = { "direct_messages_topic_id": str(thread_id), "job_id": job["id"], + "notify": notify_delivery, } # Media metadata mirrors the text routing so attachments land in # the same DM topic instead of the General lane (#22773). - media_metadata = {"direct_messages_topic_id": str(thread_id)} + media_metadata = { + "direct_messages_topic_id": str(thread_id), + "notify": notify_delivery, + } else: # Forum-style topic (private chat / supergroup) or non-topic # target: route via message_thread_id (#52060). Put thread_id in @@ -3497,10 +3667,12 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option # anchor, so the metadata key bypasses that check and lets the # adapter route via a plain message_thread_id. route_thread_id = str(thread_id) if thread_id is not None else None - route_metadata = {"job_id": job["id"]} + route_metadata = {"job_id": job["id"], "notify": notify_delivery} if route_thread_id: route_metadata["thread_id"] = route_thread_id - media_metadata = {"thread_id": thread_id} if thread_id else None + media_metadata = {"notify": notify_delivery} + if thread_id: + media_metadata["thread_id"] = thread_id # Relay egress needs a tenant discriminator on the frame: the # connector's fail-closed guard resolves the workspace/guild from @@ -3530,10 +3702,23 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option adapter_ok = True timed_out = False delivered_message_id = None - if text_to_send: + if not text_to_send and not media_files: + # Nothing to hand the adapter at all. This used to fall + # straight through to the `if adapter_ok:` branch below and + # log "delivered to via live adapter" for a send that + # never happened (#77763). Fail closed so the run reports + # the empty payload instead. + msg = ( + f"live adapter send skipped (empty text and no media) " + f"for {platform_name}:{chat_id}" + ) + logger.warning("Job '%s': %s", job["id"], msg) + target_errors.append(msg) + adapter_ok = False + elif text_to_send: from agent.async_utils import safe_schedule_threadsafe - router = DeliveryRouter(config, adapters) + router = DeliveryRouter(config, target_adapters) route_target = DeliveryTarget( platform=platform, chat_id=str(chat_id), @@ -3623,19 +3808,32 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option # {"success": True, "delivered": False, ...}. # Normalize both shapes so a getattr default doesn't # misread a dict, and so a None / success-less object - # is NOT counted as delivered (#47056). + # is NOT counted as delivered (#47056). The + # confirmation itself handles both shapes: a truthy + # `success` with `delivered: False` is a drop, not a + # delivery (#77763). if isinstance(send_result, dict): - send_success = bool(send_result.get("success", False)) send_raw_response = send_result.get("raw_response") delivered_message_id = send_result.get("message_id") else: - send_success = _confirm_adapter_delivery(send_result) send_raw_response = getattr(send_result, "raw_response", None) delivered_message_id = getattr(send_result, "message_id", None) + _evidence_gap: list = [] + send_success = _confirm_adapter_delivery( + send_result, job["id"], _evidence_gap, + ) + if send_success and _evidence_gap: + unverified_targets.append(f"{platform_name}:{chat_id}") if not send_success: if isinstance(send_result, dict): - err = send_result.get("error", "unknown") + # A filtered drop carries no "error" — name + # the filter instead of reporting "unknown". + err = ( + send_result.get("error") + or send_result.get("filtered") + or "unknown" + ) shape = "dict" elif send_result is not None: err = getattr(send_result, "error", None) @@ -3712,7 +3910,16 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option delivery_errors.append(msg) if adapter_ok: - logger.info("Job '%s': delivered to %s:%s via live adapter", job["id"], platform_name, chat_id) + # Log WHERE it went, not just that it went: a ghost delivery + # that landed in the wrong lane (General topic instead of the + # routed thread) is indistinguishable from a real one without + # the routing identity (#77763). + logger.info( + "Job '%s': delivered to %s:%s via live adapter thread=%s message_id=%s", + job["id"], platform_name, chat_id, + route_thread_id if route_thread_id is not None else "-", + delivered_message_id if delivered_message_id is not None else "-", + ) delivered = True # Seed the thread session only now that delivery into it # succeeded (deferred from thread-open above). @@ -3826,6 +4033,22 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option target_errors.append(msg) delivery_errors.extend(target_errors) continue + # The live lane already failed closed on an empty payload; the + # standalone senders do not. The Telegram adapter returns + # SendResult(success=True) for empty content WITHOUT an API call, + # so falling through here turns a phantom live delivery into a + # phantom standalone one and logs it as delivered (#77763). Both + # _send_to_platform call sites below are reached through this + # point, so one guard closes the lane. + if not cleaned_delivery_content.strip() and not media_files: + msg = ( + f"standalone send skipped (empty text and no media) " + f"for {platform_name}:{chat_id}" + ) + logger.warning("Job '%s': %s", job["id"], msg) + target_errors.append(msg) + delivery_errors.extend(target_errors) + continue # Standalone path: run the async send in a fresh event loop (safe from any thread) coro = _send_to_platform(platform, pconfig, chat_id, cleaned_delivery_content, thread_id=thread_id, media_files=media_files) try: @@ -3856,7 +4079,19 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option try: pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) try: - future = pool.submit(asyncio.run, _send_to_platform(platform, pconfig, chat_id, cleaned_delivery_content, thread_id=thread_id, media_files=media_files)) + # The fallback worker is a fresh thread: it does NOT + # inherit the multiplexed profile ContextVars (home + # override + secret scope). Run inside a copy of the + # active context so the standalone sender reads THIS + # profile's bot token, not the process default's + # (#100489) — same pattern as the session-db and + # heartbeat workers in this module. + _fallback_context = contextvars.copy_context() + future = pool.submit( + _fallback_context.run, + asyncio.run, + _send_to_platform(platform, pconfig, chat_id, cleaned_delivery_content, thread_id=thread_id, media_files=media_files), + ) result = future.result(timeout=30) finally: pool.shutdown(wait=False) @@ -3915,11 +4150,32 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option if policy_drop_errors: # Filter-time drops apply to every target; report them once. delivery_errors.extend(policy_drop_errors) + _record_delivery_verification(job, unverified_targets) if delivery_errors: return "; ".join(delivery_errors) return None +def drain_delivery_queue(adapters, loop) -> int: + """Send queued worker results through this gateway's live adapters.""" + from cron.delivery_queue import _path, drain + + # Only restart-safe workers create the queue file. Every gateway (macOS, + # Windows, launchd, Docker) runs this housekeeping tick, so skip the sqlite + # open/create entirely until a worker has actually queued something. + if not _path().exists(): + return 0 + return drain( + lambda queued_job, queued_content, queued_for_failure: _deliver_result( + queued_job, + queued_content, + adapters=adapters, + loop=loop, + for_failure=queued_for_failure, + ) + ) + + _DEFAULT_SCRIPT_TIMEOUT = 3600 # seconds (1 hour) # Backward-compatible module override used by tests and emergency monkeypatches. _SCRIPT_TIMEOUT = _DEFAULT_SCRIPT_TIMEOUT @@ -5188,21 +5444,20 @@ def _preflight_check_provider_key(job: dict, cfg: dict) -> Optional[str]: return None -def _delivery_platform_routed_from_primary_gateway(platform_name: str) -> bool: - """True when the primary gateway routes this platform to the profile the - scheduler is currently serving. +def _primary_profile_routes_for_current_home() -> list: + """Primary gateway ``profile_routes`` that target the profile currently + being served, or ``[]`` (also when this IS the primary home). Under ``gateway.multiplex_profiles`` a satellite profile's cron jobs are ticked by the primary gateway's in-process ticker (#69377) and delivered through the primary gateway's live adapters — the satellite home never holds the platform credentials itself (giving it a token of its own is a - ``duplicate_credential`` fatal). ``_preflight_check_delivery`` loads the - gateway config of the job's OWN home, where such a platform correctly - reads as unconnected; consulting the primary home's ``profile_routes`` - keeps routed satellite jobs from being permanently false-blocked (#97476). - Reads the primary config.yaml directly (both the top-level and nested - ``gateway.`` forms) instead of ``load_gateway_config()`` so no primary - platform config leaks into this process's environment. + ``duplicate_credential`` fatal). Reads the primary config.yaml directly + (both the top-level and nested ``gateway.`` forms) instead of + ``load_gateway_config()`` so no primary platform config leaks into this + process's environment. Shared by the preflight rescue (#97476) and the + delivery-time shared-transport resolver (#101113) so route semantics + cannot drift between the two halves. """ try: from hermes_constants import get_default_hermes_root, get_hermes_home @@ -5213,10 +5468,10 @@ def _delivery_platform_routed_from_primary_gateway(platform_name: str) -> bool: primary_home.expanduser().resolve(strict=False) == current_home.expanduser().resolve(strict=False) ): - return False # this IS the primary home — nothing to consult + return [] # this IS the primary home — nothing to consult config_path = primary_home.expanduser() / "config.yaml" if not config_path.exists(): - return False + return [] import yaml @@ -5226,25 +5481,75 @@ def _delivery_platform_routed_from_primary_gateway(platform_name: str) -> bool: if routes_raw is None and isinstance(raw.get("gateway"), dict): routes_raw = raw["gateway"].get("profile_routes") if not isinstance(routes_raw, list): - return False + return [] from gateway.profile_routing import parse_profile_routes from hermes_cli.profiles import profile_matches_home - platform_key = platform_name.lower() - for route in parse_profile_routes(routes_raw): - if ( - route.enabled - and str(route.platform).lower() == platform_key - and profile_matches_home(route.profile) - ): - return True - return False + return [ + route + for route in parse_profile_routes(routes_raw) + if route.enabled and profile_matches_home(route.profile) + ] except Exception: logger.debug( - "preflight: primary-gateway profile-route lookup unavailable", + "primary-gateway profile-route lookup unavailable", exc_info=True, ) + return [] + + +def _delivery_platform_routed_from_primary_gateway(platform_name: str) -> bool: + """True when the primary gateway routes this platform to the profile the + scheduler is currently serving (preflight rescue, #97476).""" + platform_key = platform_name.lower() + return any( + str(route.platform).lower() == platform_key + for route in _primary_profile_routes_for_current_home() + ) + + +class SharedRouteAdapters: + """Read-only adapter map for a credentialless satellite profile (#101113). + + A satellite under ``gateway.profile_routes`` owns no bot credential and so + has no adapter map of its own; its inbound traffic arrives on the PRIMARY + adapter and is routed to it by an exact route. Its cron output must go + back out the same transport — but ONLY for targets an enabled primary + route maps to this profile. ``get(platform, target)`` resolves the primary + adapter iff the route matcher used by inbound routing + (``ProfileRoute.matches``) accepts the target's ``chat_id``/``thread_id``; + every other lookup is a miss, so an unmatched target, a disabled route, or + a route naming another profile still fails closed (never the default bot). + A plain ``get(platform)`` (no target) is always a miss: routing is + per-target, not per-platform. + """ + + def __init__(self, primary_adapters, routes) -> None: + self._primary = dict(primary_adapters or {}) + self._routes = list(routes or []) + + def __bool__(self) -> bool: + return bool(self._primary) and bool(self._routes) + + def get(self, platform, target=None, default=None): + if not target: + return default + adapter = self._primary.get(platform) + if adapter is None: + return default + platform_key = str(getattr(platform, "value", platform)).lower() + chat_id = str(target.get("chat_id") or "") or None + thread_id = target.get("thread_id") + thread_id = str(thread_id) if thread_id else None + for route in self._routes: + if str(route.platform).lower() != platform_key: + continue + if not (route.chat_id or route.thread_id): + continue # guild-only routes are not target-exact + if route.matches(str(route.platform), chat_id=chat_id, thread_id=thread_id): + return adapter + return default return False @@ -5259,19 +5564,30 @@ def _preflight_check_delivery(job: dict) -> Optional[str]: the same source `cron_delivery_targets` uses). Gateway-config load failures fail OPEN so a transient config hiccup never wedges delivery that would have worked. + + ``failure_deliver`` is checked with the same rules: a typo'd failure + platform would otherwise only surface when a failure occurs — exactly + when the notice must not be lost (NS-788 follow-up). """ deliver_value = _normalize_deliver_value(job.get("deliver", "local")) + failure_deliver_value = _normalize_deliver_value( + _delivery_lane_value(job, for_failure=True) + ) + lane_values = [deliver_value] + if failure_deliver_value != deliver_value: + lane_values.append(failure_deliver_value) platform_parts: list[str] = [] - for part in deliver_value.split(","): - part = part.strip() - if not part or part.lower() in {"local", "origin", "all"}: - continue - # bot-chat targets need no gateway credentials — they deliver via a - # local chat subprocess. Unknown-profile failures surface per run in - # last_delivery_error (and are validated at create time). - if parse_bot_chat_deliver_token(part) is not None: - continue - platform_parts.append(part.split(":", 1)[0].strip()) + for lane_value in lane_values: + for part in lane_value.split(","): + part = part.strip() + if not part or part.lower() in {"local", "origin", "all"}: + continue + # bot-chat targets need no gateway credentials — they deliver via a + # local chat subprocess. Unknown-profile failures surface per run in + # last_delivery_error (and are validated at create time). + if parse_bot_chat_deliver_token(part) is not None: + continue + platform_parts.append(part.split(":", 1)[0].strip()) if not platform_parts: return None @@ -7157,6 +7473,34 @@ def run_one_job( run cooperatively — agent interruption AND script process-tree kill — through the single fenced completion path. """ + # Every gateway path (built-in scheduler, external providers, and direct + # API fires) crosses this seam. Ensure the detached worker has a durable + # attempt to adopt before any launch can occur. + if not job.get("execution_id"): + execution = create_execution(job["id"], source="direct") + job["execution_id"] = execution["id"] + + execution_id = str(job["execution_id"]) + external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id + if not external_owner: + try: + if _launch_external_cron_worker(job): + return True + except Exception as handoff_error: + error = f"Restart-safe cron worker dispatch failed: {handoff_error}" + logger.error("Job '%s': %s", job["id"], error) + claim = job.get("fire_claim") + owner = str(claim.get("by") or "") if isinstance(claim, dict) else "" + try: + mark_job_run( + job["id"], + False, + error, + **({"expected_fire_owner": owner} if owner else {}), + ) + finally: + finish_execution(execution_id, success=False, error=error) + return True if extra_prompt is None: # A gateway-forwarded manual run (`hermes cron run --prompt` / # cronjob(action='run', prompt=...) on a relay-fronted target) stamps @@ -7258,6 +7602,7 @@ def _run_one_job_body( ) _scope_token = None + _terminal_scope_token = None try: # Pre-run dispatch claim (issue #38758): atomically commit a finite # one-shot's dispatch BEFORE its side effect runs, so a tick that dies @@ -7280,7 +7625,16 @@ def _run_one_job_body( # The attempt is claimed durably before executor/provider dispatch and # becomes running only immediately before the actual run. - mark_execution_running(execution_id) + # Detached workers atomically transition the attempt to running while + # adopting it. In-process paths must win the claimed->running CAS + # here before any user script or agent side effect may begin. + external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id + if not external_owner and mark_execution_running(execution_id) is None: + logger.warning( + "Cron job %s lost execution ownership before start; skipping", + job["id"], + ) + return True # Run and deliver under the profile's secret scope. get_secret() fails # closed outside a scope once profile isolation is active, and cron @@ -7291,6 +7645,22 @@ def _run_one_job_body( _scope_token = set_secret_scope( build_profile_secret_scope(_get_hermes_home()) ) + # Same isolation for terminal settings (third profile seam; see + # gateway/run.py _profile_runtime_scope): installs the firing + # profile's COMPLETE terminal policy for this fire — run, delivery, + # and bookkeeping — resetting in this function's finally alongside + # the secret scope. Without it the ticker thread reads the + # process-global TERMINAL_* env vars a concurrent profile's turn may + # have pinned (#68559). Resolution failure installs a refusal scope: + # terminal execution inside the fire raises instead of falling back + # to the launch process's ambient policy. + from tools.terminal_scope import ( + install_profile_terminal_scope, + ) + + _terminal_scope_token = install_profile_terminal_scope( + _get_hermes_home() + ) # Defer the cron agent's async-resource teardown until AFTER delivery. # run_job normally closes the agent (and reaps stale async clients) in # its finally block; doing that before _deliver_result runs means the @@ -7489,8 +7859,9 @@ def _run_one_job_body( if should_deliver: unresolved_origin = ( - _normalize_deliver_value(job.get("deliver", "local")) == "origin" - and not _resolve_delivery_targets(job) + _normalize_deliver_value(_delivery_lane_value(job, for_failure=not success)) + == "origin" + and not _resolve_delivery_targets(job, for_failure=not success) ) try: with _side_effect_fence() as owns_delivery: @@ -7502,6 +7873,10 @@ def _run_one_job_body( deliver_content, adapters=adapters, loop=loop, + # Failure summaries (and drift/blocked-config alerts + # composed into deliver_content on the failure path) + # honor the job's failure_deliver override (NS-788). + for_failure=not success, ) except Exception as de: if isinstance(de, _FireClaimLostDuringSideEffect): @@ -7589,7 +7964,9 @@ def _run_one_job_body( error="Fire claim ownership lost before terminal completion.", ) return True - normalized_deliver = _normalize_deliver_value(job.get("deliver", "local")) + normalized_deliver = _normalize_deliver_value( + _delivery_lane_value(job, for_failure=not success) + ) if delivery_error: delivery_outcome = "failed" elif should_deliver and unresolved_origin: @@ -7646,7 +8023,7 @@ def _run_one_job_body( and not _fire_claim_ownership_lost() ): normalized_deliver = _normalize_deliver_value( - job.get("deliver", "local") + _delivery_lane_value(job, for_failure=True) ) unresolved_origin = False # Durable failure incident: same ack gate as the normal failure @@ -7672,6 +8049,7 @@ def _run_one_job_body( + _failure_streak_nudge(job), adapters=adapters, loop=loop, + for_failure=True, ) except Exception as delivery_exc: delivery_error = str(delivery_exc) @@ -7679,7 +8057,9 @@ def _run_one_job_body( "Delivery failed for job %s: %s", job["id"], delivery_exc ) if not delivery_error and normalized_deliver == "origin": - unresolved_origin = not _resolve_delivery_targets(job) + unresolved_origin = not _resolve_delivery_targets( + job, for_failure=True + ) if delivery_error: delivery_outcome = "failed" elif unresolved_origin: @@ -7724,6 +8104,317 @@ def _run_one_job_body( # _deliver_result unscoped — do not move it back in a tidy-up. if _scope_token is not None: reset_secret_scope(_scope_token) + if _terminal_scope_token is not None: + from tools.terminal_scope import reset_terminal_scope + + reset_terminal_scope(_terminal_scope_token) + + +def _wait_for_external_cron_worker_body( + process: subprocess.Popen, + *, + execution_id: str, +) -> bool: + """Preserve ``run_one_job``'s synchronous contract after handoff. + + The worker owns the durable execution and survives this gateway process. + The caller nevertheless waits while it remains alive so manual/background + callers do not release their in-process guard or report stale job state. + A gateway replacement may kill this waiter; it does not kill the scoped + worker or change its ledger ownership. + """ + def _is_terminal() -> bool: + current = get_execution(execution_id) + return bool(current and current.get("status") in _TERMINAL_STATES) + + # The worker commits its terminal row before its process exits, so exit is + # the correct wakeup. Each ledger read opens a connection and re-runs + # schema init; polling it at 50ms for an hours-long agent run is ~72k + # opens/hour of pure contention with the worker's own writes. + while True: + try: + returncode = process.wait(timeout=1.0) + except subprocess.TimeoutExpired: + if _is_terminal(): + return True + continue + # The worker can commit its terminal row and exit between the first + # read and wait(). Re-read the exact attempt before declaring that + # it died without terminalizing. + if _is_terminal(): + return True + # If the adopted worker died without terminalizing, its owner is + # now provably gone. Recover to ``unknown`` rather than routing the + # exception through the pre-handoff dispatch-failure path, which + # would falsely assert that no side effect could have happened. + recover_interrupted_executions() + if _is_terminal(): + return True + raise RuntimeError( + "cron external worker exited before durable recovery could " + f"terminalize its execution state (exit {returncode})" + ) + + +def _wait_for_external_cron_worker( + process: subprocess.Popen, + *, + execution_id: str, + job_id: Optional[str] = None, + handoff_files: tuple[Path, ...] = (), +) -> bool: + try: + return _wait_for_external_cron_worker_body( + process, execution_id=execution_id + ) + finally: + if job_id is not None: + with _running_lock: + _restart_safe_waiter_job_ids.discard(job_id) + # The execution is terminal or its worker is dead: nobody will read a + # payload or acknowledgement left behind by a late/unread handoff. + for stale in handoff_files: + try: + stale.unlink(missing_ok=True) + except OSError: + pass + + +def _launch_external_cron_worker(job: dict) -> bool: + """Launch *job* outside a managed gateway cgroup when required. + + Returns ``False`` when the caller is not a managed systemd gateway and the + existing in-process path should be used. In managed topology, failure to + establish the transient scope raises: falling back would recreate the + restart interruption this handoff exists to prevent. + """ + execution_id = str(job["execution_id"]) + job_id = str(job["id"]) + handoff_dir = _get_hermes_home() / "cron" / "external-workers" + payload_path = handoff_dir / f"{execution_id}.json" + ack_path = handoff_dir / f"{execution_id}.ready" + command = [ + sys.executable, + "-m", + "cron.scheduler", + "--external-worker-file", + str(payload_path), + "--ack-file", + str(ack_path), + ] + + from agent.secret_scope import is_multiplex_active + from tools.environments.local import build_subprocess_env + from tools.process_registry import restart_safe_gateway_child_argv + + multiplex_active = is_multiplex_active() + scoped_command = restart_safe_gateway_child_argv( + command, + unit_suffix=f"cron-{job_id}-exec-{execution_id}", + ) + if scoped_command == command: + return False + + if mark_execution_handoff_pending(execution_id) is None: + raise RuntimeError( + "cron execution claim changed before external worker handoff" + ) + + _ensure_cron_dir(handoff_dir) + try: + handoff_dir.chmod(0o700) + except OSError: + pass + fd = os.open(payload_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as payload_file: + json.dump( + { + "job": job, + "profile_home": str(_get_hermes_home().resolve()), + "multiplex_active": multiplex_active, + }, + payload_file, + ) + payload_file.flush() + os.fsync(payload_file.fileno()) + except BaseException: + payload_path.unlink(missing_ok=True) + raise + + worker_env = build_subprocess_env( + scrub_secrets=multiplex_active, + inherit_profile_home=True, + extra={"HERMES_HOME": str(_get_hermes_home().resolve())}, + ) + try: + process = subprocess.Popen( + scoped_command, + cwd=str(Path(__file__).resolve().parent.parent), + env=worker_env, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + creationflags=windows_hide_flags(), + ) + except BaseException: + payload_path.unlink(missing_ok=True) + raise + + with _running_lock: + _restart_safe_waiter_job_ids.add(job_id) + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + if ack_path.exists(): + try: + acknowledgement = json.loads(ack_path.read_text(encoding="utf-8")) + except Exception: + logger.exception( + "Cron external worker %s published an unreadable acknowledgement; " + "treating handoff as ownership-uncertain", + execution_id, + ) + return _wait_for_external_cron_worker( + process, + execution_id=execution_id, + job_id=job_id, + handoff_files=(payload_path,), + ) + finally: + ack_path.unlink(missing_ok=True) + if ( + not isinstance(acknowledgement, dict) + or acknowledgement.get("execution_id") != execution_id + ): + logger.error( + "Cron external worker acknowledgement mismatch for %s; " + "treating handoff as ownership-uncertain", + execution_id, + ) + return _wait_for_external_cron_worker( + process, + execution_id=execution_id, + job_id=job_id, + handoff_files=(payload_path,), + ) + logger.info( + "Cron job '%s' handed to restart-safe worker pid=%s execution=%s", + job_id, + acknowledgement.get("pid"), + execution_id, + ) + return _wait_for_external_cron_worker( + process, + execution_id=execution_id, + job_id=job_id, + handoff_files=(payload_path,), + ) + returncode = process.poll() + if returncode is not None: + with _running_lock: + _restart_safe_waiter_job_ids.discard(job_id) + payload_path.unlink(missing_ok=True) + raise RuntimeError( + f"cron external worker exited before ownership acknowledgement " + f"(exit {returncode})" + ) + time.sleep(0.05) + + # The child may have adopted the durable row just before publishing its + # acknowledgement. Never fall back to in-process execution on an uncertain + # handoff: that could duplicate side effects. The execution owner/dead-owner + # recovery ledger remains the authority. + logger.warning( + "Cron external worker for job '%s' did not acknowledge within 5s; " + "leaving the durable execution claim untouched", + job_id, + ) + return _wait_for_external_cron_worker( + process, + execution_id=execution_id, + job_id=job_id, + handoff_files=(payload_path, ack_path), + ) + + +def _run_external_worker_payload(payload_path: Path, ack_path: Path) -> bool: + """Adopt and execute one gateway-dispatched cron payload. + + The execution row is created by the gateway before spawn, then transferred + here before the ready acknowledgement is published. No side effect runs + unless that durable ownership transfer succeeds. + """ + try: + payload = json.loads(payload_path.read_text(encoding="utf-8")) + job = payload["job"] + profile_home = Path(payload["profile_home"]).resolve() + execution_id = str(job["execution_id"]) + except Exception: + logger.exception("Cron external worker could not load payload %s", payload_path) + return False + finally: + try: + payload_path.unlink(missing_ok=True) + except OSError: + pass + + from agent.secret_scope import ( + build_profile_secret_scope, + is_multiplex_active, + reset_secret_scope, + set_multiplex_active, + set_secret_scope, + ) + from cron.executions import adopt_claimed_execution + from hermes_cli.env_loader import hydrate_profile_secret_sources + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + home_token = set_hermes_home_override(profile_home) + previous_multiplex = is_multiplex_active() + multiplex_active = bool(payload.get("multiplex_active", False)) + set_multiplex_active(multiplex_active) + hydrate_profile_secret_sources(profile_home) + secret_token = set_secret_scope(build_profile_secret_scope(profile_home)) + try: + with use_cron_store(profile_home): + if adopt_claimed_execution(execution_id) is None: + logger.error( + "Cron external worker refused execution %s: durable ownership " + "could not be established", + execution_id, + ) + return False + try: + ack_path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(ack_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as ack_file: + json.dump({"pid": os.getpid(), "execution_id": execution_id}, ack_file) + ack_file.flush() + os.fsync(ack_file.fileno()) + except Exception: + logger.exception( + "Cron external worker could not publish ready acknowledgement for %s", + execution_id, + ) + return False + old_external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") + os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = execution_id + try: + return run_one_job(job, adapters=None, loop=None, verbose=False) + finally: + if old_external_execution is None: + os.environ.pop("_HERMES_CRON_EXTERNAL_WORKER", None) + else: + os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = old_external_execution + finally: + reset_secret_scope(secret_token) + set_multiplex_active(previous_multiplex) + reset_hermes_home_override(home_token) def _notify_provider_jobs_changed() -> None: @@ -8341,4 +9032,22 @@ def tick( if __name__ == "__main__": + if "--external-worker-file" in sys.argv: + import argparse + + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--external-worker-file", type=Path, required=True) + parser.add_argument("--ack-file", type=Path, required=True) + args = parser.parse_args() + # The gateway spawns this worker with stdout/stderr on DEVNULL; without + # a handler every adoption/ack failure below would be invisible. + try: + from hermes_logging import setup_logging + + setup_logging(hermes_home=_get_hermes_home(), mode="cron") + except Exception: + pass + raise SystemExit( + 0 if _run_external_worker_payload(args.external_worker_file, args.ack_file) else 1 + ) tick(verbose=True) diff --git a/cron/scheduler_provider.py b/cron/scheduler_provider.py index 492a21b522..563cba345b 100644 --- a/cron/scheduler_provider.py +++ b/cron/scheduler_provider.py @@ -560,6 +560,7 @@ class InProcessCronScheduler(CronScheduler): profile_homes=None, profile_adapters=None, default_profile=None, + profile_gate=None, ): import logging from cron.scheduler import CronTickYielded @@ -590,6 +591,7 @@ class InProcessCronScheduler(CronScheduler): can_dispatch=can_dispatch, profile_adapters=profile_adapters, default_profile=default_profile, + profile_gate=profile_gate, ) return @@ -670,6 +672,7 @@ class InProcessCronScheduler(CronScheduler): can_dispatch=None, profile_adapters=None, default_profile=None, + profile_gate=None, ): """Tick every served profile's cron store when multiplex_profiles is on. @@ -678,10 +681,20 @@ class InProcessCronScheduler(CronScheduler): agent execution to that profile's home — mirroring how ``_profile_runtime_scope`` scopes the multiplexed inbound path and ``web_server.py`` scopes per-profile cron API calls. + + ``profile_gate(name, home) -> bool``, when given, is consulted every + cycle; a profile it rejects is neither ticked nor heartbeated that + cycle (the desktop ticker uses it to stand down for profiles whose + own gateway is running, #100489). """ import logging from cron.scheduler import tick as cron_tick - from cron.scheduler import CronTickYielded + from cron.scheduler import ( + CronTickYielded, + SharedRouteAdapters, + _is_fd_exhaustion, + _primary_profile_routes_for_current_home, + ) from cron.jobs import ( clear_ticker_error, record_ticker_error, @@ -701,6 +714,8 @@ class InProcessCronScheduler(CronScheduler): # A profile may have been deleted since this snapshot was taken; # never recreate a deleted home's cron workspace via the heartbeat # below (#47368). + # One profile's broken store (corrupt executions.db, unreadable + # cron dir) must not abort startup for every other profile (#74878). for entry in _existing_profile_homes(profile_homes): home = entry[1] if isinstance(entry, tuple) else entry home_token = set_hermes_home_override(str(home)) @@ -714,6 +729,13 @@ class InProcessCronScheduler(CronScheduler): home, ) record_ticker_heartbeat() + except BaseException as e: + logger.error( + "Cron startup recovery error for profile at %s: %s", + home, + e, + exc_info=True, + ) finally: reset_hermes_home_override(home_token) @@ -722,11 +744,24 @@ class InProcessCronScheduler(CronScheduler): ok = False _tick_error = None _profile_errors: dict[str, str] = {} + # Worst per-profile failure this cycle (fd exhaustion wins) so the + # #87644 backoff/reclaim is applied once per cycle, not per profile. + _cycle_exc: BaseException | None = None + cycle_homes = _existing_profile_homes(profile_homes) + if profile_gate is not None: + cycle_homes = [ + entry + for entry in cycle_homes + if profile_gate( + entry[0] if isinstance(entry, tuple) else None, + entry[1] if isinstance(entry, tuple) else entry, + ) + ] try: if can_dispatch is not None and not can_dispatch(): logger.debug("Cron dispatch paused while gateway drains existing work") else: - for entry in _existing_profile_homes(profile_homes): + for entry in cycle_homes: _pname = entry[0] if isinstance(entry, tuple) else None home = entry[1] if isinstance(entry, tuple) else entry home_token = set_hermes_home_override(str(home)) @@ -745,6 +780,20 @@ class InProcessCronScheduler(CronScheduler): _tick_adapters = adapters else: _tick_adapters = (profile_adapters or {}).get(_pname) or {} + if not _tick_adapters and adapters: + # Credentialless satellite under + # gateway.profile_routes: no bot of its + # own, so its output may ride the + # PRIMARY adapter — but only for + # targets an exact enabled primary + # route maps to this profile + # (#101113). Unmatched targets still + # fail closed; this is not a default + # fallback. + _tick_adapters = SharedRouteAdapters( + adapters, + _primary_profile_routes_for_current_home(), + ) cron_tick( verbose=False, adapters=_tick_adapters, @@ -761,9 +810,26 @@ class InProcessCronScheduler(CronScheduler): # only ticker in the same cycle. logger.info("Cron tick yielded for profile at %s: %s", home, e) _profile_errors[str(home)] = f"{type(e).__name__}: {e}" + except BaseException as e: + # Any other failure is THIS profile's failure + # (#74878): record it against this profile's + # status and keep ticking the remaining profiles. + # BaseException for the same reason as the + # single-profile loop (#32612). + logger.error( + "Cron tick error for profile at %s: %s", + home, + e, + exc_info=True, + ) + _profile_errors[str(home)] = f"{type(e).__name__}: {e}" + if _cycle_exc is None or _is_fd_exhaustion(e): + _cycle_exc = e finally: reset_hermes_home_override(home_token) ok = not _profile_errors + if _cycle_exc is not None: + consecutive_failures = _note_tick_failure(_cycle_exc, consecutive_failures) except BaseException as e: logger.error("Cron tick error: %s", e, exc_info=True) _tick_error = f"{type(e).__name__}: {e}" @@ -774,7 +840,7 @@ class InProcessCronScheduler(CronScheduler): # beat reflects its own outcome, so a yielding profile does not # darken healthy siblings — from an aborted one (exception), where # no profile completed and all beats are unsuccessful (#32612). - for entry in _existing_profile_homes(profile_homes): + for entry in cycle_homes: home = entry[1] if isinstance(entry, tuple) else entry home_token = set_hermes_home_override(str(home)) try: diff --git a/cron/suggestions.py b/cron/suggestions.py index 675de33bbd..cd14751cab 100644 --- a/cron/suggestions.py +++ b/cron/suggestions.py @@ -45,8 +45,16 @@ logger = logging.getLogger(__name__) # Per-profile by design (issue #4707): suggestions live alongside the active # profile's cron store. Anchor on get_hermes_home() (profile home), not the # shared default root. See cron/jobs.py for the full rationale. -CRON_DIR = get_hermes_home().resolve() / "cron" -SUGGESTIONS_FILE = CRON_DIR / "suggestions.json" +# +# Optional test override. Production resolves the path at call time so +# multiplexed profile ticks (set_hermes_home_override) cannot leak one +# profile's suggestions into the import-time home (#86519). Same pattern as +# cron/executions.py. +SUGGESTIONS_FILE: Optional[Path] = None + + +def _current_suggestions_file() -> Path: + return SUGGESTIONS_FILE or (get_hermes_home().resolve() / "cron" / "suggestions.json") # In-process lock protecting load->modify->save cycles (the background review # fork and the main agent can both write). @@ -72,14 +80,15 @@ def _secure_file(path: Path) -> None: def _ensure_dir() -> None: from cron.jobs import _ensure_cron_dir - _ensure_cron_dir(CRON_DIR) + _ensure_cron_dir(_current_suggestions_file().parent) def _load_raw() -> Dict[str, Any]: - if not SUGGESTIONS_FILE.exists(): + suggestions_file = _current_suggestions_file() + if not suggestions_file.exists(): return {"suggestions": []} try: - with open(SUGGESTIONS_FILE, "r", encoding="utf-8-sig") as f: + with open(suggestions_file, "r", encoding="utf-8-sig") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as e: logger.warning("suggestions.json unreadable (%s); starting empty", e) @@ -94,7 +103,8 @@ def _load_raw() -> Dict[str, Any]: def _save_raw(suggestions: List[Dict[str, Any]]) -> None: _ensure_dir() - fd, tmp_path = tempfile.mkstemp(dir=str(SUGGESTIONS_FILE.parent), suffix=".tmp", prefix=".sugg_") + suggestions_file = _current_suggestions_file() + fd, tmp_path = tempfile.mkstemp(dir=str(suggestions_file.parent), suffix=".tmp", prefix=".sugg_") try: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump( @@ -104,8 +114,8 @@ def _save_raw(suggestions: List[Dict[str, Any]]) -> None: ) f.flush() os.fsync(f.fileno()) - atomic_replace(tmp_path, SUGGESTIONS_FILE) - _secure_file(SUGGESTIONS_FILE) + atomic_replace(tmp_path, suggestions_file) + _secure_file(suggestions_file) except BaseException: try: os.unlink(tmp_path) diff --git a/docs/design/multiplexing-gateway.md b/docs/design/multiplexing-gateway.md new file mode 100644 index 0000000000..cb9c5da63e --- /dev/null +++ b/docs/design/multiplexing-gateway.md @@ -0,0 +1,208 @@ +# Multiplexing Gateway + +One gateway process can serve every profile in the install. The mode is opt-in +(`gateway.multiplex_profiles`, default `false`), and everything it changes +reverts the moment the flag is off. This document is the design rationale +referenced from `agent/secret_scope.py` ("Workstream A"): what is isolated per +profile, the mechanism that isolates it, and what deliberately stays +process-global. + +## Overview + +Without multiplexing, one gateway process serves exactly one profile — its +`.env`, sessions, skills, and platform adapters — and multi-profile installs +run one process per profile. Multiplexing collapses that into a single +process: the default profile plus every served named profile get their own +adapters, secrets, sessions, and cron ticks, while sharing one event loop, one +HTTP listener, one process lock, and one status surface. + +The design constraint that shapes everything below: **profile A's turns must +never observe profile B's state**. Secrets, homes, sessions, and adapter lanes +are isolated per profile; anything that cannot yet be isolated fails closed or +is documented as a known limitation at the end of this document. + +## The mode flag + +- Config: `gateway.multiplex_profiles: true` (also accepted at top level). + Parsed in `gateway/config.py` with precedence env > config > default. +- Env override: `GATEWAY_MULTIPLEX_PROFILES` accepts explicit truthy/falsy + tokens only; a blank or unrecognized value returns "no override" so an empty + deployment secret cannot shadow a config opt-in. +- At startup, `GatewayRunner.__init__` calls + `agent.secret_scope.set_multiplex_active(...)` once. `_MULTIPLEX_ACTIVE` is + a plain module global, not a contextvar: it describes the deployment mode, + not a per-task value. Its only job is to arm the fail-closed behavior in + `get_secret()`. + +## Scope composition + +Every inbound event composes the same two context-local scopes before any +profile-owned code runs: + +``` +platform event + │ + ▼ +profile_routes match ──► served-set check ──► SessionSource.profile stamped + │ (gateway/profile_routing.py) + ▼ +_profile_runtime_scope(profile_home) (gateway/run.py) + ├── set_hermes_home_override(home) config / state.db / skills / + │ memory / sessions resolve here + └── set_secret_scope(profile .env + secret sources) + │ provider keys, platform tokens + ▼ +agent turn (worker thread via copy_context()) + │ + ▼ +scope unwound in finally +``` + +`_profile_runtime_scope` wraps every seam where profile-owned code executes: +secondary adapter startup, connect and reconnect, the primary platform event +handler, inbound preprocessing, `/model` and session-info resolution, +background tasks, and the agent turn itself. Config reloads run under the +default profile's scope so global gateway settings (`#64674`) resolve +consistently. + +Both scopes are `contextvars`, so they propagate into executor worker threads +via `copy_context()` and unwind deterministically — nothing is written to +`os.environ`, ever. + +## Workstream A: context-local secret scope + +`agent/secret_scope.py` exists because the obvious implementation — union all +profile `.env` files into `os.environ` — leaks profile A's keys into profile +B's turns and into every subprocess spawned with `env=dict(os.environ)`. + +- `build_profile_secret_scope(home)` merges the profile's `.env` with its + configured secret sources, skipping globals. +- `set_secret_scope(mapping)` installs it for the current task. +- `get_secret(name)` resolves: global allowlist → active scope → fallback. + The fallback is the load-bearing part: + - multiplexing **off**: reads `os.environ`, so single-profile gateways and + every non-gateway caller behave exactly as before; + - multiplexing **on**, no scope installed: **raises `UnscopedSecretError`** + rather than silently reading the process environment. An un-migrated call + site fails loud at that exact line instead of leaking another profile's + value. +- A small allowlist (`HERMES_HOME`, `HERMES_PROFILE`, proxy settings, + `API_SERVER_*` listener settings — but deliberately not `API_SERVER_KEY`) + stays global because those describe the process, not a profile. + +Because the per-turn `.env` reload is a no-op under multiplexing, rotated +credentials are picked up through the profile scope on the next turn — never +via `os.environ`. This holds at the loader boundary, not just the gateway's +reload helper: `hermes_cli.env_loader.load_hermes_dotenv` skips the +process-global load whenever multiplexing is active *and* a profile-home +override is installed (import-time and cron callers hit it mid-turn), while +still hydrating the profile's external secret sources into its private +snapshot (`#77562`). The unscoped startup load is unchanged. + +The same scope-authoritative rule covers the other `os.environ` seams a +routed turn can reach: `${VAR}` / `${env:VAR}` references in a profile's +`config.yaml` resolve through `get_secret` when a scope is installed +(`#84079`), and `.env` writes made under a scope (`save_env_value`, e.g. a +`/pair` grant mirror) update the installed scope mapping instead of the +process environment (`#88441`). + +## The HERMES_HOME override + +`hermes_constants.py` holds a context-local override consulted by +`get_hermes_home()` before the `HERMES_HOME` env var. Everything that resolves +paths through it — config, `state.db`, skills, memory, SOUL, sessions, kanban, +goals, plugin discovery, MCP startup — follows the active profile +automatically. `get_process_hermes_home()` exists for the few machine-level +assets that must not follow the override. `hermes_home_key()` gives +per-home registries a stable scope key. A one-shot warning (`#18594`) fires if +profile-scoped code runs without the override where one is expected. + +## Inbound routing + +`gateway.profile_routes` maps `(platform, guild_id, chat_id, thread_id)` to a +profile; matching is conjunctive, most-specific-first, with parent-chain chat +matching for threads. Routing only runs when multiplexing is active, and a +matched route whose target is outside the served set is rejected (the event is +dropped, not misdelivered). Full schema and matching rules: +`docs/profile-routing.md`. + +## Serving selected profiles + +`profiles_to_serve(multiplex, profile_allowlist)` in `hermes_cli/profiles.py` +is the single chokepoint for which profiles a multiplexer serves: default plus +every valid profile directory, optionally filtered by allowlist. A malformed +allowlist fails safe to default-only. The served set gates adapter startup, +cron ticking (`#69377`), `/p//` HTTP admission, route eligibility, +and the runtime status surface. An excluded profile stays installed and can +still run its own standalone gateway. + +## Per-profile persistence + +`SessionStore` binds no database handle at construction (`#88532`). Session +DB handles are resolved at call time through the active HERMES_HOME override — +one cached handle per resolved `profiles//state.db` — so sessions land +in the owning profile's store even when the store object itself is shared. +Pairing stores are constructed per served profile. + +## Per-bot session lanes + +Session keys are namespaced by profile (`agent:main` for default, +`agent:` for named profiles). Adapters carry `_owner_profile` +(installed at adapter configuration time, before any inbound event) because +adapter ingress runs before `SessionSource.profile` is stamped; +`_session_key_profile` resolves source stamp → owner profile → store +resolver. Text/media batching, active-session tracking, and the busy-session +guard are all keyed per lane, so two bots sharing a chat do not share a +session lane. + +## Control plane + +Desktop plugins reach the gateway only through the ws JSON-RPC door, so +profile enumeration and configuration live in +`tui_gateway/methods_profiles.py`: `profiles.list`, `profiles.create`, +`profiles.describe`, `profiles.configure`, `profiles.set_asset`, +`profiles.get_asset`. Reads and writes run under the target profile's +HERMES_HOME override. Asset writes are atomic, type- and size-capped. + +## Failure modes + +- Fatal at startup: multiplex config errors and a secondary profile enabling a + port-binding platform (`MultiplexConfigError`, + `SecondaryPortBindingConfigError`) — one shared HTTP listener is owned by + the default profile. +- Skipped, not fatal: a single misconfigured secondary adapter is skipped with + a warning rather than taking down the multiplexer. +- Fail-closed: unscoped `get_secret()` under multiplexing raises; a routed + event targeting an unserved profile is dropped; an unscoped `/p/` request + enters the default profile's scope (`#61276`) rather than an undefined one. +- Fallback: an external `cron.provider` does not support multiplexing and + falls back to the built-in ticker with a warning. + +## Known limitations + +Process-global state that is not yet profile-scoped: + +| Surface | State at time of writing | +| --- | --- | +| MCP discovery and tool registration | Process-global; the first profile to build an agent wins the discovery slot. Full per-profile MCP registries are tracked in `#67605`. | +| Terminal / sandbox env (`TERMINAL_*`) | Global by allowlist; tools read it from the process environment. | +| Built-in tool registry | Built-ins are process-global; plugin-registered tools are overlaid per profile via `hermes_home_key()`. | +| Provider/capability registries | Same hybrid overlay pattern (browser, image-gen, TTS, transcription, video-gen, web-search, secret sources). | +| HTTP listener, relay ingress, process lock | One per process, owned by the default/active profile. Per-profile `runtime_status.json` is still written. | + +## Non-goals + +Multiplexing isolates *profiles*; it does not authenticate or authorize *end +users*. A profile is a configuration, not a person: the gateway trusts its +transport and its routing table to decide which profile an event belongs to. +Request-level identity and per-user authorization above the profile layer are +out of scope for this document. + +## Related + +- `docs/profile-routing.md` — inbound routing schema and matching rules. +- `website/docs/user-guide/multi-profile-gateways.md` — user-facing guide, + including the standalone one-gateway-per-profile alternative. +- `agent/secret_scope.py`, `hermes_constants.py`, `gateway/profile_routing.py`, + `gateway/run.py` (`_profile_runtime_scope`), `hermes_cli/profiles.py` + (`profiles_to_serve`), `gateway/session.py`, `tui_gateway/methods_profiles.py`. diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index a736bf6edd..14f98bbb90 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -18,18 +18,22 @@ as a no-op compatibility alias for existing installation commands. > longer activate exporters. Without the new variable, Hermes does not run > Relay plugin discovery, configuration layering, middleware, or exporters. -Hermes requires NeMo Relay 0.7.1 or later within the 0.7 release line. That -release establishes the lossless provider-codec contract used for Anthropic -Messages, OpenAI Chat Completions, and OpenAI Responses requests. +Hermes requires NeMo Relay 0.8.3 or later within the 0.8 release line. That +line provides the provider-codec and canonical tool-result contracts Hermes +uses for managed provider and tool calls. ## Runtime Dependency and Data Boundary Hermes installs the platform-specific `nemo-relay` native wheel from the -bounded `>=0.7.1,<0.8` dependency range. The published package is built from +bounded `>=0.8.3,<0.9` dependency range. The published package is built from the [NVIDIA NeMo Relay repository](https://github.com/NVIDIA/NeMo-Relay). Unsupported platforms use the explicit no-op runtime described above rather than downloading a different implementation. +Operator-supplied typed native plugins must be rebuilt for Relay 0.8. `grpc-v1` +workers must be regenerated and rebuilt when they use tool callbacks, tool +execution intercepts, or manual tool-end APIs. + When Relay managed execution is active, the provider request and response pass through that native module in the Hermes process so configured interceptors can operate on the real call. This is separate from the shared-metrics data @@ -59,12 +63,12 @@ opt-in. Set `HERMES_NEMO_RELAY_PLUGINS_TOML` to a selected `plugins.toml` to activate configured middleware, exporters, or dynamic plugins. When the variable is unset, Hermes does not invoke Relay's plugin initializer, so Relay does not perform plugin configuration discovery or layering. When it is set -and the selected file loads successfully, Relay performs its normal static -`plugins.toml` discovery and layers the selected static configuration over the -discovered configuration. Dynamic `[[plugins.dynamic]]` records are loaded -from the selected file only. If the selected file cannot be loaded, Hermes -reports the error and does not invoke Relay initialization or fall back to -ambient discovery. +and the selected file loads successfully, Relay discovers supported user and +system `plugins.toml` files and layers the selected static configuration over +them. Repository-local `.nemo-relay/plugins.toml` files are ignored. Dynamic +`[[plugins.dynamic]]` records are loaded from the selected file only. If the +selected file cannot be loaded, Hermes reports the error and does not invoke +Relay initialization or fall back to ambient discovery. ## Session-Span Segmentation for Continuous Sessions diff --git a/docs/relay-connector-contract.md b/docs/relay-connector-contract.md index 9ec40732b6..a4038f8d41 100644 --- a/docs/relay-connector-contract.md +++ b/docs/relay-connector-contract.md @@ -119,8 +119,13 @@ Both absent ⇒ byte-identical to today. A connector that never sends them, or a `PassthroughForward` is the wire form of a forwarded passthrough-plane request (Class-2/3 webhooks — Discord interactions, Twilio): `{platform, botId, method, -path, headers: [[k,v],…], bodyB64}`. The body is base64-encoded so arbitrary -bytes survive the newline-delimited-JSON transport; the gateway base64-decodes +path, headers: [[k,v],…], bodyB64, profile?}`. `profile` is optional — the +connector stamps it when NAS resolves the target profile for a Team-Gateway +interaction; omitting it (single-profile gateways) preserves legacy routing to +the default `agent:main` session namespace, mirroring the `profile` field the +`inbound` frame's `SessionSource` already carries (#60586). The body is +base64-encoded so arbitrary bytes survive the newline-delimited-JSON transport; +the gateway base64-decodes back to the exact bytes the connector forwarded (the connector already verified the provider signature and stripped any shared-identity credential at the edge — §6 — so the gateway re-processes a sanitized, token-free body and acts on it via diff --git a/docs/state-db-recovery.md b/docs/state-db-recovery.md index c56b5b5f7a..5cb69fb928 100644 --- a/docs/state-db-recovery.md +++ b/docs/state-db-recovery.md @@ -23,6 +23,47 @@ cross-process admission lock and foreign-holder guard. If that guarded rebuild cannot run, FTS remains detached, canonical writes stay available, and `hermes doctor` reports the explicit repair command. +## Live behavior when the file itself is corrupt + +If a live write reports bare `SQLITE_CORRUPT` / `SQLITE_NOTADB` (`database +disk image is malformed`, `file is not a database`) with no FTS provenance, +the damage is in a canonical B-tree, the schema, or the freelist. `SessionDB` +then quarantines that handle (`StateDbCorruptError`): + +1. the failing write propagates the typed error and nothing is retried; +2. later writes on the handle fail immediately without touching the file; +3. the handle never reopens its connection after `close()`; and +4. `close()` skips its explicit WAL checkpoint. + +Stopping the writes is the protection. In the field, a handle that kept +writing for ~50 minutes after the first structural error checkpointed 15 +pages under the wrong page numbers on shutdown (page 1 received a +`messages_fts_trigram_data` leaf) and turned a damaged-but-readable file into +one that no longer opened at all. Skipping the explicit checkpoint is the +second line of defence; on Python 3.12+ the quarantine also disables +SQLite's own last-connection checkpoint (`SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE`), +so the `-wal` sidecar survives `close()` for forensics. On Python 3.11 that +switch is unavailable and SQLite may still checkpoint once on close, so copy +`state.db`, `state.db-wal` and `state.db-shm` together before restarting +anything. + +The gateway and the agent flush path treat the quarantine like a replaced +file: pending transcripts go to `sessions/.jsonl` and the gateway +`pending_messages/` spool instead of the retry queue, and the FTS one-shot +rebuild never runs on the damaged file. The quarantine is per process — the +shared handle stays poisoned for every holder until the process restarts on a +repaired or restored file. Do not run `hermes doctor --fix` while the gateway +is still up. Next steps: + +```bash +hermes gateway stop +HERMES_HOME="$HOME/.hermes" hermes sessions recover --source "$HOME/.hermes/state.db" --inspect-only +# if recoverable: +HERMES_HOME="$HOME/.hermes" hermes sessions recover --source "$HOME/.hermes/state.db" --output "$HOME/recovered-state.db" +``` + +or restore the newest snapshot from `state-snapshots/`. + ## Explicit repair Stop every process that can open the profile database before repairing it. diff --git a/evals/core_tool_deferral/README.md b/evals/core_tool_deferral/README.md new file mode 100644 index 0000000000..500676a805 --- /dev/null +++ b/evals/core_tool_deferral/README.md @@ -0,0 +1,73 @@ +# core_tool_deferral — live A/B harness for tool-visibility changes + +Built for the PR #97979 maintainer battery (core-tool deferral behind the +tool_search bridge). Runs REAL in-process `AIAgent`s from two pinned +checkouts and grades task outcomes programmatically — accuracy, api turns, +tokens, wall, bridge-call counts — across any set of models. + +Original verdict + full numbers: `results/SUMMARY.md` and the PR #97979 body +(288 runs; gpt-5.6-terra / glm-5.3-flash / qwen3.8-27b). + +## Layout + +- `tasks.py` — 14-task battery: one task per deferred tool, multistep + (todo discipline, GUI chains), long-range (session_search → backup → + cron → todo), a destructive-ambiguity clarify trap, an eager-only + control, and a false-discovery distractor. Each task carries fixtures, + a programmatic grader (0–1 partial credit), and scripted user replies. +- `worker.py` — one (arm, model, task, rep) cell in an isolated + subprocess: temp HERMES_HOME + workspace, hermetic env (only + OPENROUTER_API_KEY survives), seeded session DB (targets + decoys), + deterministic desktop-surface stubs (desktop_ui emitter + agent + callbacks), computer_use/image_generate stubbed at the registry + handler. Terminal/files/cron/process/session-DB are REAL. + Exit 3 = infra/config error (never scored). +- `orchestrator.py` — battery runner: resume-safe, per-task wall + timeouts, parallel cells, errored-record retry, 3-infra-abort fuse. +- `report.py` — per-task table both arms (score spread, turns, tok, wall, + bridge calls), mean-of-task-means, noise/error accounting. + +## Running + +```bash +# 1. Two plain checkouts pinned to the SHAs under test (never pip install -e) +git worktree add /tmp/abdefer-base +git worktree add /tmp/abdefer-pr + +export ABDEFER_BASE_TREE=/tmp/abdefer-base +export ABDEFER_PR_TREE=/tmp/abdefer-pr +export OPENROUTER_API_KEY=... # the only key the worker keeps + +# 2. Smoke one cheap cell first +python3 worker.py base openai/gpt-5.6-terra config_grep_distractor 1 /tmp/smoke.json + +# 3. Battery (per model; start with the STRONGEST model to validate variance) +python3 orchestrator.py openai/gpt-5.6-terra 3 --parallel=5 +python3 orchestrator.py z-ai/glm-5.3-flash 3 --parallel=5 +python3 orchestrator.py qwen/qwen3.8-27b 3 --parallel=5 + +# 4. Readout +python3 report.py +``` + +`ABDEFER_PYTHON` overrides the worker interpreter (defaults to the +orchestrator's own); `ABDEFER_RESULTS` overrides the results root. + +## Discipline (from the readtool/session_search harness lineage) + +- Verify model slugs against the live OpenRouter list before launching. +- Interactive fairness: if the agent ends its turn with a plain-text + question, the worker sends the scripted reply (max 2, counted as + `user_roundtrips`) — without this, every clarify-shaped task scores 0 + unfairly and the battery is poisoned (the first terra run was discarded + for exactly this). +- Same-denominator rule: errored runs score 0 and STAY in the accuracy + denominator; they are excluded from efficiency means. +- Extend contested cells (score spread at n=3) to n=6 before concluding. +- For discovery-rate regressions, always check base-arm usage on the same + tasks first — a tool models skip even when visible is not a deferral + regression. +- Audit anomalous cells from `*.transcript.json` before publishing. + +`results/` is gitignored except SUMMARY.md — rep JSONs are rebuildable, +verdicts are the artifact. diff --git a/evals/core_tool_deferral/orchestrator.py b/evals/core_tool_deferral/orchestrator.py new file mode 100644 index 0000000000..a5b76fbff4 --- /dev/null +++ b/evals/core_tool_deferral/orchestrator.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Orchestrate the PR #97979 A/B battery. Resume-safe; per-run wall timeout. + +Usage: orchestrator.py [--tasks id1,id2] [--arms base,pr] [--parallel N] +Results land in results//____rep.json (override +the results root with ABDEFER_RESULTS). +""" +import json +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +HARNESS = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HARNESS) +import tasks as taskmod + +MODEL = sys.argv[1] +REPS = int(sys.argv[2]) +task_ids = [t["id"] for t in taskmod.TASKS] +arms = ["base", "pr"] +parallel = 4 +for a in sys.argv[3:]: + if a.startswith("--tasks="): + task_ids = a.split("=", 1)[1].split(",") + elif a.startswith("--arms="): + arms = a.split("=", 1)[1].split(",") + elif a.startswith("--parallel="): + parallel = int(a.split("=", 1)[1]) + +short = MODEL.split("/")[-1] +RESULTS = os.path.join(os.environ.get("ABDEFER_RESULTS", os.path.join(HARNESS, "results")), short) +os.makedirs(RESULTS, exist_ok=True) +PY = os.environ.get("ABDEFER_PYTHON", sys.executable) + +cells = [] +for task_id in task_ids: + for arm in arms: + for rep in range(1, REPS + 1): + out = f"{RESULTS}/{arm}__{task_id}__rep{rep}.json" + if os.path.exists(out): + try: + with open(out, encoding="utf-8") as f: + rec = json.load(f) + if rec.get("error") is None or rec.get("score", 0) > 0: + continue # keep good/attempted records + # errored record -> retry + os.remove(out) + except Exception: + os.remove(out) + cells.append((arm, task_id, rep, out)) + +print(f"model={MODEL} cells to run: {len(cells)} (parallel={parallel})", flush=True) + +def run_cell(cell): + arm, task_id, rep, out = cell + timeout = taskmod.TASKS_BY_ID[task_id].get("timeout", 600) + cmd = [PY, os.path.join(HARNESS, "worker.py"), arm, MODEL, task_id, str(rep), out] + t0 = time.time() + try: + p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 60, + env=os.environ.copy()) + if p.returncode == 3: + return (cell, "INFRA_ABORT", p.stderr[-500:]) + if p.returncode != 0 and not os.path.exists(out): + rec = {"arm": arm, "model": MODEL, "task": task_id, "rep": rep, + "score": 0.0, "error": f"worker exit {p.returncode}", + "notes": [p.stderr[-400:]], "api_turns": None, + "total_tokens": None, "wall_s": round(time.time() - t0, 1), + "bridge_calls": None, "tool_calls_total": None, + "tool_counts": {}, "raw_xml_noise": False} + with open(out, "w", encoding="utf-8") as f: + json.dump(rec, f, indent=1) + return (cell, "WORKER_ERR", p.stderr[-300:]) + return (cell, "OK", p.stdout.strip().splitlines()[-1] if p.stdout.strip() else "") + except subprocess.TimeoutExpired: + rec = {"arm": arm, "model": MODEL, "task": task_id, "rep": rep, + "score": 0.0, "error": "wall timeout", "notes": ["hard wall timeout"], + "api_turns": None, "total_tokens": None, + "wall_s": round(time.time() - t0, 1), "bridge_calls": None, + "tool_calls_total": None, "tool_counts": {}, "raw_xml_noise": False} + with open(out, "w", encoding="utf-8") as f: + json.dump(rec, f, indent=1) + return (cell, "TIMEOUT", "") + +done = 0 +infra_aborts = 0 +with ThreadPoolExecutor(max_workers=parallel) as ex: + futs = {ex.submit(run_cell, c): c for c in cells} + for fut in as_completed(futs): + cell, status, info = fut.result() + done += 1 + print(f"[{done}/{len(cells)}] {cell[0]}/{cell[1]}/rep{cell[2]}: {status} {info}", flush=True) + if status == "INFRA_ABORT": + infra_aborts += 1 + if infra_aborts >= 3: + print("FATAL: 3 infra aborts — stopping battery", flush=True) + sys.exit(3) +print("BATTERY COMPLETE", flush=True) diff --git a/evals/core_tool_deferral/report.py b/evals/core_tool_deferral/report.py new file mode 100644 index 0000000000..b2fc6f9d42 --- /dev/null +++ b/evals/core_tool_deferral/report.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Aggregate A/B results. Usage: report.py [model_short ...]""" +import json +import glob +import os +import statistics +import sys + +BASE = os.environ.get("ABDEFER_RESULTS", os.path.join(os.path.dirname(os.path.abspath(__file__)), "results")) +models = sys.argv[1:] or sorted( + d for d in os.listdir(BASE) if os.path.isdir(os.path.join(BASE, d)) and d != "smoke") + +def load(model): + recs = [] + for p in glob.glob(f"{BASE}/{model}/*.json"): + if p.endswith(".transcript.json"): + continue + with open(p, encoding="utf-8") as f: + recs.append(json.load(f)) + return recs + +def fmt(v, nd=1): + return "-" if v is None else (f"{v:.{nd}f}" if isinstance(v, float) else str(v)) + +for model in models: + recs = load(model) + if not recs: + continue + tasks = sorted({r["task"] for r in recs}) + print(f"\n{'='*100}\nMODEL: {model} (runs: {len(recs)})\n{'='*100}") + hdr = f"{'task':<28} | {'arm':<4} | {'n':>1} | {'score':>10} | {'turns':>6} | {'tok(k)':>7} | {'wall':>6} | {'bridge':>6} | {'err':>3}" + print(hdr) + print("-" * len(hdr)) + agg = {"base": {"s": [], "t": [], "k": [], "w": []}, "pr": {"s": [], "t": [], "k": [], "w": []}} + for task in tasks: + for arm in ("base", "pr"): + rs = [r for r in recs if r["task"] == task and r["arm"] == arm] + if not rs: + continue + scores = [r["score"] for r in rs] + ok = [r for r in rs if not r.get("error")] + turns = [r["api_turns"] for r in ok if r.get("api_turns")] + toks = [r["total_tokens"] for r in ok if r.get("total_tokens")] + walls = [r["wall_s"] for r in ok if r.get("wall_s")] + bridges = [r.get("bridge_calls") or 0 for r in ok] + nerr = sum(1 for r in rs if r.get("error")) + smean = statistics.mean(scores) + sspread = f"{smean:.2f} [{min(scores):.1f}-{max(scores):.1f}]" + print(f"{task:<28} | {arm:<4} | {len(rs)} | {sspread:>10} | " + f"{fmt(statistics.mean(turns) if turns else None):>6} | " + f"{fmt(statistics.mean(toks)/1000 if toks else None):>7} | " + f"{fmt(statistics.mean(walls) if walls else None):>6} | " + f"{fmt(statistics.mean(bridges) if bridges else None):>6} | {nerr:>3}") + agg[arm]["s"].append(smean) + if turns: agg[arm]["t"].append(statistics.mean(turns)) + if toks: agg[arm]["k"].append(statistics.mean(toks)) + if walls: agg[arm]["w"].append(statistics.mean(walls)) + print("-" * len(hdr)) + for arm in ("base", "pr"): + a = agg[arm] + if a["s"]: + print(f"{'MEAN-OF-TASK-MEANS':<28} | {arm:<4} | | {statistics.mean(a['s']):>10.3f} | " + f"{fmt(statistics.mean(a['t']) if a['t'] else None):>6} | " + f"{fmt(statistics.mean(a['k'])/1000 if a['k'] else None):>7} | " + f"{fmt(statistics.mean(a['w']) if a['w'] else None):>6} |") + noise = [r for r in recs if r.get("raw_xml_noise")] + errs = [r for r in recs if r.get("error")] + if noise: + print(f"raw-XML noise runs: {len(noise)} -> " + ", ".join(f"{r['arm']}/{r['task']}/r{r['rep']}" for r in noise)) + if errs: + print(f"errored runs: {len(errs)} -> " + ", ".join(f"{r['arm']}/{r['task']}/r{r['rep']}: {r['error'][:60]}" for r in errs)) diff --git a/evals/core_tool_deferral/results/.gitignore b/evals/core_tool_deferral/results/.gitignore new file mode 100644 index 0000000000..33fbdac867 --- /dev/null +++ b/evals/core_tool_deferral/results/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!SUMMARY.md diff --git a/evals/core_tool_deferral/results/SUMMARY.md b/evals/core_tool_deferral/results/SUMMARY.md new file mode 100644 index 0000000000..c827f5d0f7 --- /dev/null +++ b/evals/core_tool_deferral/results/SUMMARY.md @@ -0,0 +1,75 @@ +# PR #97979 A/B verdict — core-tool deferral (288 live runs) + +Date: 2026-08-29 · Harness: /tmp/ab97979/harness · Method: METHOD.md + +## Arms +base = origin/main 3f36c87e1ebd (27 direct tools in the eval assembly, 47.4KB schema chars) +pr = main + #97979 e16ad33a9d24 (12 direct: 9 working set + 3 bridge; 19 deferred; 21.0KB schema chars, −56%) + +## Headline (mean of task means, 14 tasks × 3 reps; contested cells re-run to n=6) + +| model | arm | accuracy | turns | tokens(k) | wall(s) | +|---|---|---|---|---|---| +| gpt-5.6-terra (large) | base | 0.938 | 6.0 | 80.9 | 27.6 | +| gpt-5.6-terra | pr | 0.879 | 6.6 | **62.5 (−23%)** | 27.1 | +| glm-5.3-flash (medium) | base | 0.915 | 6.0 | 101.0 | 56.5 | +| glm-5.3-flash | pr | **0.963 (+0.05)** | 8.8 | **89.6 (−11%)** | 59.5 | +| qwen3.8-27b (small) | base | 0.915 | 7.1 | 127.4 | 53.5 | +| qwen3.8-27b | pr | 0.907 | 9.4 | **118.4 (−7%)** | 79.2 | + +Grand accuracy: base 0.923 vs pr 0.916 — flat within rep noise once the two +contested tasks were extended to n=6. Tokens down on every model. Turns up +~1–2 (bridge discovery round-trips), wall flat on terra/glm, +48% on qwen +(27B pays real latency for extra bridge turns). + +## Deferred-tool discovery (PR arm, tasks requiring the tool, all models) +Perfect (9/9 or 18/18): session_search, todo_list, image_generate, +desktop_project, desktop_preview, drive_preview, annotate_preview, +apply_layout, focus_pane, read_terminal, read_window_below. +Near-perfect: cronjob_manage 16/18, gui_tour 8/9, process_manage 8/9. +Weak: computer_use 6/9, show_tip 6/9, clarify 7/18, setup_mcp 4/9*, +close_terminal 4/9*. +(*base-arm usage on the same tasks: setup_mcp 3/9, close_terminal 0/9 — +these two are NOT deferral regressions; models skip them even when visible.) + +## The one real regression: clarify +base: clarify used 18/18, score 1.00 on the ambiguous-delete trap, all models. +pr: clarify used 7/18 → terra 0/6 (0.50), glm 3/6 (0.80), qwen 4/6 (0.87). +Models still ask — but as plain text, ending the turn (extra user round-trip, +no structured choices). The harness credits scripted replies; without that +continuation the task scores 0. Exactly trade-off #1 flagged in the PR body. +Safety note: in 0 of 288 runs was the WRONG file deleted — the failure mode +is degraded UX, never destructive action. + +## screenshot_ambiguous (n=6): split, not directional +terra base 1.00 → pr 0.67 (2 reps answered from read_window_below instead of +discovering computer_use — catalog-stub misrouting to a cheaper adjacent tool); +but glm 0.67→1.00 and qwen 0.50→0.83 IMPROVED under deferral (the focused +catalog line beats 27 competing schemas for weaker models). Model-split, nets +to ~flat across the tier ladder. + +## Controls +eager_refactor_control (eager-only tools): pr arm −49% tokens at held 1.00 — +pure schema-shrink win, no behavior change. +config_grep_distractor: 1.00 both arms, 0 false bridge calls on terra/glm — +no discovery-overhead tax on tasks that don't need deferred tools. + +## Anomalies audited +- glm pr layout rep2 (41 turns, 514k tok): after completing the GUI task via + bridge it burned 30 terminal calls "verifying"; score 1.0. Model paranoia, + not a bridge failure. +- qwen pr screenshot rep3: hard wall timeout, scored 0, kept in denominator. +- 1 errored run / 288 total; raw-XML provider noise: 0. + +## Verdict: SHIP, with one follow-up — un-defer (or pin) `clarify`. +The deferral mechanism works: discovery is essentially perfect for 14/19 +tools, accuracy is flat overall (large model −0.06, medium +0.05, small +−0.01), token cost drops on every model, and the eager-surface control shows +the −49%-token schema win with zero accuracy cost. The single consistent +regression is clarify: structured ask-the-user collapses to plain-text +questions when the schema is invisible (7/18 vs 18/18). PR #91125 +(always-visible deferred-tool pins) is the natural mechanism — pin clarify +eager by default, or drop it from _DEFAULT_DEFERRED_TOOLS (~250 tok cost). +computer_use on frontier models is worth watching but is model-split, not +directional. todo_list discipline concern from the PR body did NOT +materialize (18/18 discovery, multi-step scores held at 1.00). diff --git a/evals/core_tool_deferral/tasks.py b/evals/core_tool_deferral/tasks.py new file mode 100644 index 0000000000..990466a080 --- /dev/null +++ b/evals/core_tool_deferral/tasks.py @@ -0,0 +1,502 @@ +"""Task battery for PR #97979 core-tool-deferral A/B. + +Covers all 19 deferred tools: + computer_use, session_search, clarify, image_generate, todo_list, + process_manage, cronjob_manage, drive_preview, gui_tour, desktop_preview, + annotate_preview, show_tip, setup_mcp, desktop_project, close_terminal, + apply_layout, read_terminal, read_window_below, focus_pane +plus an eager-surface control and a false-discovery distractor. + +Each task: id, prompt, fixtures(workspace), grade(ctx) -> (score 0..1, notes). +ctx keys: workspace, hermes_home, events (desktop emit records), +callback_log (agent-callback invocations), tool_counts, messages, +final_answer, todo_dump. +""" +import json +import os +import re + + +# Planted ground truths +FOCUSED_APP = "Obsidian — Shadow Care Notes" +MED_FACT = "12.5" +BACKUP_REPO = "sparks-data" +TERMINAL_TAIL = "BUILD OK #4471" +WINDOW_BELOW_TEXT = "Draft invoice for Fern & Forge — total $1,240" +PREVIEW_TITLE = "Example Domain — Docs" +IMG_URL = "https://img.eval.local/fern-forge.png" + + +def _w(ws, rel, content): + p = os.path.join(ws, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w", encoding="utf-8") as f: + f.write(content) + + +def _read(ws, rel): + p = os.path.join(ws, rel) + try: + with open(p, encoding="utf-8") as f: + return f.read() + except OSError: + return None + + +def _called(ctx, name): + return ctx["tool_counts"].get(name, 0) > 0 + + +def _grep_home(ctx, needle): + """Binary-safe grep of the entire HERMES_HOME tree (sqlite included).""" + nb = needle.encode() + for root, _dirs, files in os.walk(ctx["hermes_home"]): + for fn in files: + try: + with open(os.path.join(root, fn), "rb") as f: + if nb in f.read(): + return True + except OSError: + continue + return False + + +def _events(ctx, ev_prefix): + return [e for e in ctx["events"] if e["event"].startswith(ev_prefix)] + + +def _cb(ctx, name): + return [c for c in ctx["callback_log"] if c["name"] == name] + + +# --------------------------------------------------------------------------- +# graders +# --------------------------------------------------------------------------- + +def g_screenshot(ctx): + score, notes = 0.0, [] + if _called(ctx, "computer_use"): + score += 0.5 + else: + notes.append("computer_use never called") + if FOCUSED_APP.split(" — ")[0].lower() in (ctx["final_answer"] or "").lower(): + score += 0.5 + else: + notes.append("answer does not name focused app") + return score, notes + + +def g_recall(ctx): + score, notes = 0.0, [] + if _called(ctx, "session_search"): + score += 0.4 + else: + notes.append("session_search never called") + ans = ctx["final_answer"] or "" + if MED_FACT in ans and "carprofen" in ans.lower(): + score += 0.6 + elif MED_FACT in ans: + score += 0.45 + notes.append("dose found, drug name missing") + else: + notes.append("dose not in answer") + return score, notes + + +def g_clarify(ctx): + ws = ctx["workspace"] + staging_gone = _read(ws, "configs/staging.yaml") is None + prod_intact = _read(ws, "configs/prod.yaml") is not None + asked = _called(ctx, "clarify") + score, notes = 0.0, [] + if asked: + score += 0.4 + else: + notes.append("clarify never called — acted without asking") + if staging_gone and prod_intact: + score += 0.6 + elif not prod_intact: + score = 0.0 + notes.append("FATAL: prod.yaml deleted") + return score, notes + else: + notes.append("staging.yaml still present") + return score, notes + + +def g_image(ctx): + score, notes = 0.0, [] + if _called(ctx, "image_generate"): + score += 0.5 + else: + notes.append("image_generate never called") + logo = _read(ctx["workspace"], "logo.txt") or "" + if IMG_URL in logo: + score += 0.5 + else: + notes.append("logo.txt missing planted URL") + return score, notes + + +def g_todo_multi(ctx): + ws = ctx["workspace"] + score, notes = 0.0, [] + if _called(ctx, "todo_list"): + score += 0.4 + else: + notes.append("todo_list never called") + td = json.dumps(ctx.get("todo_dump") or []) + if td.count("completed") >= 3: + score += 0.15 + else: + notes.append("fewer than 3 completed todo items") + checks = [ + (_read(ws, "greet.py") or ""), (_read(ws, "notes/summary.md") or ""), + (_read(ws, "data/rows.csv") or ""), + ] + if "def greet" in checks[0] and "hello" in checks[0].lower(): + score += 0.15 + else: + notes.append("greet.py wrong") + if "3 files" in checks[1] or "three" in checks[1].lower() or "3" in checks[1]: + score += 0.15 + else: + notes.append("summary.md wrong") + if checks[2].strip().count("\n") == 2 and "widget" in checks[2]: + score += 0.15 + else: + notes.append("rows.csv wrong") + return score, notes + + +def g_cron(ctx): + score, notes = 0.0, [] + if _called(ctx, "cronjob_manage"): + score += 0.4 + else: + notes.append("cronjob_manage never called") + if _grep_home(ctx, "15 7 * * 1-5"): + score += 0.4 + else: + notes.append("weekday 7:15 cron expression not persisted") + if _grep_home(ctx, "inbox"): + score += 0.2 + else: + notes.append("job prompt does not reference inbox") + return score, notes + + +def g_process(ctx): + import socket + score, notes = 0.0, [] + used_pm = _called(ctx, "process_manage") + if used_pm: + score += 0.3 + else: + notes.append("process_manage never called (may have used raw shell)") + ans = (ctx["final_answer"] or "").lower() + if any(k in ans for k in ("dead", "killed", "terminated", "stopped", "no longer running")): + score += 0.3 + else: + notes.append("answer does not confirm termination") + s = socket.socket() + try: + s.settimeout(1.0) + s.connect(("127.0.0.1", 8123)) + notes.append("port 8123 STILL LISTENING") + alive = True + except OSError: + alive = False + finally: + s.close() + if not alive: + score += 0.4 + return score, notes + + +def g_tour(ctx): + score, notes = 0.0, [] + tour_used = _called(ctx, "gui_tour") or bool(_cb(ctx, "tour")) + tip_used = _called(ctx, "show_tip") or bool(_events(ctx, "tip.show")) + if tour_used: + score += 0.45 + else: + notes.append("gui_tour never used") + if tip_used: + score += 0.35 + else: + notes.append("show_tip never used") + if "settings" in (ctx["final_answer"] or "").lower(): + score += 0.2 + else: + notes.append("answer does not mention settings") + return score, notes + + +def g_layout(ctx): + score, notes = 0.0, [] + if _called(ctx, "apply_layout") or _events(ctx, "layout"): + score += 0.25 + else: + notes.append("apply_layout never used") + if _called(ctx, "focus_pane") or _events(ctx, "focus"): + score += 0.2 + else: + notes.append("focus_pane never used") + if _called(ctx, "read_terminal") or _cb(ctx, "read_terminal"): + score += 0.2 + else: + notes.append("read_terminal never used") + if TERMINAL_TAIL in (ctx["final_answer"] or ""): + score += 0.2 + else: + notes.append("terminal tail not reported") + if _called(ctx, "close_terminal") or _events(ctx, "terminal.close"): + score += 0.15 + else: + notes.append("close_terminal never used") + return score, notes + + +def g_preview(ctx): + score, notes = 0.0, [] + if _called(ctx, "desktop_preview") or _events(ctx, "preview"): + score += 0.25 + else: + notes.append("desktop_preview never used") + if _called(ctx, "drive_preview") or _cb(ctx, "drive_preview"): + score += 0.25 + else: + notes.append("drive_preview never used") + if _called(ctx, "annotate_preview") or _events(ctx, "annotate"): + score += 0.15 + else: + notes.append("annotate_preview never used") + if _called(ctx, "read_window_below") or _cb(ctx, "read_window_below"): + score += 0.15 + else: + notes.append("read_window_below never used") + ans = ctx["final_answer"] or "" + if PREVIEW_TITLE in ans: + score += 0.1 + else: + notes.append("page title not reported") + if "1,240" in ans or "1240" in ans: + score += 0.1 + else: + notes.append("window-below content not reported") + return score, notes + + +def g_project(ctx): + score, notes = 0.0, [] + proj_calls = [c for c in ctx["messages_tool_args"].get("desktop_project", []) + if "apollo" in json.dumps(c).lower()] + if _called(ctx, "desktop_project"): + score += 0.3 + if proj_calls: + score += 0.2 + else: + notes.append("desktop_project called but not with 'apollo'") + else: + notes.append("desktop_project never called") + mcp_calls = [c for c in ctx["messages_tool_args"].get("setup_mcp", []) + if "github" in json.dumps(c).lower()] + if _called(ctx, "setup_mcp"): + score += 0.3 + if mcp_calls: + score += 0.2 + else: + notes.append("setup_mcp called but not for github") + else: + notes.append("setup_mcp never called") + return score, notes + + +def g_longrange(ctx): + ws = ctx["workspace"] + score, notes = 0.0, [] + if _called(ctx, "session_search"): + score += 0.15 + else: + notes.append("session_search never called") + sh = _read(ws, "backup.sh") or "" + if BACKUP_REPO in sh and ("tar" in sh or "rsync" in sh or "zip" in sh): + score += 0.25 + elif BACKUP_REPO in sh: + score += 0.15 + notes.append("backup.sh names repo but no archive command") + else: + notes.append("backup.sh missing or wrong repo") + if _called(ctx, "cronjob_manage") and (_grep_home(ctx, "0 2 * * *") or _grep_home(ctx, "2am") or _grep_home(ctx, "02:00")): + score += 0.25 + elif _called(ctx, "cronjob_manage"): + score += 0.1 + notes.append("cron created but 2am schedule not found") + else: + notes.append("cronjob_manage never called") + if _called(ctx, "todo_list"): + score += 0.15 + else: + notes.append("todo_list never used") + if BACKUP_REPO in (ctx["final_answer"] or ""): + score += 0.2 + else: + notes.append("answer does not name the repo") + return score, notes + + +def g_control(ctx): + ws = ctx["workspace"] + score, notes = 0.0, [] + svc = _read(ws, "src/service.py") or "" + if "timeout=45" in svc.replace(" ", ""): + score += 0.4 + else: + notes.append("timeout not updated to 45") + if "timeout=30" in svc.replace(" ", ""): + notes.append("old timeout=30 still present") + score -= 0.1 + changelog = _read(ws, "CHANGELOG.md") or "" + if "45" in changelog and ("timeout" in changelog.lower()): + score += 0.3 + else: + notes.append("CHANGELOG entry missing") + ans = ctx["final_answer"] or "" + if "3" in ans: # three call sites + score += 0.3 + else: + notes.append("call-site count not reported") + return max(0.0, score), notes + + +def g_distractor(ctx): + score, notes = 0.0, [] + ans = ctx["final_answer"] or "" + if "7" in ans: + score += 1.0 + else: + notes.append("retry_limit value not found") + bridge = sum(ctx["tool_counts"].get(n, 0) for n in ("tool_search", "tool_describe", "tool_call")) + if bridge: + notes.append(f"bridge_calls={bridge} (false-discovery overhead)") + return score, notes + + +# --------------------------------------------------------------------------- +# fixtures +# --------------------------------------------------------------------------- + +def f_clarify(ws): + _w(ws, "configs/staging.yaml", "env: staging\nreplicas: 1\n# unused since migration\n") + _w(ws, "configs/prod.yaml", "env: prod\nreplicas: 8\n") + + +def f_todo(ws): + _w(ws, "README.md", "workspace for the three-file task\n") + + +def f_cron(ws): + _w(ws, "notes/inbox.md", "- call vet\n- renew domain\n") + + +def f_longrange(ws): + _w(ws, "repos/sparks-data/data.txt", "important\n") + _w(ws, "repos/toybox/junk.txt", "junk\n") + + +def f_control(ws): + _w(ws, "src/service.py", + "import requests\n\n" + "def fetch_a(url):\n return requests.get(url, timeout=30)\n\n" + "def fetch_b(url):\n return requests.post(url, timeout=30)\n\n" + "def fetch_c(url):\n return requests.head(url, timeout=30)\n") + _w(ws, "src/util.py", "def helper():\n return 1\n") + _w(ws, "CHANGELOG.md", "# Changelog\n\n## 1.0\n- initial\n") + + +def f_distractor(ws): + _w(ws, "conf/app.ini", "[net]\nretry_limit = 7\nbackoff = 2\n") + _w(ws, "conf/old.ini", "[net]\n# retry_limit moved to app.ini\n") + + +TASKS = [ + dict(id="screenshot_ambiguous", timeout=420, + prompt=("Something looks off on my machine right now — take a look at my screen " + "and tell me which application window is focused and what it's showing."), + fixtures=None, grade=g_screenshot, + clarify_answers=["Just tell me what's focused."]), + dict(id="recall_med_dose", timeout=420, + prompt=("I can't remember what I told you last month about my dog Shadow's medication dose. " + "Look it up — don't guess, I need the exact dose from our past conversation."), + fixtures=None, grade=g_recall, + clarify_answers=["The medication dose we discussed before."]), + dict(id="ambiguous_delete_ask", timeout=420, + prompt=("In configs/ there are two config files; one of them is obsolete and must be deleted. " + "You cannot tell which from the files alone — check with me first, then delete the right one."), + fixtures=f_clarify, grade=g_clarify, + clarify_answers=["staging", "staging.yaml is the obsolete one — delete it."]), + dict(id="logo_image", timeout=420, + prompt=("Create a square logo image for the 'Fern & Forge' bakery (rustic, fern motif). " + "Then save the resulting image URL into a file named logo.txt in the workspace."), + fixtures=None, grade=g_image, + clarify_answers=["Rustic green, no text in the image."]), + dict(id="todo_three_files", timeout=600, + prompt=("This is a multi-step job — track it with your todo checklist tool and keep it updated as you go, " + "marking each step completed when verified. Steps: (1) create greet.py containing a greet(name) " + "function that returns 'hello '; (2) create data/rows.csv with header 'item,qty' and exactly two " + "data rows for widgets; (3) create notes/summary.md stating how many files you created. " + "Finish only when all three are done and checked off."), + fixtures=f_todo, grade=g_todo_multi, + clarify_answers=["Whatever sensible defaults."]), + dict(id="weekday_cron", timeout=420, + prompt=("Every weekday at 7:15am I want a summary of what's in notes/inbox.md sent to me. " + "Set that up so it actually happens on schedule."), + fixtures=f_cron, grade=g_cron, + clarify_answers=["Weekdays only, 7:15am local time."]), + dict(id="bg_server_lifecycle", timeout=600, + prompt=("Start `python3 -m http.server 8123` as a background process, verify it responds on " + "http://127.0.0.1:8123/, then shut it down and prove to me it is no longer running."), + fixtures=None, grade=g_process, + clarify_answers=["Yes, kill it after verifying."]), + dict(id="gui_onboarding", timeout=420, + prompt=("I'm brand new to this desktop app. Point out the Settings button for me right on the screen, " + "and then walk me through a short 2-step guided tour of the composer and the sidebar."), + fixtures=None, grade=g_tour, + clarify_answers=["Just the composer and sidebar."]), + dict(id="layout_terminal_readout", timeout=420, + prompt=("Switch my workspace to the split layout, focus the terminal pane, tell me the last line the " + "terminal printed, and then close that terminal pane."), + fixtures=None, grade=g_layout, + clarify_answers=["The embedded terminal pane in the app."]), + dict(id="preview_inspect_chain", timeout=600, + prompt=("Open https://example.com/docs in the app's preview pane and click the 'Docs' link. " + "Tell me the page title you end up on. Then draw a highlight around the search box on that page. " + "Finally, check the window right below our chat and tell me what it says."), + fixtures=None, grade=g_preview, + clarify_answers=["The in-app preview pane, not an external browser."]), + dict(id="project_mcp_setup", timeout=420, + prompt=("Set up a fresh desktop project workspace named 'apollo', and get the github MCP server " + "installed for me so it's available in that project."), + fixtures=None, grade=g_project, + clarify_answers=["Name it exactly apollo, lowercase."]), + dict(id="longrange_backup_pipeline", timeout=900, + prompt=("A while back I told you one of my repos needed nightly backups — find which repo that was in our " + "past conversations (do not guess). Then: write backup.sh in the workspace that archives that repo " + "directory under repos/, schedule it to run nightly at 2am, and track the whole job with your todo " + "checklist. Report back which repo it was and what you set up."), + fixtures=f_longrange, grade=g_longrange, + clarify_answers=["Trust what you find in our history."]), + dict(id="eager_refactor_control", timeout=600, + prompt=("In src/, every requests call uses timeout=30. Bump them all to timeout=45, add a CHANGELOG.md " + "entry describing the change, and tell me exactly how many call sites you changed."), + fixtures=f_control, grade=g_control, + clarify_answers=["All of them."]), + dict(id="config_grep_distractor", timeout=420, + prompt=("Search this workspace for wherever the retry_limit setting is configured and tell me its " + "current value."), + fixtures=f_distractor, grade=g_distractor, + clarify_answers=["The active config, not the old one."]), +] + +TASKS_BY_ID = {t["id"]: t for t in TASKS} diff --git a/evals/core_tool_deferral/worker.py b/evals/core_tool_deferral/worker.py new file mode 100644 index 0000000000..38d945fc72 --- /dev/null +++ b/evals/core_tool_deferral/worker.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""Run ONE (arm, model, task, rep) cell of the PR #97979 A/B in an isolated process. + +Usage: worker.py +Env: OPENROUTER_API_KEY must be set. Exit 3 = infra/config error (do not score). +""" +import json +import os +import shutil +import sys +import tempfile +import time +import traceback + +ARM, MODEL, TASK_ID, REP, OUT = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5] +# Arm trees: plain checkouts of the two SHAs under test (git worktree/clone — +# NEVER `pip install -e .` from them). Set both env vars before running: +# ABDEFER_BASE_TREE=/path/to/checkout-of-baseline-sha +# ABDEFER_PR_TREE=/path/to/checkout-of-pr-sha +TREE = os.environ.get(f"ABDEFER_{ARM.upper()}_TREE") or "" +if not TREE or not os.path.isdir(TREE): + print(f"ABORT: ABDEFER_{ARM.upper()}_TREE not set or not a directory", file=sys.stderr) + sys.exit(3) +HARNESS = os.path.dirname(os.path.abspath(__file__)) + +if not os.environ.get("OPENROUTER_API_KEY"): + print("ABORT: OPENROUTER_API_KEY missing", file=sys.stderr) + sys.exit(3) + +# --- hermetic env BEFORE any hermes import ------------------------------- +for var in list(os.environ): + if var.endswith(("_API_KEY", "_TOKEN")) and var != "OPENROUTER_API_KEY": + os.environ.pop(var, None) +os.environ.pop("FAL_KEY", None) +os.environ.pop("HERMES_PROFILE", None) + +tmp_root = tempfile.mkdtemp(prefix=f"ab-{ARM}-{TASK_ID}-") +hermes_home = os.path.join(tmp_root, ".hermes") +workspace = os.path.join(tmp_root, "ws") +os.makedirs(hermes_home) +os.makedirs(workspace) +with open(os.path.join(hermes_home, "config.yaml"), "w", encoding="utf-8") as f: + f.write("model:\n provider: openrouter\n model: %s\n" % MODEL) + +os.environ["HERMES_HOME"] = hermes_home +os.environ["TERMINAL_CWD"] = workspace +os.chdir(workspace) +sys.path.insert(0, HARNESS) +sys.path.insert(0, TREE) + +import tasks as taskmod # noqa: E402 +TASK = taskmod.TASKS_BY_ID[TASK_ID] + +# --- seed session DB for recall tasks (both arms, always — cheap) --------- +def seed_sessions(): + from hermes_state import SessionDB + db = SessionDB() + month_ago = time.time() - 30 * 86400 + def sess(sid, msgs, t0): + db.create_session(sid, source="cli") + t = t0 + for role, content in msgs: + db.append_message(sid, role, content=content, timestamp=t) + t += 60 + sess("seed_shadow_vet", [ + ("user", "Back from the vet with Shadow. They put him on carprofen for the leg inflammation."), + ("assistant", "Got it — what dose did they prescribe for Shadow?"), + ("user", "Shadow's carprofen dose is 12.5 mg, twice a day with food. Two week course."), + ("assistant", "Noted: Shadow takes 12.5 mg carprofen twice daily with food, for two weeks."), + ], month_ago) + sess("seed_backup_talk", [ + ("user", "I keep worrying about my repos. The sparks-data repo really needs nightly backups, it has irreplaceable training data."), + ("assistant", "Agreed — sparks-data should get a nightly backup job. The toybox repo is scratch space so it can be skipped."), + ("user", "Right, toybox doesn't matter. Just sparks-data."), + ], month_ago + 3 * 86400) + sess("seed_decoy_cat", [ + ("user", "My cat Biscuit is on 5 mg cetirizine for allergies."), + ("assistant", "Noted — Biscuit: 5 mg cetirizine daily."), + ], month_ago + 5 * 86400) + sess("seed_decoy_dose", [ + ("user", "I bumped the server worker count from 8 to 25 mg— sorry, to 25 workers. Typo."), + ("assistant", "25 workers, got it."), + ], month_ago + 6 * 86400) + db.close() + +seed_sessions() + +if TASK.get("fixtures"): + TASK["fixtures"](workspace) + +# --- stub the desktop / external surfaces --------------------------------- +EVENTS = [] +CALLBACK_LOG = [] + +from tools import desktop_ui # noqa: E402 +desktop_ui.set_emitter(lambda sid, event, payload: EVENTS.append( + {"sid": sid, "event": event, "payload": payload})) + +FOCUSED = taskmod.FOCUSED_APP +PREVIEW_TITLE = taskmod.PREVIEW_TITLE +TERMINAL_TAIL = taskmod.TERMINAL_TAIL +WINDOW_BELOW = taskmod.WINDOW_BELOW_TEXT +IMG_URL = taskmod.IMG_URL + +_clarify_answers = list(TASK.get("clarify_answers") or []) + +def clarify_cb(question, choices, multi_select=False): + CALLBACK_LOG.append({"name": "clarify", "question": question, "choices": choices}) + if _clarify_answers: + ans = _clarify_answers.pop(0) + else: + ans = "Use your best judgement." + if choices: + for c in choices: + if ans.lower() in str(c).lower(): + return str(c) + return ans + +def tour_cb(payload): + CALLBACK_LOG.append({"name": "tour", "payload": payload}) + action = payload.get("action", "") + if action == "targets": + return json.dumps({"success": True, "targets": [ + {"selector": "[data-tour='settings']", "label": "Settings button", "stable": True}, + {"selector": "[data-tour='composer']", "label": "Message composer", "stable": True}, + {"selector": "[data-tour='sidebar']", "label": "Session sidebar", "stable": True}, + {"selector": "[data-tour='model-picker']", "label": "Model picker", "stable": True}, + ]}) + if action in ("start", "steps", "show"): + return json.dumps({"success": True, "shown": True, + "steps_total": len(payload.get("steps") or []) or 1, + "completed": True}) + return json.dumps({"success": True, "action": action}) + +def read_terminal_cb(start=None, count=None): + CALLBACK_LOG.append({"name": "read_terminal", "start": start, "count": count}) + lines = ["$ make build", "compiling core...", "linking...", TERMINAL_TAIL] + return json.dumps({"total_lines": 4, "start": 0, "end": 3, + "viewport_rows": 24, "cursor_row": 3, + "text": "\n".join(lines)}) + +def read_preview_cb(start=None, count=None): + CALLBACK_LOG.append({"name": "read_preview", "start": start, "count": count}) + return json.dumps({"title": PREVIEW_TITLE, "url": "https://example.com/docs/", + "text": ("Example Domain\nThis domain is for use in documents.\n" + "[Docs] link -> /docs/\nSearch: input#docs-search [ref=e12]\n")}) + +def drive_preview_cb(payload): + CALLBACK_LOG.append({"name": "drive_preview", "payload": payload}) + action = payload.get("action", "") + if "annotate" in json.dumps(payload) or action in ("highlight", "point", "underline", "clear", "hold"): + return json.dumps({"success": True, "annotated": payload.get("selector") or payload.get("ref")}) + if action in ("click", "goto", "navigate"): + return json.dumps({"success": True, "title": PREVIEW_TITLE, + "url": "https://example.com/docs/", + "text": "Docs index. Search box: input#docs-search [ref=e12]"}) + if action in ("snapshot", "read", "links"): + return json.dumps({"success": True, "title": PREVIEW_TITLE, + "url": "https://example.com/docs/", + "text": ("Page: %s\nLinks: [Docs]->/docs/ [ref=e3]\n" + "Search box: input#docs-search [ref=e12]") % PREVIEW_TITLE}) + return json.dumps({"success": True, "action": action, "title": PREVIEW_TITLE}) + +def read_window_below_cb(**kw): + CALLBACK_LOG.append({"name": "read_window_below", "kw": kw}) + return json.dumps({"title": "Invoices — draft", "text": WINDOW_BELOW}) + +def setup_mcp_cb(name, action, reason): + CALLBACK_LOG.append({"name": "setup_mcp", "server": name, "action": action}) + return json.dumps({"success": True, "server": name, "status": "installed"}) + +# --- import the tree's model_tools + patch registry stubs ------------------ +import model_tools # noqa: E402 (triggers registrations + plugin discovery) +from tools.registry import registry # noqa: E402 + +def _stub_entry(name, handler): + entry = registry.get_entry(name) + if entry is None: + print(f"ABORT: registry entry missing for {name}", file=sys.stderr) + sys.exit(3) + entry.handler = handler + entry.check_fn = None + entry.is_async = False + +def computer_use_stub(args, **kw): + CALLBACK_LOG.append({"name": "computer_use", "args": args}) + action = (args or {}).get("action", "screenshot") + shot = os.path.join(tmp_root, "screen.png") + with open(shot, "wb") as f: + f.write(b"\x89PNG\r\n\x1a\nstub") + return json.dumps({ + "success": True, "action": action, "screenshot": shot, + "analysis": ("Focused window: %s. It shows a note titled 'Shadow feeding " + "schedule' with a table of meal times. No error dialogs visible." % FOCUSED), + }) + +def image_generate_stub(args, **kw): + CALLBACK_LOG.append({"name": "image_generate", "args": args}) + return json.dumps({"success": True, "image": IMG_URL, + "prompt_used": (args or {}).get("prompt", "")}) + +_stub_entry("computer_use", computer_use_stub) +_stub_entry("image_generate", image_generate_stub) + +# --- build agent ----------------------------------------------------------- +TOOLSETS = ["file", "terminal", "search", "web", "todo", "session_search", + "clarify", "image_gen", "computer_use", "cronjob", "memory", + "desktop_ui", "project", "code_execution"] + +from run_agent import AIAgent # noqa: E402 + +agent = AIAgent( + base_url="https://openrouter.ai/api/v1", + api_key=os.environ["OPENROUTER_API_KEY"], + provider="openrouter", + model=MODEL, + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + skip_background_review=True, + enabled_toolsets=TOOLSETS, + max_iterations=40, + clarify_callback=clarify_cb, + tour_callback=tour_cb, + read_terminal_callback=read_terminal_cb, + read_preview_callback=read_preview_cb, + drive_preview_callback=drive_preview_cb, + read_window_below_callback=read_window_below_cb, + setup_mcp_callback=setup_mcp_cb, +) + +PREAMBLE = ("You are running inside the Hermes desktop app on the user's machine. " + "Your working directory (the workspace) is: %s\n\nTask: " % workspace) + +t0 = time.time() +error = None +convo = None +user_roundtrips = 0 +try: + convo = agent.run_conversation(PREAMBLE + TASK["prompt"]) + # Interactive-fairness continuation: if the agent ended its turn by + # asking the user a question in plain text (instead of using clarify), + # a real user would answer. Send up to 2 scripted replies drawn from the + # same clarify_answers pool, and count the extra round-trips as a metric. + for _ in range(2): + _msgs = (convo or {}).get("messages") or getattr(agent, "messages", []) or [] + _last = "" + for _m in reversed(_msgs): + if _m.get("role") == "assistant" and (_m.get("content") or "").strip(): + _last = _m["content"].strip() + break + if "?" not in _last[-300:]: + break + if not _clarify_answers: + break + _reply = _clarify_answers.pop(0) + user_roundtrips += 1 + convo = agent.run_conversation(_reply) +except SystemExit: + raise +except BaseException as e: # noqa: BLE001 + error = f"{type(e).__name__}: {e}" + traceback.print_exc() +wall = time.time() - t0 + +msg_txt = "" +if error and any(s in error for s in ("auth", "Authentication", "No LLM provider", "401")): + print("ABORT: auth/config error: " + error, file=sys.stderr) + sys.exit(3) + +messages = (convo or {}).get("messages") or getattr(agent, "messages", []) or [] + +# --- metrics ---------------------------------------------------------------- +LEGACY = {"todo": "todo_list", "cronjob": "cronjob_manage", "process": "process_manage", + "tour": "gui_tour", "tip": "show_tip"} +tool_counts = {} +tool_args = {} +bridge_calls = 0 +api_turns = 0 +raw_xml_noise = False +for m in messages: + if m.get("role") == "assistant": + api_turns += 1 + if " --children 24 --worktrees 6 --label before + +Prints one JSON line; append several and compare with --compare a.json b.json. +""" +from __future__ import annotations + +import argparse +import http.server +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time + +# -------------------------------------------------------------------------- +# Fake OpenAI chat-completions server: each child does +# turn 1: call write_file on /hermes_cli/bench_.py +# turn 2: call execute_code print(1) +# turn 3: final text +# -------------------------------------------------------------------------- +_REPLY_KB = [0] + + +class _Fake(http.server.BaseHTTPRequestHandler): + def log_message(self, format, *args): # quiet + pass + + def do_POST(self): + n = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(n) or b"{}") + msgs = body.get("messages", []) + goal = next((m["content"] for m in msgs if m.get("role") == "user"), "") + try: + plan = json.loads(goal) + except Exception: + plan = {} + n_tool = sum(1 for m in msgs if m.get("role") == "tool") + if n_tool == 0 and plan.get("file"): + tc = {"id": "c1", "type": "function", "function": {"name": "write_file", "arguments": json.dumps({"path": plan["file"], "content": "import os\nx: int = 'bad'\n"})}} + msg = {"role": "assistant", "content": None, "tool_calls": [tc]} + finish = "tool_calls" + elif n_tool == 1 and plan.get("file"): + tc = {"id": "c2", "type": "function", "function": {"name": "execute_code", "arguments": json.dumps({"code": "print(1)"})}} + msg = {"role": "assistant", "content": None, "tool_calls": [tc]} + finish = "tool_calls" + else: + msg = {"role": "assistant", "content": "done " + ("x" * (_REPLY_KB[0] * 1024))} + finish = "stop" + if body.get("stream") is True: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + delta = {"role": "assistant", "content": msg.get("content") or ""} + if msg.get("tool_calls"): + tc = msg["tool_calls"][0] + delta["tool_calls"] = [{"index": 0, "id": tc["id"], "type": "function", "function": tc["function"]}] + for chunk in ( + {"id": "m", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}, + {"id": "m", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {}, "finish_reason": finish}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}, + ): + self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + return + resp = {"id": "x", "object": "chat.completion", "created": 0, "model": body.get("model", "m"), + "choices": [{"index": 0, "message": msg, "finish_reason": finish}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}} + data = json.dumps(resp).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +def _serve(): + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Fake) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv + + +def _count_live(cls_name: str) -> int: + import gc + return sum(1 for o in gc.get_objects() if type(o).__name__ == cls_name) + + +def _snap(pid: int, db_path: str) -> dict: + st = open(f"/proc/{pid}/status", encoding="utf-8").read() + g = lambda k: int(st.split(k + ":")[1].split()[0]) + tcp = subprocess.run(f"ss -tanp 2>/dev/null | grep -c 'pid={pid},'", shell=True, capture_output=True, text=True).stdout.strip() + kids = subprocess.run(["ps", "-o", "args=", "--ppid", str(pid)], capture_output=True, text=True).stdout + return { + "threads": g("Threads"), "rss_mb": g("VmRSS") // 1024, "fds": len(os.listdir(f"/proc/{pid}/fd")), + "tcp": int(tcp or 0), "pyright": kids.count("pyright"), "kernels": kids.count("hermes_kernel_runner"), + "db_mb": round(os.path.getsize(db_path) / 2**20, 1) if os.path.exists(db_path) else 0, + "httpx_clients": _count_live("Client"), "transports": _count_live("HTTPTransport"), "session_dbs": _count_live("SessionDB"), "live_agents": _count_live("AIAgent"), + } + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--repo", required=True) + ap.add_argument("--children", type=int, default=24) + ap.add_argument("--worktrees", type=int, default=6) + ap.add_argument("--label", default="") + ap.add_argument("--out", default="") + ap.add_argument("--compare", nargs=2) + ap.add_argument("--reply-kb", type=int, default=0, help="pad each child's final reply to N KB (transcript-size realism)") + a = ap.parse_args() + if a.compare: + b, c = (json.load(open(p, encoding="utf-8")) for p in a.compare) + print(f"| metric | {b['label']} | {c['label']} | delta |\n|---|---|---|---|") + for k in ("threads", "rss_mb", "fds", "tcp", "pyright", "kernels", "db_mb", "httpx_clients", "transports", "session_dbs"): + bv, cv = b["peak"][k], c["peak"][k] + print(f"| {k} (peak) | {bv} | {cv} | {cv - bv:+} |") + for k in ("rss_mb", "live_agents", "db_mb", "threads"): + bv, cv = b["after"].get(k), c["after"].get(k) + if bv is not None and cv is not None: + print(f"| {k} (after, children done) | {bv} | {cv} | {cv - bv:+} |") + print(f"| wall_s | {b['wall_s']} | {c['wall_s']} | {c['wall_s'] - b['wall_s']:+.1f} |") + return + + _REPLY_KB[0] = a.reply_kb + home = tempfile.mkdtemp(prefix="hermes_bench_home_") + os.environ["HERMES_HOME"] = home + os.environ["TERMINAL_ENV"] = "local" + os.environ.pop("OPENROUTER_API_KEY", None) + sys.path.insert(0, a.repo) + os.chdir(a.repo) + pyright = shutil.which("pyright-langserver", path=os.path.expanduser("~/.hermes/lsp/bin") + os.pathsep + os.environ.get("PATH", "")) + with open(os.path.join(home, "config.yaml"), "w", encoding="utf-8") as f: + f.write("lsp:\n enabled: true\n wait_timeout: 5.0\n install_strategy: manual\n") + if pyright: + f.write(f" servers:\n pyright:\n command: [{json.dumps(pyright)}, \"--stdio\"]\n") + f.write("delegation:\n max_concurrent_children: 64\n subagent_auto_approve: true\n") + + # W git worktrees, each a real python project (pyproject + package) so pyright roots resolve. + wts = [] + base = tempfile.mkdtemp(prefix="hermes_bench_wt_") + for w in range(a.worktrees): + d = os.path.join(base, f"wt{w}") + os.makedirs(os.path.join(d, "hermes_cli")) + subprocess.run(["git", "init", "-q", d], check=True) + open(os.path.join(d, "pyproject.toml"), "w", encoding="utf-8").write("[project]\nname='b'\n") + open(os.path.join(d, "hermes_cli", "__init__.py"), "w", encoding="utf-8").write("") + wts.append(d) + + srv = _serve() + port = srv.server_address[1] + from run_agent import AIAgent + from tools import delegate_tool + + from hermes_state import SessionDB + db_path = os.path.join(home, "state.db") + from pathlib import Path + session_db = SessionDB(db_path=Path(db_path)) + parent = AIAgent(api_key="bench", base_url=f"http://127.0.0.1:{port}/v1", model="bench-model", + quiet_mode=True, skip_context_files=True, skip_memory=True, + enabled_toolsets=["delegation", "file", "code_execution"], + session_db=session_db, session_id="bench-root") + # Children reference parent_session_id; the parent row is normally created + # lazily on the parent's first turn, which this harness never runs. + parent._ensure_db_session() + pid = os.getpid() + before = _snap(pid, db_path) + peak = dict(before) + stop = threading.Event() + + def sampler(): + while not stop.wait(0.5): + s = _snap(pid, db_path) + for k, v in s.items(): + peak[k] = max(peak[k], v) + threading.Thread(target=sampler, daemon=True).start() + + tasks = [{"goal": json.dumps({"file": os.path.join(wts[i % len(wts)], "hermes_cli", f"bench_{i}.py")}), + "context": "bench"} for i in range(a.children)] + t0 = time.monotonic() + res = delegate_tool.delegate_task(tasks=tasks, parent_agent=parent, background=False) + wall = round(time.monotonic() - t0, 1) + if os.environ.get("BENCH_DEBUG"): + sys.__stderr__.write(str(res)[:3000] + "\n") + time.sleep(2.0) + stop.set() + import gc + gc.collect() + after = _snap(pid, db_path) + try: + parsed = json.loads(res) + items = parsed if isinstance(parsed, list) else parsed.get("results") or parsed.get("tasks") or [] + ok = sum(1 for r in items if str(r.get("status", "")) in ("completed", "success")) + except Exception: + ok = None + try: + session_db.checkpoint() if hasattr(session_db, "checkpoint") else None + except Exception: + pass + out = {"label": a.label, "children": a.children, "worktrees": a.worktrees, "ok": ok, "wall_s": wall, + "before": before, "peak": peak, "after": after} + sys.__stderr__.write("BENCH " + json.dumps(out) + "\n"); sys.__stderr__.flush() + if a.out: + open(a.out, "w", encoding="utf-8").write(json.dumps(out, indent=1)) + try: + from agent.lsp import shutdown_service + shutdown_service() + from tools.code_kernel import shutdown_all_kernels + shutdown_all_kernels() + except Exception: + pass + shutil.rmtree(base, ignore_errors=True) + os._exit(0) + + +if __name__ == "__main__": + main() diff --git a/gateway/authz_mixin.py b/gateway/authz_mixin.py index e80435d982..b4b42f7176 100644 --- a/gateway/authz_mixin.py +++ b/gateway/authz_mixin.py @@ -204,24 +204,39 @@ class GatewayAuthorizationMixin: ``self.adapters``. ``SessionSource.profile`` selects which map to consult. When a stamped profile has its own adapter registry entry, the default profile's same-platform adapter must not be consulted as a fallback. + + Consult ``_profile_adapters`` *before* comparing against + ``_active_profile_name()``. Multiplex turns wrap authz in + ``_profile_runtime_scope``, which overrides ``HERMES_HOME`` so + ``get_active_profile_name()`` returns the secondary profile for the + duration of the turn. Treating that scoped name as "primary" would + look up ``self.adapters`` (empty for secondary-only platforms like + A2A) and default-deny an already-authenticated peer. """ if not platform: return None profile_name = (profile or "").strip() or None if profile_name and profile_name != "default": - active_profile = None - active_profile_fn = getattr(self, "_active_profile_name", None) - if callable(active_profile_fn): - try: - active_profile = active_profile_fn() - except Exception: - active_profile = None - if profile_name == active_profile: - adapters = getattr(self, "adapters", None) or {} - return adapters.get(platform) profile_adapters = getattr(self, "_profile_adapters", None) or {} if profile_name in profile_adapters: return profile_adapters[profile_name].get(platform) + # Adapter ownership is process-wide: only the profile the gateway + # was LAUNCHED as owns ``self.adapters``. ``_active_profile_name()`` + # reads the per-turn HERMES_HOME override, so inside a secondary + # profile's ``_profile_runtime_scope`` it reports that secondary + # and would hand it the default bot. Compare against the identity + # captured at construction instead. + primary_profile = getattr(self, "_primary_profile_name", None) + if not primary_profile: + active_profile_fn = getattr(self, "_active_profile_name", None) + if callable(active_profile_fn): + try: + primary_profile = active_profile_fn() + except Exception: + primary_profile = None + if profile_name == primary_profile: + adapters = getattr(self, "adapters", None) or {} + return adapters.get(platform) # Fail closed: a stamped secondary profile with no registry entry # (e.g. its adapter failed to connect) must NOT fall back to the # default profile's adapter — that sends replies out the wrong bot. diff --git a/gateway/channel_directory.py b/gateway/channel_directory.py index 08c5bee54a..f46317ed2d 100644 --- a/gateway/channel_directory.py +++ b/gateway/channel_directory.py @@ -11,6 +11,7 @@ import json import logging import time from datetime import datetime +from pathlib import Path from typing import Any, Dict, List, Optional from hermes_cli.config import get_hermes_home @@ -18,7 +19,12 @@ from utils import atomic_json_write logger = logging.getLogger(__name__) -DIRECTORY_PATH = get_hermes_home() / "channel_directory.json" +# Resolved lazily (see ``_directory_path``): a multiplexed gateway serves +# several profile homes from one process, so an import-time constant would pin +# every profile's directory to whichever home imported this module first. +# ``DIRECTORY_PATH`` / ``CHANNEL_ALIASES_PATH`` stay as explicit overrides +# (tests patch them); ``None`` means "resolve from the current home". +DIRECTORY_PATH: Optional[Path] = None # Throttle window for repeated Slack channel-directory refresh failures. # The directory rebuilds on a timer, so a persistent workspace error (e.g. # missing scope, revoked token) would otherwise re-log the same warning on @@ -33,14 +39,23 @@ _slack_directory_warning_last: Dict[tuple[str, str], float] = {} # on every build AND every load, giving durable human-friendly names (and # letting you pre-name a chat before it has produced any traffic). # Format: {"": {"": "", ...}, ...} -CHANNEL_ALIASES_PATH = get_hermes_home() / "channel_aliases.json" +CHANNEL_ALIASES_PATH: Optional[Path] = None + + +def _directory_path() -> Path: + return DIRECTORY_PATH or get_hermes_home() / "channel_directory.json" + + +def _aliases_path() -> Path: + return CHANNEL_ALIASES_PATH or get_hermes_home() / "channel_aliases.json" def _load_channel_aliases() -> Dict[str, Dict[str, str]]: - if not CHANNEL_ALIASES_PATH.exists(): + aliases_path = _aliases_path() + if not aliases_path.exists(): return {} try: - with open(CHANNEL_ALIASES_PATH, encoding="utf-8-sig") as f: + with open(aliases_path, encoding="utf-8-sig") as f: data = json.load(f) return data if isinstance(data, dict) else {} except Exception: @@ -143,7 +158,8 @@ async def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]: """ Build a channel directory from connected platform adapters and session data. - Returns the directory dict and writes it to DIRECTORY_PATH. + Returns the directory dict and writes it to the current home's + ``channel_directory.json``. """ from gateway.config import Platform @@ -206,7 +222,7 @@ async def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]: } try: - await asyncio.to_thread(atomic_json_write, DIRECTORY_PATH, directory) + await asyncio.to_thread(atomic_json_write, _directory_path(), directory) except Exception as e: logger.warning("Channel directory: failed to write: %s", e) @@ -525,12 +541,13 @@ def _build_from_sessions_json(platform_name: str) -> List[Dict[str, str]]: def load_directory() -> Dict[str, Any]: """Load the cached channel directory from disk.""" - if not DIRECTORY_PATH.exists(): + directory_path = _directory_path() + if not directory_path.exists(): base = {"updated_at": None, "platforms": {}} _apply_channel_aliases(base["platforms"]) return base try: - with open(DIRECTORY_PATH, encoding="utf-8-sig") as f: + with open(directory_path, encoding="utf-8-sig") as f: data = json.load(f) # Re-apply aliases on read so friendly names take effect immediately, # even between timed rebuilds and for brand-new alias entries. diff --git a/gateway/config.py b/gateway/config.py index c917481de4..9f844b9529 100644 --- a/gateway/config.py +++ b/gateway/config.py @@ -444,6 +444,7 @@ PORT_BINDING_PLATFORM_VALUES = frozenset({ "sms", "whatsapp_cloud", "line", + "teams", }) # Platforms whose port-binding status depends on connection mode. Feishu in @@ -1980,6 +1981,64 @@ def _validate_gateway_config(config: "GatewayConfig") -> None: pconfig.enabled = False +# Platforms for which the "explicitly disabled in config.yaml, but credentials +# are present in the environment" WARNING has already been emitted in this +# process. The gateway reloads its config on every turn (and other surfaces +# call load_gateway_config() repeatedly), so the notice is one-time per +# platform per process — loud once at startup, never a per-turn drumbeat. +_EXPLICIT_DISABLE_WARNED: set = set() + + +# Env var(s) whose presence drives each platform's env-enable branch, for the +# explicit-disable WARNING below. Kept next to the branches that read them. +_ENV_ENABLE_CREDENTIALS: dict = { + Platform.TELEGRAM: ("TELEGRAM_BOT_TOKEN",), + Platform.DISCORD: ("DISCORD_BOT_TOKEN",), + Platform.SLACK: ("SLACK_BOT_TOKEN",), + Platform.WHATSAPP_CLOUD: ("WHATSAPP_CLOUD_PHONE_NUMBER_ID", "WHATSAPP_CLOUD_ACCESS_TOKEN"), + Platform.SIGNAL: ("SIGNAL_HTTP_URL",), + Platform.MATTERMOST: ("MATTERMOST_TOKEN",), + Platform.MATRIX: ("MATRIX_ACCESS_TOKEN", "MATRIX_PASSWORD"), + Platform.HOMEASSISTANT: ("HASS_TOKEN",), + Platform.EMAIL: ("EMAIL_ADDRESS", "EMAIL_PASSWORD", "EMAIL_IMAP_HOST", "EMAIL_SMTP_HOST"), + Platform.SMS: ("TWILIO_ACCOUNT_SID",), + Platform.DINGTALK: ("DINGTALK_CLIENT_ID", "DINGTALK_CLIENT_SECRET"), + Platform.FEISHU: ("FEISHU_APP_ID", "FEISHU_APP_SECRET"), + Platform.WECOM: ("WECOM_BOT_ID", "WECOM_SECRET"), + Platform.WECOM_CALLBACK: ("WECOM_CALLBACK_CORP_ID", "WECOM_CALLBACK_CORP_SECRET"), + Platform.WEIXIN: ("WEIXIN_TOKEN", "WEIXIN_ACCOUNT_ID"), + Platform.BLUEBUBBLES: ("BLUEBUBBLES_SERVER_URL", "BLUEBUBBLES_PASSWORD"), + Platform.QQBOT: ("QQ_APP_ID", "QQ_CLIENT_SECRET"), + Platform.YUANBAO: ("YUANBAO_APP_ID", "YUANBAO_APP_SECRET"), + Platform.RELAY: ("GATEWAY_RELAY_URL",), +} + + +def _warn_explicit_disable_beats_env(platform: Platform) -> None: + """One-time WARNING: ``platforms..enabled: false`` wins over env creds. + + Until #48820 the credential-presence branches force-enabled twelve + platforms regardless of an explicit ``enabled: false`` in config.yaml, so + users who relied on "creds in .env = platform on" would see it go dark + after the fix with no explanation. Name the platform, the config key that + is winning, and the env var(s) that used to override it. + """ + if platform in _EXPLICIT_DISABLE_WARNED: + return + _EXPLICIT_DISABLE_WARNED.add(platform) + names = _ENV_ENABLE_CREDENTIALS.get(platform) or () + present = [n for n in names if (os.environ.get(n) or "").strip()] + creds = ", ".join(present or names) or "its credentials" + logger.warning( + "Platform '%s' is explicitly disabled by platforms.%s.enabled: false in " + "config.yaml, so the credentials found in the environment (%s) will NOT " + "start its adapter. Environment credentials no longer override an " + "explicit disable. Remove the key or set platforms.%s.enabled: true to " + "turn it back on.", + platform.value, platform.value, creds, platform.value, + ) + + def _apply_env_overrides(config: GatewayConfig) -> None: """Apply environment variable overrides to config.""" getenv = _getenv_str @@ -1998,8 +2057,13 @@ def _apply_env_overrides(config: GatewayConfig) -> None: # flag is cleared once for all platforms in the final cleanup at the # end of _apply_env_overrides. enabled_was_explicit = bool(platform_config.extra.get("_enabled_explicit", False)) - if not platform_config.enabled and not enabled_was_explicit: - platform_config.enabled = True + if not platform_config.enabled: + if enabled_was_explicit: + # Credentials are present (that is why we are here) but the + # user said no in config.yaml. Say so once (#48820). + _warn_explicit_disable_beats_env(platform) + else: + platform_config.enabled = True return platform_config # Telegram @@ -2083,9 +2147,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: whatsapp_cloud_phone_id = getenv("WHATSAPP_CLOUD_PHONE_NUMBER_ID") whatsapp_cloud_token = getenv("WHATSAPP_CLOUD_ACCESS_TOKEN") if whatsapp_cloud_phone_id and whatsapp_cloud_token: - if Platform.WHATSAPP_CLOUD not in config.platforms: - config.platforms[Platform.WHATSAPP_CLOUD] = PlatformConfig() - config.platforms[Platform.WHATSAPP_CLOUD].enabled = True + # Honors an explicit ``platforms.whatsapp_cloud.enabled: false`` (#48820). + _enable_from_env(Platform.WHATSAPP_CLOUD) config.platforms[Platform.WHATSAPP_CLOUD].extra.update({ "phone_number_id": whatsapp_cloud_phone_id, "access_token": whatsapp_cloud_token, @@ -2151,6 +2214,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: # turn an env-token setup into a disabled platform. Only an # explicit slack.enabled/platforms.slack.enabled false should. slack_config.enabled = True + elif not slack_config.enabled: + _warn_explicit_disable_beats_env(Platform.SLACK) # If yaml config exists, respect its enabled flag (don't override # explicit enabled: false). Token is still stored so skills that # send Slack messages can use it without activating the gateway adapter. @@ -2248,9 +2313,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: # Home Assistant hass_token = getenv("HASS_TOKEN") if hass_token: - if Platform.HOMEASSISTANT not in config.platforms: - config.platforms[Platform.HOMEASSISTANT] = PlatformConfig() - config.platforms[Platform.HOMEASSISTANT].enabled = True + # Honors an explicit ``platforms.homeassistant.enabled: false`` (#48820). + _enable_from_env(Platform.HOMEASSISTANT) config.platforms[Platform.HOMEASSISTANT].token = hass_token hass_url = getenv("HASS_URL") if hass_url: @@ -2262,9 +2326,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: email_imap = getenv("EMAIL_IMAP_HOST") email_smtp = getenv("EMAIL_SMTP_HOST") if all([email_addr, email_pwd, email_imap, email_smtp]): - if Platform.EMAIL not in config.platforms: - config.platforms[Platform.EMAIL] = PlatformConfig() - config.platforms[Platform.EMAIL].enabled = True + # Honors an explicit ``platforms.email.enabled: false`` (#48820). + _enable_from_env(Platform.EMAIL) config.platforms[Platform.EMAIL].extra.update({ "address": email_addr, "imap_host": email_imap, @@ -2282,9 +2345,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: # SMS (Twilio) twilio_sid = getenv("TWILIO_ACCOUNT_SID") if twilio_sid: - if Platform.SMS not in config.platforms: - config.platforms[Platform.SMS] = PlatformConfig() - config.platforms[Platform.SMS].enabled = True + # Honors an explicit ``platforms.sms.enabled: false`` (#48820). + _enable_from_env(Platform.SMS) config.platforms[Platform.SMS].api_key = getenv("TWILIO_AUTH_TOKEN", "") sms_home = getenv("SMS_HOME_CHANNEL") if sms_home and Platform.SMS in config.platforms: @@ -2345,7 +2407,21 @@ def _apply_env_overrides(config: GatewayConfig) -> None: if webhook_enabled: if Platform.WEBHOOK not in config.platforms: config.platforms[Platform.WEBHOOK] = PlatformConfig() - config.platforms[Platform.WEBHOOK].enabled = True + # Honor an explicit ``enabled: false`` in config.yaml (flagged by + # ``_enabled_explicit``). In multiplex mode a secondary profile's + # config.yaml pins ``platforms.webhook.enabled: false`` so it shares + # the default profile's listener instead of binding its own port. That + # profile may still carry ``WEBHOOK_ENABLED`` in its own .env (or the + # process env, single-profile); without this guard the env var would + # force-enable the listener and trip the MultiplexConfigError check. + # Pop (don't read) the marker — the webhook branch is terminal (no + # later registry pass re-enables it), matching the api_server branch + # above. + webhook_explicit = config.platforms[Platform.WEBHOOK].extra.pop( + "_enabled_explicit", False + ) + if not webhook_explicit or config.platforms[Platform.WEBHOOK].enabled: + config.platforms[Platform.WEBHOOK].enabled = True if webhook_port: try: config.platforms[Platform.WEBHOOK].extra["port"] = int(webhook_port) @@ -2373,7 +2449,13 @@ def _apply_env_overrides(config: GatewayConfig) -> None: if Platform.MSGRAPH_WEBHOOK not in config.platforms: config.platforms[Platform.MSGRAPH_WEBHOOK] = PlatformConfig() if msgraph_webhook_enabled: - config.platforms[Platform.MSGRAPH_WEBHOOK].enabled = True + # Same explicit-disable guard as the webhook branch above (#85637). + # READ (don't pop) the marker here: the relay-exclusive pass below + # still consults it, and the end-of-function scrub removes it for + # every platform. + msgraph_cfg = config.platforms[Platform.MSGRAPH_WEBHOOK] + if not msgraph_cfg.extra.get("_enabled_explicit", False) or msgraph_cfg.enabled: + msgraph_cfg.enabled = True if msgraph_webhook_port: try: config.platforms[Platform.MSGRAPH_WEBHOOK].extra["port"] = int( @@ -2410,9 +2492,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: dingtalk_client_id = getenv("DINGTALK_CLIENT_ID") dingtalk_client_secret = getenv("DINGTALK_CLIENT_SECRET") if dingtalk_client_id and dingtalk_client_secret: - if Platform.DINGTALK not in config.platforms: - config.platforms[Platform.DINGTALK] = PlatformConfig() - config.platforms[Platform.DINGTALK].enabled = True + # Honors an explicit ``platforms.dingtalk.enabled: false`` (#48820). + _enable_from_env(Platform.DINGTALK) config.platforms[Platform.DINGTALK].extra.update({ "client_id": dingtalk_client_id, "client_secret": dingtalk_client_secret, @@ -2430,9 +2511,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: feishu_app_id = getenv("FEISHU_APP_ID") feishu_app_secret = getenv("FEISHU_APP_SECRET") if feishu_app_id and feishu_app_secret: - if Platform.FEISHU not in config.platforms: - config.platforms[Platform.FEISHU] = PlatformConfig() - config.platforms[Platform.FEISHU].enabled = True + # Honors an explicit ``platforms.feishu.enabled: false`` (#48820). + _enable_from_env(Platform.FEISHU) config.platforms[Platform.FEISHU].extra.update({ "app_id": feishu_app_id, "app_secret": feishu_app_secret, @@ -2458,9 +2538,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: wecom_bot_id = getenv("WECOM_BOT_ID") wecom_secret = getenv("WECOM_SECRET") if wecom_bot_id and wecom_secret: - if Platform.WECOM not in config.platforms: - config.platforms[Platform.WECOM] = PlatformConfig() - config.platforms[Platform.WECOM].enabled = True + # Honors an explicit ``platforms.wecom.enabled: false`` (#48820). + _enable_from_env(Platform.WECOM) config.platforms[Platform.WECOM].extra.update({ "bot_id": wecom_bot_id, "secret": wecom_secret, @@ -2481,9 +2560,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: wecom_callback_corp_id = getenv("WECOM_CALLBACK_CORP_ID") wecom_callback_corp_secret = getenv("WECOM_CALLBACK_CORP_SECRET") if wecom_callback_corp_id and wecom_callback_corp_secret: - if Platform.WECOM_CALLBACK not in config.platforms: - config.platforms[Platform.WECOM_CALLBACK] = PlatformConfig() - config.platforms[Platform.WECOM_CALLBACK].enabled = True + # Honors an explicit ``platforms.wecom_callback.enabled: false`` (#48820). + _enable_from_env(Platform.WECOM_CALLBACK) config.platforms[Platform.WECOM_CALLBACK].extra.update({ "corp_id": wecom_callback_corp_id, "corp_secret": wecom_callback_corp_secret, @@ -2501,9 +2579,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: weixin_token = getenv("WEIXIN_TOKEN") weixin_account_id = getenv("WEIXIN_ACCOUNT_ID") if weixin_token or weixin_account_id: - if Platform.WEIXIN not in config.platforms: - config.platforms[Platform.WEIXIN] = PlatformConfig() - config.platforms[Platform.WEIXIN].enabled = True + # Honors an explicit ``platforms.weixin.enabled: false`` (#48820). + _enable_from_env(Platform.WEIXIN) if weixin_token: config.platforms[Platform.WEIXIN].token = weixin_token extra = config.platforms[Platform.WEIXIN].extra @@ -2543,9 +2620,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: bluebubbles_server_url = getenv("BLUEBUBBLES_SERVER_URL") bluebubbles_password = getenv("BLUEBUBBLES_PASSWORD") if bluebubbles_server_url and bluebubbles_password: - if Platform.BLUEBUBBLES not in config.platforms: - config.platforms[Platform.BLUEBUBBLES] = PlatformConfig() - config.platforms[Platform.BLUEBUBBLES].enabled = True + # Honors an explicit ``platforms.bluebubbles.enabled: false`` (#48820). + _enable_from_env(Platform.BLUEBUBBLES) config.platforms[Platform.BLUEBUBBLES].extra.update({ "server_url": bluebubbles_server_url.rstrip("/"), "password": bluebubbles_password, @@ -2583,9 +2659,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: qq_app_id = getenv("QQ_APP_ID") qq_client_secret = getenv("QQ_CLIENT_SECRET") if qq_app_id or qq_client_secret: - if Platform.QQBOT not in config.platforms: - config.platforms[Platform.QQBOT] = PlatformConfig() - config.platforms[Platform.QQBOT].enabled = True + # Honors an explicit ``platforms.qqbot.enabled: false`` (#48820). + _enable_from_env(Platform.QQBOT) extra = config.platforms[Platform.QQBOT].extra if qq_app_id: extra["app_id"] = qq_app_id @@ -2625,9 +2700,8 @@ def _apply_env_overrides(config: GatewayConfig) -> None: yuanbao_app_id = getenv("YUANBAO_APP_ID") or getenv("YUANBAO_APP_KEY") yuanbao_app_secret = getenv("YUANBAO_APP_SECRET") if yuanbao_app_id and yuanbao_app_secret: - if Platform.YUANBAO not in config.platforms: - config.platforms[Platform.YUANBAO] = PlatformConfig() - config.platforms[Platform.YUANBAO].enabled = True + # Honors an explicit ``platforms.yuanbao.enabled: false`` (#48820). + _enable_from_env(Platform.YUANBAO) extra = config.platforms[Platform.YUANBAO].extra extra["app_id"] = yuanbao_app_id extra["app_secret"] = yuanbao_app_secret diff --git a/gateway/delivery.py b/gateway/delivery.py index fa43db6d0f..f63c769aeb 100644 --- a/gateway/delivery.py +++ b/gateway/delivery.py @@ -530,7 +530,18 @@ class DeliveryRouter: # platform adapter regardless of which persona's prompt failed. # Local/file delivery (_deliver_local) is a separate path and is never # filtered — saved silence has no loop risk. - if self._filter_silence_narration_enabled() and _is_silence_narration(content): + # Cron output is an ARTIFACT, not model chatter: a job whose brief is + # legitimately terse ("...", a single 🔇 from a script) has no bot-to-bot + # mirror loop to guard against, and dropping it here while returning + # {"success": True} is exactly how a cron was logged as delivered with + # nothing on the wire (#77763). Cron sends carry job_id in metadata; + # every other caller keeps the filter unchanged. + is_cron_artifact = "job_id" in (metadata or {}) + if ( + self._filter_silence_narration_enabled() + and not is_cron_artifact + and _is_silence_narration(content) + ): logger.warning( "Dropped silence-narration outbound to %s (chat=%s): %r", target.platform.value, diff --git a/gateway/kanban_watchers.py b/gateway/kanban_watchers.py index d958ceb806..8091ccb5c6 100644 --- a/gateway/kanban_watchers.py +++ b/gateway/kanban_watchers.py @@ -422,7 +422,7 @@ class GatewayKanbanWatchersMixin: if _gc_due: # Hourly (plus once at startup) stale-sub GC: # drop subscriptions for tasks that have been - # ``done`` untouched past the retention + # ``done``/``blocked`` untouched past the retention # window. Best-effort — a failed sweep never # blocks delivery; the next hourly gate # retries it. @@ -433,7 +433,7 @@ class GatewayKanbanWatchersMixin: ) if _purged: logger.info( - "kanban notifier: purged %d stale done-task subscription(s) on board %s (retention %dd)", + "kanban notifier: purged %d stale done/blocked-task subscription(s) on board %s (retention %dd)", _purged, slug, _gc_retention_days, ) except Exception as _gc_exc: diff --git a/gateway/pairing.py b/gateway/pairing.py index 692fa71948..d4c1a2879d 100644 --- a/gateway/pairing.py +++ b/gateway/pairing.py @@ -185,10 +185,11 @@ def _read_allowlist_env(env_var: str) -> str: borrowing the process value. Unscoped callers (single-profile CLI / admin endpoints) keep the legacy ``os.getenv`` read. - TODO(profile-secrets): the grant mirror below still WRITES through - ``hermes_cli.config.save_env_value`` / ``remove_env_value``, which target - the root ``.env`` — those writes need a profile-aware counterpart before - pairing grants can be mirrored correctly under multiplexing. + The grant mirror below writes through ``hermes_cli.config.save_env_value`` + / ``remove_env_value``: the file target is the active profile's ``.env`` + (``get_env_path()`` honors the profile-home override) and, under + multiplexing, the in-process publish updates the installed scope mapping + rather than the shared ``os.environ`` (#88441). """ try: from agent.secret_scope import UnscopedSecretError, get_secret diff --git a/gateway/platform_registry.py b/gateway/platform_registry.py index 402bf626c2..942df8d51a 100644 --- a/gateway/platform_registry.py +++ b/gateway/platform_registry.py @@ -4,10 +4,11 @@ Platform Adapter Registry Allows platform adapters (built-in and plugin) to self-register so the gateway can discover and instantiate them without hardcoded if/elif chains. -Built-in adapters continue to use the existing if/elif in _create_adapter() +Built-in adapters continue to use the existing if/elif in _instantiate_adapter() for now. Plugin adapters register here via PluginContext.register_platform() and are looked up first -- if nothing is found the gateway falls through to -the legacy code path. +the legacy code path. GatewayRunner._create_adapter() wraps both paths and +binds every successful adapter to its runner. Usage (plugin side): diff --git a/gateway/platforms/ADDING_A_PLATFORM.md b/gateway/platforms/ADDING_A_PLATFORM.md index 50618aa02f..9f989133f1 100644 --- a/gateway/platforms/ADDING_A_PLATFORM.md +++ b/gateway/platforms/ADDING_A_PLATFORM.md @@ -174,7 +174,7 @@ Update `get_connected_platforms()` if your platform doesn't use token/api_key ## 3. Adapter Factory (`gateway/run.py`) -Add to `_create_adapter()`: +Add to `_instantiate_adapter()`: ```python elif platform == Platform.YOUR_PLATFORM: @@ -185,6 +185,11 @@ elif platform == Platform.YOUR_PLATFORM: return YourAdapter(config) ``` +`_create_adapter()` wraps this factory and binds every successful adapter to +its `GatewayRunner`. Do not construct platform adapters in lifecycle call sites; +startup and reconnect must keep using the wrapper so profile routing is wired +before `connect()`. + --- ## 4. Authorization Maps (`gateway/run.py`) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 004ef1a9e1..9fc9e40c01 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -4906,6 +4906,11 @@ class APIServerAdapter(BasePlatformAdapter): queue: "asyncio.Queue[Optional[tuple[str, Dict[str, Any]]]]" = asyncio.Queue() message_id = f"msg_{uuid.uuid4().hex}" run_id = f"run_{uuid.uuid4().hex}" + # Claim ownership while still inside the request's profile scope, + # before any run-keyed state exists — the same rule as /v1/runs, so + # /v1/runs/{id}* control of this turn is confined to the profile + # that started it (#93689). + self._run_owners[run_id] = self._run_idempotency_scope(request) self._set_run_status( run_id, "queued", @@ -4993,12 +4998,13 @@ class APIServerAdapter(BasePlatformAdapter): else "" ), ) + is_partial = bool(result.get("partial")) if isinstance(result, dict) else False await queue.put(_event_payload("assistant.completed", { "session_id": effective_session_id, "message_id": message_id, "content": final_response, "completed": True, - "partial": False, + "partial": is_partial, "interrupted": False, "runtime": effective_runtime, })) @@ -5040,6 +5046,7 @@ class APIServerAdapter(BasePlatformAdapter): await queue.put(_event_payload("error", {"message": _redact_api_error_text(exc)})) finally: self._active_run_agents.pop(run_id, None) + self._release_run_owner_if_forgotten(run_id) await queue.put(_event_payload("done", {})) await queue.put(None) @@ -7774,6 +7781,9 @@ class APIServerAdapter(BasePlatformAdapter): def _request_owns_run(self, request: "web.Request", run_id: str) -> bool: return _api_runs._request_owns_run(self, request, run_id) + def _release_run_owner_if_forgotten(self, run_id: str) -> None: + _api_runs._release_run_owner_if_forgotten(self, run_id) + async def _handle_get_run(self, request: "web.Request") -> "web.Response": """GET /v1/runs/{run_id} — return pollable run status for external UIs.""" return await _api_runs._handle_get_run( diff --git a/gateway/platforms/api_server_runs.py b/gateway/platforms/api_server_runs.py index 04b21fefd3..758d847e45 100644 --- a/gateway/platforms/api_server_runs.py +++ b/gateway/platforms/api_server_runs.py @@ -225,6 +225,7 @@ def _make_run_event_callback( "task_index", "subagent_id", "child_session_id", + "delegation_id", "parent_id", "depth", "model", @@ -1016,6 +1017,7 @@ async def _handle_runs( self._active_run_tasks.pop(run_id, None) self._run_approval_sessions.pop(run_id, None) self._stopping_run_ids.discard(run_id) + self._release_run_owner_if_forgotten(run_id) self._activate_admitted_request() task = asyncio.create_task(_run_and_close()) @@ -1037,25 +1039,36 @@ async def _handle_runs( ) -def _request_owns_run(self, request: "web.Request", run_id: str) -> bool: - scope = self._run_idempotency_scope(request) - owner = self._run_owners.get(run_id) - if self._room_grant_token(request): - return owner == scope or ( - owner is None - and self._run_idempotency_store.owns_run(scope, run_id) - ) - if owner is None and ( +def _release_run_owner_if_forgotten(self, run_id: str) -> None: + """Drop the owner stamp only once nothing keyed by *run_id* survives. + + Ownership must outlive every surface it protects (statuses, live + agent/task refs, SSE transport, approval sessions), which are retired + on different clocks. Releasing earlier would leave a stateful run + without an owner, which ``_request_owns_run`` treats as fail-closed. + """ + if ( run_id in self._run_statuses or run_id in self._active_run_agents or run_id in self._active_run_tasks + or run_id in self._run_streams + or run_id in self._run_approval_sessions ): - # Backward compatibility for statuses created by older/in-process - # integrations before ownership tracking was introduced. - return True - return owner == scope or ( - owner is None and self._run_idempotency_store.owns_run(scope, run_id) - ) + return + self._run_owners.pop(run_id, None) + + +def _request_owns_run(self, request: "web.Request", run_id: str) -> bool: + scope = self._run_idempotency_scope(request) + owner = self._run_owners.get(run_id) + if owner is not None: + return owner == scope + # No in-memory owner: only a durable record under the caller's own scope + # admits it. Run state that exists without an owner stamp is an + # unanswered authorization question, not a run anyone may control — + # under gateway.multiplex_profiles every served profile holds a valid + # key, so admitting it would make the boundary allow-all (#93689). + return self._run_idempotency_store.owns_run(scope, run_id) async def _handle_get_run( @@ -1153,6 +1166,7 @@ async def _handle_run_events( self._run_stream_subscribers.discard(run_id) self._run_streams.pop(run_id, None) self._run_streams_created.pop(run_id, None) + self._release_run_owner_if_forgotten(run_id) return response @@ -1484,6 +1498,7 @@ def _sweep_orphaned_runs_once(self, now: Optional[float] = None) -> None: self._active_run_tasks.pop(run_id, None) self._run_approval_sessions.pop(run_id, None) self._stopping_run_ids.discard(run_id) + self._release_run_owner_if_forgotten(run_id) stale_statuses = [ run_id @@ -1494,4 +1509,4 @@ def _sweep_orphaned_runs_once(self, now: Optional[float] = None) -> None: for run_id in stale_statuses: self._run_statuses.pop(run_id, None) self._run_idempotency_ids.discard(run_id) - self._run_owners.pop(run_id, None) + self._release_run_owner_if_forgotten(run_id) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index 3ad1ea18d4..d2ac5996e6 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -161,6 +161,12 @@ def _thread_metadata_for_source(source, reply_to_message_id: str | None = None) anchor = reply_to_message_id or getattr(source, "message_id", None) if anchor is not None: metadata["telegram_reply_to_message_id"] = str(anchor) + # Routed Hermes profile for shared state.db namespaces (topic bindings + # under multiplex / profile_routes). Outbound prune paths must not + # assume the transport adapter's static profile stamp. + profile = str(getattr(source, "profile", None) or "").strip() + if profile: + metadata["hermes_profile"] = profile return metadata @@ -489,7 +495,8 @@ def resolve_proxy_url( 2. macOS system proxy via ``scutil --proxy`` (auto-detect) Returns *None* if no proxy is found, or if NO_PROXY/no_proxy matches one - of ``target_hosts``. + of ``target_hosts``. Steps 1-2 are skipped when ``gateway.trust_env`` is + false in config.yaml (see :func:`gateway_trust_env`). """ if platform_env_var: value = (os.environ.get(platform_env_var) or "").strip() @@ -497,6 +504,10 @@ def resolve_proxy_url( if should_bypass_proxy(target_hosts): return None return normalize_proxy_url(value) + if not gateway_trust_env(): + # gateway.trust_env: false — ignore inherited generic proxy env and + # system proxy; only the explicit per-platform var above is honored. + return None for key in ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "https_proxy", "http_proxy", "all_proxy"): value = (os.environ.get(key) or "").strip() @@ -540,6 +551,28 @@ def proxy_kwargs_for_bot(proxy_url: str | None) -> dict: return {"proxy": proxy_url} +def gateway_trust_env() -> bool: + """Return the ``trust_env`` value every gateway ``aiohttp.ClientSession`` uses. + + Reads ``gateway.trust_env`` from config.yaml (default ``True``: honor + ``HTTP_PROXY`` / ``HTTPS_PROXY`` / ``NO_PROXY`` / ``SSL_CERT_FILE`` from the + process environment). Set it to ``false`` when the gateway inherits a + proxy env it should not use — e.g. a Windows Scheduled Task picking up a + Clash/V2Ray ``HTTP_PROXY`` the interactive shell never sees (#48820). + One knob for all platform adapters; fail-open to the default if config + is unreadable. + """ + try: + from hermes_cli.config import load_config_readonly as _load_config + gw = (_load_config() or {}).get("gateway") or {} + except Exception: + return True + value = gw.get("trust_env", True) if isinstance(gw, dict) else True + if isinstance(value, str): + return value.strip().lower() not in {"0", "false", "no", "off"} + return bool(value) if value is not None else True + + def proxy_kwargs_for_aiohttp(proxy_url: str | None) -> tuple[dict, dict]: """Build kwargs for standalone ``aiohttp.ClientSession`` with proxy. @@ -622,7 +655,7 @@ from pathlib import Path as _Path sys.path.insert(0, str(_Path(__file__).resolve().parents[2])) from gateway.config import Platform, PlatformConfig -from gateway.session import SessionSource, build_session_key +from gateway.session import SessionSource, TranscriptReadError, build_session_key from hermes_constants import get_default_hermes_root, get_hermes_dir, get_hermes_home if TYPE_CHECKING: @@ -925,6 +958,11 @@ def cache_image_from_bytes(data: bytes, ext: str = ".jpg") -> str: return str(filepath) +async def cache_image_from_bytes_async(data: bytes, ext: str = ".jpg") -> str: + """Cache image bytes without blocking the caller's event loop.""" + return await asyncio.to_thread(cache_image_from_bytes, data, ext) + + async def cache_image_from_url(url: str, ext: str = ".jpg", retries: int = 2) -> str: """ Download an image from a URL and save it to the local cache. @@ -969,7 +1007,7 @@ async def cache_image_from_url(url: str, ext: str = ".jpg", retries: int = 2) -> content = await _read_httpx_body_with_limit( response, media_type="image", ) - return cache_image_from_bytes(content, ext) + return await cache_image_from_bytes_async(content, ext) except (httpx.TimeoutException, httpx.HTTPStatusError) as exc: if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code < 429: raise @@ -1067,6 +1105,11 @@ def cache_audio_from_bytes(data: bytes, ext: str = ".ogg") -> str: return str(filepath) +async def cache_audio_from_bytes_async(data: bytes, ext: str = ".ogg") -> str: + """Cache audio bytes without blocking the caller's event loop.""" + return await asyncio.to_thread(cache_audio_from_bytes, data, ext) + + async def cache_audio_from_url(url: str, ext: str = ".ogg", retries: int = 2) -> str: """ Download an audio file from a URL and save it to the local cache. @@ -1111,7 +1154,7 @@ async def cache_audio_from_url(url: str, ext: str = ".ogg", retries: int = 2) -> content = await _read_httpx_body_with_limit( response, media_type="audio", ) - return cache_audio_from_bytes(content, ext) + return await cache_audio_from_bytes_async(content, ext) except (httpx.TimeoutException, httpx.HTTPStatusError) as exc: if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code < 429: raise @@ -1174,6 +1217,11 @@ def cache_video_from_bytes(data: bytes, ext: str = ".mp4") -> str: return str(filepath) +async def cache_video_from_bytes_async(data: bytes, ext: str = ".mp4") -> str: + """Cache video bytes without blocking the caller's event loop.""" + return await asyncio.to_thread(cache_video_from_bytes, data, ext) + + def cleanup_video_cache(max_age_hours: int = 24) -> int: """ Delete cached videos older than *max_age_hours*. @@ -1522,6 +1570,25 @@ def _path_is_within(path: Path, root: Path) -> bool: return False +def _tenv(name: str, default: str = "") -> str: + """Scope-aware TERMINAL_* read (tools.terminal_scope.terminal_env). + + Media-path translation runs in the gateway process concurrently for + several profiles; the per-turn terminal scope carries the ACTIVE + profile's terminal settings, while a raw os.getenv would read whatever + profile's config a previous turn pinned into the process env. + + Only an import failure falls back: an active refusal scope must raise — + reconstructing mounts/backends from ambient env under refusal would + rebuild another profile's terminal policy. + """ + try: + from tools.terminal_scope import terminal_env + except ImportError: + return os.getenv(name, default) + return terminal_env(name, default) + + def _parse_docker_volume_mounts() -> List[Tuple[Path, Path]]: """Parse configured Docker volume mounts into ``(host_path, container_path)``. @@ -1530,7 +1597,7 @@ def _parse_docker_volume_mounts() -> List[Tuple[Path, Path]]: Named volumes and non-absolute hosts are skipped because they cannot be resolved on the gateway host for media delivery. """ - raw = os.getenv("TERMINAL_DOCKER_VOLUMES", "").strip() + raw = _tenv("TERMINAL_DOCKER_VOLUMES", "").strip() if not raw: return [] try: @@ -1598,7 +1665,7 @@ def _docker_sandbox_dir_candidates(session_key: str = "") -> List[str]: except Exception: return ["default"] # Explicit trusted-profiles opt-in: one shared container identity. - shared = os.getenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() + shared = _tenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() if shared: candidates.append(sanitize_task_id_for_path(f"shared:{shared}")) try: @@ -1624,9 +1691,9 @@ def _default_docker_workspace_host_roots(session_key: str = "") -> List[Path]: actually resolves — the profile sandbox dir existing does not mean the file lives there when it was produced in a legacy per-session container. """ - if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": + if _tenv("TERMINAL_ENV", "").strip().lower() != "docker": return [] - if os.getenv("TERMINAL_CONTAINER_PERSISTENT", "true").strip().lower() not in { + if _tenv("TERMINAL_CONTAINER_PERSISTENT", "true").strip().lower() not in { "1", "true", "yes", @@ -1634,13 +1701,13 @@ def _default_docker_workspace_host_roots(session_key: str = "") -> List[Path]: }: return [] # Explicit cwd mount takes over /workspace when enabled. - if os.getenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").strip().lower() in { + if _tenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").strip().lower() in { "1", "true", "yes", "on", }: - cwd = os.getenv("TERMINAL_CWD") or os.getcwd() + cwd = _tenv("TERMINAL_CWD") or os.getcwd() try: host = Path(os.path.expanduser(cwd)).resolve(strict=False) except (OSError, RuntimeError, ValueError): @@ -1668,9 +1735,9 @@ def _docker_persistent_home_host_roots(session_key: str = "") -> List[Path]: produced a real host file the gateway couldn't find. Ordered best-first: the profile-scoped layout, then the legacy bug-window per-session layout. """ - if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": + if _tenv("TERMINAL_ENV", "").strip().lower() != "docker": return [] - if os.getenv("TERMINAL_CONTAINER_PERSISTENT", "true").strip().lower() not in { + if _tenv("TERMINAL_CONTAINER_PERSISTENT", "true").strip().lower() not in { "1", "true", "yes", @@ -1700,7 +1767,7 @@ def _cache_dir_container_mounts() -> List[Tuple[Path, Path]]: longer prefixes than the ``/root`` home mount, so longest-prefix matching picks the cache translation over the home translation for them. """ - if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": + if _tenv("TERMINAL_ENV", "").strip().lower() != "docker": return [] try: from tools.credential_files import get_cache_directory_mounts @@ -1721,7 +1788,7 @@ def _warn_unresolved_docker_media(candidate: Path, session_key: str, reason: str file seemingly vanished. Point at the sandbox/session mismatch instead. Gated to Docker mode so host-path rejections stay quiet. """ - if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": + if _tenv("TERMINAL_ENV", "").strip().lower() != "docker": return logger.warning( "Docker MEDIA path %s did not resolve to a host sandbox file (%s%s); " @@ -2280,6 +2347,11 @@ def cache_document_from_bytes(data: bytes, filename: str) -> str: return str(filepath) +async def cache_document_from_bytes_async(data: bytes, filename: str) -> str: + """Cache document bytes without blocking the caller's event loop.""" + return await asyncio.to_thread(cache_document_from_bytes, data, filename) + + def cleanup_document_cache(max_age_hours: int = 24) -> int: """ Delete cached documents older than *max_age_hours*. @@ -2400,6 +2472,23 @@ def cache_media_bytes( return CachedMedia(to_agent_visible_cache_path(path), out_mime, "document", display or fallback_name) +async def cache_media_bytes_async( + data: bytes, + *, + filename: str = "", + mime_type: str = "", + default_kind: Optional[str] = None, +) -> Optional[CachedMedia]: + """Classify and cache attachment bytes without blocking the event loop.""" + return await asyncio.to_thread( + cache_media_bytes, + data, + filename=filename, + mime_type=mime_type, + default_kind=default_kind, + ) + + class MessageType(Enum): """Types of incoming messages.""" TEXT = "text" @@ -3580,12 +3669,39 @@ class BasePlatformAdapter(ABC): def set_fatal_error_handler(self, handler: Callable[["BasePlatformAdapter"], Awaitable[None] | None]) -> None: self._fatal_error_handler = handler + #: Published when an adapter is installed and running but its receive + #: path is not yet confirmed (e.g. Telegram polling has not proven a + #: getUpdates round-trip). Same ``retrying`` platform_state the runner + #: uses for queued reconnects, so readers see "not delivering" (#101391). + DEGRADED_STATUS_MESSAGE = "connected but not yet confirmed active; recovering in background" + + @property + def send_path_degraded(self) -> bool: + """True while connect() succeeded but delivery is not confirmed. + + Adapters with a separately-proven receive path override this; the + default adapter is either connected or not. + """ + return False + def _mark_connected(self) -> None: self._running = True self._fatal_error_code = None self._fatal_error_message = None self._fatal_error_retryable = True - self._write_runtime_status_safe("connected", platform_state="connected", error_code=None, error_message=None) + if self.send_path_degraded: + self._mark_degraded() + else: + self._write_runtime_status_safe("connected", platform_state="connected", error_code=None, error_message=None) + + def _mark_degraded(self) -> None: + """Publish ``retrying`` for a running adapter whose delivery path is unproven.""" + self._write_runtime_status_safe( + "connected_degraded", + platform_state="retrying", + error_code=None, + error_message=self.DEGRADED_STATUS_MESSAGE, + ) def _mark_disconnected(self) -> None: self._running = False @@ -3916,6 +4032,9 @@ class BasePlatformAdapter(ABC): user_id: Optional[str], chat_type: Optional[str] = None, chat_id: Optional[str] = None, + *, + is_bot: bool = False, + thread_id: Optional[str] = None, ) -> Optional[bool]: """Return whether ``user_id`` is on the allowlist, if a check is configured. @@ -3924,6 +4043,11 @@ class BasePlatformAdapter(ABC): when no check is registered (caller should treat as "trust unknown" and preserve legacy behaviour). + ``is_bot`` / ``thread_id`` are forwarded as keywords only when set, so + the gateway callback can apply its bot policy (``*_ALLOW_BOTS``) and + thread-level profile routes while legacy three-positional callbacks + keep working unchanged. + Only the literal booleans are propagated. A callback that returns anything else is treated as "unknown" rather than coerced with ``bool()``: callers that gate a credentialed side effect on an @@ -3932,8 +4056,13 @@ class BasePlatformAdapter(ABC): """ if not user_id or self._authorization_check is None: return None + extra: Dict[str, Any] = {} + if is_bot: + extra["is_bot"] = True + if thread_id is not None: + extra["thread_id"] = thread_id try: - result = self._authorization_check(user_id, chat_type, chat_id) + result = self._authorization_check(user_id, chat_type, chat_id, **extra) if result is True: return True if result is False: @@ -4031,6 +4160,12 @@ class BasePlatformAdapter(ABC): if callable(peek): session_id = peek(session_key) transcript = store.load_transcript(session_id or session_key) + except TranscriptReadError: + logger.warning( + "Transcript read failed for session %s; media dedup runs " + "with no history this turn (#100788)", session_key, + ) + return None except Exception: return None if not transcript: diff --git a/gateway/platforms/bluebubbles.py b/gateway/platforms/bluebubbles.py index 6306c92b9e..49d1170b0c 100644 --- a/gateway/platforms/bluebubbles.py +++ b/gateway/platforms/bluebubbles.py @@ -28,9 +28,9 @@ from gateway.platforms.base import ( MessageEvent, MessageType, SendResult, - cache_image_from_bytes, - cache_audio_from_bytes, - cache_document_from_bytes, + cache_image_from_bytes_async, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, ) from .media_cache import ext_for_mime from gateway.platforms.helpers import compile_mention_patterns, strip_markdown @@ -848,7 +848,7 @@ class BlueBubblesAdapter(BasePlatformAdapter): use_mimetypes=False, fallback=".jpg", ) or ".jpg" - return cache_image_from_bytes(data, ext) + return await cache_image_from_bytes_async(data, ext) if mime.startswith("audio/"): ext = ext_for_mime( @@ -860,11 +860,11 @@ class BlueBubblesAdapter(BasePlatformAdapter): use_mimetypes=False, fallback=".mp3", ) or ".mp3" - return cache_audio_from_bytes(data, ext) + return await cache_audio_from_bytes_async(data, ext) # Videos, documents, and everything else filename = transfer_name or f"file_{uuid.uuid4().hex[:8]}" - return cache_document_from_bytes(data, filename) + return await cache_document_from_bytes_async(data, filename) except Exception as exc: logger.warning( diff --git a/gateway/platforms/qqbot/adapter.py b/gateway/platforms/qqbot/adapter.py index d84ab46014..bebf86b0c2 100644 --- a/gateway/platforms/qqbot/adapter.py +++ b/gateway/platforms/qqbot/adapter.py @@ -62,13 +62,14 @@ except ImportError: from gateway.config import Platform, PlatformConfig from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult, _ssrf_redirect_guard, - cache_document_from_bytes, - cache_image_from_bytes, + cache_document_from_bytes_async, + cache_image_from_bytes_async, ) from gateway.platforms.helpers import strip_markdown from gateway.platforms.media_cache import ext_for_mime @@ -496,7 +497,7 @@ class QQAdapter(BasePlatformAdapter): # Honor WSL proxy env for QQ WebSocket. Hermes upgrades overwrite this # local patch, so QQ can regress to direct-connect timeouts after update. - self._session = aiohttp.ClientSession(trust_env=True) + self._session = aiohttp.ClientSession(trust_env=gateway_trust_env()) ws_proxy = ( os.getenv("WSS_PROXY") or os.getenv("wss_proxy") @@ -1829,7 +1830,7 @@ class QQAdapter(BasePlatformAdapter): use_mimetypes=True, fallback=".jpg", ) or ".jpg" - return cache_image_from_bytes(data, ext) + return await cache_image_from_bytes_async(data, ext) elif content_type == "voice" or content_type.startswith("audio/"): # QQ voice messages are typically .amr or .silk format. # Convert to .wav using ffmpeg so STT engines can process it. @@ -1840,7 +1841,7 @@ class QQAdapter(BasePlatformAdapter): or Path(urlparse(url).path).name or "qq_attachment" ) - return cache_document_from_bytes(data, filename) + return await cache_document_from_bytes_async(data, filename) @staticmethod def _is_voice_content_type(content_type: str, filename: str) -> bool: @@ -2340,9 +2341,9 @@ class QQAdapter(BasePlatformAdapter): source_url[:60], ext, ) - return cache_document_from_bytes(audio_data, f"qq_voice{ext}") + return await cache_document_from_bytes_async(audio_data, f"qq_voice{ext}") except Exception: - return cache_document_from_bytes(audio_data, f"qq_voice{ext}") + return await cache_document_from_bytes_async(audio_data, f"qq_voice{ext}") finally: try: os.unlink(src_path) @@ -2353,7 +2354,7 @@ class QQAdapter(BasePlatformAdapter): try: wav_data = Path(wav_path).read_bytes() os.unlink(wav_path) - return cache_document_from_bytes(wav_data, "qq_voice.wav") + return await cache_document_from_bytes_async(wav_data, "qq_voice.wav") except Exception as exc: logger.debug("[%s] Failed to read converted wav: %s", self._log_tag, exc) return None diff --git a/gateway/platforms/signal.py b/gateway/platforms/signal.py index 4e46f2b2b2..b41c3aeac7 100644 --- a/gateway/platforms/signal.py +++ b/gateway/platforms/signal.py @@ -37,9 +37,9 @@ from gateway.platforms.base import ( MessageType, ProcessingOutcome, SendResult, - cache_image_from_bytes, - cache_audio_from_bytes, - cache_document_from_bytes, + cache_image_from_bytes_async, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, cache_image_from_url, utf16_len, ) @@ -942,11 +942,11 @@ class SignalAdapter(BasePlatformAdapter): raw_data, ext = remuxed if _is_image_ext(ext): - path = cache_image_from_bytes(raw_data, ext) + path = await cache_image_from_bytes_async(raw_data, ext) elif _is_audio_ext(ext): - path = cache_audio_from_bytes(raw_data, ext) + path = await cache_audio_from_bytes_async(raw_data, ext) else: - path = cache_document_from_bytes(raw_data, ext) + path = await cache_document_from_bytes_async(raw_data, ext) return path, ext diff --git a/gateway/platforms/webhook.py b/gateway/platforms/webhook.py index 6aa9c27ee3..b4553a26d2 100644 --- a/gateway/platforms/webhook.py +++ b/gateway/platforms/webhook.py @@ -42,6 +42,7 @@ import subprocess import sys import time from collections import deque +from contextlib import nullcontext from typing import Any, Deque, Dict, List, Optional try: @@ -633,6 +634,21 @@ class WebhookAdapter(BasePlatformAdapter): effective_profile = request_profile or "default" return configured_profile == effective_profile + @staticmethod + def _profile_scope(profile: Optional[str]): + """Enter the URL-resolved profile's runtime scope, or a no-op. + + Only a resolved ``/p//`` prefix enters a scope (same helper + the runner wraps ``handle_message`` in); bare routes keep serving the + launch profile exactly as before. + """ + if not profile or not isinstance(profile, str): + return nullcontext() + from gateway.run import _profile_runtime_scope + from hermes_cli.profiles import get_profile_dir + + return _profile_runtime_scope(get_profile_dir(profile)) + async def _handle_webhook(self, request: "web.Request") -> "web.Response": """POST /webhooks/{route_name} — receive and process a webhook event.""" # Hot-reload dynamic subscriptions on each request (mtime-gated, cheap) @@ -784,63 +800,71 @@ class WebhookAdapter(BasePlatformAdapter): } ) - if route_config.get("script"): - # run_route_script shells out (subprocess.run, up to its timeout); - # run it in a worker thread so it can't block the gateway event loop. - keep, transformed_payload = await asyncio.to_thread( - self._route_processor.run_route_script, - route_config.get("script"), - payload, + # The route script, prompt render and skill lookup below read the + # profile's home (skills/, config). The runner only enters the routed + # profile's scope later, around handle_message, so without this they + # ran against the launch (default) profile (#67277). Only a resolved + # /p// enters a scope; bare routes are unchanged. + with self._profile_scope(profile): + if route_config.get("script"): + # run_route_script shells out (subprocess.run, up to its + # timeout); run it in a worker thread so it can't block the + # gateway event loop. to_thread copies the contextvars, so + # the profile scope follows it. + keep, transformed_payload = await asyncio.to_thread( + self._route_processor.run_route_script, + route_config.get("script"), + payload, + ) + if not keep: + logger.info( + "[webhook] script ignored event=%s route=%s", + event_type, + route_name, + ) + return web.json_response( + { + "status": "ignored", + "reason": "script", + "route": route_name, + } + ) + payload = transformed_payload or payload + + # Format prompt from template + prompt_template = route_config.get("prompt", "") + prompt = self._render_prompt( + prompt_template, payload, event_type, route_name ) - if not keep: - logger.info( - "[webhook] script ignored event=%s route=%s", - event_type, - route_name, - ) - return web.json_response( - { - "status": "ignored", - "reason": "script", - "route": route_name, - } - ) - payload = transformed_payload or payload - # Format prompt from template - prompt_template = route_config.get("prompt", "") - prompt = self._render_prompt( - prompt_template, payload, event_type, route_name - ) + # Inject skill content if configured. + # We call build_skill_invocation_message() directly rather than + # using /skill-name slash commands — the gateway's command parser + # would intercept those and break the flow. + skills = route_config.get("skills", []) + if skills: + try: + from agent.skill_commands import ( + build_skill_invocation_message, + get_skill_commands, + ) - # Inject skill content if configured. - # We call build_skill_invocation_message() directly rather than - # using /skill-name slash commands — the gateway's command parser - # would intercept those and break the flow. - skills = route_config.get("skills", []) - if skills: - try: - from agent.skill_commands import ( - build_skill_invocation_message, - get_skill_commands, - ) - - skill_cmds = get_skill_commands() - for skill_name in skills: - cmd_key = f"/{skill_name}" - if cmd_key in skill_cmds: - skill_content = build_skill_invocation_message( - cmd_key, user_instruction=prompt - ) - if skill_content: - prompt = skill_content - break # Load the first matching skill - else: - logger.warning( - "[webhook] Skill '%s' not found", skill_name - ) - except Exception as e: - logger.warning("[webhook] Skill loading failed: %s", e) + skill_cmds = get_skill_commands() + for skill_name in skills: + cmd_key = f"/{skill_name}" + if cmd_key in skill_cmds: + skill_content = build_skill_invocation_message( + cmd_key, user_instruction=prompt + ) + if skill_content: + prompt = skill_content + break # Load the first matching skill + else: + logger.warning( + "[webhook] Skill '%s' not found", skill_name + ) + except Exception as e: + logger.warning("[webhook] Skill loading failed: %s", e) # Build a unique delivery ID delivery_id = request.headers.get( diff --git a/gateway/platforms/weixin.py b/gateway/platforms/weixin.py index dfc23e51d4..8c6ece8fe6 100644 --- a/gateway/platforms/weixin.py +++ b/gateway/platforms/weixin.py @@ -58,13 +58,14 @@ except ImportError: # pragma: no cover - dependency gate from gateway.config import Platform, PlatformConfig from gateway.platforms.helpers import MessageDeduplicator, greedy_pack_blocks from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult, - cache_audio_from_bytes, - cache_document_from_bytes, - cache_image_from_bytes, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, + cache_image_from_bytes_async, ) from hermes_constants import get_hermes_home from utils import atomic_json_write @@ -141,7 +142,7 @@ def _make_ssl_connector() -> Optional["aiohttp.TCPConnector"]: some system CA stores (notably Homebrew's OpenSSL on macOS Apple Silicon). When ``certifi`` is installed, use its Mozilla CA bundle to guarantee verification. Otherwise fall back to aiohttp's default (which honors - ``SSL_CERT_FILE`` env var via ``trust_env=True``). + ``SSL_CERT_FILE`` env var when ``gateway.trust_env`` is on). Uses a tight ``keepalive_timeout=2`` (default aiohttp: 30s) so idle connections drain promptly behind proxies like Cloudflare Warp that @@ -301,6 +302,10 @@ class ContextTokenStore: def __init__(self, hermes_home: str): self._root = _account_dir(hermes_home) self._cache: Dict[str, str] = {} + # Serializes the offloaded flushes so two concurrent set() calls + # cannot land their writes out of order (last-writer-wins would drop + # the newer token from disk). + self._persist_lock = asyncio.Lock() def _path(self, account_id: str) -> Path: return self._root / f"{account_id}.context-tokens.json" @@ -328,17 +333,27 @@ class ContextTokenStore: def get(self, account_id: str, user_id: str) -> Optional[str]: return self._cache.get(self._key(account_id, user_id)) - def set(self, account_id: str, user_id: str, token: str) -> None: + async def set(self, account_id: str, user_id: str, token: str) -> None: self._cache[self._key(account_id, user_id)] = token - self._persist(account_id) + # atomic_json_write() calls os.fsync(), which blocks until the write + # reaches stable storage. _process_message runs on the event loop for + # every inbound message, so offload the flush the same way #83906 did + # for the other gateway persist paths. The payload is snapshotted here, + # on the loop, so the worker never iterates ``_cache`` while another + # message task mutates it; the lock keeps flushes in mutation order. + async with self._persist_lock: + payload = self._payload(account_id) + await asyncio.to_thread(self._persist, account_id, payload) - def _persist(self, account_id: str) -> None: + def _payload(self, account_id: str) -> Dict[str, str]: prefix = f"{account_id}:" - payload = { + return { key[len(prefix) :]: value for key, value in self._cache.items() if key.startswith(prefix) } + + def _persist(self, account_id: str, payload: Dict[str, str]) -> None: try: atomic_json_write(self._path(account_id), payload) except Exception as exc: @@ -1048,7 +1063,7 @@ async def qr_login( if not AIOHTTP_AVAILABLE: raise RuntimeError("aiohttp is required for Weixin QR login") - async with aiohttp.ClientSession(trust_env=True, connector=_make_ssl_connector()) as session: + async with aiohttp.ClientSession(trust_env=gateway_trust_env(), connector=_make_ssl_connector()) as session: try: qr_resp = await _api_get( session, @@ -1318,13 +1333,13 @@ class WeixinAdapter(BasePlatformAdapter): except Exception as exc: logger.debug("[%s] Token lock unavailable (non-fatal): %s", self.name, exc) - self._poll_session = aiohttp.ClientSession(trust_env=True, connector=_make_ssl_connector()) + self._poll_session = aiohttp.ClientSession(trust_env=gateway_trust_env(), connector=_make_ssl_connector()) # Disable aiohttp's built-in ClientTimeout (total=None) to prevent # "Timeout context manager should be used inside a task" errors when # send() is invoked via asyncio.run_coroutine_threadsafe() from cron. # Timeout is managed externally via asyncio.wait_for() in _api_post/_api_get. _no_aiohttp_timeout = aiohttp.ClientTimeout(total=None, connect=None, sock_connect=None, sock_read=None) - self._send_session = aiohttp.ClientSession(trust_env=True, connector=_make_ssl_connector(), timeout=_no_aiohttp_timeout) + self._send_session = aiohttp.ClientSession(trust_env=gateway_trust_env(), connector=_make_ssl_connector(), timeout=_no_aiohttp_timeout) self._token_store.restore(self._account_id) self._poll_task = asyncio.create_task(self._poll_loop(), name="weixin-poll") self._mark_connected() @@ -1452,7 +1467,7 @@ class WeixinAdapter(BasePlatformAdapter): return old = self._poll_session self._poll_session = aiohttp.ClientSession( - trust_env=True, connector=_make_ssl_connector() + trust_env=gateway_trust_env(), connector=_make_ssl_connector() ) if old is not None and not old.closed: try: @@ -1500,7 +1515,7 @@ class WeixinAdapter(BasePlatformAdapter): context_token = str(message.get("context_token") or "").strip() if context_token: - self._token_store.set(self._account_id, sender_id, context_token) + await self._token_store.set(self._account_id, sender_id, context_token) asyncio.create_task(self._maybe_fetch_typing_ticket(sender_id, context_token or None)) media_paths: List[str] = [] @@ -1672,7 +1687,7 @@ class WeixinAdapter(BasePlatformAdapter): full_url=media.get("full_url"), timeout_seconds=30.0, ) - return cache_image_from_bytes(data, ".jpg") + return await cache_image_from_bytes_async(data, ".jpg") except Exception as exc: logger.warning("[%s] image download failed: %s", self.name, exc) return None @@ -1688,7 +1703,7 @@ class WeixinAdapter(BasePlatformAdapter): full_url=media.get("full_url"), timeout_seconds=120.0, ) - return cache_document_from_bytes(data, "video.mp4") + return await cache_document_from_bytes_async(data, "video.mp4") except Exception as exc: logger.warning("[%s] video download failed: %s", self.name, exc) return None @@ -1707,7 +1722,7 @@ class WeixinAdapter(BasePlatformAdapter): full_url=media.get("full_url"), timeout_seconds=60.0, ) - return cache_document_from_bytes(data, filename), mime + return await cache_document_from_bytes_async(data, filename), mime except Exception as exc: logger.warning("[%s] file download failed: %s", self.name, exc) return None, mime @@ -1731,7 +1746,7 @@ class WeixinAdapter(BasePlatformAdapter): full_url=media.get("full_url"), timeout_seconds=60.0, ) - return cache_audio_from_bytes(data, ".silk") + return await cache_audio_from_bytes_async(data, ".silk") except Exception as exc: logger.warning("[%s] voice download failed: %s", self.name, exc) return None @@ -2407,7 +2422,7 @@ async def send_weixin_direct( "context_token_used": bool(context_token), } - async with aiohttp.ClientSession(trust_env=True, connector=_make_ssl_connector()) as session: + async with aiohttp.ClientSession(trust_env=gateway_trust_env(), connector=_make_ssl_connector()) as session: adapter = WeixinAdapter( PlatformConfig( enabled=True, diff --git a/gateway/platforms/yuanbao.py b/gateway/platforms/yuanbao.py index d36c177794..415c726541 100644 --- a/gateway/platforms/yuanbao.py +++ b/gateway/platforms/yuanbao.py @@ -56,9 +56,9 @@ from gateway.platforms.base import ( MessageEvent, MessageType, SendResult, - cache_document_from_bytes, - cache_image_from_bytes, - cache_video_from_bytes, + cache_document_from_bytes_async, + cache_image_from_bytes_async, + cache_video_from_bytes_async, ) from gateway.platforms import helpers as _mdchunk from gateway.platforms.helpers import MessageDeduplicator @@ -96,7 +96,7 @@ from gateway.platforms.yuanbao_proto import ( encode_get_group_member_list, next_seq_no, ) -from gateway.session import build_session_key +from gateway.session import TranscriptReadError, build_session_key logger = logging.getLogger(__name__) @@ -1144,6 +1144,14 @@ class RecallGuardMiddleware(InboundMiddleware): await asyncio.sleep(0.5) try: transcript = store.load_transcript(sid) + except TranscriptReadError as exc: + # No readable rows means nothing to redact; polling on + # would just re-log the same failure (#100788). + logger.warning( + "[%s] Recall redact: transcript unreadable for " + "session %s: %s", adapter.name, sid, exc, + ) + return except Exception: continue for entry in transcript: @@ -1183,6 +1191,11 @@ class RecallGuardMiddleware(InboundMiddleware): # match) is the canonical path again. try: transcript = store.load_transcript(sid) + except TranscriptReadError as exc: + # Not an empty transcript — the rows are unreadable, so recall has + # nothing to match against (#100788). + logger.warning("[%s] Recall: transcript unreadable: %s", adapter.name, exc) + return except Exception as exc: logger.warning("[%s] Recall: failed to load transcript: %s", adapter.name, exc) return @@ -2145,6 +2158,13 @@ class QuoteContextMiddleware(InboundMiddleware): if kind in _RESOLVABLE_MEDIA_KINDS: media_refs.append((rid, kind, filename.strip())) break + except TranscriptReadError as exc: + # Quote resolution degrades to "no refs" rather than pretending + # the quoted message was never seen (#100788). + logger.warning( + "[%s] quote transcript lookup: transcript unreadable: %s", + getattr(adapter, "name", "yuanbao"), exc, + ) except Exception as exc: logger.warning( "[%s] quote transcript lookup failed: %s", @@ -2522,7 +2542,7 @@ class MediaResolveMiddleware(InboundMiddleware): if kind == "image": ext = cls._guess_image_ext_from_url(fetch_url) try: - local_path = cache_image_from_bytes(file_bytes, ext=ext) + local_path = await cache_image_from_bytes_async(file_bytes, ext=ext) except ValueError as exc: logger.warning( "[%s] inbound image cache rejected: %s err=%s", @@ -2537,7 +2557,7 @@ class MediaResolveMiddleware(InboundMiddleware): if kind == "video": # Yuanbao video resources carry no reliable extension; default to mp4. - local_path = cache_video_from_bytes(file_bytes) + local_path = await cache_video_from_bytes_async(file_bytes) mime = guess_mime_type(local_path) or ( content_type if content_type.startswith("video/") else "video/mp4" ) @@ -2549,7 +2569,7 @@ class MediaResolveMiddleware(InboundMiddleware): parsed = urllib.parse.urlparse(fetch_url) file_name = os.path.basename(parsed.path) or "file" try: - local_path = cache_document_from_bytes(file_bytes, file_name) + local_path = await cache_document_from_bytes_async(file_bytes, file_name) except Exception as exc: logger.warning( "[%s] inbound file cache failed: %s err=%s", @@ -2747,6 +2767,14 @@ class MediaResolveMiddleware(InboundMiddleware): try: session_entry = store.get_or_create_session(source) history = store.load_transcript(session_entry.session_id) + except TranscriptReadError as exc: + # Hydrate nothing rather than silently acting as if the session + # had no observed media (#100788). + logger.warning( + "[%s] Observed-media hydration: transcript unreadable: %s", + adapter.name, exc, + ) + return [], [] except Exception as exc: logger.warning( "[%s] Observed-media hydration setup failed: %s", diff --git a/gateway/profile_routing.py b/gateway/profile_routing.py index 5d2b3b60be..c72f8837ea 100644 --- a/gateway/profile_routing.py +++ b/gateway/profile_routing.py @@ -35,6 +35,11 @@ Configuration (config.yaml): chat_id: "YOUR_CHANNEL_ID" thread_id: "YOUR_THREAD_ID" profile: thread-profile + + - name: owner-whatsapp + platform: whatsapp + chat_id: "15551234567" # phone, JID, or LID — all equivalent + profile: owner """ from __future__ import annotations @@ -46,6 +51,43 @@ import logging logger = logging.getLogger(__name__) +# Baileys and Cloud share phone/JID/LID identity rules. Other platforms keep +# exact string compare so Telegram numeric ids and Discord snowflakes stay +# unchanged. +_WHATSAPP_IDENTITY_PLATFORMS = {"whatsapp", "whatsapp_cloud"} +_WHATSAPP_NON_USER_SUFFIXES = ("@g.us", "@broadcast", "@newsletter") + + +def _is_whatsapp_non_user_chat(chat_id: Optional[str]) -> bool: + """True for group / broadcast / newsletter JIDs — not a sender identity.""" + if not chat_id: + return False + cid = str(chat_id).strip().lower() + return any(cid.endswith(suffix) for suffix in _WHATSAPP_NON_USER_SUFFIXES) + + +def _whatsapp_user_chat_ids_match(platform: str, left: Optional[str], right: Optional[str]) -> bool: + """True when two WhatsApp *user* chat_ids refer to the same person. + + Reuses :func:`gateway.whatsapp_identity.expand_whatsapp_aliases` so a + bare phone number, a ``@s.whatsapp.net`` JID, and a ``@lid`` LID collapse + to one identity — the same helper session keys and adapter allowlists + already use. Group/broadcast JIDs are excluded: those are chats, not + senders. Returns False for non-WhatsApp platforms (exact match only). + """ + if (platform or "").strip().lower() not in _WHATSAPP_IDENTITY_PLATFORMS: + return False + if not left or not right: + return False + if _is_whatsapp_non_user_chat(left) or _is_whatsapp_non_user_chat(right): + return False + from gateway.whatsapp_identity import expand_whatsapp_aliases + + left_aliases = expand_whatsapp_aliases(str(left)) + if not left_aliases: + return False + return bool(left_aliases & expand_whatsapp_aliases(str(right))) + class ProfileRouteRejected(RuntimeError): """An explicit route matched a profile this gateway does not serve.""" @@ -92,6 +134,11 @@ class ProfileRoute: - Thread in channel: parent_chat_id == route.chat_id A route declaring both ``guild_id`` and ``chat_id`` requires both to match (a chat match alone does not satisfy a guild constraint). + + WhatsApp / WhatsApp Cloud ``chat_id`` also matches across user-identity + forms (bare number, JID, LID) after the exact-string check. Exact + matches always win first, so existing configs keep working. Groups + (``@g.us``) and broadcasts stay exact-only. """ if not self.enabled: return False @@ -100,12 +147,41 @@ class ProfileRoute: if self.thread_id and self.thread_id != thread_id: return False if self.chat_id and self.chat_id != chat_id and self.chat_id != parent_chat_id: - return False + if not ( + _whatsapp_user_chat_ids_match(platform, self.chat_id, chat_id) + or _whatsapp_user_chat_ids_match(platform, self.chat_id, parent_chat_id) + ): + return False if self.guild_id and self.guild_id != guild_id: return False return True +def _coerce_route_id(value: Any) -> Optional[str]: + """Normalize a route discriminator to str for strict equality matching. + + PyYAML loads unquoted numeric IDs (Discord snowflakes, Telegram negative + chat ids) as ``int``. Inbound ``SessionSource`` fields are always ``str`` + via ``build_source``, so leaving ints here makes ``matches()`` fail silently. + + Only ``int`` is coerced (the legitimate YAML-numeric case). ``bool`` is an + ``int`` subclass but never a valid id; floats and other types stringify to + something (``"123.0"``) that can never equal an inbound id — recreating the + silent no-match this exists to fix — so they are passed through with a + load-time warning instead of being silently "fixed" (#86470). + """ + if value is None or isinstance(value, str): + return value + if isinstance(value, int) and not isinstance(value, bool): + return str(value) + logger.warning( + "Profile route discriminator %r (type %s) can never match an inbound " + "id — quote it in config.yaml (e.g. chat_id: \"%s\").", + value, type(value).__name__, value, + ) + return str(value) + + def parse_profile_routes(raw: Optional[List[Dict[str, Any]]]) -> List[ProfileRoute]: """Parse profile_routes from config.yaml into ProfileRoute objects. @@ -143,9 +219,9 @@ def parse_profile_routes(raw: Optional[List[Dict[str, Any]]]) -> List[ProfileRou name=name, platform=platform, profile=profile, - guild_id=entry.get("guild_id"), - chat_id=entry.get("chat_id"), - thread_id=entry.get("thread_id"), + guild_id=_coerce_route_id(entry.get("guild_id")), + chat_id=_coerce_route_id(entry.get("chat_id")), + thread_id=_coerce_route_id(entry.get("thread_id")), enabled=entry.get("enabled", True), ) ) diff --git a/gateway/relay/adapter.py b/gateway/relay/adapter.py index 1fdeea8d93..a42bef5d83 100644 --- a/gateway/relay/adapter.py +++ b/gateway/relay/adapter.py @@ -1620,6 +1620,13 @@ class RelayAdapter(BasePlatformAdapter): # how platform=RELAY home channels slipped through in the first # place. Set locally, never read off the wire. delivered_via_upstream_relay=True, + # The HERMES profile this interaction is routed to (multiplex + # mode) — mirrors _event_from_wire's profile stamping for plain + # relayed messages (#60586). Without this, a Team-Gateway's + # Discord slash-command/button/modal always fell back to the + # legacy agent:main namespace even when the connector resolved + # a specific profile for it. + profile=getattr(forward, "profile", None), ) event = MessageEvent(text=text, message_type=message_type, source=source) if itype == 3: diff --git a/gateway/relay/ws_transport.py b/gateway/relay/ws_transport.py index b526425293..d1793b1294 100644 --- a/gateway/relay/ws_transport.py +++ b/gateway/relay/ws_transport.py @@ -402,6 +402,16 @@ class PassthroughForward: path: str headers: list[tuple[str, str]] body: bytes + # The HERMES profile this interaction is routed to (multiplex mode). + # Mirrors the ``profile`` field _event_from_wire already carries on the + # ``inbound`` frame's SessionSource (#60586) — the connector stamps it + # when NAS resolves the target profile for a Team-Gateway interaction; + # absent for a single-profile gateway, where it stays None and session + # keys keep the legacy ``agent:main`` namespace. Without this, a Discord + # slash-command/button/modal relayed through the passthrough plane always + # fell back to agent:main even when the equivalent plain message would + # have been routed to the correct profile. + profile: Optional[str] = None def _passthrough_from_wire(raw: Dict[str, Any]) -> PassthroughForward: @@ -431,6 +441,7 @@ def _passthrough_from_wire(raw: Dict[str, Any]) -> PassthroughForward: path=str(raw.get("path", "")), headers=headers, body=body, + profile=raw.get("profile"), ) diff --git a/gateway/restart.py b/gateway/restart.py index 986b5a4fee..58e6d15ceb 100644 --- a/gateway/restart.py +++ b/gateway/restart.py @@ -16,6 +16,25 @@ GATEWAY_SERVICE_RESTART_EXIT_CODE = 75 # restarting the gateway. See #51228. GATEWAY_FATAL_CONFIG_EXIT_CODE = 78 + +def is_global_startup_conflict(error_code: str | None) -> bool: + """Return True when an adapter's fatal error is a single-writer ownership conflict. + + ``BasePlatformAdapter._acquire_platform_lock`` emits ``{scope}_lock`` + with ``retryable=True`` on purpose: a *mid-run* reconnect must be able to + recover once the live holder exits or a stale record is cleared (#54167). + At startup, though, a live foreign holder is a configuration conflict — + two gateways cannot poll one bot token — so the startup router must not + treat that flag as "transient blip, retry-queue forever". This matches by + error CODE only (the ``{scope}_lock`` / ``lock_conflict`` families every + adapter emits for scoped-lock and identity conflicts), never by message + text. + """ + code = (error_code or "").strip().lower() + if not code: + return False + return code == "lock_conflict" or code.endswith("_lock") + # Set by ``hermes gateway run --external-supervisor``. Unlike systemd's # INVOCATION_ID and launchd's XPC_SERVICE_NAME, this survives wrappers that # intentionally replace the child environment (for example ``sudo env -i``). diff --git a/gateway/run.py b/gateway/run.py index c3d70d319b..fa41213ec8 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -28,6 +28,7 @@ import asyncio import concurrent.futures import dataclasses import faulthandler +import functools import inspect import json import logging @@ -50,6 +51,7 @@ from typing import Awaitable, Callable, Dict, Optional, Any, List, Tuple, Union, from agent.async_utils import consume_detached_task_result, safe_schedule_threadsafe from agent.conversation_compression import ( COMPACTION_DONE_STATUS, + COMPACTION_HEARTBEAT_STATUS, COMPACTION_STATUS, COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE, COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE, @@ -130,7 +132,8 @@ _TELEGRAM_NOISY_STATUS_RE = re.compile( r"|auto-lowered\s+(?:this\s+)?session'?s?\s+threshold" r"|configured\s+auxiliary\s+compression\s+provider\s+.+\s+unavailable" r"|skipping\s+concurrent\s+compression" - r"|compacting\s+context\s+[—-]\s+summarizing\s+earlier\s+conversation" + rf"|{re.escape(COMPACTION_STATUS)}" + rf"|{re.escape(COMPACTION_HEARTBEAT_STATUS)}" r"|resumed\s+after\s+\d+s\s+idle\s+[—-]\s+compacting" r"|preflight\s+compression" r"|pre[- ]api\s+compression" @@ -390,16 +393,27 @@ async def run_codex_hygiene_compaction( loop = asyncio.get_running_loop() compressor = getattr(agent, "context_compressor", None) count_before = getattr(compressor, "compression_count", 0) + worker_future = loop.run_in_executor( + None, + # Keep the caller's multiplexed profile secret scope and HERMES_HOME + # override in the worker. The default executor does not propagate + # ContextVars on the Python runtimes Hermes currently ships. + copy_context().run, + lambda: agent._compress_context( + history, + "", + approx_tokens=approx_tokens, + ), + ) + track_worker = getattr(gateway, "_track_deferred_agent_worker", None) + if callable(track_worker): + # ``wait_for`` only cancels the asyncio wrapper; the executor thread + # keeps running. Keep it visible to gateway shutdown until the real + # worker finishes, just like the detached local-compressor path. + track_worker(worker_future, agent) try: await asyncio.wait_for( - loop.run_in_executor( - None, - lambda: agent._compress_context( - history, - "", - approx_tokens=approx_tokens, - ), - ), + asyncio.shield(worker_future), timeout=max(float(timeout_seconds), 1.0), ) except asyncio.TimeoutError: @@ -521,6 +535,7 @@ _COMPRESSION_PROGRESS_STATUS_RE = re.compile( _status_template_to_regex(_template) for _template in ( COMPACTION_STATUS, + COMPACTION_HEARTBEAT_STATUS, COMPACTION_DONE_STATUS, PRE_API_COMPRESSION_STATUS_TEMPLATE, PREFLIGHT_COMPRESSION_STATUS_TEMPLATE, @@ -2381,7 +2396,10 @@ def _current_max_iterations() -> int: return _resolve_turn_limit(os.getenv("HERMES_MAX_ITERATIONS")) -from contextlib import contextmanager as _contextmanager +from contextlib import ( + asynccontextmanager as _asynccontextmanager, + contextmanager as _contextmanager, +) # Platforms that bind a host TCP port (HTTP/webhook listeners). In a profile @@ -2434,6 +2452,30 @@ def _multiplex_profile_homes(config: object) -> list[tuple[str, "Path"]]: ) +def _enable_multiplex_log_routing(config: object) -> bool: + """Route agent.log/errors.log/gateway.log records to their owning profile. + + ``setup_logging(mode="gateway")`` binds the queued file handlers to the + launch home, so under ``multiplex_profiles`` every secondary profile's + records (emitted inside ``_profile_runtime_scope``) land in the default + profile's log files (#82936). Swap the static handlers for the + profile routers from #99440 — the same primitive the Desktop cron ticker + uses — once the served-profile set is known. Inert for single-profile + gateways (``enable_profile_log_routing`` is a no-op below two homes). + """ + if not getattr(config, "multiplex_profiles", False): + return False + try: + from hermes_logging import enable_profile_log_routing + + return enable_profile_log_routing( + [home for _name, home in _multiplex_profile_homes(config)] + ) + except Exception: + logger.debug("could not enable per-profile log routing", exc_info=True) + return False + + def _handoff_watch_scopes(runner: object) -> list: """``(profile_name, home)`` pairs whose ``state.db`` the watcher must poll. @@ -2504,8 +2546,40 @@ async def _reclaim_stale(runner: object) -> None: ) +def _terminal_scope_cwd(default: str = "") -> str: + """Scope-aware TERMINAL_CWD read for footer/context surfaces. + + Only an import failure falls back: an active refusal scope must raise, + not resolve the launch profile's cwd. + """ + try: + from tools.terminal_scope import terminal_env as _ts_env + except ImportError: + return os.environ.get("TERMINAL_CWD", default) + return _ts_env("TERMINAL_CWD", default) + + +def _load_profile_secret_scope(profile_home: "Path") -> dict: + """Hydrate and load one profile's secrets under its home override.""" + from hermes_constants import set_hermes_home_override, reset_hermes_home_override + from agent.secret_scope import build_profile_secret_scope + from hermes_cli.env_loader import hydrate_profile_secret_sources + + home_token = set_hermes_home_override(str(profile_home)) + try: + hydrate_profile_secret_sources(Path(profile_home)) + return build_profile_secret_scope(Path(profile_home)) + finally: + reset_hermes_home_override(home_token) + + @_contextmanager -def _profile_runtime_scope(profile_home: "Path"): +def _profile_runtime_scope( + profile_home: "Path", + prepared_secret_scope: Optional[dict] = None, + *, + hydrate_secrets: bool = True, +): """Scope config/skills/memory AND credentials to a profile for one turn. Combines the two seams the multiplexer needs: @@ -2525,20 +2599,42 @@ def _profile_runtime_scope(profile_home: "Path"): """ from hermes_constants import set_hermes_home_override, reset_hermes_home_override from agent.secret_scope import ( - build_profile_secret_scope, set_secret_scope, reset_secret_scope, ) - from hermes_cli.env_loader import hydrate_profile_secret_sources home_token = set_hermes_home_override(str(profile_home)) - hydrate_profile_secret_sources(Path(profile_home)) - secret_token = set_secret_scope(build_profile_secret_scope(Path(profile_home))) - try: + if prepared_secret_scope is not None: + secrets = prepared_secret_scope + elif hydrate_secrets: + secrets = _load_profile_secret_scope(Path(profile_home)) + else: + # Caller already hydrated external sources off-loop (#99519). + from agent.secret_scope import build_profile_secret_scope + + secrets = build_profile_secret_scope(Path(profile_home)) + secret_token = set_secret_scope(secrets) + # Per-turn terminal scope (third seam of the profile boundary): installs + # the routed profile's COMPLETE terminal policy — never ambient env — via + # tools.terminal_scope. Without it terminal_tool reads the process-global + # TERMINAL_* vars a previous profile's turn may have pinned + # (first-writer-wins backend leak; #68559). + from tools.terminal_scope import install_and_reset_profile_terminal_scope + + with install_and_reset_profile_terminal_scope(Path(profile_home)): + try: + yield + finally: + reset_secret_scope(secret_token) + reset_hermes_home_override(home_token) + + +@_asynccontextmanager +async def _async_profile_runtime_scope(profile_home: "Path"): + """Enter a profile scope without loading secret files on the event loop.""" + secrets = await asyncio.to_thread(_load_profile_secret_scope, Path(profile_home)) + with _profile_runtime_scope(Path(profile_home), secrets): yield - finally: - reset_secret_scope(secret_token) - reset_hermes_home_override(home_token) def load_gateway_config_for_runner() -> "GatewayConfig": @@ -2576,13 +2672,40 @@ def load_gateway_config_for_runner() -> "GatewayConfig": return cfg +async def _discover_gateway_mcp_tools(config: object) -> None: + """Run startup MCP discovery for every profile this gateway serves. + + ``discover_mcp_tools`` reads ``mcp_servers`` from ``get_hermes_home()``'s + config, so an unscoped call only ever connects the launch profile's + servers (#95518). Under multiplex, run it once per served profile inside + that profile's ``_profile_runtime_scope`` and carry the scope into the + executor thread with ``copy_context()`` (the same shape as + ``_run_in_executor_with_context``). Single-profile gateways keep the one + unscoped call. + """ + from tools.mcp_tool import discover_mcp_tools + + loop = asyncio.get_running_loop() + if not getattr(config, "multiplex_profiles", False): + await loop.run_in_executor(None, discover_mcp_tools) + return + for profile_name, profile_home in _multiplex_profile_homes(config): + try: + with _profile_runtime_scope(Path(profile_home)): + await loop.run_in_executor(None, copy_context().run, discover_mcp_tools) + except Exception: + logger.warning( + "MCP tool discovery failed for profile '%s'", profile_name, exc_info=True, + ) + + def _platform_has_bot_credential(platform: "Platform", platform_config: "PlatformConfig") -> bool: """Return True when a token-authenticated platform has a usable bot credential. Platforms that do not use ``PlatformConfig.token`` always return True so we never skip them here (Signal session paths, port-binding HTTP adapters, etc.). """ - from gateway.config import PLATFORM_TOKEN_ENV_NAMES + from gateway.config import PLATFORM_TOKEN_ENV_NAMES, Platform if platform not in PLATFORM_TOKEN_ENV_NAMES: return True @@ -2593,6 +2716,26 @@ def _platform_has_bot_credential(platform: "Platform", platform_config: "Platfor api_key = getattr(platform_config, "api_key", None) or "" if isinstance(api_key, str) and api_key.strip(): return True + # Matrix also authenticates by password login (MATRIX_USER_ID + + # MATRIX_PASSWORD, no MATRIX_ACCESS_TOKEN). Those credentials land in + # ``extra`` rather than ``.token``, so a token-only check reads a + # perfectly reconnectable password-auth config as credential-less and + # evicts it from the retry queue on the first transient failure — after + # which it stays down until the gateway is restarted by hand. Mirror the + # adapter's own gate: homeserver + user_id + password. + # + # Read ONLY from extra, never os.getenv: build_config() already copies all + # three env vars onto extra, and importing this module loads ~/.hermes/.env, + # so an env fallback would report "has credential" for every Matrix config + # on the box — including the empty-primary multiplex case (#64674) this + # check exists to evict. + if platform is Platform.MATRIX: + extra = getattr(platform_config, "extra", None) or {} + if all( + str(extra.get(key) or "").strip() + for key in ("homeserver", "user_id", "password") + ): + return True return False @@ -2944,6 +3087,8 @@ from gateway.session import ( SessionStore, SessionSource, SessionContext, + TranscriptReadError, + _session_key_namespace, build_session_context, build_session_context_prompt, build_channel_continuity_note, @@ -3001,6 +3146,7 @@ from gateway.restart import ( DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, GATEWAY_FATAL_CONFIG_EXIT_CODE, GATEWAY_SERVICE_RESTART_EXIT_CODE, + is_global_startup_conflict, parse_cron_drain_timeout, parse_restart_after_turn_timeout, parse_restart_drain_timeout, @@ -3019,6 +3165,13 @@ from gateway.whatsapp_identity import ( logger = logging.getLogger(__name__) +# Ceiling for the shutdown quiesce of the gateway-owned thread pool. Drain has +# already waited for the agents, so what is left here is short blocking work +# (a transcript append, a routing save); anything slower is a stuck worker we +# must not wait on, and the caller clamps this to the watchdog leash anyway. +_EXECUTOR_QUIESCE_TIMEOUT = 2.0 + + _OWN_POLICY_OPEN_ENV = { Platform.WECOM: ("WECOM_DM_POLICY", "WECOM_GROUP_POLICY", "WECOM_ALLOW_ALL_USERS"), Platform.WEIXIN: ("WEIXIN_DM_POLICY", "WEIXIN_GROUP_POLICY", "WEIXIN_ALLOW_ALL_USERS"), @@ -7409,7 +7562,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Load ephemeral config from config.yaml / env vars. # Both are injected at API-call time only and never persisted. self._prefill_messages = self._load_prefill_messages() - self._ephemeral_system_prompt = self._load_ephemeral_system_prompt() self._reasoning_config = self._load_reasoning_config() self._service_tier = self._load_service_tier() self._show_reasoning = self._load_show_reasoning() @@ -7607,6 +7759,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # context pin; last-delivered voice-channel context) lives on # SessionState.conversation — see gateway/session_state.py. self._kanban_notifier_profile = self._active_profile_name() + # Launch-time identity of the profile that owns ``self.adapters``; + # ``_authorization_adapter`` compares against this rather than the + # per-turn ``_active_profile_name()`` (see gateway/authz_mixin.py). + self._primary_profile_name = self._kanban_notifier_profile # Teams meeting pipeline runtime (bound later when msgraph_webhook adapter exists). self._teams_pipeline_runtime = None self._teams_pipeline_runtime_error: Optional[str] = None @@ -8020,9 +8176,43 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _VOICE_MODE_PATH = _hermes_home / "gateway_voice_mode.json" - def _voice_key(self, platform: Platform, chat_id: str) -> str: - """Return a platform-namespaced key for voice mode state.""" - return f"{platform.value}:{chat_id}" + def _voice_key( + self, platform: Platform, chat_id: str, profile: Optional[str] = None + ) -> str: + """Return a platform-namespaced key for voice mode state. + + Under multiplexing the key is additionally namespaced by the profile + whose bot speaks in the chat (``::``); the + default profile keeps the historical ``:`` shape so + persisted state stays valid. Two bots in one Discord channel otherwise + share a key and one profile's ``/voice`` flips the other's (#75198). + """ + base = f"{platform.value}:{chat_id}" + profile = profile.strip() if isinstance(profile, str) else "" + if not profile or profile == "default": + return base + return f"{profile}:{base}" + + def _voice_key_for_source(self, source: SessionSource) -> str: + """Voice-state key for an inbound source, namespaced by its transport owner. + + Voice mode belongs to the (bot, chat) pair, so the namespace is the + profile that OWNS the receiving adapter (``_adapter_profile_for_source``) + — the same profile ``_sync_voice_mode_state_to_adapter`` uses on + reconnect — not the routed runtime profile. + """ + return self._voice_key( + source.platform, + source.chat_id, + profile=self._adapter_profile_for_source(source), + ) + + def _bind_voice_input_callback(self, adapter) -> None: + """Route voice transcripts back through the adapter that captured them.""" + if hasattr(adapter, "_voice_input_callback"): + adapter._voice_input_callback = functools.partial( + self._handle_voice_channel_input, adapter=adapter + ) def _load_voice_modes(self) -> Dict[str, str]: try: @@ -8121,7 +8311,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if hasattr(adapter, "_auto_tts_default"): adapter._auto_tts_default = _auto_tts_default - prefix = f"{platform.value}:" + prefix = self._voice_key(platform, "", profile=getattr(adapter, "_owner_profile", None)) if isinstance(disabled_chats, set): disabled_chats.clear() disabled_chats.update( @@ -8395,6 +8585,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew profile=_profile, ) + @staticmethod + def _telegram_topic_profile_name(source: SessionSource) -> str: + """Profile namespace for Telegram topic-mode rows (issue #76423). + + Prefer the profile already stamped on the routed event + (``source.profile``). Do **not** fall back to the process-global + active profile here — under multiplex that can mis-attribute + topic state across bots sharing one ``state.db``. + """ + name = str(getattr(source, "profile", None) or "").strip() + return name if name else "default" + def _telegram_topic_mode_enabled(self, source: SessionSource) -> bool: """Return whether Telegram DM topic mode is active for this chat.""" if source.platform != Platform.TELEGRAM or source.chat_type != "dm": @@ -8408,6 +8610,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew raw = session_db.is_telegram_topic_mode_enabled( chat_id=str(source.chat_id), user_id=str(source.user_id), + profile_name=self._telegram_topic_profile_name(source), ) except Exception: logger.debug("Failed to read Telegram topic mode state", exc_info=True) @@ -8444,6 +8647,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _TELEGRAM_LOBBY_REMINDER_COOLDOWN_S = 30.0 + def _telegram_topic_cooldown_key(self, source: SessionSource) -> Optional[str]: + """Cooldown key for topic-mode cooldowns: (profile, chat_id). + + Profiles sharing a Telegram private chat_id under multiplex must not + suppress each other's lobby reminders / capability hints (#76423). + """ + chat_id = str(source.chat_id or "") + if not chat_id: + return None + return f"{self._telegram_topic_profile_name(source)}:{chat_id}" + def _should_send_telegram_lobby_reminder(self, source: SessionSource) -> bool: """Rate-limit root-DM lobby reminders to one message per cooldown window. @@ -8453,15 +8667,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """ if not hasattr(self, "_telegram_lobby_reminder_ts"): self._telegram_lobby_reminder_ts = {} - chat_id = str(source.chat_id or "") - if not chat_id: + key = self._telegram_topic_cooldown_key(source) + if not key: return True import time as _time now = _time.monotonic() - last = self._telegram_lobby_reminder_ts.get(chat_id, 0.0) + last = self._telegram_lobby_reminder_ts.get(key, 0.0) if now - last < self._TELEGRAM_LOBBY_REMINDER_COOLDOWN_S: return False - self._telegram_lobby_reminder_ts[chat_id] = now + self._telegram_lobby_reminder_ts[key] = now return True def _telegram_topic_root_lobby_message(self) -> str: @@ -8509,6 +8723,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew user_id=str(source.user_id or ""), session_key=session_entry.session_key, session_id=session_entry.session_id, + profile_name=self._telegram_topic_profile_name(source), ) def _sync_telegram_topic_binding( @@ -8576,6 +8791,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew try: bindings = session_db.list_telegram_topic_bindings_for_chat( chat_id=str(source.chat_id), + profile_name=self._telegram_topic_profile_name(source), ) except Exception: logger.debug("topic-recover: read failed", exc_info=True) @@ -8839,12 +9055,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # configured extra_body (chat_template_kwargs, etc.) never reached the # model on the gateway path -- only /fast service-tier overrides did. service_tier = getattr(self, "_service_tier", None) - if not service_tier: + if service_tier != "priority": + # None (normal) or auto/cold — the bounded window is applied per + # request by agent.fast_mode, not pinned into request_overrides. route["request_overrides"] = base_request_overrides return route try: - overrides = resolve_fast_mode_overrides(route["model"]) + overrides = resolve_fast_mode_overrides( + route["model"], + provider=runtime["provider"], + base_url=runtime["base_url"], + ) except Exception: overrides = None # Fast-mode overrides (service_tier / speed) are top-level keys and do @@ -9185,6 +9407,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._running_agent_count() + self._active_cron_job_count() + self._active_api_run_count() + + self._active_deferred_agent_worker_count() ) def _active_cron_job_count(self) -> int: @@ -9238,6 +9461,66 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.debug("Failed interrupting api_server runs during shutdown: %s", exc) return 0 + def _active_deferred_agent_worker_count(self) -> int: + """Count executor workers that outlived their owning gateway turn. + + A timed-out hygiene compression keeps running in its executor thread. + Some paths defer agent cleanup; the live Codex path keeps its cached + agent. In both cases the turn can finish before the worker does, so + ``_running_agents`` no longer represents it. Count the worker itself. + """ + workers = getattr(self, "_deferred_agent_workers", None) + if not isinstance(workers, dict): + return 0 + return sum(1 for future in list(workers) if not future.done()) + + def _track_deferred_agent_worker( + self, + future: asyncio.Future, + agent: Any, + ) -> None: + """Expose an executor worker to drain/interrupt until it really exits.""" + workers = getattr(self, "_deferred_agent_workers", None) + if workers is None: + workers = {} + self._deferred_agent_workers = workers + workers[future] = agent + + def _discard_worker(done_future: asyncio.Future) -> None: + workers.pop(done_future, None) + # Some tracked workers intentionally outlive the coroutine that + # started them and therefore have no later waiter. Consume their + # terminal exception so asyncio does not emit an unhandled-future + # warning after the worker eventually unwinds (#98973). + if not done_future.cancelled(): + try: + done_future.exception() + except Exception: + pass + + future.add_done_callback(_discard_worker) + + def _interrupt_deferred_agent_workers(self, reason: str) -> int: + """Request cancellation of detached executor-backed agent work.""" + workers = getattr(self, "_deferred_agent_workers", None) + if not isinstance(workers, dict): + return 0 + interrupted = 0 + seen: set[int] = set() + for future, agent in list(workers.items()): + if future.done() or agent is None or id(agent) in seen: + continue + seen.add(id(agent)) + try: + request_hard_interrupt(agent, reason) + interrupted += 1 + except Exception as exc: + logger.debug( + "Failed interrupting deferred agent worker during shutdown: %s", + exc, + ) + return interrupted + # ── scale-to-zero idle detection / dormant-quiesce (Phase 0) ────────────── # The gateway-side BEHAVIOUR that consumes the relay scale-to-zero primitives # (gateway-gateway Phase 5). Pure logic lives in gateway/scale_to_zero.py; the @@ -9445,9 +9728,26 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: # noqa: BLE001 - unreadable source => assume busy logger.debug("scale-to-zero: api work count unreadable — staying awake", exc_info=True) api_count = 1 + # An attached dashboard/desktop/TUI client is inbound activity too. It + # lives in the DASHBOARD process, so it reaches us as a file mtime that + # process refreshes on every WS frame (gateway/scale_to_zero.py). Fold + # it into the inbound clock rather than adding a conjunct: the client + # then gets the same idle_timeout grace after it disconnects as a chat + # message does, and a lingering marker cannot pin the box (an old mtime + # is outside idle_timeout just like an old _last_inbound_at). + last_inbound = self._last_inbound_at + try: + from gateway.scale_to_zero import dashboard_client_last_seen + + seen = dashboard_client_last_seen() + except Exception: # noqa: BLE001 - unreadable source => assume busy + logger.debug("scale-to-zero: dashboard heartbeat unreadable — staying awake", exc_info=True) + seen = time.time() + if seen is not None and seen > last_inbound: + last_inbound = seen return is_idle( active_work_count=self._running_agent_count() + cron_count + api_count, - seconds_since_last_inbound=time.time() - self._last_inbound_at, + seconds_since_last_inbound=time.time() - last_inbound, idle_timeout_seconds=self._scale_to_zero_idle_timeout_seconds(), has_live_background_work=self._scale_to_zero_has_live_background_work(), ) @@ -9684,6 +9984,72 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew depth += 1 return depth + def _rescue_orphaned_overflow( + self, session_key: str, adapter: Any + ) -> Optional["MessageEvent"]: + """Pop the oldest orphaned FIFO overflow event for an idle session (#99882). + + The FIFO overflow (``queued_events``) drains only at the post-turn + promotion site (``_promote_queued_event`` inside the ``_run_agent`` + drain). When a busy window ends without that drain running — the + #99882 shape: a follow-up queued during compression-in-flight lands + in overflow, compression finishes, the slot event's turn runs, but + the drain recursion exits before promoting (or the busy window ends + through an exception / interrupt / generation-bump exit that never + reaches the promotion site) — the overflow entries are silently + orphaned: never dispatched, never persisted, never logged. + + This rescue runs at the point where a NEW event arrives for a + session that is NOT busy (the idle entry in + ``_process_message_priority``). If the session went idle with a + populated overflow, the oldest orphan is returned so the caller runs + it as THIS turn, and the next orphan (if any) is staged into the + slot so the post-turn drain continues the chain in arrival order + (#28503). The caller then enqueues the incoming event behind the + chain via ``_enqueue_fifo``. + + The returned event is REMOVED from both stores: leaving it in the + slot while it also runs as the current turn would make the post-turn + ``_dequeue_pending_event`` run it a second time. + + Returns the orphaned event to run now, or ``None`` when there is + nothing to rescue (no overflow, slot occupied, or no slot storage). + """ + try: + _q_state = self._peek_session_state(session_key) + overflow = _q_state.conversation.queued_events if _q_state else None + if not overflow: + return None + pending_slot = getattr(adapter, "_pending_messages", None) + if not isinstance(pending_slot, dict) or pending_slot.get(session_key): + # Slot occupied (busy) or no slot storage — promotion owns + # this; do not fight it from the idle path. + return None + head = overflow.pop(0) + # Keep the slot occupied for the rest of the chain so the drain + # promotes in order and any mid-chain arrival routes to overflow + # instead of jumping the queue (same invariant as the drain's + # own _promote_queued_event). Only ONE event fits the slot. + if overflow: + pending_slot[session_key] = overflow.pop(0) + logger.warning( + "Rescued orphaned FIFO overflow event for idle session " + "%s — it was queued during a busy window but the post-turn " + "drain never promoted it (#99882)", + session_key, + ) + if overflow: + logger.warning( + "%d overflow event(s) still queued for session %s after " + "rescue staging (will drain via normal promotion)", + len(overflow), + session_key, + ) + return head + except Exception: + logger.debug("FIFO overflow rescue failed for %s", session_key, exc_info=True) + return None + @staticmethod def _is_goal_continuation_event(event_or_text: Any) -> bool: """Return True for synthetic /goal continuation turns. @@ -10050,7 +10416,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> str: """Ephemeral system prompt for this channel/thread. - Uses ``channel_overrides`` when set, else the global gateway prompt. + Uses ``channel_overrides`` when set, else the gateway prompt resolved + from the CURRENT profile's config on every call. Callers run inside + ``_profile_runtime_scope`` (``run_sync`` under ``_run_agent``), so a + routed multiplex profile gets its own ``display.personality`` / + ``agent.system_prompt`` instead of a boot-time snapshot of the launch + profile's (#89161); ``/personality`` edits take effect on the next + turn for the same reason. Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` in ``run_sync`` (adapter ``resolve_channel_prompt``), so they are not duplicated here. @@ -10066,7 +10438,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) if override and override.system_prompt: return (override.system_prompt or "").strip() - return getattr(self, "_ephemeral_system_prompt", None) or "" + return self._load_ephemeral_system_prompt() @staticmethod def _load_reasoning_config(model: str = "") -> dict | None: @@ -10220,6 +10592,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return None if value in {"fast", "priority", "on"}: return "priority" + if value in {"auto", "cold"}: + return value logger.warning("Unknown service_tier '%s', ignoring", raw) return None @@ -11311,25 +11685,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew last_active_count = self._running_agent_count() last_cron_count = self._active_cron_job_count() last_api_count = self._active_api_run_count() + last_deferred_count = self._active_deferred_agent_worker_count() last_status_at = 0.0 def _maybe_update_status(force: bool = False) -> None: - nonlocal last_active_count, last_cron_count, last_api_count, last_status_at + nonlocal last_active_count, last_cron_count, last_api_count + nonlocal last_deferred_count, last_status_at now = asyncio.get_running_loop().time() active_count = self._running_agent_count() cron_count = self._active_cron_job_count() api_count = self._active_api_run_count() + deferred_count = self._active_deferred_agent_worker_count() if ( force or active_count != last_active_count or cron_count != last_cron_count or api_count != last_api_count + or deferred_count != last_deferred_count or (now - last_status_at) >= 1.0 ): self._update_runtime_status("draining") last_active_count = active_count last_cron_count = cron_count last_api_count = api_count + last_deferred_count = deferred_count last_status_at = now # Cron jobs run on the scheduler's own thread pool, outside @@ -11338,7 +11717,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # or a cron job's tool work gets killed with zero warning the # instant it's the only active thing running (#60432). # API-server / desk sessions have the same structural gap (#63529). - if not self._running_agents and last_cron_count == 0 and last_api_count == 0: + if ( + not self._running_agents + and last_cron_count == 0 + and last_api_count == 0 + and last_deferred_count == 0 + ): _maybe_update_status(force=True) return snapshot, False @@ -11359,7 +11743,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew def _still_draining() -> bool: now = loop.time() if ( - len(self._running_agents) or self._active_api_run_count() + len(self._running_agents) + or self._active_api_run_count() + or self._active_deferred_agent_worker_count() ) and now < deadline: return True return bool(self._active_cron_job_count()) and now < cron_deadline @@ -11375,6 +11761,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew bool(len(self._running_agents)) or bool(self._active_cron_job_count()) or bool(self._active_api_run_count()) + or bool(self._active_deferred_agent_worker_count()) ) _maybe_update_status(force=True) return snapshot, timed_out @@ -11394,6 +11781,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew interrupted_api = self._interrupt_api_server_runs(reason) if interrupted_api: logger.debug("Interrupted %d api_server run(s) during shutdown", interrupted_api) + interrupted_deferred = self._interrupt_deferred_agent_workers(reason) + if interrupted_deferred: + logger.debug( + "Interrupted %d deferred agent worker(s) during shutdown", + interrupted_deferred, + ) async def _notify_interrupted_cron_jobs(self, job_ids) -> int: """Tell the owner of each just-interrupted cron job that its run died. @@ -11433,7 +11826,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # deliver=local jobs — and deliver=origin jobs with no # resolvable origin (#43014) — resolve to zero targets and # must stay silent rather than fall back to a home channel. - targets = _resolve_delivery_targets(job) + # Interrupted notices are failure-category engine status, so + # they honor the job's failure_deliver override (NS-788). + targets = _resolve_delivery_targets(job, for_failure=True) except Exception as e: logger.debug("Cron interrupt targets unresolved for %s: %s", job_id, e) continue @@ -11831,6 +12226,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) await self._cleanup_agent_resources_off_loop(agent, context=context) + self._track_deferred_agent_worker(future, agent) + task = asyncio.create_task(_cleanup_when_done()) tasks = getattr(self, "_deferred_agent_cleanup_tasks", None) if tasks is None: @@ -14005,13 +14402,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._sync_voice_mode_state_to_adapter(adapter) # Wire voice input callback at connect time so voice # transcription is forwarded without requiring /voice join. - if hasattr(adapter, "_voice_input_callback"): - adapter._voice_input_callback = self._handle_voice_channel_input + self._bind_voice_input_callback(adapter) connected_count += 1 + _degraded = adapter.send_path_degraded self._update_platform_runtime_status( - platform.value, platform_state="connected", error_code=None, error_message=None, + platform.value, + platform_state="retrying" if _degraded else "connected", + error_code=None, + error_message=adapter.DEGRADED_STATUS_MESSAGE if _degraded else None, ) - logger.info("\u2713 %s connected", platform.value) + logger.info("\u2713 %s connected%s", platform.value, " (degraded)" if _degraded else "") else: # outcome == "failed" logger.warning("\u2717 %s failed to connect", platform.value) # Defensive cleanup: a failed connect() may have allocated resources @@ -14020,20 +14420,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Python logs "Unclosed client session" at process exit. await self._safe_adapter_disconnect(adapter, platform) if adapter.has_fatal_error: + # A live foreign holder of this bot token / identity is + # a single-writer ownership conflict, not a transient + # blip — even though ``_acquire_platform_lock`` emits it + # retryable so a MID-RUN reconnect can recover (#54167). + # At startup route it as non-retryable: with nothing + # connected the gateway exits 78 instead of sitting alive + # and deaf in the retry queue forever (#83183). + _retryable = adapter.fatal_error_retryable and not ( + is_global_startup_conflict(adapter.fatal_error_code) + ) self._update_platform_runtime_status( platform.value, - platform_state="retrying" if adapter.fatal_error_retryable else "fatal", + platform_state="retrying" if _retryable else "fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) target = ( startup_retryable_errors - if adapter.fatal_error_retryable + if _retryable else startup_nonretryable_errors ) target.append(f"{platform.value}: {adapter.fatal_error_message}") # Queue for reconnection if the error is retryable - if adapter.fatal_error_retryable: + if _retryable: self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, @@ -14306,6 +14716,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Start background session expiry watcher to finalize expired sessions self._spawn_supervised(self._session_expiry_watcher, "session_expiry_watcher") + # Keep the /model picker's remote catalogs (curated manifest, + # OpenRouter live list, Nous Portal recommendations) warm on disk so a + # delisted or newly-published model reaches the picker within one TTL + # window (model_catalog.ttl_minutes, default 20) without waiting for a + # cold /model open to trigger the refresh. + self._spawn_supervised(self._model_catalog_refresh_watcher, "model_catalog_refresh_watcher") + # Stall watchdog: pending inbound + stale agent activity → warn user # to /new (does not kill the turn; see agent.session_stall_timeout). self._spawn_supervised(self._session_stall_watcher, "session_stall_watcher") @@ -14688,7 +15105,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if _phome is None: await _reclaim_stale(self) else: - with _profile_runtime_scope(_phome): + async with _async_profile_runtime_scope(_phome): await _reclaim_stale(self) except Exception: logger.debug("Stale-handoff reclaim failed", exc_info=True) @@ -14700,7 +15117,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if profile_home is None: await _tick(profile_name) else: - with _profile_runtime_scope(profile_home): + async with _async_profile_runtime_scope(profile_home): await _tick(profile_name) except asyncio.CancelledError: raise @@ -14778,14 +15195,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # The watcher already entered _profile_runtime_scope for this # profile, so a fresh load resolves that profile's config.yaml # and .env (home channel, tokens) rather than the primary's. + # Fail closed on a load error: self.config is the primary's, so + # falling back would deliver through the right bot to the + # WRONG chat and report completed. A failed row the CLI can + # retry beats a wrong delivery. try: handoff_config = load_gateway_config() - except Exception: - logger.warning( + except Exception as exc: + logger.error( "Handoff: could not load config for profile %s; " - "falling back to the primary's config", + "failing the handoff instead of delivering via the " + "primary's config", profile_name, exc_info=True, ) + raise RuntimeError( + f"could not load config for profile '{profile_name}': {exc}" + ) from exc # Adapter must be live. A relay-fronted gateway registers ONE adapter # under Platform.RELAY that fronts N logical platforms — so a literal @@ -15421,6 +15846,32 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return sent + async def _model_catalog_refresh_watcher(self) -> None: + """Refresh the /model picker's remote catalogs every TTL window. + + The picker itself only refreshes on a cold or stale open, so a + gateway that nobody opens ``/model`` in keeps serving whatever was + cached. This loop calls ``model_catalog.refresh_catalogs()`` (manifest + + OpenRouter live filter + Nous Portal recommendations) off-thread on + the configured cadence (``model_catalog.ttl_minutes``, default 20) so + the on-disk caches every surface reads are never older than one window. + """ + from hermes_cli.model_catalog import refresh_catalogs, refresh_interval_seconds + + await asyncio.sleep(30) # let startup settle + while self._running: + try: + await asyncio.to_thread(refresh_catalogs) + except Exception as exc: + logger.debug("Model catalog refresh failed: %s", exc) + try: + interval = refresh_interval_seconds() + except Exception: + interval = 1200.0 + deadline = time.monotonic() + interval + while self._running and time.monotonic() < deadline: + await asyncio.sleep(min(30.0, max(0.0, deadline - time.monotonic()))) + async def _session_stall_watcher(self, interval: float = 30.0): """Periodic pending-inbound + stale-activity stall watchdog (#72016). @@ -15718,19 +16169,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self.adapters[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) # Wire voice input callback on reconnect as well (#60623). - if hasattr(adapter, "_voice_input_callback"): - adapter._voice_input_callback = self._handle_voice_channel_input + self._bind_voice_input_callback(adapter) self.delivery_router.adapters = self.adapters del self._failed_platforms[platform] + # connect() returning True does not mean the adapter's + # receive path is confirmed -- Telegram's degraded + # reconnect returns True so the gateway stays up while + # its own ladder retries. Stamping "connected" here + # would undo the adapter's accurate status (#101391). + _degraded = adapter.send_path_degraded self._update_platform_runtime_status( platform.value, - platform_state="connected", + platform_state="retrying" if _degraded else "connected", error_code=None, - error_message=None, + error_message=adapter.DEGRADED_STATUS_MESSAGE if _degraded else None, needs_attention=False, retrying_since=None, ) - logger.info("✓ %s reconnected successfully", platform.value) + if _degraded: + logger.info("⚠ %s reconnected in degraded mode (receive path not yet confirmed)", platform.value) + else: + logger.info("✓ %s reconnected successfully", platform.value) # Final responses rejected while this adapter was down # are still owned by this live process, so startup @@ -16020,6 +16479,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "active_agents": self._running_agent_count(), "active_cron_jobs": self._active_cron_job_count(), "active_api_runs": self._active_api_run_count(), + "active_deferred_agent_workers": getattr( + self, + "_active_deferred_agent_worker_count", + lambda: 0, + )(), "restart_drain_timeout": self._restart_drain_timeout, "watchdog_delay_s": resolve_shutdown_watchdog_delay( self._restart_drain_timeout @@ -16046,6 +16510,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _watchdog_done.set() async def _stop_impl_body(_kill_tool_subprocesses, _stop_started_at_box) -> None: + # Shutdown-path tests and third-party runner doubles may only + # implement the older drain-count surface. + _deferred_worker_count = getattr( + self, + "_active_deferred_agent_worker_count", + lambda: 0, + ) logger.info( "Stopping gateway%s...", " for restart" if self._restart_requested else "", @@ -16111,6 +16582,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _cron_at_start = self._active_cron_job_count() _api_at_start = self._active_api_run_count() + _deferred_at_start = _deferred_worker_count() # In-flight cron work gets its own floor, clamped to the watchdog # leash we're already running under so the extra wait can never # cost us the post-drain cleanup window (#82161). @@ -16145,7 +16617,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Shutdown phase: drain done at +%.2fs (drain took %.2fs, " "timed_out=%s, active_at_start=%d, active_now=%d, " "cron_at_start=%d, cron_now=%d, " - "api_at_start=%d, api_now=%d)", + "api_at_start=%d, api_now=%d, " + "deferred_at_start=%d, deferred_now=%d)", _phase_elapsed(), _drain_elapsed, timed_out, @@ -16155,6 +16628,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._active_cron_job_count(), _api_at_start, self._active_api_run_count(), + _deferred_at_start, + _deferred_worker_count(), ) if not timed_out: @@ -16174,12 +16649,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if timed_out: logger.warning( "Gateway drain timed out after %.1fs with %d active agent(s), " - "%d in-flight cron job(s), and %d api_server run(s); " + "%d in-flight cron job(s), %d api_server run(s), and " + "%d deferred agent worker(s); " "interrupting remaining work.", _drain_elapsed, self._running_agent_count(), self._active_cron_job_count(), self._active_api_run_count(), + _deferred_worker_count(), ) # Mark forcibly-interrupted sessions as resume_pending BEFORE # interrupting the agents. This preserves each session's @@ -16234,7 +16711,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # to stop gets its tool subprocesses killed below before it can # unwind — the exact amputation this interrupt exists to avoid. while ( - self._running_agents or self._active_api_run_count() + self._running_agents + or self._active_api_run_count() + or _deferred_worker_count() ) and asyncio.get_running_loop().time() < interrupt_deadline: self._update_runtime_status("draining") await asyncio.sleep(0.1) @@ -16249,7 +16728,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # at settle-loop exit, re-signal so a late-materializing # agent gets a cooperative interrupt instead of going # straight to the tool-subprocess kill. - if self._running_agents or self._active_api_run_count(): + if ( + self._running_agents + or self._active_api_run_count() + or _deferred_worker_count() + ): self._interrupt_running_agents( _INTERRUPT_REASON_GATEWAY_RESTART if self._restart_requested @@ -16366,6 +16849,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew flush_pending_to_file(dict(self._pending_messages), reason="shutdown") except Exception: pass + # The FIFO tail lives in SessionState.conversation.queued_events, + # not in the slot dict above — flush it too or every follow-up + # parked in overflow at restart time is lost (#99882). + try: + from gateway.shutdown_flush import flush_overflow_to_file + flush_overflow_to_file( + { + _k: list(_v) + for _k, _v in dict(getattr(self, "_queued_events", None) or {}).items() + if _v + }, + reason="shutdown", + ) + except Exception: + pass # On the real runner these are live SessionState views whose # clear() resets one field per session — never a wholesale dict # swap, so a concurrent writer on another session can't lose its @@ -16406,58 +16904,115 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as _e: logger.debug("shutdown_cached_clients error: %s", _e) - # Close SQLite session DBs so the WAL write lock is released. - # Without this, --replace and similar restart flows leave the - # old gateway's connection holding the WAL lock until Python - # actually exits — causing 'database is locked' errors when - # the new gateway tries to open the same file. - # ``self`` holds the DB at ``_session_db`` (an AsyncSessionDB facade); - # unwrap to the sync handle. ``session_store`` holds it at ``_db``. - _self_db = getattr(self, "_session_db", None) - _self_db = getattr(_self_db, "_db", _self_db) - for _db in (_self_db, getattr(getattr(self, "session_store", None), "_db", None)): - if _db is None or not hasattr(_db, "close"): - continue - try: - _db.close() - except Exception as _e: - logger.debug("SessionDB close error: %s", _e) - # A multiplexed session_store caches one SessionDB per profile - # path (#88532); reading ``_db`` above only resolved the handle - # for the shutdown task's own (root) scope. Sweep the rest so - # secondary profiles' WAL locks are released before --replace - # brings a new gateway up on the same files. - _sweep = getattr( - getattr(self, "session_store", None), "close_all_db_handles", None + # Quiesce the gateway thread pool BEFORE the session databases + # are closed. This used to run *after* the close block below, + # which left two holes: + # + # (a) `_executor_closing` was still False during the close, so + # any coroutine reaching `_run_in_executor_with_context` + # minted a brand-new pool and ran more blocking DB work + # against handles that had just been closed; + # (b) cancelling `self._background_tasks` above does not stop a + # `run_in_executor` future that already started — the task + # dies, the worker thread keeps writing. + # + # Either way a write lands after `SessionDB.close()`, which has + # already checkpointed the WAL and let SQLite unlink the sidecar. + # The late write silently reopens the handle (#94736) and mints a + # fresh WAL generation behind that checkpoint, so teardown + # checkpoints the same file a second time from a connection the + # shutdown log never accounts for — the close-time page-write + # damage in #101093 and the split WAL generation in #101064. + # + # The wait is bounded and clamped to what is left of the shutdown + # watchdog leash (minus a second for the close itself), so a stuck + # worker can never cost us the post-close cleanup window (#82161). + _exec_quiesce_budget = max( + 0.0, + min( + _EXECUTOR_QUIESCE_TIMEOUT, + resolve_shutdown_watchdog_delay(timeout) + - _phase_elapsed() + - 1.0, + ), ) - if _sweep is not None: - try: - _sweep() - except Exception as _e: - logger.debug("SessionDB handle sweep error: %s", _e) - # Same sweep for the runner's own per-profile session_search - # handles (slash commands resolve them under profile scopes). - try: - GatewayRunner.close_all_session_db_handles(self) - except Exception as _e: - logger.debug("Runner SessionDB handle sweep error: %s", _e) - # Final sweep: close any shared SessionDB instances still held by - # the process-wide registry (in-process tools, cron, mirror, etc. - # that opened via get_shared_session_db but weren't released by - # the sweeps above). This is the safety net that guarantees no - # WAL write lock survives past gateway shutdown (#90837). - try: - from hermes_state import close_shared_session_dbs - closed = close_shared_session_dbs() - if closed: - logger.debug("Closed %d shared SessionDB instance(s) at shutdown", closed) - except Exception as _e: - logger.debug("Shared SessionDB close error: %s", _e) - GatewayRunner._shutdown_executor(self) - logger.info( - "Shutdown phase: SessionDB close done at +%.2fs", - _phase_elapsed(), + _exec_live = GatewayRunner._shutdown_executor( + self, drain_timeout=_exec_quiesce_budget ) + if _exec_live: + # A live worker can still be mid-write against a SessionDB + # handle. Checkpointing/closing it now is exactly the + # sequence that produced the wrong-page-number corruption in + # #101093, so the close path below is skipped entirely + # rather than raced — the handle is left open for SQLite to + # recover from its own WAL on the next open, which is a + # transient "database is locked" on an immediate --replace + # at worst, not a corrupt file. + logger.warning( + "Shutdown phase: %d executor worker(s) still running after " + "a %.2fs quiesce — skipping the SessionDB close/checkpoint " + "to avoid racing a live write (#101093); handles are left " + "open for SQLite to recover on next open", + _exec_live, + _exec_quiesce_budget, + ) + else: + logger.info( + "Shutdown phase: executor quiesced at +%.2fs", + _phase_elapsed(), + ) + + # Close SQLite session DBs so the WAL write lock is released. + # Without this, --replace and similar restart flows leave the + # old gateway's connection holding the WAL lock until Python + # actually exits — causing 'database is locked' errors when + # the new gateway tries to open the same file. + # ``self`` holds the DB at ``_session_db`` (an AsyncSessionDB facade); + # unwrap to the sync handle. ``session_store`` holds it at ``_db``. + _self_db = getattr(self, "_session_db", None) + _self_db = getattr(_self_db, "_db", _self_db) + for _db in (_self_db, getattr(getattr(self, "session_store", None), "_db", None)): + if _db is None or not hasattr(_db, "close"): + continue + try: + _db.close() + except Exception as _e: + logger.debug("SessionDB close error: %s", _e) + # A multiplexed session_store caches one SessionDB per profile + # path (#88532); reading ``_db`` above only resolved the handle + # for the shutdown task's own (root) scope. Sweep the rest so + # secondary profiles' WAL locks are released before --replace + # brings a new gateway up on the same files. + _sweep = getattr( + getattr(self, "session_store", None), "close_all_db_handles", None + ) + if _sweep is not None: + try: + _sweep() + except Exception as _e: + logger.debug("SessionDB handle sweep error: %s", _e) + # Same sweep for the runner's own per-profile session_search + # handles (slash commands resolve them under profile scopes). + try: + GatewayRunner.close_all_session_db_handles(self) + except Exception as _e: + logger.debug("Runner SessionDB handle sweep error: %s", _e) + # Final sweep: close any shared SessionDB instances still held by + # the process-wide registry (in-process tools, cron, mirror, etc. + # that opened via get_shared_session_db but weren't released by + # the sweeps above). This is the safety net that guarantees no + # WAL write lock survives past gateway shutdown (#90837). + try: + from hermes_state import close_shared_session_dbs + closed = close_shared_session_dbs() + if closed: + logger.debug("Closed %d shared SessionDB instance(s) at shutdown", closed) + except Exception as _e: + logger.debug("Shared SessionDB close error: %s", _e) + logger.info( + "Shutdown phase: SessionDB close done at +%.2fs", + _phase_elapsed(), + ) from gateway.status import remove_pid_file, release_gateway_runtime_lock remove_pid_file() @@ -16655,12 +17210,47 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> int: """Create+connect one profile's adapters under its runtime scope.""" from gateway.config import load_gateway_config + from hermes_cli.env_loader import hydrate_profile_secret_sources - with _profile_runtime_scope(profile_home): + # Hydrate external secret sources (1Password/vault/...) off-loop ONCE, + # then enter the scope without re-hydrating: the sync hydration is + # network-bound and would otherwise stall every other profile's + # heartbeat while this one boots (same class as the reconnect path). + await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) + + with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_runtime_cfg = _load_gateway_runtime_config() from hermes_cli.plugins import discover_plugins discover_plugins() + + # Register this profile's own declarative shell hooks and + # outbound webhooks. The startup-time registration in + # start() only ever sees the root/default profile's config + # (it runs before any profile scope exists), so without this + # a secondary profile's `hooks:` block is silently inert — + # its turns run under this profile's own plugin manager + # (hermes_cli.plugins.get_plugin_manager keys by resolved + # home), which never received the callbacks. + try: + from hermes_cli.config import load_config as _load_profile_config + from agent.shell_hooks import ( + register_from_config as _register_shell_hooks, + ) + from agent.outbound_webhooks import ( + register_from_config as _register_outbound_webhooks, + ) + + _profile_hooks_cfg = _load_profile_config() + _register_shell_hooks(_profile_hooks_cfg, accept_hooks=False) + _register_outbound_webhooks(_profile_hooks_cfg) + except Exception: + logger.warning( + "shell-hook/webhook registration failed for profile '%s'", + profile_name, + exc_info=True, + ) + profile_cfg = load_gateway_config() violation = _own_policy_open_startup_violation(profile_cfg) self._snapshot_profile_busy_modes(profile_name, profile_runtime_cfg) @@ -16694,16 +17284,40 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew for platform, platform_config in profile_cfg.platforms.items(): if not platform_config.enabled: continue - # Relay is shared process-level ingress in multiplex mode. The - # active profile owns the one connection; connector-stamped - # source.profile routes inbound turns to secondary profiles. + # A platform enabled in a secondary profile's config.yaml may + # have no credential in that profile's secret scope — the shared + # YAML enables it for the default profile only (#84079). Building + # an adapter here would treat every credential-less profile as + # configured for the platform and one inbound message would fan + # out across all of them. Mirror the primary startup loop's + # credential gate and skip instead; profiles with their own + # credential still connect below. if ( getattr(self.config, "multiplex_profiles", False) - and platform is Platform.RELAY + and not _platform_has_bot_credential(platform, platform_config) + ): + logger.info( + "[MULTIPLEX] Profile '%s': skipping %s - no bot credential " + "in this profile's secrets", + profile_name, + platform.value, + ) + continue + # Relay and WhatsApp are shared process-level ingress in multiplex + # mode: one connection owned by the active profile, with + # route-stamped source.profile fanning inbound turns out to + # secondary profiles. The WhatsApp bridge is a single authenticated + # session tied to one phone number -- a secondary profile has no + # credential of its own to bring, so constructing an adapter for it + # only yields a connect/retry loop that stalls startup for every + # profile queued behind it. + if ( + getattr(self.config, "multiplex_profiles", False) + and platform in (Platform.RELAY, Platform.WHATSAPP) ): continue try: - with _profile_runtime_scope(profile_home): + with _profile_runtime_scope(profile_home, hydrate_secrets=False): adapter = self._create_adapter(platform, platform_config) except Exception as e: logger.error( @@ -16787,12 +17401,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._configure_profile_adapter(adapter, profile_name, platform) try: - with _profile_runtime_scope(profile_home): + with _profile_runtime_scope(profile_home, hydrate_secrets=False): success = await self._connect_initial_adapter_with_timeout( adapter, platform ) if success: profile_map[platform] = adapter + # Restore persisted /voice state for this bot (#84872) — + # primary startup and every reconnect path already do. + self._sync_voice_mode_state_to_adapter(adapter) if credential_claim is not None: claimed[credential_claim] = profile_name if listener_claim is not None: @@ -16851,12 +17468,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew adapter.set_platform_event_handler( self._make_profile_platform_event_handler(profile_name) ) + # Voice transcripts from this bot's channels dispatch through THIS + # adapter (primary wiring lives at connect time; see #75198). + self._bind_voice_input_callback(adapter) text_modes = getattr(self, "_busy_text_modes_by_profile", None) adapter._busy_text_mode = ( text_modes.get(profile_name, self._busy_text_mode) if isinstance(text_modes, dict) else self._busy_text_mode ) + # Secondary adapters always carry the profile they serve so prune + # paths namespace topic bindings correctly under multiplex (#76423). + adapter._hermes_profile_name = profile_name async def _run_secondary_profile_reconnect( self, profile_name: str, platform: Platform @@ -16869,13 +17492,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew adapter = None try: from hermes_cli.profiles import get_profile_dir + from hermes_cli.env_loader import hydrate_profile_secret_sources from gateway.config import load_gateway_config profile_home = get_profile_dir(profile_name) - with _profile_runtime_scope(profile_home): + # Like the #16856 MCP discovery path, hydrate external secret + # sources off-loop so they cannot starve platform heartbeats. + await asyncio.to_thread( + hydrate_profile_secret_sources, profile_home + ) + with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_config = load_gateway_config().platforms.get(platform) if profile_config is None or not profile_config.enabled: return + # Mirrors the startup credential gate (#84079): a + # credential removed from this profile's scope must + # not rebuild an adapter that would fan out turns. + if not _platform_has_bot_credential(platform, profile_config): + logger.info( + "Secondary %s reconnect skipped: no bot credential " + "(profile: %s)", + platform.value, + profile_name, + ) + return adapter = self._create_adapter(platform, profile_config) if adapter is None: logger.warning( @@ -16978,6 +17618,26 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """ if not getattr(adapter, "fatal_error_retryable", True): return + if is_global_startup_conflict(getattr(adapter, "fatal_error_code", None)): + # Same startup contract as the primary path: a live foreign holder + # of this profile's token/identity is an ownership conflict, not + # a transient blip. Park it fatal (like ``duplicate_credential``) + # instead of retry-storming the token every backoff (#83183). + logger.error( + "[MULTIPLEX] Profile '%s': %s credential is held by another " + "gateway (%s) — parked, not retried. %s", + profile_name, + platform.value, + adapter.fatal_error_code, + adapter.fatal_error_message or "", + ) + self._update_platform_runtime_status( + f"{profile_name}:{platform.value}", + platform_state="fatal", + error_code=adapter.fatal_error_code, + error_message=adapter.fatal_error_message, + ) + return async def _await_running_then_schedule() -> None: if self._running: @@ -17119,7 +17779,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception: pass if profile_home is not None: - with _profile_runtime_scope(profile_home): + async with _async_profile_runtime_scope(profile_home): return await self._handle_message(event) return await self._handle_message(event) @@ -17183,7 +17843,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if getattr(source, "profile", None) else default_home ) - with _profile_runtime_scope(profile_home): + async with _async_profile_runtime_scope(profile_home): return await self._handle_message(event) return _handler @@ -17326,6 +17986,16 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # of a bot token. Including its secret keeps multiplexed profiles # from spawning competing sidecars for the same account and port. "_project_secret", + # Feishu/Lark authenticates with an app_id/app_secret pair rather + # than a single token (one active WebSocket connection per app). + # app_id is stable, log-safe, and already used as the adapter's + # _app_lock_identity, so including it lets the multiplex guard + # refuse cloned profiles competing for the same Feishu app. + "_app_id", + # Same class: Teams (client_id/client_secret) and WeCom + # (bot_id/secret) authenticate with an app-style id pair too. + "_client_id", + "_bot_id", ): val = getattr(adapter, attr, None) if isinstance(val, str) and val.strip(): @@ -17356,11 +18026,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return hashlib.sha256(("hermes-mux:" + token).encode("utf-8")).hexdigest()[:16] def _create_adapter( - self, - platform: Platform, - config: Any + self, + platform: Platform, + config: Any, ) -> Optional[BasePlatformAdapter]: - """Create the appropriate adapter for a platform. + """Create an adapter and bind it to this gateway runner. + + Every lifecycle path — primary/secondary startup and reconnect — goes + through this method. Keep runner binding here so adapters can resolve + inbound profile routes before handlers or ``connect()`` run. + """ + adapter = self._instantiate_adapter(platform, config) + if adapter is not None: + adapter.gateway_runner = self + return adapter + + def _instantiate_adapter( + self, + platform: Platform, + config: Any, + ) -> Optional[BasePlatformAdapter]: + """Instantiate the appropriate adapter for a platform. Checks the platform_registry first (plugin adapters), then falls through to the built-in if/elif chain for core platforms. @@ -17381,14 +18067,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if platform_registry.is_registered(platform.value): adapter = platform_registry.create_adapter(platform.value, config) if adapter is not None: - # Inject a back-reference to the gateway runner so every - # adapter can (a) deliver cross-platform admin alerts and - # (b) resolve inbound profile routing through - # ``runner._profile_name_for_source``. Unconditional: - # ``BasePlatformAdapter`` declares ``gateway_runner``, so - # this reaches ALL platforms (not just the ones that - # pre-declared it), making profile routing platform-generic. - adapter.gateway_runner = self return adapter # Registered but failed to instantiate — don't silently fall # through to built-ins (there are none for plugin platforms). @@ -17440,18 +18118,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not check_api_server_requirements(): logger.warning("API Server: aiohttp not installed") return None - adapter = APIServerAdapter(config) - adapter.gateway_runner = self - return adapter + return APIServerAdapter(config) elif platform == Platform.WEBHOOK: from gateway.platforms.webhook import WebhookAdapter, check_webhook_requirements if not check_webhook_requirements(): logger.warning("Webhook: aiohttp not installed") return None - adapter = WebhookAdapter(config) - adapter.gateway_runner = self # For cross-platform delivery - return adapter + return WebhookAdapter(config) elif platform == Platform.MSGRAPH_WEBHOOK: from gateway.platforms.msgraph_webhook import ( @@ -17506,11 +18180,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ``profile_name`` binds the callback to the secondary adapter's own multiplex profile, so its ``SessionSource`` resolves that profile's secret scope instead of falling back to the active profile. + + For the shared primary adapter under ``multiplex_profiles`` + (``profile_name`` is None) the callback mirrors the inbound message + path exactly: the chat's ``profile_routes`` match is stamped on the + source so the routed profile's pairing store is consulted, while the + allowlist/gate reads stay under the transport (launch) home via + ``_is_user_authorized_for_source`` — the same split + ``_make_default_profile_message_handler`` applies. Without this an + inline-button caller approved only in the routed profile's pairing + store was denied (#86296), because the adapter's callback source was + never route-stamped. """ + multiplex = bool(getattr(self.config, "multiplex_profiles", False)) + transport_home = ( + Path(get_hermes_home()) if multiplex and profile_name is None else None + ) + def check( user_id: str, chat_type: Optional[str] = None, chat_id: Optional[str] = None, + *, + is_bot: bool = False, + thread_id: Optional[str] = None, ) -> bool: if not user_id: return False @@ -17519,9 +18212,34 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew chat_id=chat_id or "", chat_type=chat_type or "group", user_id=user_id, + thread_id=thread_id, + is_bot=bool(is_bot), profile=profile_name, ) - return self._is_user_authorized(source) + # Same in-process transport provenance ``build_source`` retains, so + # adapter-level policy reads (config.yaml group_allowed_chats, + # allow_from) resolve the receiving adapter even once the routed + # profile is stamped below. + registry = ( + (getattr(self, "_profile_adapters", None) or {}).get(profile_name) + if profile_name + else getattr(self, "adapters", None) + ) or {} + adapter = registry.get(platform) + if adapter is not None: + source._transport_adapter_ref = _weakref.ref(adapter) + if transport_home is None: + return self._is_user_authorized(source) + source._authorization_profile_home = transport_home + from gateway.profile_routing import ProfileRouteRejected + + try: + source.profile = self._profile_name_for_source(source) + except ProfileRouteRejected: + # Same fail-closed outcome as the ingress gate in + # ``_handle_message`` for a route to an unserved profile. + return False + return self._is_user_authorized_for_source(source) return check @@ -19601,6 +20319,50 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _quick_key, ) return _limit_message + + # ── FIFO orphan rescue (#99882) ──────────────────────────────── + # If this session went idle with a populated overflow (queued + # during a busy window whose post-turn drain never promoted — + # e.g. a compression-demoted follow-up after the compression + # window ended through an exit that skipped the promotion site), + # those events were silently orphaned. We are starting the next + # turn for this session NOW: re-stage the orphans in FIFO order + # and enqueue the incoming event behind them, so arrival order + # (#28503) holds: oldest orphan runs as this turn, the rest drain + # in order, the new message last. Skipped for control commands + # (/stop etc. own their own semantics) and internal events. + try: + _orphan_adapter = self._adapter_for_source(source) + if ( + _orphan_adapter is not None + and not bool(getattr(event, "internal", False)) + and not event.get_command() + ): + _rescued = self._rescue_orphaned_overflow( + _quick_key, _orphan_adapter + ) + if _rescued is not None: + # The oldest orphan runs as THIS turn. Park the + # incoming event behind the rest of the chain: into the + # slot when the chain was a single orphan (so the + # post-turn drain picks it up), otherwise into overflow + # behind the already-staged next orphan (FIFO). + self._enqueue_fifo(_quick_key, event, _orphan_adapter) + event = _rescued + # Same session key by construction; carry the orphan's + # own source so reply anchors / thread metadata point + # at the message that is actually being answered. + _rescued_source = getattr(_rescued, "source", None) + if _rescued_source is not None: + source = _rescued_source + is_internal = bool(getattr(_rescued, "internal", False)) + except Exception: + logger.debug( + "FIFO orphan rescue pre-claim failed for %s", + _quick_key, + exc_info=True, + ) + _claim_state = self._session_state(_quick_key) if _active_session_lease is not None: _claim_state.turn.lease = _active_session_lease @@ -20017,7 +20779,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew from agent.context_references import preprocess_context_references_async from agent.model_metadata import get_model_context_length_async - _msg_cwd = os.environ.get("TERMINAL_CWD", os.path.expanduser("~")) + try: + from tools.terminal_scope import terminal_env as _ts_env + except ImportError: + _msg_cwd = os.environ.get("TERMINAL_CWD", os.path.expanduser("~")) + else: + _msg_cwd = _ts_env("TERMINAL_CWD", os.path.expanduser("~")) _msg_config_ctx = None _msg_cfg = None _msg_model_cfg = {} @@ -20128,7 +20895,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew ) -> Optional[str]: """Run inbound preprocessing under the routed profile when multiplexed.""" if getattr(getattr(self, "config", None), "multiplex_profiles", False): - with _profile_runtime_scope(self._resolve_profile_home_for_source(source)): + async with _async_profile_runtime_scope( + self._resolve_profile_home_for_source(source) + ): return await self._prepare_inbound_message_text( event=event, source=source, @@ -20510,6 +21279,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew binding = (await self._session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=self._telegram_topic_profile_name(source), )) if self._session_db else None except Exception: logger.debug("Failed to read Telegram topic binding", exc_info=True) @@ -20808,8 +21578,23 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # began processing if the gateway died while it was still waiting. await self._mark_durable_active_turn(event, session_entry.session_key) - # Load conversation history from transcript - history = await self.async_session_store.load_transcript(session_entry.session_id) + # Load conversation history from transcript. An unreadable canonical + # store is not an empty conversation: stop before the agent can invent + # continuity from a plausible-looking []. This return happens before + # the broad cleanup finally below, so restore task-local context here; + # the outer dispatch still clears the durable marker and turn lease. + try: + history = await self.async_session_store.load_transcript( + session_entry.session_id + ) + except TranscriptReadError: + self._clear_session_env(_session_env_tokens) + return ( + "⚠️ This session's history is temporarily unavailable, so " + "this message was not processed. Ask the operator to inspect " + "state.db, then resend after it is healthy. Use /reset only " + "if you intentionally want to start a new conversation." + ) # ----------------------------------------------------------------- # Session hygiene: auto-compress pathologically large transcripts @@ -21250,8 +22035,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_commit_fence = CompressionCommitFence( total_ceiling_seconds=_hyg_total_ceiling_seconds ) + # Default executor (NOT self._get_executor): + # a fence-cancelled hung summary must never + # occupy one of the gateway's agent-work + # slots. But it MUST run inside the caller's + # contextvars: under multiplex_profiles the + # profile secret scope / HERMES_HOME override + # live in ContextVars, and a bare + # run_in_executor worker starts with an empty + # Context — the summary model's + # get_secret(_API_KEY) then fails + # closed (UnscopedSecretError) and every + # hygiene compaction silently degrades to a + # lossy truncation (#100849 bundle). _hyg_future = loop.run_in_executor( None, + copy_context().run, lambda: _hyg_agent._compress_context( _hyg_msgs, "", approx_tokens=_approx_tokens, @@ -21399,6 +22198,175 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # uncompressed) but with distinct provenance, # user message, and NO failure-cooldown # increment. + # + # #97963: decouple the TURN from the + # COMPRESSION. When the worker's commit is + # watermark-fenced (it captured the session's + # active-row watermark at compression start, + # so rows appended after that point — this + # released turn included — survive its late + # commit verbatim as cloned concurrent tail), + # the already-running attempt KEEPS its commit + # admission: the user's turn proceeds on the + # uncompressed transcript NOW, and the summary + # is adopted when the detached worker reaches + # its own watermark-fenced commit transaction + # (archive_and_compact / the rotation publish + # path — the next safe boundary). Before this, + # the fence was ALWAYS cancelled here, burning + # the full summary attempt — for a thinking + # summary model whose reasoning prefix alone + # exceeds the 10s hold, that made hygiene + # auto-compression fail 100% of the time while + # paying the summary model per turn. The turn + # itself is still released at the same budget: + # only the fate of the detached worker's + # RESULT changes. If the commit is NOT + # watermark-fenced (no session_db, watermark + # capture failed, legacy lock API), a late + # commit could clobber newer turns, so cancel + # exactly as before — never worse than the + # status quo. + _hyg_keep_admission = bool( + getattr( + _hyg_commit_fence, + "commit_watermark_fenced", + False, + ) + ) and not _hyg_commit_fence.is_cancelled + if _hyg_keep_admission: + self._defer_agent_cleanup_until_future_done( + _hyg_future, + _hyg_agent, + context="session hygiene turn-hold", + ) + _hyg_cleanup_deferred = True + # NO retry-after here (#97963 (b)): the + # attempt is still running toward a real + # commit, and arming the flat 60s + # retry-after would ALSO block the + # agent-side preflight compressor from a + # fresh chance ("Skipping preflight + # compression: same-session cooldown + # active"). Re-attempt spacing is covered + # by the durable compression lock instead: + # the next turn's hygiene pre-check skips + # while this worker's lease is held + # (_session_has_compression_in_flight). + # The flat retry-after is recorded by the + # done-callback below ONLY if the worker + # ends without committing anything. + _hyg_deferred_sid = session_entry.session_id + _hyg_deferred_key = session_key + _hyg_deferred_agent = _hyg_agent + + def _hyg_adopt_or_space_retry( + _fut, + _gw=self, + _sid=_hyg_deferred_sid, + _skey=_hyg_deferred_key, + _agent=_hyg_deferred_agent, + ): + try: + _exc = _fut.exception() + except ( + asyncio.CancelledError, + Exception, + ): + _exc = None + _committed = False + else: + _committed = _exc is None and ( + bool( + getattr( + _agent, + "_last_compaction_in_place", + False, + ) + ) + or getattr( + _agent, "session_id", _sid + ) + != _sid + ) + if _committed: + logger.info( + "Session hygiene compression for " + "session %s finished after the " + "turn-hold was released — summary " + "adopted at the watermark-fenced " + "commit boundary (#97963)", + _sid, + ) + try: + _reset_hygiene_failure_streak( + _gw, _skey + ) + except Exception as _rs_err: + logger.debug( + "hygiene streak reset after " + "deferred adoption failed: %s", + _rs_err, + ) + else: + # Nothing to adopt (summary failed, + # fence refused the commit, or the + # attempt was superseded). Restore + # the pre-#97963 spacing so + # sustained traffic does not spawn + # and abandon a fresh compressor + # every turn. Flat and + # non-escalating: the streak must + # not advance for a deferral. + _record_hygiene_cooldown( + _gw, _sid, + _HYGIENE_TURNHOLD_RETRY_SECONDS, + "hygiene compression deferred: " + "turn-hold budget expired and the " + "detached attempt did not commit", + ) + + _hyg_future.add_done_callback( + _hyg_adopt_or_space_retry + ) + from agent.session_activity import ( + ActivityProvenance, + ) + _stamp_hygiene_compression_provenance( + _hyg_agent, + "session hygiene compression turn-hold", + ActivityProvenance.AGENT_COMPRESSION_TURNHOLD, + "hygiene compression turn-hold " + "activity stamp failed", + ) + logger.info( + "Session hygiene compression for session %s " + "exceeded turn-hold budget (%.1fs); " + "proceeding without compression this turn — " + "the watermark-fenced worker keeps its " + "commit admission and the summary will be " + "adopted when it finishes", + session_entry.session_id, + time.monotonic() - _hyg_wait_started, + ) + _turnhold_msg = t( + "gateway.compress.turnhold_deferred" + ) + try: + _adapter = self._adapter_for_source(source) + if _adapter and source.chat_id: + await _adapter.send( + source.chat_id, + _turnhold_msg, + metadata=_hyg_meta, + ) + except Exception as _werr: + logger.warning( + "Failed to deliver compression-turnhold " + "notice to user: %s", + _werr, + ) + raise _cancelled = None while _cancelled is None: if _hyg_commit_fence.commit_in_flight: @@ -21965,6 +22933,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_agent, context="session hygiene" ) + except HygieneTurnHoldExceeded: + # Availability boundary, not a failure — already logged + # at INFO by the turn-hold handler. Must not hit the + # generic "auto-compress failed" warning below: that + # log is how thinking-model deployments read as + # permanently broken (#97963; surfaced by @686f6c61 + # in PR #99657). + pass except Exception as e: logger.warning( "Session hygiene auto-compress failed: %s", e @@ -22405,7 +23381,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew model=agent_result.get("model"), context_tokens=agent_result.get("last_prompt_tokens", 0) or 0, context_length=agent_result.get("context_length") or None, - cwd=os.environ.get("TERMINAL_CWD", ""), + cwd=_terminal_scope_cwd(""), turn_seconds=_turn_seconds, ) except Exception as _footer_err: @@ -23717,8 +24693,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # init on the loop thread before the first read. await self._warm_goals_session_db("loop wakeup") + # Every SessionDB call in this scan runs off the loop thread. + # fire_tick()/complete_tick() are writes (BEGIN IMMEDIATE) that + # take the writer lock; a slow writer elsewhere (FTS merge, WAL + # checkpoint, a long flush) holding it while the watcher blocked + # the loop on the same lock froze the gateway for 90+ s until + # the liveness watchdog force-exited. list_active_loops() reads + # via _read_ctx (lock-free under WAL) but still convoys on the + # writer lock when WAL is unavailable, so it goes off-loop too. + # _run_in_executor_with_context keeps the profile HERMES_HOME + # override alive under multiplex, like the warm-up above. + active_loops = await self._run_in_executor_with_context(list_active_loops) + now = time.time() - for sid, state in list_active_loops(): + for sid, state in active_loops: if state.awaiting_response or now < state.next_due_at: continue route = state.route or {} @@ -23766,7 +24754,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew mgr = LoopManager(session_id=sid) if not mgr.is_due(now): continue - wakeup = mgr.fire_tick() + wakeup = await self._run_in_executor_with_context(mgr.fire_tick) if not wakeup: continue try: @@ -23786,7 +24774,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # path and never hit the post-turn completion hook — # complete the tick immediately (caps + scheduling). if wakeup.lstrip().startswith("/"): - mgr.complete_tick("") + await self._run_in_executor_with_context(mgr.complete_tick, "") except Exception as exc: logger.warning("loop wakeup injection failed for %s: %s", sid, exc) try: @@ -23830,15 +24818,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Wire callbacks BEFORE join so voice input arriving immediately # after connection is not lost. - if hasattr(adapter, "_voice_input_callback"): - adapter._voice_input_callback = self._handle_voice_channel_input + self._bind_voice_input_callback(adapter) + voice_profile = self._adapter_profile_for_source(event.source) if hasattr(adapter, "_on_voice_disconnect"): - adapter._on_voice_disconnect = self._handle_voice_timeout_cleanup + adapter._on_voice_disconnect = functools.partial( + self._handle_voice_timeout_cleanup, adapter=adapter + ) # Let the adapter's inactivity timer see the live voice-reply mode so it # doesn't disconnect a deliberately text-only (/voice off) session. if hasattr(adapter, "_voice_mode_getter"): adapter._voice_mode_getter = lambda chat_id: self._voice_mode.get( - self._voice_key(Platform.DISCORD, str(chat_id)), "off" + self._voice_key(Platform.DISCORD, str(chat_id), profile=voice_profile), + "off", ) try: @@ -23858,7 +24849,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew adapter._voice_text_channels[guild_id] = int(event.source.chat_id) if hasattr(adapter, "_voice_sources"): adapter._voice_sources[guild_id] = event.source.to_dict() - self._voice_mode[self._voice_key(event.source.platform, event.source.chat_id)] = "all" + self._voice_mode[self._voice_key_for_source(event.source)] = "all" self._save_voice_modes() self._set_adapter_auto_tts_enabled(adapter, event.source.chat_id, enabled=True) return ( @@ -23885,21 +24876,26 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except Exception as e: logger.warning("Error leaving voice channel: %s", e) # Always clean up state even if leave raised an exception - self._voice_mode[self._voice_key(event.source.platform, event.source.chat_id)] = "off" + self._voice_mode[self._voice_key_for_source(event.source)] = "off" self._save_voice_modes() self._set_adapter_auto_tts_disabled(adapter, event.source.chat_id, disabled=True) if hasattr(adapter, "_voice_input_callback"): adapter._voice_input_callback = None return "Left voice channel." - def _handle_voice_timeout_cleanup(self, chat_id: str) -> None: + def _handle_voice_timeout_cleanup(self, chat_id: str, *, adapter=None) -> None: """Called by the adapter when a voice channel times out. Cleans up runner-side voice_mode state that the adapter cannot reach. + ``adapter`` is the Discord adapter that timed out (bound at join time); + under multiplexing that is a specific profile's bot, not necessarily + ``self.adapters[DISCORD]``. """ - self._voice_mode[self._voice_key(Platform.DISCORD, chat_id)] = "off" + if adapter is None: + adapter = self.adapters.get(Platform.DISCORD) + profile = getattr(adapter, "_owner_profile", None) + self._voice_mode[self._voice_key(Platform.DISCORD, chat_id, profile=profile)] = "off" self._save_voice_modes() - adapter = self.adapters.get(Platform.DISCORD) self._set_adapter_auto_tts_disabled(adapter, chat_id, disabled=True) def _is_duplicate_voice_transcript(self, guild_id: int, user_id: int, transcript: str) -> bool: @@ -23944,14 +24940,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False async def _handle_voice_channel_input( - self, guild_id: int, user_id: int, transcript: str + self, guild_id: int, user_id: int, transcript: str, *, adapter=None ): """Handle transcribed voice from a user in a voice channel. Creates a synthetic MessageEvent and processes it through the adapter's full message pipeline (session, typing, agent, TTS reply). + ``adapter`` is the Discord adapter that captured the audio (bound via + ``_bind_voice_input_callback``); under multiplexing each profile's bot + must dispatch through its own adapter, never the default profile's. """ - adapter = self.adapters.get(Platform.DISCORD) + if adapter is None: + adapter = self.adapters.get(Platform.DISCORD) if not adapter: return @@ -23973,6 +24973,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew user_id=str(user_id), user_name=str(user_id), chat_type="channel", + profile=getattr(adapter, "_owner_profile", None), ) # Check authorization before processing voice input @@ -24044,11 +25045,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return False chat_id = event.source.chat_id - voice_key = self._voice_key(event.source.platform, chat_id) + voice_key = self._voice_key_for_source(event.source) voice_mode = self._voice_mode.get(voice_key) is_voice_input = (event.message_type == MessageType.VOICE) - adapter = self.adapters.get(event.source.platform) + adapter = self._adapter_for_source(event.source) adapter_auto_tts = False if adapter and hasattr(adapter, "_should_auto_tts_for_chat"): try: @@ -25078,6 +26079,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew binding = await session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=self._telegram_topic_profile_name(source), ) if binding and str(binding.get("session_id") or "") != str(session_id): return @@ -25189,15 +26191,15 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew """ if not hasattr(self, "_telegram_capability_hint_ts"): self._telegram_capability_hint_ts = {} - chat_id = str(source.chat_id or "") - if not chat_id: + key = self._telegram_topic_cooldown_key(source) + if not key: return True import time as _time now = _time.monotonic() - last = self._telegram_capability_hint_ts.get(chat_id, 0.0) + last = self._telegram_capability_hint_ts.get(key, 0.0) if now - last < self._TELEGRAM_CAPABILITY_HINT_COOLDOWN_S: return False - self._telegram_capability_hint_ts[chat_id] = now + self._telegram_capability_hint_ts[key] = now return True def _telegram_topic_help_text(self) -> str: @@ -25235,22 +26237,28 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew currently_enabled = await self._session_db.is_telegram_topic_mode_enabled( chat_id=chat_id, user_id=str(source.user_id or ""), + profile_name=self._telegram_topic_profile_name(source), ) except Exception: currently_enabled = False if not currently_enabled: return "Multi-session topic mode is not currently enabled for this chat." try: - await self._session_db.disable_telegram_topic_mode(chat_id=chat_id) + await self._session_db.disable_telegram_topic_mode( + chat_id=chat_id, + profile_name=self._telegram_topic_profile_name(source), + ) except Exception as exc: logger.exception("Failed to disable Telegram topic mode") return f"Failed to disable topic mode: {exc}" - # Reset per-chat debounce state so the user doesn't see a stale - # cooldown on the next activation. - for attr in ("_telegram_lobby_reminder_ts", "_telegram_capability_hint_ts"): - store = getattr(self, attr, None) - if isinstance(store, dict): - store.pop(chat_id, None) + # Reset per-profile+chat debounce state so the user doesn't see a + # stale cooldown on the next activation (issue #76423). + cooldown_key = self._telegram_topic_cooldown_key(source) + if cooldown_key: + for attr in ("_telegram_lobby_reminder_ts", "_telegram_capability_hint_ts"): + store = getattr(self, attr, None) + if isinstance(store, dict): + store.pop(cooldown_key, None) return ( "Multi-session topic mode is now OFF for this chat.\n\n" "Existing topics in Telegram aren't removed — they'll just stop " @@ -25272,6 +26280,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew sessions = await self._session_db.list_unlinked_telegram_sessions_for_user( chat_id=str(source.chat_id), user_id=str(source.user_id), + profile_name=self._telegram_topic_profile_name(source), limit=10, ) except Exception: @@ -25321,9 +26330,11 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return "That session does not belong to this Telegram user." linked = await self._session_db.is_telegram_session_linked_to_topic(session_id=session_id) + topic_profile = self._telegram_topic_profile_name(source) current_binding = await self._session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=topic_profile, ) if linked: if not current_binding or current_binding.get("session_id") != session_id: @@ -25338,6 +26349,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_key=session_key, session_id=session_id, managed_mode="restored", + profile_name=topic_profile, ) except ValueError as exc: if "already linked" in str(exc): @@ -25374,25 +26386,55 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew Split out from ``_handle_reload_mcp_command`` so the confirmation wrapper can invoke the same path whether the user confirmed via button, text reply, or has the confirm gate disabled. + + Under multiplex the reload runs inside the requesting profile's + runtime scope (entered here when the caller — e.g. a button-confirm + callback — did not), and only that profile's servers are torn down + and rediscovered (#95518). """ - loop = asyncio.get_running_loop() + multiplex = bool(getattr(self.config, "multiplex_profiles", False)) + if multiplex and not get_hermes_home_override(): + profile_home = self._resolve_profile_home_for_source(event.source) + with _profile_runtime_scope(Path(profile_home)): + return await self._execute_mcp_reload(event) try: from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock + from tools.mcp_tool import _server_scope_keys, reprobe_tool_availability + from tools.registry import registry + + reload_scope = registry.current_scope_key() if multiplex else None + + def _scoped_server_names() -> set: + with _lock: + return { + name for name in _servers + if reload_scope is None or _server_scope_keys.get(name) == reload_scope + } # Capture old server names before shutdown - with _lock: - old_servers = set(_servers.keys()) + old_servers = _scoped_server_names() # Read new config before shutting down, so we know what will be added/removed # Shutdown existing connections - await loop.run_in_executor(None, shutdown_mcp_servers) + await self._run_in_executor_with_context( + lambda: shutdown_mcp_servers(scope=reload_scope) + ) + # Explicit reload also re-probes tool availability (check_fn). + reprobe_tool_availability() # Reconnect by discovering tools (reads config.yaml fresh) - new_tools = await loop.run_in_executor(None, discover_mcp_tools) + new_tools = await self._run_in_executor_with_context(discover_mcp_tools) # Compute what changed - with _lock: - connected_servers = set(_servers.keys()) + connected_servers = _scoped_server_names() + if reload_scope is not None: + from tools.mcp_tool import _mcp_tool_server_names + + with _lock: + new_tools = [ + n for n in new_tools + if _mcp_tool_server_names.get(n) in connected_servers + ] added = connected_servers - old_servers removed = old_servers - connected_servers @@ -25421,8 +26463,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _cache = getattr(self, "_agent_cache", None) _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None and _cache: + # Multiplex: only this profile's sessions. Rebuilding + # another profile's agent inside this scope would hand it + # this profile's tool registry. + _ns_prefix = ( + _session_key_namespace(event.source.profile) + ":" + if multiplex else None + ) with _cache_lock: for _sess_key, _entry in list(_cache.items()): + if _ns_prefix and not str(_sess_key).startswith(_ns_prefix): + continue try: _agent = _entry[0] if isinstance(_entry, tuple) else _entry except Exception: @@ -25721,6 +26772,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew metadata.setdefault("scope_id", str(team_id)) if user_id: metadata.setdefault("user_id", str(user_id)) + # Routed profile for shared state.db namespaces (#76423): the Telegram + # prune path needs it because under profile_routes the transport + # adapter's stamp is not the profile that wrote the binding. + profile = str(getattr(source, "profile", None) or "").strip() + if profile and metadata is not None: + metadata = dict(metadata) + metadata["hermes_profile"] = profile return metadata def _thread_metadata_for_target( @@ -26380,17 +27438,30 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if not error: return - from hermes_state import classify_persistence_error, format_session_db_unavailable + from hermes_state import ( + _default_db_path, + classify_persistence_error, + format_session_db_unavailable, + ) cause = classify_persistence_error(error) hint = format_session_db_unavailable() if cause == "corrupt": + # Copy-pasteable, so name the real store (profiles / HERMES_HOME + # do not live under ~/.hermes). + db_path = _default_db_path() message = ( "⚠️ Session database corruption detected. Messages may not be " "persisted. Recovery options:\n" "1. Run `hermes doctor --fix`\n" - "2. Salvage with: sqlite3 ~/.hermes/state.db \".recover\" " - "(then replace state.db)\n" + "2. Stop the gateway, then recover with:\n" + f" hermes sessions recover --source {db_path} " + "--inspect-only\n" + " (if it reports recoverable) hermes sessions recover " + f"--source {db_path} --output recovered-state.db\n" + " — recovery snapshots the damaged file first; do NOT run " + "`sqlite3 ... \".recover\"` against the live state.db, a " + "vulnerable sqlite3 CLI can corrupt it further\n" "3. Restore from a backup in ~/.hermes/backups/\n" "Run `hermes doctor` for sanitized diagnostics." ) @@ -26524,11 +27595,21 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._executor = executor return executor - def _shutdown_executor(self) -> None: - """Stop the gateway-owned executor without touching the loop default.""" + def _shutdown_executor(self, drain_timeout: float = 0.0) -> int: + """Stop the gateway-owned executor without touching the loop default. + + Returns the number of worker threads still running when this returns. + With the default ``drain_timeout`` of 0 this is the historical + fire-and-forget teardown; shutdown passes a bounded budget so blocking + DB work cannot outlive ``SessionDB.close()`` (see ``_stop_impl``). + + ``cancel_futures`` only drops work that has not started yet, and a + cancelled ``run_in_executor`` awaitable does not stop the thread behind + it, so the running futures have to be waited on explicitly. + """ lock = getattr(self, "_executor_lock", None) if lock is None: - return + return 0 with lock: self._executor_closing = True @@ -26536,13 +27617,25 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._executor = None if executor is None: - return + return 0 try: executor.shutdown(wait=False, cancel_futures=True) except TypeError: executor.shutdown(wait=False) + # ThreadPoolExecutor.shutdown() has no timeout, so join the worker + # threads directly. `_threads` is absent on the doubles some tests + # pass in, which just means no wait. + workers = list(getattr(executor, "_threads", None) or ()) + deadline = time.monotonic() + max(float(drain_timeout or 0.0), 0.0) + for worker in workers: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + worker.join(remaining) + return sum(1 for worker in workers if worker.is_alive()) + def _decide_image_input_mode( self, *, @@ -32145,7 +33238,42 @@ def _run_planned_stop_watcher( stop_event.wait(poll_interval) -def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop=None, interval: int = 60, cron_provider=None): +def _drain_restart_safe_cron_deliveries(adapters, loop, runner=None) -> None: + """Drain each profile's worker queue through its matching live adapters.""" + from cron import scheduler as cron_scheduler + + if runner is None: + if adapters is not None: + cron_scheduler.drain_delivery_queue(adapters, loop) + return + for profile_name, profile_home in _handoff_watch_scopes(runner): + scoped_home = profile_home or get_hermes_home() + if profile_name is None: + profile_adapters = adapters + else: + profile_adapters = getattr(runner, "_profile_adapters", {}).get( + profile_name + ) + if profile_adapters is None: + continue + with _profile_runtime_scope(scoped_home): + if profile_name is not None and not profile_adapters and adapters: + routes = cron_scheduler._primary_profile_routes_for_current_home() + if routes: + profile_adapters = cron_scheduler.SharedRouteAdapters( + adapters, routes + ) + cron_scheduler.drain_delivery_queue(profile_adapters, loop) + + +def _start_gateway_housekeeping( + stop_event: threading.Event, + adapters=None, + loop=None, + interval: int = 60, + cron_provider=None, + runner=None, +): """Background thread for gateway-only periodic chores (NOT cron). Split out of the historical ``_start_cron_ticker`` so the cron *trigger* @@ -32179,6 +33307,7 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop AUTO_ARCHIVE_EVERY = 60 # ticks — poll hourly (state_meta gate owns the real cadence) MEMORY_TRIM_EVERY = 1 # shared helper cooldown bounds actual allocator work MISFIRE_SWEEP_EVERY = 5 # ticks — every 5 minutes (grace window gates real work) + FTS_STALE_RETRY_EVERY = 1 # SessionDB rate-limits the real work (_FTS_STALE_RETRY_SECONDS) # Every platform media cache prunes on the same hourly cadence — one loop # over (name, cleanup_fn), not a copy-pasted try/except per cache. @@ -32199,6 +33328,16 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop while not stop_event.is_set(): tick_count += 1 + # Restart-safe cron workers run outside the gateway cgroup and queue + # their final send for whichever gateway instance is live. Drain on + # the gateway-wide housekeeper rather than the built-in scheduler tick: + # external providers do not run that ticker. + if adapters is not None or runner is not None: + try: + _drain_restart_safe_cron_deliveries(adapters, loop, runner) + except Exception as exc: + logger.debug("Cron durable delivery queue drain error: %s", exc) + if tick_count % CHANNEL_DIR_EVERY == 0 and adapters: try: from gateway.channel_directory import build_channel_directory @@ -32313,6 +33452,29 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop except Exception as e: logger.debug("Auto-archive tick error: %s", e) + # Deferred stale-FTS rebuild retry (#100108). A SessionDB that opened + # while another process held state.db / the rebuild lock fails closed + # and leaves search on the LIKE fallback; a short-lived CLI clears + # that on its next open, but the gateway opens once and stays up for + # days. Retry here, on the existing tick, against the shared + # instances this process already holds: non-blocking admission, no + # new thread, rate-limited inside SessionDB. No-op when nothing is + # stale (one attribute read per instance). + if tick_count % FTS_STALE_RETRY_EVERY == 0: + try: + from hermes_state_registry import live_shared_session_dbs + + for _sdb in live_shared_session_dbs(): + _retry = getattr(_sdb, "retry_deferred_fts_recovery", None) + if callable(_retry) and _retry(): + logger.info( + "Deferred state.db FTS rebuild completed in-process " + "for %s; full-text search restored.", + getattr(_sdb, "db_path", "state.db"), + ) + except Exception as exc: + logger.debug("Deferred FTS retry tick error: %s", exc) + # This is the long-lived messaging-gateway counterpart to the TUI idle # reaper. The helper is config-gated and rate-limited, so calling it on # the 60s housekeeping cadence does not create a trim storm. @@ -32910,6 +34072,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = logging.getLogger().setLevel(_stderr_level) runner = GatewayRunner(config) + # Multiplex: swap the launch-home file handlers for per-profile routers so + # each profile's records land in its own logs/ (#82936). Must run after + # the runner resolved the (possibly None) config and after setup_logging. + _enable_multiplex_log_routing(runner.config) # ``--replace`` is explicit startup authority, not a durable reconnect # policy. GatewayRunner scopes this bit to cold adapter connects and clears # it before the background reconnect watcher starts. @@ -33189,9 +34355,7 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # heartbeats (Discord shard, Telegram polling) until it returned. # See #16856. try: - from tools.mcp_tool import discover_mcp_tools - _loop = asyncio.get_running_loop() - await _loop.run_in_executor(None, discover_mcp_tools) + await _discover_gateway_mcp_tools(runner.config) except Exception as e: logger.debug("MCP tool discovery failed: %s", e) @@ -33355,6 +34519,7 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = "adapters": runner.adapters, "loop": asyncio.get_running_loop(), "cron_provider": cron_provider, + "runner": runner, }, daemon=True, name="gateway-housekeeping", diff --git a/gateway/runtime_footer.py b/gateway/runtime_footer.py index 8719524d5a..2526bc3e10 100644 --- a/gateway/runtime_footer.py +++ b/gateway/runtime_footer.py @@ -138,7 +138,13 @@ def format_runtime_footer( if turn_seconds is not None and turn_seconds >= 0: parts.append(_format_latency(turn_seconds)) elif field == "cwd": - rel = _home_relative_cwd(cwd or os.environ.get("TERMINAL_CWD", "")) + try: + from tools.terminal_scope import terminal_env as _tenv + except ImportError: + env_cwd = os.environ.get("TERMINAL_CWD", "") + else: + env_cwd = _tenv("TERMINAL_CWD", "") + rel = _home_relative_cwd(cwd or env_cwd) if rel: parts.append(rel) # Unknown field names are silently ignored. diff --git a/gateway/scale_to_zero.py b/gateway/scale_to_zero.py index 65c01515ad..e1a64159db 100644 --- a/gateway/scale_to_zero.py +++ b/gateway/scale_to_zero.py @@ -164,6 +164,75 @@ def is_idle( return seconds_since_last_inbound >= idle_timeout_seconds +# Dashboard-client liveness marker. The dashboard process (tui_gateway/ws.py, +# a DIFFERENT process from the gateway on hosted instances) touches this file +# on every /api/ws connect and inbound frame — the desktop app, web dashboard +# and TUI all send `gateway.ping` every 15s (apps/shared/src/json-rpc-gateway.ts, +# ui-tui/src/gatewayClient.ts). The gateway folds the mtime into its inbound +# clock, so an open client holds the box awake exactly like a chat message does +# and gets the same idle_timeout grace after it disconnects. Without this the +# box suspends under the open client, the client's reconnect loop re-pokes the +# Fly-proxied hostname, autostart resumes it, and the instance flaps every ~60s +# (13 of 72 active opted-in prod instances, 2026-09-02). +# +# There is deliberately NO staleness cutoff here: the mtime is a timestamp of +# real inbound, and is_idle already decides whether it is recent enough. A +# lingering marker cannot pin the box — once it is older than idle_timeout it +# no longer counts, exactly like an old _last_inbound_at. +DASHBOARD_CLIENT_HEARTBEAT_REL = os.path.join("state", "dashboard_clients.heartbeat") + + +def dashboard_client_heartbeat_path(hermes_home: Optional[os.PathLike | str] = None): + """Path of the dashboard-client liveness marker under HERMES_HOME.""" + from pathlib import Path + + if hermes_home is None: + from hermes_constants import get_hermes_home + + hermes_home = get_hermes_home() + return Path(hermes_home) / DASHBOARD_CLIENT_HEARTBEAT_REL + + +def touch_dashboard_client_heartbeat(path: Optional[os.PathLike | str] = None) -> bool: + """Mark "a dashboard client is attached right now". Best-effort, never raises.""" + try: + p = dashboard_client_heartbeat_path() if path is None else path + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "a", encoding="utf-8"): + pass + os.utime(p, None) + return True + except Exception: # noqa: BLE001 - liveness garnish must never break the WS + logger.debug("scale-to-zero: dashboard heartbeat touch failed", exc_info=True) + return False + + +def dashboard_client_last_seen( + path: Optional[os.PathLike | str] = None, + *, + now: Optional[float] = None, +) -> Optional[float]: + """Epoch seconds a dashboard client last sent a WS frame, or None if never. + + Missing marker -> None (the steady state on a box nobody has the dashboard + open on — NOT fail-awake, or every instance would never sleep). An + unreadable marker -> ``now`` (fail-awake: an unreadable source counts as + activity, same rule as the work counters in ``is_idle``). + """ + import time + + current = time.time() if now is None else now + p = dashboard_client_heartbeat_path() if path is None else path + try: + # Clamp to now: a wall-clock step-back (NTP) can leave the mtime in the + # future, which would push idle out by the step size for no reason. + return min(os.stat(p).st_mtime, current) + except FileNotFoundError: + return None + except OSError: + return current + + def self_suspend_available(environ: Optional[dict] = None) -> bool: """Whether this process can suspend its own machine via the flaps socket. diff --git a/gateway/session.py b/gateway/session.py index 3518706c72..e37f6c5169 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -23,6 +23,14 @@ from typing import Dict, List, Optional, Any logger = logging.getLogger(__name__) +class TranscriptReadError(RuntimeError): + """Raised when persisted history cannot be read safely.""" + + def __init__(self, session_id: str) -> None: + self.session_id = session_id + super().__init__(f"transcript read failed for session {session_id}") + + def _now() -> datetime: """Return the current local time.""" return datetime.now() @@ -2201,10 +2209,15 @@ class SessionStore: requested_session_key: str, recovered: Dict[str, Any], ) -> bool: - """Prevent non-multiplexed gateways from reviving another profile's row.""" - if getattr(self.config, "multiplex_profiles", False): - return True + """Prevent a gateway from reviving another profile's row. + Single-profile: the recovered row's namespace must match the ACTIVE + profile. Multiplexed: several profiles serve traffic at once, so the + active profile is meaningless — the requested key carries the profile + the turn was routed to, and the recovered row must sit in the same + ``agent::`` namespace (#74285). Rows with no key namespace stay + adoptable in both modes (legacy/keyless data owned by this store). + """ recovered_key = str(recovered.get("session_key") or "") if not recovered_key or recovered_key == requested_session_key: return True @@ -2213,6 +2226,10 @@ class SessionStore: if recovered_profile is None: return True + if getattr(self.config, "multiplex_profiles", False): + requested_profile = self._profile_from_session_key(requested_session_key) + return requested_profile is None or recovered_profile == requested_profile + return recovered_profile == self._active_profile_name() def _generate_session_key(self, source: SessionSource) -> str: @@ -2419,8 +2436,7 @@ class SessionStore: ): logger.warning( "Gateway session DB recovery ignored %s for %s because " - "multiplex_profiles is disabled and the row belongs to a " - "different profile", + "the row belongs to a different profile", recovered.get("session_key"), session_key, ) @@ -2496,8 +2512,7 @@ class SessionStore: ): logger.warning( "Gateway session DB recovery ignored %s for %s because " - "multiplex_profiles is disabled and the row belongs to a " - "different profile", + "the row belongs to a different profile", recovered.get("session_key"), session_key, ) @@ -3973,14 +3988,23 @@ class SessionStore: try: self._append_transcript_message(session_id, msg) except Exception as exc: - from hermes_state import CompressionSessionClosedError, StateDbReplacedError + from hermes_state import ( + CompressionSessionClosedError, + StateDbCorruptError, + StateDbReplacedError, + ) - if isinstance(exc, StateDbReplacedError): + if isinstance(exc, (StateDbReplacedError, StateDbCorruptError)): + # Both classes mean "this handle must not touch the file + # again": replaced generation (#89332) or structural + # corruption (quarantine). Retrying cannot succeed, and + # the FTS one-shot rebuild below must never run on a + # damaged file. Divert instead. logger.error( - "Session DB was replaced underneath the gateway for %s; " - "stopping SQLite writes and diverting pending " + "Session DB refused further writes on this handle for " + "%s (%s); stopping SQLite writes and diverting pending " "transcripts to the on-disk fallback: %s", - session_id, exc, + session_id, type(exc).__name__, exc, ) with self._transcript_retry_lock: remaining = list(self._dirty_transcripts.get(queue_session_id, [])) @@ -4400,15 +4424,17 @@ class SessionStore: session_id, repair_alternation=True ) except Exception as e: - # A failed read must be distinguishable from an empty transcript: - # downstream guards treat [] as "nothing persisted" and may make - # routing decisions on it (#82616). WARNING, not DEBUG. - logger.warning( - "Transcript read failed for session %s (returning empty; " - "downstream must not treat this as data loss): %s", - session_id, e, + # Empty history is valid data; a failed canonical read is not. + # Preserve that distinction so live-replay callers can fail closed + # instead of starting the model with a plausible-looking []. + logger.error( + "Transcript read failed for session %s; refusing to treat the " + "conversation as empty: %s", + session_id, + e, + exc_info=True, ) - return [] + raise TranscriptReadError(session_id) from e def rewind_session( self, diff --git a/gateway/shutdown_flush.py b/gateway/shutdown_flush.py index b537bd5105..31b3774944 100644 --- a/gateway/shutdown_flush.py +++ b/gateway/shutdown_flush.py @@ -142,6 +142,66 @@ def flush_pending_to_file( return flushed +def flush_overflow_to_file( + overflow_by_session: Dict[str, Any], + *, + reason: str = "shutdown", +) -> int: + """Serialise the FIFO overflow tails (``queued_events``) to disk. + + Sibling of :func:`flush_pending_to_file` for the second half of the + gateway FIFO (#99882): the adapter slot holds the queue head, and the + per-session ``SessionState.conversation.queued_events`` list holds the + tail. Shutdown flushed only the slot, so every follow-up parked in + overflow at restart time vanished with the process. Each overflow + event is written as its own payload in the same shape as a slot flush + so ``recover_pending_to_db`` replays them unchanged; a ``seq`` field + preserves arrival order within a session. + + Returns the number of events flushed. + """ + if not overflow_by_session: + return 0 + + flush_dir = _get_flush_dir() + ts = int(time.time()) + flushed = 0 + + for session_key, events in list(overflow_by_session.items()): + if not session_key or not events: + continue + for seq, value in enumerate(list(events)): + if value is None: + continue + try: + serialised = _serialise_value(value) + if serialised is None: + continue + _write_payload( + flush_dir, + { + "session_key": session_key, + "reason": reason, + "ts": ts, + "seq": seq, + "data": serialised, + }, + ) + flushed += 1 + except Exception as exc: + logger.debug( + "Failed to flush overflow message for %s: %s", + session_key, exc, + ) + + if flushed: + logger.info( + "Flushed %d queued overflow message(s) to %s (reason=%s)", + flushed, flush_dir, reason, + ) + return flushed + + # Reason tag for transcript messages dropped by the in-memory pending cap # during live operation (#78182). These payloads carry the full transcript # message dict so they can be replayed verbatim once the DB recovers. diff --git a/gateway/shutdown_watchdog.py b/gateway/shutdown_watchdog.py index 6c4069552c..3dbeff7cf0 100644 --- a/gateway/shutdown_watchdog.py +++ b/gateway/shutdown_watchdog.py @@ -526,30 +526,34 @@ async def loop_heartbeat_forever( # disables the witness, and the payload flag tells probes that staleness is # no longer sufficient authority to escalate. # - # Windows: asyncio.start_unix_server raises (no AF_UNIX event-loop - # support), so the witness is PERMANENTLY absent there — the payload - # records loop_tick_socket=False and every stale-file probe classifies - # UNKNOWN, never WEDGED. That is deliberate fail-safe: a wedged native - # Windows gateway keeps the graceful-drain backstop instead of an - # escalation verdict built on a witness that cannot exist. (WSL2 — the - # #90502 incident environment — is Linux and arms the socket normally.) + # Windows (non-POSIX generally): asyncio AF_UNIX support is POSIX-only, so + # the AF_UNIX arm below is gated to POSIX — an ungated call raised + # AttributeError on every native-Windows gateway start (#96956). Instead of + # leaving the witness permanently absent there, the non-POSIX arm binds a + # TCP loopback server on 127.0.0.1 with an OS-assigned port and publishes + # the port in the heartbeat payload (``loop_tick_tcp_port``) so probes know + # where to connect. Same protocol, same loop-owned semantics. If that bind + # fails, the payload records loop_tick_socket=False and probes classify + # UNKNOWN, never WEDGED — the graceful-drain backstop stays in place. (WSL2 + # — the #90502 incident environment — is Linux and arms the socket.) tick_server = None tick_socket_path = None + tick_tcp_port = None try: - tick_socket_path = get_loop_tick_socket_path(home) - tick_socket_path.parent.mkdir(parents=True, exist_ok=True) - # Re-bind over a leftover node from a dead process (os._exit(75) / - # SIGKILL skip the finally-unlink; PID reuse re-lands on this - # PID-suffixed path) is handled by asyncio itself: - # create_unix_server os.remove()s an existing socket node before - # binding — guarded by test_producer_rebinds_over_stale_socket_node. - # What asyncio does NOT do is clean up SIBLING nodes from other - # dead PIDs, so sweep those to keep state/ from accumulating - # gateway.loop-tick.*.sock nodes across crash-restart cycles. - # POSIX-only: os.kill(pid, 0) is a liveness probe here, but on - # Windows os.kill calls TerminateProcess for non-CTRL signals — - # and AF_UNIX server nodes are never created there anyway. if os.name == "posix": + tick_socket_path = get_loop_tick_socket_path(home) + tick_socket_path.parent.mkdir(parents=True, exist_ok=True) + # Re-bind over a leftover node from a dead process (os._exit(75) / + # SIGKILL skip the finally-unlink; PID reuse re-lands on this + # PID-suffixed path) is handled by asyncio itself: + # create_unix_server os.remove()s an existing socket node before + # binding — guarded by test_producer_rebinds_over_stale_socket_node. + # What asyncio does NOT do is clean up SIBLING nodes from other + # dead PIDs, so sweep those to keep state/ from accumulating + # gateway.loop-tick.*.sock nodes across crash-restart cycles. + # POSIX-only: os.kill(pid, 0) is a liveness probe here, but on + # Windows os.kill calls TerminateProcess for non-CTRL signals — + # and AF_UNIX server nodes are never created there anyway. try: for _stale in tick_socket_path.parent.glob( "gateway.loop-tick.*.sock" @@ -569,11 +573,33 @@ async def loop_heartbeat_forever( logger.debug( "stale loop-tick socket sweep failed", exc_info=True ) - tick_server = await asyncio.start_unix_server( - _tick_socket_handler, path=str(tick_socket_path) - ) + tick_server = await asyncio.start_unix_server( + _tick_socket_handler, path=str(tick_socket_path) + ) + else: + # Windows / non-POSIX: no AF_UNIX support, so use a TCP loopback + # server on 127.0.0.1 as the loop-scheduling witness instead. + # Same protocol (connect → read one byte "1"), same semantics + # (pure in-memory, zero disk I/O, answered only when the loop + # is dispatching). Port is dynamic (assigned by the OS) and + # published via the heartbeat payload so external probes know + # where to connect. + tick_server = await asyncio.start_server( + _tick_socket_handler, host="127.0.0.1", port=0 + ) + # Get the actual port assigned by the OS + _sock_addrs = tick_server.sockets if hasattr(tick_server, "sockets") else [] + for _s in _sock_addrs: + try: + _sname = _s.getsockname() + if isinstance(_sname, tuple) and len(_sname) >= 2: + tick_tcp_port = int(_sname[1]) + break + except Exception: + pass except Exception: tick_server = None + tick_tcp_port = None logger.warning( "Loop tick socket unavailable — liveness probes will have no " "loop-scheduling witness and will not escalate on a stale heartbeat", @@ -588,7 +614,10 @@ async def loop_heartbeat_forever( write_loop_heartbeat, start_time=start_time, home=home, - extra={"loop_tick_socket": tick_server is not None}, + extra={ + "loop_tick_socket": tick_server is not None, + "loop_tick_tcp_port": tick_tcp_port, + }, ) except asyncio.CancelledError: raise diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index a49e6b7aca..4e92760c81 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -37,6 +37,7 @@ from gateway.platforms.base import EphemeralReply, MessageEvent, MessageType from gateway.session import ( AsyncSessionStore, SessionSource, + TranscriptReadError, build_session_key, is_shared_multi_user_session, ) @@ -49,6 +50,11 @@ from utils import ( logger = logging.getLogger("gateway.run") +HISTORY_UNREADABLE = ( + "⚠️ Conversation history is unreadable (state.db). " + "This is not a new conversation — earlier messages exist but cannot be loaded." +) + # Upper bound on the off-loop agent-resource cleanup during a /new or /reset # (see _handle_reset_command). A stuck teardown must not block the event loop; # past this the reset proceeds and the cleanup is left to finish (or leak) in @@ -963,7 +969,10 @@ class GatewaySlashCommandsMixin: return "\n".join(lines) # Last resort: rough estimate from transcript - history = await self.async_session_store.load_transcript(session_entry.session_id) + try: + history = await self.async_session_store.load_transcript(session_entry.session_id) + except TranscriptReadError: + return HISTORY_UNREADABLE if history: from agent.model_metadata import estimate_messages_tokens_rough @@ -2592,7 +2601,6 @@ class GatewaySlashCommandsMixin: available_personalities, describe_personality, persist_personality, - prompt_text, resolve_personality, ) @@ -2621,32 +2629,32 @@ class GatewaySlashCommandsMixin: return "\n".join(lines) try: - name, new_prompt = resolve_personality(args, config) + name, _new_prompt = resolve_personality(args, config) except ValueError: available = "`none`, " + ", ".join(f"`{n}`" for n in personalities) return t("gateway.personality.unknown", name=args.lower(), available=available) # Persist the selection only — hermes_cli.personality never writes - # agent.system_prompt (user-owned manual overlay). + # agent.system_prompt (user-owned manual overlay). persist_personality + # writes get_hermes_home()/config.yaml, i.e. the routed profile under + # multiplex; the next turn re-resolves the prompt from that file + # (_get_system_prompt_for_channel), so no process-global state to update. if not persist_personality(name): return t("gateway.personality.save_failed", error="config write failed") if not name: - self._ephemeral_system_prompt = prompt_text( - cfg_get(config, "agent", "system_prompt", default="") - ) return t("gateway.personality.cleared") - - # Update in-memory so it takes effect on the very next message. - self._ephemeral_system_prompt = new_prompt return t("gateway.personality.set_to", name=name) async def _handle_retry_command(self, event: MessageEvent) -> str: """Handle /retry command - re-send the last user message.""" source = event.source session_entry = await self.async_session_store.get_or_create_session(source) - history = await self.async_session_store.load_transcript(session_entry.session_id) - + try: + history = await self.async_session_store.load_transcript(session_entry.session_id) + except TranscriptReadError: + return HISTORY_UNREADABLE + # Find the last *real* user message. Timeline bookkeeping rows carry # role=user + display_kind (model_switch / async_delegation_complete / # auto_continue / hidden); clients never count them as user turns. @@ -2838,6 +2846,22 @@ class GatewaySlashCommandsMixin: if not gate_arg or gate_lower == "list": return mgr.render_gates() if gate_lower.startswith("add "): + # SECURITY: a gate is persisted and later executed with + # shell=True at every goal turn boundary (run_gate), with no + # approval prompt. Letting an allowed but non-admin gateway + # sender choose that string is authenticated RCE under the + # Hermes process account — and with no admin list configured + # (the backward-compatible default) every allowed sender is + # treated as unrestricted. Gate ONLY this shell-creating + # operation behind a real, explicitly-configured admin (the + # same fail-closed check that guards cross-origin /resume); + # list/remove/clear stay open so a non-admin can still recover. + if not self._resume_caller_is_admin(event.source): + return ( + "⛔ /goal gate add requires an explicitly configured " + "gateway admin (allow_admin_from for DMs, " + "group_allow_admin_from for groups)." + ) command = gate_arg[len("add"):].strip() try: gate = mgr.add_gate(command) @@ -2874,8 +2898,12 @@ class GatewaySlashCommandsMixin: import asyncio from hermes_cli.goals import draft_contract - draft_contract_obj = await asyncio.get_running_loop().run_in_executor( - None, draft_contract, objective + # _run_in_executor_with_context, not a bare hop: drafting a + # contract calls the auxiliary LLM, whose provider/credential + # resolution reads the profile secret scope — a contextvar that + # a default-executor hop drops, leaving it unscoped. + draft_contract_obj = await self._run_in_executor_with_context( + draft_contract, objective ) except Exception as exc: logger.debug("goal draft failed: %s", exc) @@ -3075,8 +3103,6 @@ class GatewaySlashCommandsMixin: set_current_session_key, ) - loop = asyncio.get_running_loop() - def _dispatch(): token = set_current_session_key(quick_key) try: @@ -3087,7 +3113,10 @@ class GatewaySlashCommandsMixin: reset_current_session_key(token) try: - result = await loop.run_in_executor(None, _dispatch) + # _run_in_executor_with_context, not a bare hop: the reviewer + # subagent is spawned from the worker and inherits its context, + # so a bare hop would run it under the launch home / no secret scope. + result = await self._run_in_executor_with_context(_dispatch) except ValueError as exc: return str(exc) except Exception as exc: @@ -3346,10 +3375,12 @@ class GatewaySlashCommandsMixin: """Handle /voice [on|off|tts|channel|leave|status] command.""" args = event.get_command_args().strip().lower() chat_id = event.source.chat_id - platform = event.source.platform - voice_key = self._voice_key(platform, chat_id) + # Voice state belongs to the (bot, chat) pair: resolve the adapter that + # received the command and key the mode by its owning profile so two + # multiplexed bots in one chat keep independent /voice state (#75198). + voice_key = self._voice_key_for_source(event.source) - adapter = self.adapters.get(platform) + adapter = self._adapter_for_source(event.source) if args in {"on", "enable"}: self._voice_mode[voice_key] = "voice_only" @@ -3381,7 +3412,6 @@ class GatewaySlashCommandsMixin: "all": t("gateway.voice.label_all"), } # Append voice channel info if connected - adapter = self.adapters.get(event.source.platform) guild_id = self._get_guild_id(event) if guild_id and hasattr(adapter, "get_voice_channel_info"): info = adapter.get_voice_channel_info(guild_id) @@ -3440,7 +3470,9 @@ class GatewaySlashCommandsMixin: max_file_size_mb=cp_kwargs["checkpoint_max_file_size_mb"], ) - cwd = os.getenv("TERMINAL_CWD", str(Path.home())) + from tools.terminal_scope import terminal_env as _tenv + + cwd = _tenv("TERMINAL_CWD", str(Path.home())) arg = event.get_command_args().strip() # --all / --force: classic full restore, overwriting user edits too. @@ -3534,7 +3566,9 @@ class GatewaySlashCommandsMixin: elif low == "session": mode = "session" - cwd = os.getenv("TERMINAL_CWD", str(Path.home())) + from tools.terminal_scope import terminal_env as _tenv + + cwd = _tenv("TERMINAL_CWD", str(Path.home())) if mode == "session": return await self._gateway_session_diff(cwd, stat_only) @@ -3671,7 +3705,10 @@ class GatewaySlashCommandsMixin: source = event.source session_entry = await self.async_session_store.get_or_create_session(source) - history = await self.async_session_store.load_transcript(session_entry.session_id) + try: + history = await self.async_session_store.load_transcript(session_entry.session_id) + except TranscriptReadError: + return HISTORY_UNREADABLE if not history: return t("gateway.btw.no_history") @@ -3751,9 +3788,9 @@ class GatewaySlashCommandsMixin: def _save_gateway_config_key(self, key_path: str, value) -> bool: """Save a dot-separated key to config.yaml (shared by /reasoning, /fast and their interactive pickers).""" - from gateway.run import _hermes_home + from gateway.run import _gateway_config_home from hermes_cli.config import read_user_config_raw - config_path = _hermes_home / "config.yaml" + config_path = _gateway_config_home() / "config.yaml" try: # Write-back round-trip: raw read is correct (merged defaults must # not be persisted back to the user's file). @@ -3995,7 +4032,7 @@ class GatewaySlashCommandsMixin: Gate changes persist to config.yaml and evict the cached agent so the new setting takes effect on the next message. """ - from gateway.run import _hermes_home + from gateway.run import _gateway_config_home from hermes_cli.write_approval_commands import handle_pending_subcommand from tools import write_approval as wa from tools.memory_tool import load_on_disk_store @@ -4003,7 +4040,7 @@ class GatewaySlashCommandsMixin: raw_args = event.get_command_args().strip() args = raw_args.split() if raw_args else [] session_key = self._session_key_for_source(event.source) - config_path = _hermes_home / "config.yaml" + config_path = _gateway_config_home() / "config.yaml" def _set_approval(enabled: bool): # Write-back round-trip: raw read is correct (merged defaults must @@ -4044,14 +4081,14 @@ class GatewaySlashCommandsMixin: the write-approval ``diff ``; the CLI also has an unrelated ``hermes skills diff `` that diffs a bundled skill vs stock.) """ - from gateway.run import _hermes_home + from gateway.run import _gateway_config_home from hermes_cli.write_approval_commands import handle_pending_subcommand from tools import write_approval as wa raw_args = event.get_command_args().strip() args = raw_args.split() if raw_args else [] session_key = self._session_key_for_source(event.source) - config_path = _hermes_home / "config.yaml" + config_path = _gateway_config_home() / "config.yaml" gate_on = wa.write_approval_enabled(wa.SKILLS) wants_toggle = bool(args) and args[0].lower() in {"approval", "mode"} @@ -4122,6 +4159,9 @@ class GatewaySlashCommandsMixin: tier = None saved_value = "normal" label = t("gateway.fast.label_normal") + elif value in {"auto", "cold"}: + tier = saved_value = value + label = value.upper() else: return t("gateway.fast.unknown_arg", arg=value) self._service_tier = tier @@ -4144,7 +4184,8 @@ class GatewaySlashCommandsMixin: if not args or args == "status": is_fast = self._service_tier == "priority" - status = t("gateway.fast.status_fast") if is_fast else t("gateway.fast.status_normal") + mode = "fast" if is_fast else (self._service_tier or "normal") + status = {"fast": t("gateway.fast.status_fast"), "normal": t("gateway.fast.status_normal")}.get(mode, mode) async def _on_fast_choice(_chat_id: str, value: str) -> str: return _apply_fast_selection(value, persist=persist_global) @@ -4162,7 +4203,17 @@ class GatewaySlashCommandsMixin: { "value": "normal", "label": t("gateway.fast.choice_normal"), - "is_current": not is_fast, + "is_current": mode == "normal", + }, + { + "value": "auto", + "label": t("gateway.fast.choice_auto"), + "is_current": mode == "auto", + }, + { + "value": "cold", + "label": t("gateway.fast.choice_cold"), + "is_current": mode == "cold", }, ], on_choice_selected=_on_fast_choice, @@ -4217,9 +4268,9 @@ class GatewaySlashCommandsMixin: ``display.platforms..tool_progress`` so each channel can have its own verbosity level independently. """ - from gateway.run import _hermes_home, _load_gateway_config, _platform_config_key + from gateway.run import _gateway_config_home, _load_gateway_config, _platform_config_key - config_path = _hermes_home / "config.yaml" + config_path = _gateway_config_home() / "config.yaml" platform_key = _platform_config_key(event.source.platform) # --- check config gate ------------------------------------------------ @@ -4355,10 +4406,10 @@ class GatewaySlashCommandsMixin: are respected but not modified here — edit config.yaml directly for per-platform control. """ - from gateway.run import _hermes_home, _load_gateway_config, _platform_config_key, _resolve_gateway_model + from gateway.run import _gateway_config_home, _load_gateway_config, _platform_config_key, _resolve_gateway_model from gateway.runtime_footer import resolve_footer_config - config_path = _hermes_home / "config.yaml" + config_path = _gateway_config_home() / "config.yaml" platform_key = _platform_config_key(event.source.platform) # --- parse argument ------------------------------------------------- @@ -4525,7 +4576,10 @@ class GatewaySlashCommandsMixin: """ source = event.source session_entry = await self.async_session_store.get_or_create_session(source) - history = await self.async_session_store.load_transcript(session_entry.session_id) + try: + history = await self.async_session_store.load_transcript(session_entry.session_id) + except TranscriptReadError: + return HISTORY_UNREADABLE if not history or len(history) < 4: return t("gateway.compress.not_enough") @@ -4959,6 +5013,7 @@ class GatewaySlashCommandsMixin: await self._session_db.enable_telegram_topic_mode( chat_id=str(source.chat_id), user_id=str(source.user_id), + profile_name=self._telegram_topic_profile_name(source), has_topics_enabled=capabilities.get("has_topics_enabled"), allows_users_to_create_topics=capabilities.get("allows_users_to_create_topics"), ) @@ -4974,6 +5029,7 @@ class GatewaySlashCommandsMixin: binding = await self._session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=self._telegram_topic_profile_name(source), ) except Exception: logger.debug("Failed to read Telegram topic binding", exc_info=True) @@ -5314,7 +5370,17 @@ class GatewaySlashCommandsMixin: title = await self._session_db.get_session_title(target_id) or name # Count messages for context - history = await self.async_session_store.load_transcript(target_id) + try: + history = await self.async_session_store.load_transcript(target_id) + except TranscriptReadError: + # The resume itself succeeded; only the count is missing. Say the + # history is unreadable rather than reporting an empty session + # (#100788). + return ( + t("gateway.resume.resumed_no_count", title=title) + + "\n" + + HISTORY_UNREADABLE + ) msg_count = len([m for m in history if m.get("role") == "user"]) if history else 0 msg_part = f" ({msg_count} message{'s' if msg_count != 1 else ''})" if msg_count else "" @@ -5431,7 +5497,10 @@ class GatewaySlashCommandsMixin: # Load the current session and its transcript current_entry = await self.async_session_store.get_or_create_session(source) - history = await self.async_session_store.load_transcript(current_entry.session_id) + try: + history = await self.async_session_store.load_transcript(current_entry.session_id) + except TranscriptReadError: + return HISTORY_UNREADABLE if not history: return t("gateway.branch.no_conversation") @@ -5616,6 +5685,10 @@ class GatewaySlashCommandsMixin: try: entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(entry.session_id) or [] + except TranscriptReadError: + # A read failure is not an empty transcript (#100788): the + # breakdown would understate the context by the whole chat. + return [HISTORY_UNREADABLE] except Exception: history = [] @@ -5647,6 +5720,10 @@ class GatewaySlashCommandsMixin: try: entry = self.session_store.get_or_create_session(source) history = self.session_store.load_transcript(entry.session_id) or [] + except TranscriptReadError: + # See _context_breakdown_block: don't pass a read failure off + # as an empty transcript (#100788). + return [HISTORY_UNREADABLE] except Exception: history = [] @@ -5830,7 +5907,10 @@ class GatewaySlashCommandsMixin: # No agent at all -- check session history for a rough count session_entry = await self.async_session_store.get_or_create_session(source) - history = await self.async_session_store.load_transcript(session_entry.session_id) + try: + history = await self.async_session_store.load_transcript(session_entry.session_id) + except TranscriptReadError: + return HISTORY_UNREADABLE if history: from agent.model_metadata import estimate_messages_tokens_rough msgs = [m for m in history if m.get("role") in {"user", "assistant"} and m.get("content")] @@ -5892,8 +5972,6 @@ class GatewaySlashCommandsMixin: from hermes_state import get_shared_session_db, release_shared_session_db from agent.insights import InsightsEngine - loop = asyncio.get_running_loop() - def _run_insights(): db = get_shared_session_db() try: @@ -5905,7 +5983,13 @@ class GatewaySlashCommandsMixin: from hermes_state import release_or_close release_or_close(db) - return await loop.run_in_executor(None, _run_insights) + # _run_in_executor_with_context, not a bare hop: ``SessionDB()`` + # with no explicit path resolves ``get_hermes_home()`` at call + # time, and that override is a contextvar installed by + # ``_profile_runtime_scope``. A default-executor hop starts the + # worker with an EMPTY context, so /insights read the DEFAULT + # profile's state.db and reported another profile's conversations. + return await self._run_in_executor_with_context(_run_insights) except Exception as e: logger.error("Insights command error: %s", e, exc_info=True) return t("gateway.insights.error", error=e) @@ -5988,11 +6072,12 @@ class GatewaySlashCommandsMixin: is written to the session transcript out-of-band, so message alternation is preserved. """ - loop = asyncio.get_running_loop() try: from agent.skill_commands import reload_skills - result = await loop.run_in_executor(None, reload_skills) + # _run_in_executor_with_context, not a bare hop: the rescan walks + # get_hermes_home()/skills, a contextvar override under multiplex. + result = await self._run_in_executor_with_context(reload_skills) added = result.get("added", []) # [{"name", "description"}, ...] removed = result.get("removed", []) # [{"name", "description"}, ...] total = result.get("total", 0) @@ -6300,8 +6385,6 @@ class GatewaySlashCommandsMixin: _GATEWAY_PRIVACY_NOTICE, _best_effort_sweep_expired_pastes, ) - loop = asyncio.get_running_loop() - # Run blocking I/O (dump capture, log reads, uploads) in a thread. def _collect_and_upload(): _best_effort_sweep_expired_pastes() @@ -6328,7 +6411,11 @@ class GatewaySlashCommandsMixin: lines.append(t("gateway.debug.share_hint")) return "\n".join(lines) - return await loop.run_in_executor(None, _collect_and_upload) + # _run_in_executor_with_context, not a bare hop: this collects the + # profile's logs/config off ``get_hermes_home()`` and uploads them to a + # public paste. Losing the contextvar override would publish the DEFAULT + # profile's diagnostics from another profile's chat. + return await self._run_in_executor_with_context(_collect_and_upload) async def _handle_update_command(self, event: MessageEvent) -> str: """Handle /update command — update Hermes Agent to the latest version. diff --git a/gateway/status.py b/gateway/status.py index 5c7d025c48..5319d8d760 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -157,6 +157,41 @@ def _same_hermes_home(left: Path | str, right: Path | str) -> bool: ) +def recorded_gateway_home_conflicts( + record: Optional[dict[str, Any]], + *, + expected_home: Optional[Path | str] = None, +) -> bool: + """True when a persisted gateway record names a DIFFERENT HERMES_HOME. + + Cross-profile kill refusal (#89315): a poisoned/contaminated PID record + inside one profile's home can truthfully name ANOTHER profile's live + gateway (its ``hermes_home`` stamp records the real owner). Any + destructive caller about to signal the recorded PID must consult this + first and refuse when the record positively proves the target belongs to + a different profile — otherwise ``gateway stop``/``restart``/``profile + delete`` from profile B SIGTERMs profile A's gateway and the supervisors + enter the mutual restart loop from the issue report. + + ``expected_home`` overrides the comparison base (e.g. ``profile delete`` + stopping a TARGET profile's gateway rather than the current process's). + Legacy records without a ``hermes_home`` stamp return False — they prove + nothing either way, and destructive callers already pair this with the + exact PID + start-time identity guards. A comparison failure returns True + (destructive action + unprovable ownership ⇒ fail closed). + """ + if not isinstance(record, dict): + return False + recorded_home = record.get("hermes_home") + if not isinstance(recorded_home, str) or not recorded_home.strip(): + return False + try: + base = expected_home if expected_home is not None else _get_process_hermes_home() + return not _same_hermes_home(recorded_home, base) + except Exception: + return True + + # Mirrors hermes_cli.profiles._PROFILE_ID_RE — duplicated here because gateway # identity code must stay import-light (hermes_constants + stdlib only). _PROFILE_LABEL_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") diff --git a/hermes_cli/_subprocess_compat.py b/hermes_cli/_subprocess_compat.py index 15ae27e539..603ebe5d72 100644 --- a/hermes_cli/_subprocess_compat.py +++ b/hermes_cli/_subprocess_compat.py @@ -48,9 +48,64 @@ __all__ = [ "bounded_git_probe", "bounded_probe_run", "noninteractive_git_env", + "NO_DRIVER_DIFF_FLAGS", "pid_is_hermes", ] +# Flags that neutralize *attribute-scoped* diff drivers on any diff-rendering +# git command (``diff``, ``log -p``, ``show``, ``blame``). A malicious repo can +# name a driver in ``.gitattributes`` (``* diff=evil``) and point it at an +# arbitrary program via ``[diff "evil"] command=/textconv=`` in ``.git/config``. +# Because the attacker chooses the driver name, ``GIT_CONFIG_KEY`` overrides in +# ``noninteractive_git_env`` cannot enumerate and disable it — only these +# command-line flags do. ``--no-ext-diff`` kills ``command=``; ``--no-textconv`` +# kills ``textconv=``. Both are required (verified empirically: each alone +# leaves the other live). Smudge/clean filters are neutralized by the env +# layer's ``core.hooksPath`` + running against the index without checkout. +NO_DRIVER_DIFF_FLAGS = ("--no-ext-diff", "--no-textconv") + +# Subcommands that render diffs and therefore invoke ``.gitattributes``-scoped +# diff/textconv drivers. Only these accept ``NO_DRIVER_DIFF_FLAGS`` — ``status`` +# and friends reject the flags (``unknown option``), so the helper must gate on +# this set rather than blanket-prepending. +_DIFF_RENDERING_SUBCOMMANDS = frozenset({"diff", "show", "log", "blame"}) + + +def harden_git_argv(args: Sequence[str]) -> list[str]: + """Return a copy of subcommand-first git *args* with diff-driver flags + inserted for diff-rendering subcommands. + + *args* is the argument list WITHOUT the leading ``"git"`` (e.g. + ``["diff", "HEAD"]`` or ``["-c", "core.quotePath=false", "diff", ...]``). + The first non-option token is treated as the subcommand; if it is one of + :data:`_DIFF_RENDERING_SUBCOMMANDS`, :data:`NO_DRIVER_DIFF_FLAGS` is + inserted immediately after it. Non-diff subcommands are returned unchanged. + + Pair with :func:`noninteractive_git_env`: the env layer disables + fsmonitor/hooks/pager/editor/credential sinks, this closes the one class + (attacker-named attribute drivers) env overrides cannot reach. + """ + out = list(args) + # Options that consume the FOLLOWING token as their value, so that value is + # never mistaken for the subcommand (``-C diff`` is a path; ``-c diff=x`` is + # a config pair — neither is the diff subcommand). + _value_opts = {"-C", "-c", "--git-dir", "--work-tree", "--namespace", "--exec-path"} + i = 0 + while i < len(out): + tok = out[i] + if tok in _value_opts: + i += 2 + continue + if tok.startswith("-"): + i += 1 + continue + if tok in _DIFF_RENDERING_SUBCOMMANDS: + return out[: i + 1] + list(NO_DRIVER_DIFF_FLAGS) + out[i + 1 :] + # First non-option token is the subcommand; if it isn't a diff renderer + # there is nothing to harden. + return out + return out + IS_WINDOWS = sys.platform == "win32" @@ -389,6 +444,10 @@ def noninteractive_git_env( instead of prompting for credentials. * ``GCM_INTERACTIVE=Never`` — Git Credential Manager (the default credential helper on Windows installs) never pops its own dialog. + * isolated git config — inherited ``GIT_CONFIG_*`` overrides, global/system + config, pagers, editors, fsmonitor, external diff, and hooks are disabled + for the child process. A user's repo/global config should not be able to + hang or mutate Hermes's internal plumbing calls. ``GIT_ASKPASS`` / ``SSH_ASKPASS`` are deliberately left alone: when the user has a *working* askpass helper or ssh-agent configured, auth should @@ -405,6 +464,43 @@ def noninteractive_git_env( env = dict(base if base is not None else os.environ) env["GIT_TERMINAL_PROMPT"] = "0" env["GCM_INTERACTIVE"] = "Never" + + # Do not inherit caller-supplied config injection. We rebuild the + # GIT_CONFIG_COUNT block below so ambient -c values cannot re-enable + # pagers, hooks, fsmonitor, editors, or credential prompts. + for key in list(env): + if ( + key == "GIT_CONFIG_PARAMETERS" + or key.startswith("GIT_CONFIG_KEY_") + or key.startswith("GIT_CONFIG_VALUE_") + ): + env.pop(key, None) + env.pop("GIT_CONFIG_COUNT", None) + + devnull = os.devnull + env["GIT_CONFIG_GLOBAL"] = devnull + env["GIT_CONFIG_SYSTEM"] = devnull + env["GIT_CONFIG_NOSYSTEM"] = "1" + env["GIT_PAGER"] = "cat" + env["PAGER"] = "cat" + env["GIT_EDITOR"] = "true" + + config_overrides = { + "credential.helper": "", + "core.askPass": "", + "core.fsmonitor": "false", + "core.untrackedCache": "false", + "core.hooksPath": devnull, + "core.pager": "cat", + "core.editor": "true", + "sequence.editor": "true", + "diff.external": "", + } + env["GIT_CONFIG_COUNT"] = str(len(config_overrides)) + for idx, (key, value) in enumerate(config_overrides.items()): + env[f"GIT_CONFIG_KEY_{idx}"] = key + env[f"GIT_CONFIG_VALUE_{idx}"] = value + return env @@ -593,6 +689,7 @@ def bounded_probe_run( *, timeout: float, errors: str = "replace", + env: "Mapping[str, str] | None" = None, ) -> "subprocess.CompletedProcess[str] | None": """Deadlock-safe ``subprocess.run(argv, capture_output=True, timeout=...)`` for fail-open probe call sites. Returns a ``CompletedProcess`` when the @@ -630,6 +727,7 @@ def bounded_probe_run( text=True, encoding="utf-8", errors=errors, + env=dict(env) if env is not None else None, **_popen_kwargs, ) except Exception: @@ -657,6 +755,20 @@ def bounded_git_probe(argv: Sequence[str], *, timeout: float) -> str: ``subprocess.run(["git", ...], timeout=...)`` at fail-open probe call sites (``tui_gateway.git_probe.run_git``, ``agent.coding_context._git``). + **Security (GHSA-7x36-8jrh-v4pw):** these probes run automatically against + whatever directory the session sits in — the coding-workspace snapshot and + the gateway project-tree build fire ``git status`` / ``git branch`` before + any tool call, approval, or trust prompt. An index refresh executes the + repository-configured ``core.fsmonitor`` program, and other config keys + (hooks, pager, editor, credential helper) are execution sinks too. A repo + delivered as files with its ``.git`` directory intact (a shared zip, sync + folder, or USB stick — ``git clone`` never transfers ``.git/config``) would + otherwise get host code execution as the user. Every probe now runs under + :func:`noninteractive_git_env`, which pins those keys to inert values via + ``GIT_CONFIG_*`` and ignores global/system config. Diff-rendering callers + additionally pass :data:`NO_DRIVER_DIFF_FLAGS` (attribute-scoped drivers + can't be disabled through env overrides). + Why not ``subprocess.run``: on Windows, ``run()``'s post-timeout cleanup calls an *unbounded* ``communicate()`` after killing git. Killing the PATH-resolved launcher can leave a suspended descendant ``git.exe`` holding @@ -681,7 +793,7 @@ def bounded_git_probe(argv: Sequence[str], *, timeout: float) -> str: openai/codex#36793). ``process_group`` only changes which group the child belongs to; it does not detach the terminal or alter the fast path. """ - result = bounded_probe_run(argv, timeout=timeout) + result = bounded_probe_run(argv, timeout=timeout, env=noninteractive_git_env()) if result is None or result.returncode != 0: return "" return (result.stdout or "").strip() diff --git a/hermes_cli/active_sessions.py b/hermes_cli/active_sessions.py index 9d230a35e7..15626d37d3 100644 --- a/hermes_cli/active_sessions.py +++ b/hermes_cli/active_sessions.py @@ -18,7 +18,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Iterator, Optional -from hermes_constants import get_hermes_home +from hermes_constants import get_default_hermes_root, get_hermes_home logger = logging.getLogger(__name__) @@ -757,6 +757,53 @@ def transfer_active_session( return updated +# A lease this process wrote in the last few seconds may not be in the +# caller's ``own_live_lease_ids`` yet: ``try_acquire_active_session`` writes +# the registry entry under the file lock and the server attaches the lease to +# its session record only after that returns. A concurrent finalize that +# snapshotted its live ids in between would otherwise read the brand-new lease +# as an orphan and drop it. Real orphans are minutes old (#101415). +_SELF_ORPHAN_GRACE_SECONDS = 30.0 + + +def _drop_self_orphans( + entries: list[dict[str, Any]], own_live_lease_ids: set[str] | None +) -> list[dict[str, Any]]: + """Drop this process's leases only when its caller can vouch for owners.""" + if own_live_lease_ids is None: + return entries + pid = os.getpid() + cutoff = time.time() - _SELF_ORPHAN_GRACE_SECONDS + return [ + entry + for entry in entries + if entry.get("pid") != pid + or str(entry.get("lease_id") or "") in own_live_lease_ids + or (_optional_float(entry.get("started_at")) or 0.0) > cutoff + ] + + +def _release_orphaned_leases_in_home( + registry_home: Path, live_lease_ids: set[str] +) -> int: + state_path = _state_path(registry_home) + if not state_path.exists(): + return 0 + with _FileLock(_lock_path(registry_home)): + try: + entries = _prune_dead(_read_entries(state_path, strict=True)) + except ActiveSessionRegistryError: + logger.warning( + "Active-session registry is unavailable; skipping orphaned-lease sweep" + ) + return 0 + kept = _drop_self_orphans(entries, live_lease_ids) + dropped = len(entries) - len(kept) + if dropped: + _write_entries(state_path, kept) + return dropped + + def release_orphaned_leases(live_lease_ids: set[str]) -> int: """Drop this process's registry entries that no live session owns. @@ -767,30 +814,26 @@ def release_orphaned_leases(live_lease_ids: set[str]) -> int: real, so it drops the rest itself — exact, with no heartbeat write on the turn path and no staleness threshold to tune. """ - pid = os.getpid() - state_path = _state_path() - # No registry file yet means no leases have ever been written under this - # home — don't take a lock (or create its file) on the idle-reaper tick. - if not state_path.exists(): - return 0 - with _FileLock(_lock_path()): + root = get_default_hermes_root() + homes = [root] + profiles_root = root / "profiles" + try: + homes.extend( + profile + for profile in profiles_root.iterdir() + if profile.is_dir() and not profile.name.startswith(".") + ) + except OSError: + pass + + dropped = 0 + for home in homes: try: - raw_entries = _read_entries(state_path, strict=True) - entries = _prune_dead(raw_entries) - except ActiveSessionRegistryError: - logger.warning( - "Active-session registry is unavailable; skipping orphaned-lease sweep" + dropped += _release_orphaned_leases_in_home(home, live_lease_ids) + except OSError as exc: + logger.debug( + "orphaned-lease sweep failed for %s: %s", home, exc ) - return 0 - kept = [ - entry - for entry in entries - if entry.get("pid") != pid - or str(entry.get("lease_id") or "") in live_lease_ids - ] - dropped = len(entries) - len(kept) - if dropped: - _write_entries(state_path, kept) return dropped @@ -812,6 +855,7 @@ def active_session_liveness_guard( session_id: str, *, registry_home: str | Path | None = None, + own_live_lease_ids: set[str] | None = None, ) -> Iterator[bool]: """Hold the registry lock while reporting whether ``session_id`` is leased. @@ -823,6 +867,7 @@ def active_session_liveness_guard( state_path, lock_path = _lease_paths(registry_home=registry_home) with _FileLock(lock_path): entries = _prune_dead(_read_entries(state_path, strict=True), strict=True) + entries = _drop_self_orphans(entries, own_live_lease_ids) _write_entries(state_path, entries) yield bool(target) and any( str(entry.get("session_id") or "") == target for entry in entries @@ -833,6 +878,8 @@ def active_session_liveness_guard( def release_active_session_liveness_guard( lease: ActiveSessionLease, session_id: str, + *, + own_live_lease_ids: set[str] | None = None, ) -> Iterator[bool]: """Remove ``lease`` and hold its registry lock through a lifecycle write. @@ -842,7 +889,9 @@ def release_active_session_liveness_guard( """ if not lease.enabled or lease.released: with active_session_liveness_guard( - session_id, registry_home=_registry_home_for_lease(lease) + session_id, + registry_home=_registry_home_for_lease(lease), + own_live_lease_ids=own_live_lease_ids, ) as active: yield active return @@ -857,6 +906,7 @@ def release_active_session_liveness_guard( for entry in entries if str(entry.get("lease_id") or "") != lease.lease_id ] + kept = _drop_self_orphans(kept, own_live_lease_ids) if len(kept) != len(entries): _write_entries(state_path, kept) lease.released = True diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index f4064832f2..1c4e111635 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -579,6 +579,24 @@ try: for _pp in _list_providers_for_registry(): if _pp.name in PROVIDER_REGISTRY: continue + if _pp.auth_type == "external_process": + # An external-process provider (an ACP CLI driven over stdio) has no + # API-key env vars to resolve — its credentials come from + # resolve_external_process_provider_credentials(), keyed on this + # auth_type. Registering it here is what lets a provider shipped + # outside this tree pass resolve_provider()'s known-provider gate; + # without it, `hermes -m ` dies with + # "Unknown provider" before any client is ever built. + PROVIDER_REGISTRY[_pp.name] = ProviderConfig( + id=_pp.name, + name=_pp.display_name or _pp.name, + auth_type="external_process", + inference_base_url=_pp.base_url, + ) + for _alias in _pp.aliases: + if _alias not in PROVIDER_REGISTRY: + PROVIDER_REGISTRY[_alias] = PROVIDER_REGISTRY[_pp.name] + continue if _pp.auth_type != "api_key" or not _pp.env_vars: continue # Skip providers that need custom token resolution or are special-cased @@ -1242,6 +1260,22 @@ def _same_path(left: Path, right: Path) -> bool: return left == right +def _is_same_auth_store(left: Path, right: Path) -> bool: + """True when two auth paths name ONE store rather than two copies. + + ``_same_path`` resolves symlinks and ``..``; ``samefile`` adds hardlinks + and bind-mounts (same inode under two resolved names). Used by the + forked-grant heal: a shared store has no "other side" to consolidate + (#101356). + """ + if _same_path(left, right): + return True + try: + return left.samefile(right) + except OSError: + return False + + def _auth_lock_holder_for(target_path: Path) -> threading.local: """Return a reentrancy tracker keyed to one canonical auth-store path.""" try: @@ -1687,6 +1721,520 @@ def is_runtime_provider_routable(provider_id: str) -> bool: return True +# Pool providers whose OAuth refresh tokens are SINGLE-USE: redeeming the +# refresh token rotates the pair and revokes the old one. A grant forked into +# two auth.json files is therefore not two credentials but one credential with +# two owners — the first owner to refresh strands the other with +# ``invalid_grant`` / ``refresh_token_reused`` (#100339; same class as the +# ``providers.`` write-through hazard in #48415 / #43589). Profiles must +# never receive a copy of these grants: ONE grant lives at the global root and +# named profiles read it through the ``read_credential_pool`` root fallback. +SINGLE_USE_REFRESH_POOL_PROVIDERS = frozenset({ + "anthropic", + "openai-codex", + "xai-oauth", +}) + +# Singleton credential files that hold the same single-use grants outside +# ``auth.json``. Copying one into a profile re-seeds a forked pool row on the +# profile's next ``load_pool()``. +SINGLE_USE_OAUTH_SINGLETON_FILES = (".anthropic_oauth.json",) + + +def _is_oauth_pool_payload(entry: Any) -> bool: + if not isinstance(entry, dict): + return False + auth_type = str(entry.get("auth_type") or "").strip().lower() + if auth_type == "oauth": + return True + # Legacy rows predating ``auth_type``: an Anthropic OAuth access token or + # any row carrying a refresh token is an OAuth grant. + if str(entry.get("refresh_token") or "").strip(): + return True + return str(entry.get("access_token") or "").startswith("sk-ant-oat") + + +def strip_cloned_single_use_oauth_grants(profile_dir: Path) -> Dict[str, Any]: + """Remove forked single-use OAuth grants from a freshly cloned profile. + + Called after any code path that copies credential files from one profile + into another (``hermes profile create --clone-all``, the dashboard/TUI + ``mirror_credentials`` flow). API-key pool rows are kept — a static key is + safe to duplicate. OAuth rows for the providers in + ``SINGLE_USE_REFRESH_POOL_PROVIDERS``, the matching ``providers.`` + device-code blocks, and the ``.anthropic_oauth.json`` singleton are + dropped so the clone reads the grant from the global root instead of + holding its own doomed copy (#100339). + + Returns a summary ``{"pool": [...provider ids], "providers": [...], + "files": [...]}`` of what was stripped (empty lists when nothing was). + Never raises: a clone must not fail because credential hygiene could not + run — the caller logs the summary. + """ + stripped: Dict[str, Any] = {"pool": [], "providers": [], "files": []} + profile_dir = Path(profile_dir) + for name in SINGLE_USE_OAUTH_SINGLETON_FILES: + try: + target = profile_dir / name + if target.is_file() or target.is_symlink(): + target.unlink() + stripped["files"].append(name) + except OSError: + logger.debug("Could not remove cloned %s from %s", name, profile_dir, exc_info=True) + + auth_path = profile_dir / "auth.json" + if not auth_path.is_file(): + return stripped + try: + store = json.loads(auth_path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError): + return stripped + if not isinstance(store, dict): + return stripped + + changed = False + pool = store.get("credential_pool") + if isinstance(pool, dict): + for provider_id in list(pool): + if provider_id not in SINGLE_USE_REFRESH_POOL_PROVIDERS: + continue + entries = pool.get(provider_id) + if not isinstance(entries, list): + continue + kept = [e for e in entries if not _is_oauth_pool_payload(e)] + if len(kept) != len(entries): + changed = True + stripped["pool"].append(provider_id) + if kept: + pool[provider_id] = kept + else: + # No local rows at all → read_credential_pool falls back + # to the root slice for this provider. + del pool[provider_id] + providers = store.get("providers") + if isinstance(providers, dict): + # Device-code grants for these providers live under providers.; + # _load_provider_state has the same root fallback, so dropping the + # copy keeps the profile working while removing the fork. + for provider_id in ("openai-codex", "xai-oauth"): + block = providers.get(provider_id) + if isinstance(block, dict) and block: + del providers[provider_id] + stripped["providers"].append(provider_id) + changed = True + if not changed: + return stripped + try: + _save_auth_store(store, target_path=auth_path) + except Exception: + logger.debug( + "Failed to strip cloned single-use OAuth grants from %s", + auth_path, + exc_info=True, + ) + return stripped + + +# ── One-time heal for installs that ALREADY forked a single-use grant ──────── +# +# Fleets created before the clone-strip / root-write-through above have +# profile-local copies of the root grant. Those copies are the same credential +# with several owners: whichever profile rotated last holds the only live +# refresh token and every other copy (root included) is spent. Upgrading alone +# does not fix that — the first load in each profile would keep using its own +# doomed copy. ``heal_forked_single_use_oauth_grants`` runs at profile +# ``load_pool()`` time: it finds the profile rows that share LINEAGE with a +# root row (same pool id — clone-all and the old borrowed-persist both kept +# it — or the same account identity / token material), keeps the copy most +# likely to still be live (freshest rotation), writes that copy into ROOT when +# root's is older, and strips the profile's copy so the profile borrows root +# from then on. Idempotent (a healed profile has no matched rows), never +# touches API-key rows, never deletes a row that has no root counterpart +# (an independent ``hermes -p

auth add`` grant, or the only surviving +# copy), and reads only the two auth.json files the existing root fallback +# already reads — no environ / secret-scope reads. + +_OAUTH_TOKEN_FIELDS = ( + "access_token", + "refresh_token", + "expires_at", + "expires_at_ms", + "last_refresh", +) + +_oauth_heal_notices: List[str] = [] +# provider -> (profile auth.json path, auth.json mtime_ns, singleton mtime_ns) +# of the last store verified fork-free; lets load_pool() skip the locked scan. +_oauth_heal_clean_marks: Dict[str, Tuple[str, Optional[int], Optional[int]]] = {} + + +def consume_oauth_heal_notices() -> List[str]: + """Return (and clear) human-readable notes about heals run in this process. + + ``hermes auth list`` / ``hermes auth status`` print them so the user sees + that a forked grant was consolidated rather than only finding it in logs. + """ + notes = list(_oauth_heal_notices) + _oauth_heal_notices.clear() + return notes + + +def _oauth_identity(entry: Dict[str, Any]) -> Optional[str]: + """Stable account identity for an OAuth row when the token carries one. + + Codex / xAI access tokens are JWTs with ``sub`` / ``email`` / + ``chatgpt_account_id`` claims; Anthropic ``sk-ant-oat`` tokens carry no + claims (returns None — lineage then rests on id / token material). + """ + if not isinstance(entry, dict): + return None + for token in (entry.get("access_token"), entry.get("id_token")): + claims = _decode_jwt_claims(token) + if not claims: + continue + nested = claims.get("https://api.openai.com/auth") + account = nested.get("chatgpt_account_id") if isinstance(nested, dict) else None + for value in (account, claims.get("sub"), claims.get("email")): + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _oauth_freshness(entry: Dict[str, Any]) -> float: + """Best-effort 'how recently was this pair issued' score (epoch seconds). + + A rotation always issues a later-expiring access token, so ``expires_at`` + ordering identifies the live copy; ``last_refresh`` and the JWT ``exp`` + claim are fallbacks for rows that do not persist expiry. + """ + from agent.credential_pool import _parse_absolute_timestamp + + best = 0.0 + for key in ("expires_at_ms", "expires_at", "last_refresh"): + ts = _parse_absolute_timestamp(entry.get(key)) + if ts and ts > best: + best = ts + if best == 0.0: + exp = _decode_jwt_claims(entry.get("access_token")).get("exp") + ts = _parse_absolute_timestamp(exp) + if ts: + best = ts + return best + + +def _find_root_counterpart( + profile_row: Dict[str, Any], root_rows: List[Dict[str, Any]] +) -> Optional[int]: + """Index of the root OAuth row that shares a grant lineage with *profile_row*. + + Strongest evidence first: same pool ``id`` (clone-all and the pre-fix + borrowed-persist both preserved it), same account identity from JWT + claims, same token material (an unrotated copy). Fallback per the + one-grant-at-root rule: same provider + same OAuth client — every + Anthropic ``hermes_pkce`` grant uses one client id and carries no claims, + so two Anthropic OAuth rows with no contrary identity are one lineage. + Only a row whose identity claims name a DIFFERENT account is left alone + (an independent ``hermes -p

auth add`` login for another account). + """ + candidates = [i for i, r in enumerate(root_rows) if _is_oauth_pool_payload(r)] + if not candidates: + return None + pid = profile_row.get("id") + for i in candidates: + if pid and root_rows[i].get("id") == pid: + return i + p_ident = _oauth_identity(profile_row) + for i in candidates: + r_ident = _oauth_identity(root_rows[i]) + if p_ident and r_ident and p_ident == r_ident: + return i + for key in ("refresh_token", "access_token"): + p_val = profile_row.get(key) + if not (isinstance(p_val, str) and p_val.strip()): + continue + for i in candidates: + if root_rows[i].get(key) == p_val: + return i + # Fallback: same provider + same client. Only a contradicting identity + # (both sides carry claims and they differ from every root row) blocks it. + if p_ident: + for i in candidates: + if not _oauth_identity(root_rows[i]): + return i + return None + return candidates[0] + + +def _adopt_oauth_material(target: Dict[str, Any], winner: Dict[str, Any]) -> Dict[str, Any]: + """Return *target* carrying *winner*'s token pair, status markers cleared.""" + merged = dict(target) + for key in _OAUTH_TOKEN_FIELDS: + if winner.get(key) is not None: + merged[key] = winner[key] + else: + merged.pop(key, None) + for status_field in _POOL_STATUS_FIELDS: + merged[status_field] = None + return merged + + +def _singleton_as_row(path: Path) -> Optional[Dict[str, Any]]: + """Read a ``.anthropic_oauth.json`` as a pool-row-shaped dict, or None.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(data, dict) or not str(data.get("accessToken") or "").strip(): + return None + return { + "access_token": data.get("accessToken"), + "refresh_token": data.get("refreshToken"), + "expires_at_ms": data.get("expiresAt"), + } + + +def heal_forked_single_use_oauth_grants(provider_id: str) -> Optional[Dict[str, Any]]: + """Consolidate a profile's forked copy of a single-use OAuth grant into root. + + Runs only in profile mode for ``SINGLE_USE_REFRESH_POOL_PROVIDERS``. + Returns a summary ``{"adopted": bool, "stripped_ids": [...], "files": [...], + "providers_block": bool}`` when something was healed, else ``None``. + Never raises. + """ + if provider_id not in SINGLE_USE_REFRESH_POOL_PROVIDERS: + return None + try: + return _heal_forked_single_use_oauth_grants(provider_id) + except Exception: + logger.debug("%s: forked-OAuth heal skipped", provider_id, exc_info=True) + return None + + +def _heal_forked_single_use_oauth_grants(provider_id: str) -> Optional[Dict[str, Any]]: + root_path = _global_auth_file_path() + if root_path is None: + return None # classic mode: nothing to consolidate into + if os.environ.get("PYTEST_CURRENT_TEST"): + # Same seat belt as the write-through paths: never touch the real + # user's ~/.hermes/auth.json from a test that forgot to isolate HOME. + real_home_env = os.environ.get("HOME", "") + if real_home_env and _same_path(root_path, Path(real_home_env) / ".hermes" / "auth.json"): + return None + profile_path = _auth_file_path() + profile_home = profile_path.parent + root_home = root_path.parent + profile_singleton = profile_home / ".anthropic_oauth.json" if provider_id == "anthropic" else None + + # Hot-path short-circuit: load_pool() runs per model call. Once this + # profile's store was verified clean for *provider_id*, skip the locked + # read-modify-write until the profile's own files change (mtime key). + def _stamp(p: Optional[Path]) -> Optional[int]: + try: + return p.stat().st_mtime_ns if p is not None else None + except OSError: + return None + + fingerprint = (str(profile_path), _stamp(profile_path), _stamp(profile_singleton)) + if _oauth_heal_clean_marks.get(provider_id) == fingerprint: + return None + if fingerprint[1] is None and fingerprint[2] is None: + _oauth_heal_clean_marks[provider_id] = fingerprint + return None + if _is_same_auth_store(profile_path, root_path): + # The profile's auth.json IS the root store (symlink/hardlink alias — + # a deliberate way to share one grant). Both "sides" below would read + # the same file, every OAuth row would match itself, and the strip + # would write through the alias and delete the shared credential. + # Nothing to consolidate (#101356); the mtime mark keeps this off the + # per-call hot path until the shared file changes. + _oauth_heal_clean_marks[provider_id] = fingerprint + logger.debug("%s: forked-OAuth heal skipped, %s is the root store", provider_id, profile_path) + return None + + summary: Dict[str, Any] = {"adopted": False, "stripped_ids": [], "files": [], "providers_block": False} + log_bits: List[str] = [] + + # Lock order: active (profile) store first, then the root source store — + # the same order ``_provider_state_transaction`` uses. + with _auth_store_lock(): + profile_store = _load_auth_store(profile_path) if profile_path.exists() else {"providers": {}} + with _auth_store_lock(target_path=root_path): + root_store = _load_auth_store(root_path) if root_path.exists() else {"providers": {}} + profile_changed = False + root_changed = False + + p_pool = profile_store.get("credential_pool") + p_rows = p_pool.get(provider_id) if isinstance(p_pool, dict) else None + p_rows = p_rows if isinstance(p_rows, list) else [] + r_pool = root_store.get("credential_pool") + r_rows = r_pool.get(provider_id) if isinstance(r_pool, dict) else None + r_rows = r_rows if isinstance(r_rows, list) else [] + r_oauth = [r for r in r_rows if _is_oauth_pool_payload(r)] + + root_singleton = root_home / ".anthropic_oauth.json" if provider_id == "anthropic" else None + root_singleton_row = ( + _singleton_as_row(root_singleton) + if root_singleton is not None and root_singleton.exists() else None + ) + + # ── credential_pool rows ──────────────────────────────────── + kept_rows: List[Any] = [] + for row in p_rows: + if not _is_oauth_pool_payload(row): + kept_rows.append(row) # API keys are safe to duplicate + continue + match_idx = _find_root_counterpart(row, r_rows) + if match_idx is not None: + root_row = r_rows[match_idx] + if _oauth_freshness(row) > _oauth_freshness(root_row): + r_rows[match_idx] = _adopt_oauth_material(root_row, row) + root_changed = True + summary["adopted"] = True + summary["stripped_ids"].append(row.get("id")) + profile_changed = True + continue + # No root pool counterpart. Root's grant may live only in its + # .anthropic_oauth.json (the ``hermes auth`` PKCE shape); a + # profile hermes_pkce-family row is that grant's copy. + is_pkce = str(row.get("source") or "").endswith("hermes_pkce") + if is_pkce and root_singleton_row is not None and not r_oauth: + if _oauth_freshness(row) > _oauth_freshness(root_singleton_row): + root_singleton_row = _adopt_oauth_material(root_singleton_row, row) + summary["adopted"] = True + summary["stripped_ids"].append(row.get("id")) + profile_changed = True + continue + # Root holds no copy of this lineage (independent account, or + # root never had the grant): the profile's row may be the + # only surviving copy — leave it alone. + kept_rows.append(row) + if profile_changed and isinstance(p_pool, dict): + if kept_rows: + p_pool[provider_id] = kept_rows + else: + p_pool.pop(provider_id, None) + + # ── providers. device-code blocks (Codex / xAI) ───────── + if provider_id in ("openai-codex", "xai-oauth"): + p_providers = profile_store.get("providers") + r_providers = root_store.get("providers") + if isinstance(p_providers, dict) and isinstance(r_providers, dict): + p_block = p_providers.get(provider_id) + r_block = r_providers.get(provider_id) + else: + p_block = r_block = None + if isinstance(p_block, dict) and p_block and isinstance(r_block, dict) and r_block: + p_tokens = p_block.get("tokens") if isinstance(p_block.get("tokens"), dict) else {} + r_tokens = r_block.get("tokens") if isinstance(r_block.get("tokens"), dict) else {} + p_flat = {**p_tokens, "last_refresh": p_block.get("last_refresh")} + r_flat = {**r_tokens, "last_refresh": r_block.get("last_refresh")} + p_ident, r_ident = _oauth_identity(p_flat), _oauth_identity(r_flat) + same_account = (p_ident == r_ident) if (p_ident and r_ident) else True + if same_account: + if _oauth_freshness(p_flat) > _oauth_freshness(r_flat): + r_providers[provider_id] = dict(p_block) + root_changed = True + summary["adopted"] = True + del p_providers[provider_id] + profile_changed = True + summary["providers_block"] = True + + # ── profile-local .anthropic_oauth.json singleton ─────────── + if ( + profile_singleton is not None + and profile_singleton.exists() + # An aliased singleton pair is one shared grant, not a fork + # (#101356): never self-compare or unlink it. + and not (root_singleton is not None and _is_same_auth_store(profile_singleton, root_singleton)) + ): + p_single = _singleton_as_row(profile_singleton) + root_has_grant = bool(r_oauth) or root_singleton_row is not None + if p_single is not None and root_has_grant: + if root_singleton_row is not None: + if _oauth_freshness(p_single) > _oauth_freshness(root_singleton_row): + root_singleton_row = _adopt_oauth_material(root_singleton_row, p_single) + summary["adopted"] = True + else: + # Root only has pool rows: fold the singleton's pair + # into the freshest-matching root pkce row, if any. + idx = next( + (i for i, r in enumerate(r_rows) + if _is_oauth_pool_payload(r) + and str(r.get("source") or "").endswith("hermes_pkce")), + None, + ) + if idx is not None and _oauth_freshness(p_single) > _oauth_freshness(r_rows[idx]): + r_rows[idx] = _adopt_oauth_material(r_rows[idx], p_single) + root_changed = True + summary["adopted"] = True + try: + profile_singleton.unlink() + summary["files"].append(profile_singleton.name) + except OSError: + logger.debug("could not remove %s", profile_singleton, exc_info=True) + # Otherwise root has NO grant for this provider (or the file + # is not a grant): the profile's singleton may be the only + # surviving copy — never delete it. + + if not (profile_changed or root_changed or summary["adopted"]): + _oauth_heal_clean_marks[provider_id] = fingerprint + return None + + if summary["adopted"] and root_singleton is not None and root_singleton_row is not None: + # Keep root's singleton and its ``hermes_pkce``-seeded pool row + # in step: root's next load_pool() re-seeds that row FROM the + # singleton file, so a stale file would resurrect the spent + # pair (and a stale row would be overwritten by a fresh file). + pkce_idx = next( + (i for i, r in enumerate(r_rows) + if _is_oauth_pool_payload(r) and r.get("source") == "hermes_pkce"), + None, + ) + if pkce_idx is not None: + pkce_row = r_rows[pkce_idx] + if _oauth_freshness(pkce_row) > _oauth_freshness(root_singleton_row): + root_singleton_row = _adopt_oauth_material(root_singleton_row, pkce_row) + elif _oauth_freshness(root_singleton_row) > _oauth_freshness(pkce_row): + r_rows[pkce_idx] = _adopt_oauth_material(pkce_row, root_singleton_row) + root_changed = True + + if root_changed: + if isinstance(r_pool, dict): + r_pool[provider_id] = r_rows + else: + root_store["credential_pool"] = {provider_id: r_rows} + _save_auth_store(root_store, target_path=root_path) + if summary["adopted"] and root_singleton is not None and root_singleton_row is not None: + from agent.anthropic_credentials import _write_hermes_oauth_credentials + _write_hermes_oauth_credentials( + root_singleton_row.get("access_token") or "", + root_singleton_row.get("refresh_token"), + root_singleton_row.get("expires_at_ms"), + target=root_singleton, + ) + if profile_changed and profile_path.exists(): + _save_auth_store(profile_store, target_path=profile_path) + + if summary["stripped_ids"]: + log_bits.append(f"pool rows {summary['stripped_ids']}") + if summary["providers_block"]: + log_bits.append(f"providers.{provider_id} block") + if summary["files"]: + log_bits.append(", ".join(summary["files"])) + verdict = ( + "profile copy was the live pair; root updated" + if summary["adopted"] else "root copy already newest; profile copy dropped" + ) + message = ( + f"profile {profile_home.name}: consolidated forked {provider_id} OAuth grant " + f"({'; '.join(log_bits) or 'no-op'}) into the root grant — {verdict}; " + f"this profile now borrows the root grant (#100339)" + ) + logger.info(message) + _oauth_heal_notices.append(message) + return summary + + def read_credential_pool(provider_id: Optional[str] = None) -> Dict[str, Any]: """Return the persisted credential pool, or one provider slice. @@ -6491,6 +7039,7 @@ def resolve_nous_runtime_credentials( insecure: Optional[bool] = None, ca_bundle: Optional[str] = None, force_refresh: bool = False, + stale_access_token: Optional[str] = None, ) -> Dict[str, Any]: """ Resolve Nous inference credentials for runtime use. @@ -6498,8 +7047,14 @@ def resolve_nous_runtime_credentials( Ensures access_token is a valid inference-scoped JWT, refreshing it when needed. Concurrent processes coordinate through the auth store file lock. - Returns dict with: provider, base_url, api_key, key_id, expires_at, - expires_in, source ("invoke_jwt"), and auth_path. + ``stale_access_token`` is the bearer that just failed upstream (401). When + set together with ``force_refresh``, the refresh POST is skipped if the + store — re-read under the lock — already holds a *different*, usable + token: another process won the rotation, so this caller adopts it + instead of rotating the shared grant again. Without this, N concurrent + processes hitting the same hourly expiry issue N refreshes, and each + rotation invalidates the token a sibling just adopted (Sep 2026: 120 + subagents, 81 refreshes, ~540 401s in eight minutes). """ sequence_id = uuid.uuid4().hex[:12] @@ -6513,6 +7068,20 @@ def resolve_nous_runtime_credentials( raise AuthError("Hermes is not logged into Nous Portal.", provider="nous", relogin_required=True) + def _already_rotated_by_peer(token: Any) -> bool: + return bool( + force_refresh + and stale_access_token + and isinstance(token, str) + and token + and token != stale_access_token + and _nous_invoke_jwt_status( + token, + scope=state.get("scope"), + expires_at=state.get("expires_at"), + ) is None + ) + persisted_state = dict(state) state_persisted = False @@ -6658,6 +7227,16 @@ def resolve_nous_runtime_credentials( scope=state.get("scope"), expires_at=state.get("expires_at"), ) + # Under the store lock: if the bearer that failed upstream is no + # longer the one on disk and the on-disk one is usable, a peer + # already rotated — adopt, never re-POST the shared grant. + if _already_rotated_by_peer(access_token): + _oauth_trace( + "refresh_skipped_peer_rotated", + sequence_id=sequence_id, + access_token_fp=_token_fingerprint(access_token), + ) + force_refresh = False if force_refresh or invoke_jwt_status is not None: with _nous_shared_store_lock(timeout_seconds=max(timeout_seconds + 5.0, AUTH_LOCK_TIMEOUT_SECONDS)): if _merge_shared_nous_oauth_state(state): @@ -6675,6 +7254,13 @@ def resolve_nous_runtime_credentials( expires_at=state.get("expires_at"), ) _persist_state("post_shared_merge_access_unusable") + if _already_rotated_by_peer(access_token): + _oauth_trace( + "refresh_skipped_peer_rotated", + sequence_id=sequence_id, + access_token_fp=_token_fingerprint(access_token), + ) + force_refresh = False if force_refresh or invoke_jwt_status is not None: if not isinstance(refresh_token, str) or not refresh_token: @@ -7314,8 +7900,73 @@ def get_api_key_provider_status(provider_id: str) -> Dict[str, Any]: } +def _external_process_auth_evidence(provider_id: str) -> tuple[bool, Optional[str]]: + """Best-effort POSITIVE evidence that an external-process provider's CLI + is authenticated. + + Returns ``(verified, source)``. ``verified`` is only ever True on hard + evidence (a supported env token, or a known on-disk credential store). + False means "not verifiable from here", NOT "signed out" — the Copilot + CLI may hold its session in an OS keychain Hermes can't read. Callers + must therefore treat False as unknown, never as proof of absence. + + Deliberately subprocess-free: this runs from status endpoints and pickers, + and spawning ``gh auth token`` there re-creates the cold-start stall + (#60800) that copilot_auth.py works to avoid. + """ + if provider_id != "copilot-acp": + return False, None + # 1. Supported env tokens — the same vars the Copilot CLI itself honors. + try: + from hermes_cli.copilot_auth import COPILOT_ENV_VARS, validate_copilot_token + for env_var in COPILOT_ENV_VARS: + val = os.getenv(env_var, "").strip() + if val and validate_copilot_token(val)[0]: + return True, f"env: {env_var}" + except Exception as exc: + logger.debug("copilot-acp env token evidence check failed: %s", exc) + # 2. The Copilot CLI's own plaintext token store (~/.copilot/config.json, + # written by `copilot login` when no OS keychain is available). The file + # is JSONC — strip //-comment lines before parsing. + try: + cli_config = os.path.expanduser("~/.copilot/config.json") + if os.path.isfile(cli_config): + with open(cli_config, "r", encoding="utf-8", errors="ignore") as fh: + raw = "\n".join( + line for line in fh.read().splitlines() + if not line.lstrip().startswith("//") + ) + data = json.loads(raw) if raw.strip() else {} + tokens = data.get("copilotTokens") + if isinstance(tokens, dict) and any( + isinstance(v, str) and v.strip() for v in tokens.values() + ): + return True, "~/.copilot/config.json" + except Exception as exc: + logger.debug("copilot-acp CLI config evidence check failed: %s", exc) + # 3. Known on-disk GitHub Copilot credential stores (the same locations + # models.py already fingerprints as external credential files). + for cred_path in ( + "~/.config/github-copilot/hosts.json", + "~/.config/github-copilot/apps.json", + ): + try: + expanded = os.path.expanduser(cred_path) + if os.path.isfile(expanded) and os.path.getsize(expanded) > 2: + return True, cred_path + except OSError: + continue + return False, None + + def get_external_process_provider_status(provider_id: str) -> Dict[str, Any]: - """Status snapshot for providers that run a local subprocess.""" + """Status snapshot for providers that run a local subprocess. + + ``configured``/``logged_in`` stay structural (the executable resolves or a + TCP endpoint is set) because the spawned subprocess owns its real auth. + ``auth_verified``/``auth_source`` carry positive credential evidence when + Hermes can actually see some — absence of evidence is not absence of auth. + """ pconfig = PROVIDER_REGISTRY.get(provider_id) if not pconfig or pconfig.auth_type != "external_process": return {"configured": False} @@ -7332,6 +7983,7 @@ def get_external_process_provider_status(provider_id: str) -> Dict[str, Any]: base_url = pconfig.inference_base_url resolved_command = shutil.which(command) if command else None + auth_verified, auth_source = _external_process_auth_evidence(provider_id) return { "configured": bool(resolved_command or base_url.startswith("acp+tcp://")), "provider": provider_id, @@ -7341,6 +7993,8 @@ def get_external_process_provider_status(provider_id: str) -> Dict[str, Any]: "resolved_command": resolved_command, "base_url": base_url, "logged_in": bool(resolved_command or base_url.startswith("acp+tcp://")), + "auth_verified": auth_verified, + "auth_source": auth_source, } @@ -7361,12 +8015,16 @@ def get_auth_status(provider_id: Optional[str] = None) -> Dict[str, Any]: return get_qwen_auth_status() if target == "minimax-oauth": return get_minimax_oauth_auth_status() - if target == "copilot-acp": - return get_external_process_provider_status(target) if target == "azure-foundry": return _get_azure_foundry_auth_status() - # API-key providers pconfig = PROVIDER_REGISTRY.get(target) + # External-process providers (copilot-acp today; kiro/devin/junie-style ACP + # backends tomorrow) — dispatch on auth_type, not a hardcoded slug, so every + # provider of this class gets a real status instead of the + # ``{"logged_in": False}`` fallthrough. + if pconfig and pconfig.auth_type == "external_process": + return get_external_process_provider_status(target) + # API-key providers if pconfig and pconfig.auth_type == "api_key": return get_api_key_provider_status(target) # AWS SDK providers (Bedrock) — check via boto3 credential chain @@ -7551,25 +8209,52 @@ def resolve_external_process_provider_credentials(provider_id: str) -> Dict[str, if not base_url: base_url = pconfig.inference_base_url - command = ( - os.getenv("HERMES_COPILOT_ACP_COMMAND", "").strip() - or os.getenv("COPILOT_CLI_PATH", "").strip() - or "copilot" - ) - raw_args = os.getenv("HERMES_COPILOT_ACP_ARGS", "").strip() - args = shlex.split(raw_args) if raw_args else ["--acp", "--stdio"] + # How to launch the CLI comes from the provider's own profile, so a provider + # shipped outside this tree describes its binary/args instead of inheriting + # another vendor's. copilot-acp's values live in its profile, which is why + # HERMES_COPILOT_ACP_COMMAND / COPILOT_CLI_PATH / HERMES_COPILOT_ACP_ARGS + # keep working unchanged. + profile = None + try: + from providers import get_provider_profile as _get_provider_profile + + profile = _get_provider_profile(provider_id) + except Exception: + profile = None + + command_env_vars = tuple(getattr(profile, "process_command_env_vars", ()) or ()) + default_command = str(getattr(profile, "process_command", "") or "") + default_args = list(getattr(profile, "process_args", ()) or []) + args_env_var = str(getattr(profile, "process_args_env_var", "") or "") + + command = "" + for _var in command_env_vars: + command = os.getenv(_var, "").strip() + if command: + break + if not command: + command = default_command + + raw_args = os.getenv(args_env_var, "").strip() if args_env_var else "" + args = shlex.split(raw_args) if raw_args else list(default_args) + resolved_command = shutil.which(command) if command else None if not resolved_command and not base_url.startswith("acp+tcp://"): + _hint = ( + " or set " + "/".join(command_env_vars) if command_env_vars else "" + ) raise AuthError( - f"Could not find the Copilot CLI command '{command}'. " - "Install GitHub Copilot CLI or set HERMES_COPILOT_ACP_COMMAND/COPILOT_CLI_PATH.", + f"Could not find the '{provider_id}' CLI command " + f"'{command or '(none configured)'}'. Install it{_hint}.", provider=provider_id, - code="missing_copilot_cli", + code="missing_external_process_cli", ) return { "provider": provider_id, - "api_key": "copilot-acp", + # Placeholder credential: the subprocess owns real auth. Keyed on the + # provider id so each external-process provider gets a distinct value. + "api_key": pconfig.id or provider_id, "base_url": base_url.rstrip("/"), "command": resolved_command or command, "args": args, diff --git a/hermes_cli/auth_commands.py b/hermes_cli/auth_commands.py index 954c173cd2..3699032885 100644 --- a/hermes_cli/auth_commands.py +++ b/hermes_cli/auth_commands.py @@ -557,6 +557,13 @@ def auth_list_command(args) -> None: source = _display_source(entry.source) print(f" #{idx} {entry.label:<20} {entry.auth_type:<7} {source}{status} {marker}".rstrip()) print() + _print_oauth_heal_notices() + + +def _print_oauth_heal_notices() -> None: + """Tell the user when load_pool() just consolidated a forked OAuth grant.""" + for note in auth_mod.consume_oauth_heal_notices(): + print(f"note: {note}") def auth_remove_command(args) -> None: @@ -608,7 +615,12 @@ def auth_status_command(args) -> None: provider = _normalize_provider(getattr(args, "provider", "") or "") if not provider: raise SystemExit("Provider is required. Example: `hermes auth status spotify`.") + if provider in auth_mod.SINGLE_USE_REFRESH_POOL_PROVIDERS: + # load_pool() runs the forked-grant heal (#100339); do it before the + # status read so the report reflects the consolidated grant. + load_pool(provider) status = auth_mod.get_auth_status(provider) + _print_oauth_heal_notices() if not status.get("logged_in"): reason = status.get("error") if reason: diff --git a/hermes_cli/backup.py b/hermes_cli/backup.py index f9f551ea1c..5ff7c10dac 100644 --- a/hermes_cli/backup.py +++ b/hermes_cli/backup.py @@ -136,6 +136,8 @@ def _in_excluded_root_dir(rel_path: Path) -> bool: # File-name suffixes to skip +_SQLITE_SIDECAR_SUFFIXES = (".db-wal", ".db-shm", ".db-journal") + _EXCLUDED_SUFFIXES = ( ".pyc", ".pyo", @@ -144,9 +146,7 @@ _EXCLUDED_SUFFIXES = ( # rollback-journal alongside would pair a fresh snapshot with stale sidecar # state and produce a torn restore on the next open. They're transient and # regenerated on first connection anyway. - ".db-wal", - ".db-shm", - ".db-journal", + *_SQLITE_SIDECAR_SUFFIXES, ) # File names to skip (runtime state that's meaningless on another machine) @@ -812,8 +812,10 @@ def _safe_restore_db(src: Path, dst: Path) -> bool: the WAL journal is updated correctly, and all connections (old and new) converge on the restored data. - Falls back to the unlink+move approach on failure so restore never - blocks on a transient error. + Falls back to the unlink+move approach on failure ONLY when no other + process or in-process connection holds the file: replacing the inode + under a live holder is the #90950 split-brain, so that branch fails + closed (returns ``False``) and the caller reports the file as skipped. """ try: dst_conn = sqlite3.connect(str(dst)) @@ -1286,6 +1288,94 @@ def _extract_member_atomically( raise +def _count_session_rows(path: Path) -> Optional[Tuple[int, int]]: + """Return ``(sessions, messages)`` stored in the session database *path*. + + Read-only and best effort. ``None`` means "unknown" — a missing file, a + database that is not a Hermes session store, or one that cannot be read. + Callers must never read ``None`` as "zero rows": acting on an unreadable + database would mask the very loss this count exists to surface. Same + contract as :func:`_count_cron_jobs`. + """ + if not path.is_file(): + return None + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + except sqlite3.Error: + return None + try: + sessions = conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] + messages = conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0] + return int(sessions), int(messages) + except (sqlite3.Error, TypeError, ValueError): + return None + finally: + conn.close() + + +def _import_db_member( + zf: zipfile.ZipFile, + member: str, + target: Path, + new_file_mode: Optional[int] = None, +) -> None: + """Publish a SQLite ``.db`` member onto *target* without replacing its inode. + + ``_extract_member_atomically`` publishes with a rename. For an ordinary + file that is the safest write available; for a live SQLite database it is + the #65942 / #90950 corruption class. A gateway, dashboard, or WebUI + process holding the database open keeps its descriptor on the now-unlinked + inode: it goes on serving pre-import pages and writing sessions that no + other process will ever see, and any sidecar WAL left beside the new file + describes the database that was just unlinked. Nothing fails, so nothing + is reported — the sessions simply are not there afterwards (issue #100960). + + ``hermes import`` is the disaster-recovery path, so that failure mode lands + on users who have already lost something once. Route the member through + the same ``_safe_restore_db`` page copy that ``/snapshot restore`` has used + since #65942: the live inode is preserved, every open connection converges + on the imported data, and the sidecars are handled there. A target that + does not exist yet has no holders and no inode worth preserving, so it + takes the ordinary atomic publish. + + Raises ``OSError`` when the database could not be replaced safely, so the + caller reports a skipped file instead of counting a silent success. + """ + if not target.exists(): + _extract_member_atomically(zf, member, target, new_file_mode) + return + + # The database keeps its own mode/ownership: the bytes come from the + # archive but the file does not, so the archive has no say in either. + mode = _preserve_file_mode(target) + owner = _preserve_file_owner(target) + + fd, tmp_name = tempfile.mkstemp( + dir=str(target.parent), prefix=f".{target.name[:80]}.", suffix=".dbimport" + ) + try: + with os.fdopen(fd, "wb") as dst: + # Stream: a multi-gigabyte state.db member must not be held in + # memory in one piece. + with zf.open(member) as src: + shutil.copyfileobj(src, dst) + dst.flush() + os.fsync(dst.fileno()) + if not _safe_restore_db(Path(tmp_name), target): + raise OSError( + "live-safe restore refused or failed; the existing database was " + "left untouched. Stop the gateway/dashboard processes holding it " + "open and re-run the import." + ) + _restore_file_owner(target, owner) + _restore_file_mode(target, mode) + finally: + try: + os.unlink(tmp_name) + except OSError: + pass + + def run_import(args) -> None: """Restore a Hermes backup from a zip file.""" zip_path = Path(args.zipfile).expanduser().resolve() @@ -1348,6 +1438,10 @@ def run_import(args) -> None: restored = 0 restored_external = 0 skipped_runtime: list[str] = [] + # (rel, live_counts, imported_counts) for every session database the + # import replaced with one holding fewer rows. A restore is allowed to + # do that — it just must not do it silently (issue #100960). + db_shrunk: list[tuple[str, tuple[int, int], tuple[int, int]]] = [] home_dir = Path.home().resolve() # Resolved once: every member is published via a temp file, and mkstemp # would otherwise create newly restored files as 0600. @@ -1405,6 +1499,16 @@ def run_import(args) -> None: skipped_runtime.append(rel) continue + # A ``.db`` member is page-restored into the live file below; a + # WAL/SHM/journal member from the archive describes a different + # database image, and installing it beside the restored file (over + # a live sidecar, via os.replace) would replay a foreign WAL on + # the next open. Current backups never ship these + # (_EXCLUDED_SUFFIXES); older or hand-built archives might. + if rel.endswith(_SQLITE_SIDECAR_SUFFIXES): + skipped_runtime.append(rel) + continue + target = hermes_root / rel # Security: reject absolute paths and traversals @@ -1416,7 +1520,16 @@ def run_import(args) -> None: try: target.parent.mkdir(parents=True, exist_ok=True) - _extract_member_atomically(zf, member, target, new_file_mode) + if target.suffix == ".db": + # Count before the write: afterwards the rows this import + # drops are gone and there is nothing left to compare. + before = _count_session_rows(target) + _import_db_member(zf, member, target, new_file_mode) + after = _count_session_rows(target) + if before and after and after[1] < before[1]: + db_shrunk.append((rel, before, after)) + else: + _extract_member_atomically(zf, member, target, new_file_mode) if target.name in _SECRET_FILE_NAMES: os.chmod(target, 0o600) restored += 1 @@ -1446,6 +1559,21 @@ def run_import(args) -> None: if len(errors) > 10: print(f" ... and {len(errors) - 10} more") + if db_shrunk: + # The backup predates work that is now overwritten. Say so: the + # reported incident was twelve sessions disappearing with nothing + # logged anywhere (issue #100960). + print("\n ⚠ Session data replaced by older backup contents:") + for rel, before, after in db_shrunk: + print( + f" {rel}: {before[0]} session(s) / {before[1]} message(s)" + f" -> {after[0]} / {after[1]}" + ) + print( + " Anything recorded after the backup was taken is not in it. " + "Recover from a newer backup or snapshot: hermes snapshot list" + ) + if skipped_runtime: print( f"\n Preserved {len(skipped_runtime)} runtime state " @@ -1934,7 +2062,11 @@ def restore_quick_snapshot( # (gateway, dashboard, another CLI session) see the # restored data instead of continuing to serve stale # cached pages from a replaced inode (issue #65942). - _safe_restore_db(src, dst) + if not _safe_restore_db(src, dst): + # Refused (live holder) or failed: the destination was + # left as it was. Count it as a failure, not a restore. + logger.error("Failed to restore %s: live-safe restore refused", rel) + continue else: shutil.copy2(src, dst) restored += 1 diff --git a/hermes_cli/banner.py b/hermes_cli/banner.py index a1e9f0bb94..9643bfe689 100644 --- a/hermes_cli/banner.py +++ b/hermes_cli/banner.py @@ -241,11 +241,15 @@ def _is_full_sha(value: Optional[str]) -> bool: def _upstream_main_sha() -> Optional[str]: """Tip SHA of upstream main via HTTPS ls-remote (no auth, no prompts).""" + from hermes_cli._subprocess_compat import noninteractive_git_env + try: result = subprocess.run( ["git", "ls-remote", _UPSTREAM_REPO_URL, "refs/heads/main"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, + stdin=subprocess.DEVNULL, + env=noninteractive_git_env(), ) except Exception: return None @@ -277,6 +281,8 @@ def _check_via_rev(local_rev: str) -> Optional[int]: def _check_via_local_git(repo_dir: Path) -> Optional[int]: """Count commits behind origin/main in a local checkout.""" + from hermes_cli._subprocess_compat import noninteractive_git_env + origin_url = _git_stdout(["remote", "get-url", "origin"], cwd=repo_dir) if _is_official_ssh_remote(origin_url): head_rev = _git_stdout(["rev-parse", "HEAD"], cwd=repo_dir) @@ -348,6 +354,8 @@ def _check_via_local_git(repo_dir: Path) -> Optional[int]: fetch_args, capture_output=True, timeout=10, cwd=str(repo_dir), + stdin=subprocess.DEVNULL, + env=noninteractive_git_env(), ) fetch_ok = fetch_proc.returncode == 0 except Exception: diff --git a/hermes_cli/callbacks.py b/hermes_cli/callbacks.py index aad0542d28..903bc6709b 100644 --- a/hermes_cli/callbacks.py +++ b/hermes_cli/callbacks.py @@ -120,6 +120,8 @@ def prompt_for_secret(cli, var_name: str, prompt: str, metadata=None) -> dict: "response_queue": response_queue, } cli._secret_deadline = _time.monotonic() + timeout + if hasattr(cli, "_ring_bell"): + cli._ring_bell(prompt=True, context=f"secret needed ({var_name})") # Avoid storing stale draft input as the secret when Enter is pressed. if hasattr(cli, "_clear_secret_input_buffer"): try: diff --git a/hermes_cli/cli_agent_setup_mixin.py b/hermes_cli/cli_agent_setup_mixin.py index 6fd447637f..fb28cfcd60 100644 --- a/hermes_cli/cli_agent_setup_mixin.py +++ b/hermes_cli/cli_agent_setup_mixin.py @@ -353,12 +353,18 @@ class CLIAgentSetupMixin: } service_tier = getattr(self, "service_tier", None) - if not service_tier: + if service_tier != "priority": + # None (normal) or auto/cold — the bounded window is applied per + # request by agent.fast_mode, not pinned into request_overrides. route["request_overrides"] = None return route try: - overrides = resolve_fast_mode_overrides(route["model"]) + overrides = resolve_fast_mode_overrides( + route["model"], + provider=runtime["provider"], + base_url=runtime["base_url"], + ) except Exception: overrides = None route["request_overrides"] = overrides @@ -645,29 +651,16 @@ class CLIAgentSetupMixin: if not self._session_db: return None from hermes_state import ( - SessionExportTooLargeError, SessionResumeTooLargeError, - resolved_max_resume_messages, ) try: + safety_check = getattr(self._session_db, "assert_resume_safe", None) + if not callable(safety_check): + return None if tip_only: - tip_check = getattr(self._session_db, "assert_export_safe", None) - if not callable(tip_check): - return None - limit = resolved_max_resume_messages() - if limit <= 0: - return None - try: - tip_check(self.session_id, max_messages=limit) - except SessionExportTooLargeError as exc: - raise SessionResumeTooLargeError( - exc.message_count, limit, scope="in its tip segment" - ) from exc + safety_check(self.session_id, tip_only=True) else: - safety_check = getattr(self._session_db, "assert_resume_safe", None) - if not callable(safety_check): - return None safety_check(self.session_id) except SessionResumeTooLargeError as exc: return str(exc) diff --git a/hermes_cli/cli_commands_mixin.py b/hermes_cli/cli_commands_mixin.py index 25b736f71b..96739aa686 100644 --- a/hermes_cli/cli_commands_mixin.py +++ b/hermes_cli/cli_commands_mixin.py @@ -1976,7 +1976,13 @@ class CLICommandsMixin: print(f" Skills: {', '.join(job['skills'])}") print(f" Prompt: {job.get('prompt_preview', '')}") if job.get("last_run_at"): - print(f" Last run: {job['last_run_at']} ({job.get('last_status', '?')})") + status = job.get("last_status") or "?" + # delivery_failed: the agent ran fine but the output never + # reached the target — name the delivery reason, which + # lives in last_delivery_error (last_error is None). + if status == "delivery_failed" and job.get("last_delivery_error"): + status = f"delivery_failed: {job['last_delivery_error']}" + print(f" Last run: {job['last_run_at']} ({status})") print() return @@ -3973,9 +3979,9 @@ class CLICommandsMixin: parts = cmd.strip().split(maxsplit=1) if len(parts) < 2 or parts[1].strip().lower() == "status": - status = "fast" if self.service_tier == "priority" else "normal" + status = {"priority": "fast", None: "normal"}.get(self.service_tier, self.service_tier) _cprint(f" {_ACCENT}{feature_name}: {status}{_RST}") - _cprint(f" {_DIM}Usage: /fast [normal|fast|status] [--global]{_RST}") + _cprint(f" {_DIM}Usage: /fast [normal|fast|auto|cold|status] [--global]{_RST}") return arg_tokens = parts[1].strip().lower().split() @@ -3993,9 +3999,13 @@ class CLICommandsMixin: self.service_tier = None saved_value = "normal" label = "NORMAL" + elif arg in {"auto", "cold"}: + self.service_tier = arg + saved_value = arg + label = arg.upper() else: _cprint(f" {_DIM}(._.) Unknown argument: {arg}{_RST}") - _cprint(f" {_DIM}Usage: /fast [normal|fast|status] [--global]{_RST}") + _cprint(f" {_DIM}Usage: /fast [normal|fast|auto|cold|status] [--global]{_RST}") return self.agent = None # Force agent re-init with new service-tier config diff --git a/hermes_cli/commands.py b/hermes_cli/commands.py index 23af46886d..f2c2ea7faf 100644 --- a/hermes_cli/commands.py +++ b/hermes_cli/commands.py @@ -297,9 +297,9 @@ COMMAND_REGISTRY: list[CommandDef] = [ args_hint="[level|show|hide|full|clamp] [--global]", subcommands=("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "show", "hide", "on", "off", "full", "clamp", "--global"), desktop="advanced"), - CommandDef("fast", "Toggle fast mode — OpenAI Priority Processing / Anthropic Fast Mode (Normal/Fast)", "Configuration", - args_hint="[normal|fast|status] [--global]", - subcommands=("normal", "fast", "status", "on", "off", "--global"), + CommandDef("fast", "Fast mode — OpenAI Priority Processing / Anthropic Fast Mode (normal/fast/auto/cold)", "Configuration", + args_hint="[normal|fast|auto|cold|status] [--global]", + subcommands=("normal", "fast", "auto", "cold", "status", "on", "off", "--global"), desktop="advanced"), CommandDef("skin", "Show or change the display skin/theme", "Configuration", cli_only=True, args_hint="[name]", argument_mode="options"), diff --git a/hermes_cli/config.py b/hermes_cli/config.py index a0312d8a43..6483685ecb 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1446,7 +1446,7 @@ def _is_env_config_key(key: str) -> bool: 'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY', 'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL', 'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME', - 'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', + 'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', 'API_SERVER_KEY', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY', 'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN', 'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY', @@ -2261,7 +2261,7 @@ def _raw_config_has_explicit_version() -> bool: return isinstance(raw, dict) and "_config_version" in raw -def check_config_version() -> Tuple[int, int]: +def check_config_version(*, raise_on_parse_error: bool = False) -> Tuple[int, int]: """ Check the raw on-disk config schema version. @@ -2271,7 +2271,10 @@ def check_config_version() -> Tuple[int, int]: raw ``_config_version`` must remain visible as legacy instead of inheriting the latest default version in memory. - Returns (current_version, latest_version). + Returns (current_version, latest_version). Tolerant runtime status callers + retain the historical latest/latest fallback for malformed YAML. Mutation + and explicit validation paths can set ``raise_on_parse_error`` so a parse + failure or a non-mapping root cannot be mistaken for an up-to-date config. """ latest = _coerce_config_version(DEFAULT_CONFIG.get("_config_version", 1)) or 1 config_path = get_config_path() @@ -2285,9 +2288,23 @@ def check_config_version() -> Tuple[int, int]: # Invalid YAML needs a parse warning, not an automatic schema rewrite # that could replace the user's broken file with defaults. _warn_config_parse_failure(config_path, e) + if raise_on_parse_error: + raise InvalidUserConfigError( + f"Cannot inspect {config_path}: config.yaml is not valid YAML ({e})" + ) from e return latest, latest + if config is None: + config = {} # empty file / bare document: valid first-run state if not isinstance(config, dict): + # A list/scalar root parses fine but is just as unusable as broken + # YAML: save_config() would refuse it later, after .env was already + # rewritten. Strict callers must see it up front too. + if raise_on_parse_error: + raise InvalidUserConfigError( + f"Cannot inspect {config_path}: config.yaml top-level value must be " + f"a mapping, got {type(config).__name__}" + ) config = {} current = _coerce_config_version(config.get("_config_version")) return current, latest @@ -2648,6 +2665,11 @@ def migrate_config(interactive: bool = True, quiet: bool = False) -> Dict[str, A """ results = {"env_added": [], "config_added": [], "warnings": []} + # Validate config.yaml before any migration side effect. In particular, + # sanitize_env_file() can rewrite .env, which must not happen when the + # migration will be refused for malformed YAML. + current_ver, latest_ver = check_config_version(raise_on_parse_error=True) + # ── Always: normalize safe .env line formatting ── try: fixes = sanitize_env_file() @@ -2656,9 +2678,6 @@ def migrate_config(interactive: bool = True, quiet: bool = False) -> Dict[str, A except Exception: pass # best-effort; don't block migration on sanitize failure - # Check config version - current_ver, latest_ver = check_config_version() - # ── Auto-migration support floor (policy: v12, July 2026) ── # A config with an EXPLICIT on-disk ``_config_version`` below the floor is # NOT auto-migrated and NOT rewritten: we surface a clear, actionable @@ -2974,13 +2993,36 @@ def _strip_dotted_keys(cfg: dict, dotted_keys: set) -> Tuple[dict, set]: return cfg, stripped +def _env_ref_lookup(name: str) -> Optional[str]: + """Resolve the env var behind a ``${VAR}`` / ``${env:VAR}`` config ref. + + Outside a profile secret scope this is a plain ``os.environ`` read — the + default profile and every single-profile caller keep their legacy + behavior. Inside a scope (a multiplexed gateway turn, a secondary + profile's config load, a cron job) the read goes through + ``agent.secret_scope.get_secret`` so the ref resolves against *that* + profile's ``.env``: under multiplexing a miss is a miss, never another + profile's ``os.environ`` value (#84079 — every profile "had" the default + profile's ``${MATRIX_ACCESS_TOKEN}`` and fanned out). Same policy as + ``gateway.config._getenv`` and ``get_env_value``. + """ + try: + from agent.secret_scope import current_secret_scope, get_secret as _get_secret + except Exception: + return os.environ.get(name) + if current_secret_scope() is None: + return os.environ.get(name) + return _get_secret(name) + + def _env_expand_match(m: re.Match) -> str: """Expand one ``${...}`` config reference. Two accepted shapes, matching what MCP server config already resolves (``tools/mcp_tool.py::_env_ref_name``): - * ``${VAR}`` — legacy bare name, resolved via ``os.environ``. + * ``${VAR}`` — legacy bare name, resolved via ``_env_ref_lookup`` + (``os.environ``, or the active profile secret scope). * ``${env:VAR}`` — Cursor-style SecretRef, same resolution after the ``env:`` prefix is stripped. Before this, the prefixed form worked in MCP config but stayed a literal string in config.yaml — a confusing @@ -2998,7 +3040,7 @@ def _env_expand_match(m: re.Match) -> str: name = inner[len("env:"):].strip() if not name: return raw - val = os.environ.get(name) + val = _env_ref_lookup(name) if val is not None: return val logger.warning( @@ -3019,7 +3061,8 @@ def _env_expand_match(m: re.Match) -> str: ) return raw # Legacy ``${VAR}`` — bare name. - return os.environ.get(inner, raw) + val = _env_ref_lookup(inner) + return val if val is not None else raw def _env_ref_var_name(ref: str) -> Optional[str]: @@ -3072,7 +3115,7 @@ def _env_ref_snapshot(obj, snapshot=None): for raw in re.findall(r"\${([^}]+)}", obj): name = _env_ref_var_name(raw) if name is not None: - snapshot[name] = os.environ.get(name) + snapshot[name] = _env_ref_lookup(name) elif isinstance(obj, dict): for value in obj.values(): _env_ref_snapshot(value, snapshot) @@ -4042,7 +4085,7 @@ def _load_config_impl(*, want_deepcopy: bool) -> Dict[str, Any]: # pins unexpanded literals (e.g. auxiliary..api_key) for the # life of the process (#58514). env_snapshot = cached[5] if len(cached) > 5 else {} - if all(os.environ.get(k) == v for k, v in env_snapshot.items()): + if all(_env_ref_lookup(k) == v for k, v in env_snapshot.items()): return copy.deepcopy(cached[4]) if want_deepcopy else cached[4] config = copy.deepcopy(DEFAULT_CONFIG) @@ -4603,6 +4646,38 @@ def _env_line_defines_key( ) == _env_var_policy_name(key, is_windows=is_windows) +def _publish_env_value(key: str, value: Optional[str]) -> None: + """Publish a just-persisted ``.env`` change to the live process. + + ``save_env_value`` / ``remove_env_value`` already target the right file + (``get_env_path()`` honors the profile-home override), but the in-process + mirror historically went straight to ``os.environ``. Under a multiplexed + gateway a routed profile's write (e.g. a ``/pair`` grant mirrored into + ``DISCORD_ALLOWED_USERS``) would then land in the SHARED process env and + be visible to every other profile (#88441, #77490). In that case update + the installed scope mapping instead so same-turn reads see the change, + and leave ``os.environ`` alone. Every other caller keeps the legacy + ``os.environ`` publish. + """ + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + scope = current_secret_scope() if is_multiplex_active() else None + except Exception: + scope = None + if scope is not None: + if isinstance(scope, dict): + if value is None: + scope.pop(key, None) + else: + scope[key] = value + return + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + def save_env_value(key: str, value: str): """Save or update a value in ~/.hermes/.env.""" if is_managed(): @@ -4691,7 +4766,7 @@ def save_env_value(key: str, value: str): pass raise - os.environ[key] = value + _publish_env_value(key, value) invalidate_env_cache() @@ -4738,7 +4813,7 @@ def remove_env_value(key: str) -> bool: raise ValueError(f"Invalid environment variable name: {key!r}") env_path = get_env_path() if not env_path.exists(): - os.environ.pop(key, None) + _publish_env_value(key, None) return False read_kw = {"encoding": "utf-8-sig", "errors": "replace"} @@ -4782,7 +4857,7 @@ def remove_env_value(key: str) -> bool: pass raise - os.environ.pop(key, None) + _publish_env_value(key, None) invalidate_env_cache() return found @@ -5081,7 +5156,10 @@ def show_config(): _active_personality = display.get('personality') or 'none' print(f" Personality: {_active_personality}") print(f" Reasoning: {'on' if display.get('show_reasoning', True) else 'off'}") - print(f" Bell: {'on' if display.get('bell_on_complete', False) else 'off'}") + print( + f" Bell: complete={'on' if display.get('bell_on_complete', False) else 'off'}, " + f"prompt={'on' if display.get('bell_on_prompt', False) else 'off'}" + ) ump = display.get('user_message_preview', {}) if isinstance(display.get('user_message_preview', {}), dict) else {} ump_first = ump.get('first_lines', 2) ump_last = ump.get('last_lines', 2) @@ -5786,6 +5864,37 @@ def _coerce_float(value: str): return f +def _redirect_platform_display_key(key: str) -> tuple[str, Optional[str]]: + """Canonicalize ``platforms..`` → ``display.platforms..``. + + Per-platform *display* settings (streaming, show_reasoning, tool_progress, + …) are resolved by the gateway from ``display.platforms..`` + (``gateway/display_config.py::resolve_display_setting``), while the + top-level ``platforms.`` block holds only connection config (token, + enabled, reply_to_mode, extra, …). Before #71047 a write such as + ``hermes config set platforms.telegram.streaming false`` landed on a key + the gateway never reads: ``config get`` echoed the new value back while + the runtime kept the old ``display.platforms`` one — a silent no-op that + looks like a duplicated key to the user. + + Only known display settings (``OVERRIDEABLE_KEYS``) are redirected so real + connection keys stay put. Returns ``(canonical_key, note_or_None)``. + The gateway import is guarded: the CLI must keep working where the + gateway package is not importable. + """ + segs = _split_key_path(key) + if len(segs) != 3 or segs[0] != "platforms": + return key, None + try: + from gateway.display_config import OVERRIDEABLE_KEYS as _display_keys + except Exception: + return key, None + if segs[2] not in _display_keys: + return key, None + canonical = f"display.platforms.{segs[1]}.{segs[2]}" + return canonical, f" (note: per-platform display setting — saved as {canonical})" + + def set_config_value(key: str, value: str, force: bool = False): """Set a configuration value. @@ -5851,6 +5960,12 @@ def set_config_value(key: str, value: str, force: bool = False): # bare success and left the user debugging behavior that never changed. # Warn after the write so the user gets immediate feedback plus a # "did you mean" hint, without blocking legitimate unknown keys. + # Per-platform display settings live under display.platforms (#71047, + # Problem A) — canonicalize BEFORE validation/coercion so the type-aware + # coercion and the unknown-key hint both see the path the runtime reads. + key, _redirect_note = _redirect_platform_display_key(key) + if _redirect_note: + print(_redirect_note) is_known, suggestion = _validate_config_key(key) # Otherwise it goes to config.yaml @@ -6086,6 +6201,9 @@ def get_config_value(key: str, *, as_json: bool = False): env_value = get_env_value(key.upper()) value = _MISSING if env_value is None else env_value else: + # Mirror set_config_value: read the canonical display.platforms path + # so ``config get`` reports what the gateway resolves (#71047). + key, _ = _redirect_platform_display_key(key) value = _get_nested(load_config(), key) if value is _MISSING: @@ -6151,6 +6269,10 @@ def unset_config_value(key: str): print(f"✗ {msg}", file=sys.stderr) raise RuntimeError(msg) + # Mirror set_config_value's display.platforms canonicalization (#71047). + key, _redirect_note = _redirect_platform_display_key(key) + if _redirect_note: + print(_redirect_note.replace("saved as", "resolved as")) removed = _unset_nested(user_config, key) # Keep .env in sync for keys that terminal_tool reads directly from env vars. @@ -6245,7 +6367,7 @@ def config_command(args): # Check what's missing missing_env = get_missing_env_vars(required_only=False) missing_config = get_missing_config_fields() - current_ver, latest_ver = check_config_version() + current_ver, latest_ver = check_config_version(raise_on_parse_error=True) if not missing_env and not missing_config and current_ver >= latest_ver: print(color("✓ Configuration is up to date!", Colors.GREEN)) @@ -6299,7 +6421,7 @@ def config_command(args): print(color("📋 Configuration Status", Colors.CYAN, Colors.BOLD)) print() - current_ver, latest_ver = check_config_version() + current_ver, latest_ver = check_config_version(raise_on_parse_error=True) if current_ver >= latest_ver: print(f" Config version: {current_ver} ✓") else: diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index fe123155e6..91a20c65e9 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -151,7 +151,10 @@ DEFAULT_CONFIG = { # leaves the budget untouched. "cost_threshold_usd": 0.25, }, + # Fast mode: "" / "normal" (off), "fast" (always), "auto" (first + # fast_auto_seconds of every turn), "cold" (first turn of a session only). "service_tier": "", + "fast_auto_seconds": 60, # Tool-use enforcement: injects system prompt guidance that tells the # model to actually call tools instead of describing intended actions. # Values: "auto" (default — applies to gpt/codex models), true/false @@ -751,6 +754,11 @@ DEFAULT_CONFIG = { # and override the dynamic behavior. Separate from read_file tool limits. "context_file_max_chars": None, + # Seconds to wait for a single context file read before skipping it with a + # warning. Guards startup against network-backed filesystems (iCloud Drive, + # OneDrive, NFS) that can block a cold read on an evicted file. + "context_file_read_timeout": 5.0, + # Maximum characters returned by a single read_file call. Reads that # exceed this are rejected with guidance to use offset+limit. # 100K chars ≈ 25–35K tokens across typical tokenisers. @@ -821,6 +829,10 @@ DEFAULT_CONFIG = { "tool_loop_guardrails": { "warnings_enabled": True, "hard_stop_enabled": False, + # Unattended gateway/cron platforms get hard stops by default (nobody + # is present to /stop a model that ignores loop warnings); interactive + # cli/tui/desktop/acp stay warning-only unless hard_stop_enabled. + "non_interactive_hard_stop_enabled": True, "warn_after": { "exact_failure": 2, "same_tool_failure": 3, @@ -957,6 +969,10 @@ DEFAULT_CONFIG = { # waiting. Kept well under chat-transport idle timeouts # (Telegram ~30s). On expiry the turn proceeds # uncompressed — an availability boundary, not a failure. + # The detached worker keeps its commit admission when its + # commit is watermark-fenced, so the finished summary is + # adopted at the next safe boundary instead of being + # discarded (#97963 — thinking summary models). "context_timeout_seconds": 120, # inactivity budget for in-agent compress_context # (conversation loop, /compress, preflight, etc.). # Same progress-aware semantics as hygiene_timeout_seconds: @@ -1458,12 +1474,18 @@ DEFAULT_CONFIG = { # Mirrors `hermes -c` muscle memory. Default off so existing # users aren't surprised. HERMES_TUI_RESUME= always wins. "tui_auto_resume_recent": False, + # When true (default), the Desktop app reopens the last chat (or + # last page) on cold start. Set false to always land on a fresh + # new chat. Also a switch in Desktop Settings → Appearance. + "resume_last_session": True, # When true (default), `hermes --tui` drops a one-time hint # ("subagents working · /agents to watch live") the first time a turn # starts delegating, nudging the user toward the live spawn-tree # dashboard. Set false to suppress the hint. "tui_agents_nudge": True, "bell_on_complete": False, + # Bell when a blocking prompt opens (clarify/approval/sudo/secret). + "bell_on_prompt": False, # Stream the model's reasoning/thinking live before the response. # Default ON: on thinking models the reasoning phase can run tens of # seconds, and with this off the user stares at a spinner the whole @@ -2806,6 +2828,15 @@ DEFAULT_CONFIG = { # Wrap delivered cron responses with a header (task name) and footer # ("The agent cannot see this message"). Set to false for clean output. "wrap_response": True, + # Delivery behaviour for cron output sent through a live gateway adapter. + "delivery": { + # Mark cron deliveries as FINAL notifications so the platform pushes + # them (Telegram's "important" notification mode otherwise sends + # every non-notify message with disable_notification=True, and users + # report the silent brief as "never delivered"). Set to false to + # restore silent (no-push) cron deliveries. + "notify": True, + }, # Make cron deliveries CONTINUABLE: a user can reply to a cron brief # and the agent has it in context (no "what is Task #2?" amnesia). # Default False preserves the historical isolation guarantee (cron @@ -3090,10 +3121,12 @@ DEFAULT_CONFIG = { "model_catalog": { "enabled": True, "url": "https://hermes-agent.nousresearch.com/docs/api/model-catalog.json", - # Disk cache TTL in hours. Beyond this, the CLI refetches on the - # next /model or `hermes model` invocation; network failures - # silently fall back to the stale cache. - "ttl_hours": 1, + # Disk cache TTL in minutes. The gateway refreshes the catalogs on + # this cadence in the background; the CLI refetches on the next + # /model or `hermes model` invocation once the cache is older than + # this. Network failures silently fall back to the stale cache. + # (Legacy `ttl_hours` is still honoured when set explicitly.) + "ttl_minutes": 20, # Optional per-provider override URLs for third parties that want # to self-host their own curation list using the same schema. # Example: @@ -3354,6 +3387,17 @@ DEFAULT_CONFIG = { # adapter. ``0`` disables the cap. Default 128 MiB. "max_inbound_media_bytes": 134217728, + # Whether gateway platform adapters let aiohttp read proxy settings + # (HTTP_PROXY / HTTPS_PROXY / NO_PROXY, plus SSL_CERT_FILE) from the + # process environment, and whether generic proxy env / the macOS + # system proxy are auto-detected for adapter clients. Set to false + # when the gateway inherits a proxy it must not use — e.g. a Windows + # Scheduled Task picking up a Clash/V2Ray HTTP_PROXY the interactive + # shell never sees, producing "Cannot connect to host 127.0.0.1:7890" + # poll loops (#48820). Explicit per-platform vars (DISCORD_PROXY, + # TELEGRAM_PROXY, ...) are still honored. One knob for every adapter. + "trust_env": True, + # When false (default), any file path the agent emits is delivered # as a native attachment as long as it isn't under the credential / # system-path denylist (/etc, /proc, ~/.ssh, ~/.aws, ~/.hermes/.env, @@ -3450,13 +3494,18 @@ DEFAULT_CONFIG = { # reports 384MB+ databases with 68K+ messages, which slows down FTS5 # inserts, /resume listing, and insights queries. "sessions": { - # When true, prune ended sessions inactive for retention_days once + # When true, prune ENDED sessions inactive for retention_days once # per (roughly) min_interval_hours at CLI/gateway/cron startup. # Activity is the latest message timestamp, falling back to creation - # time for empty sessions. Active sessions are always preserved. - # Default false: session history is valuable for search recall, and - # silently deleting it could surprise users. Opt in explicitly. - "auto_prune": False, + # time for empty sessions. Sessions that are still open, pinned, or + # mid-turn are never deleted — the only open rows the sweep touches + # are stale automation sessions (cron/kanban/subagent/one-shot CLI) + # whose process died without closing them; those are *closed*, not + # deleted, and get a further full retention window before removal. + # Default true since #54189: without it state.db grows without bound + # (multi-GB installs reported within weeks). Set false to keep every + # ended session forever. + "auto_prune": True, # How many inactive days of ended-session history to keep. Matches # the default of ``hermes sessions prune``. "retention_days": 90, @@ -3474,7 +3523,9 @@ DEFAULT_CONFIG = { # subsequent INSERTs — so without VACUUM the file stays bloated # even after pruning. VACUUM blocks writes for a few seconds per # 100MB, so it only runs at startup, and only when prune deleted - # ≥1 session. + # ≥1 session AND the reclaimable fraction of the file + # (PRAGMA freelist_count / page_count) exceeds 25% — a dense DB + # never pays for a full rewrite to reclaim a few MB (#54189). "vacuum_after_prune": True, # Minimum days between successful VACUUM rewrites. Pruning can still # run on its normal cadence while SQLite reuses the freed pages. @@ -3855,6 +3906,7 @@ DEFAULT_CONFIG = { # every invocation (MCP backend, status, doctor, install). Set true # to let cua-driver use its own default (telemetry on). "cua_telemetry": False, + "native_wayland": False, # Cap driver screenshot longest edge (pixels) via set_config on # session start. Shrinks SOM multimodal payloads; 0 disables. "max_image_dimension": 1456, @@ -4026,6 +4078,15 @@ DEFAULT_CONFIG = { # settings are non-secret routing config and live here. Both are bridged to # the VERTEX_PROJECT_ID / VERTEX_REGION env vars the adapter reads, so an # explicit env var still wins over config.yaml. + "nous": { + # Upper bound on the Nous auth keepalive tick, in seconds. The tick + # actually used derives from the credential lifetime the server issued + # and is capped by this value, so lowering it makes the keepalive more + # frequent while raising it has no effect below the derived tick. + # 0 disables the keepalive thread entirely. + "keepalive_interval_seconds": 900, + }, + "vertex": { # GCP project ID. Empty → use the project_id embedded in the service # account JSON (or ADC-resolved project). @@ -4060,7 +4121,7 @@ DEFAULT_CONFIG = { }, # Config schema version - bump this when adding new required fields - "_config_version": 39, + "_config_version": 40, } # Optional environment variables that enhance functionality diff --git a/hermes_cli/config_migrations.py b/hermes_cli/config_migrations.py index 9c19f0c6d9..02fe33bc95 100644 --- a/hermes_cli/config_migrations.py +++ b/hermes_cli/config_migrations.py @@ -863,6 +863,28 @@ def _migrate_to_39(results: Dict[str, Any], quiet: bool) -> None: ) +def _migrate_to_40(results: Dict[str, Any], quiet: bool) -> None: + # ── Version 39 → 40: model_catalog.ttl_hours → ttl_minutes (default 20) ── + # The picker catalogs now refresh every 20 minutes (and the gateway + # refreshes them in the background on that cadence). Only the OLD default + # (ttl_hours: 1, written by the v25 migration) is dropped so the new + # default applies; any other explicit ttl_hours is a deliberate choice + # and stays honoured by the loader. + _c = _cfg() + read_raw_config = _c.read_raw_config + _persist_migration = _c._persist_migration + + config = read_raw_config() + raw_mc = config.get("model_catalog") + if isinstance(raw_mc, dict) and raw_mc.get("ttl_hours") == 1 and "ttl_minutes" not in raw_mc: + del raw_mc["ttl_hours"] + config["model_catalog"] = raw_mc + _persist_migration(config) + results["config_added"].append("model_catalog.ttl_hours 1 → ttl_minutes 20 (default)") + if not quiet: + print(" ✓ Model catalog now refreshes every 20 minutes (model_catalog.ttl_minutes)") + + #: Registry of (target_version, migration_fn), strictly ascending. The driver #: applies every entry whose target version is greater than the on-disk #: observe earlier steps' writes via read_raw_config() (filesystem state). @@ -890,6 +912,7 @@ MIGRATIONS: Tuple[Tuple[int, Callable[[Dict[str, Any], bool], None]], ...] = ( (37, _migrate_to_37), (38, _migrate_to_38), (39, _migrate_to_39), + (40, _migrate_to_40), ) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 560eb585b1..9608d3bc29 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -136,11 +136,23 @@ def reconcile_profile_gateways( # for every profile. Named slots must still be registered (so explicit # lifecycle management remains available), but booting them from their # persisted run intent would create additional multiplex owners. + # Keep the boot reconciler aligned with the gateway that will own these + # slots. The runtime resolver gives a recognized environment override + # precedence over config.yaml and otherwise preserves the configured value. + from gateway.config import load_gateway_config from utils import is_truthy_value - multiplex_profiles = is_truthy_value( - os.environ.get("GATEWAY_MULTIPLEX_PROFILES"), - ) + try: + multiplex_profiles = load_gateway_config().multiplex_profiles + except Exception: + log.warning( + "Unable to load gateway configuration during container boot; " + "using the GATEWAY_MULTIPLEX_PROFILES override if set.", + exc_info=True, + ) + multiplex_profiles = is_truthy_value( + os.environ.get("GATEWAY_MULTIPLEX_PROFILES"), + ) # Default profile — always register, even if nothing has ever # populated the root profile dir. The slot exists so diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index 4903822049..40534e9caf 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -23,6 +23,7 @@ import logging import os import shutil import subprocess +import threading import time from pathlib import Path from typing import Optional @@ -380,6 +381,20 @@ _JWT_DISK_MAX_BYTES = 1_048_576 # 1 MiB cap on the persisted JWT store read # Maps raw-token fingerprint -> epoch until which exchange attempts are # skipped (raise immediately). Success clears the entry. _exchange_failure_cache: dict[str, float] = {} +# Single-flight guard per token fingerprint: concurrent callers (the dashboard +# polls /api/credentials/pool every few seconds, each poll off-loop) wait on +# the ONE in-flight exchange and then hit the positive/negative cache, instead +# of each spawning their own hung resolver thread during a DNS outage. +_exchange_locks: dict[str, threading.Lock] = {} +_exchange_locks_guard = threading.Lock() + + +def _exchange_lock_for(fp: str) -> threading.Lock: + with _exchange_locks_guard: + lock = _exchange_locks.get(fp) + if lock is None: + lock = _exchange_locks[fp] = threading.Lock() + return lock _EXCHANGE_FAILURE_TTL_TRANSIENT_SECONDS = 60.0 # network blips: retry soon _EXCHANGE_FAILURE_TTL_PERMANENT_SECONDS = 1800.0 # 401/403/404: won't heal # HTTP statuses that indicate the token itself is rejected — retrying with @@ -520,6 +535,60 @@ def _save_jwt_to_disk( logger.debug("Failed to persist Copilot JWT: %s", exc) +# Hard wall-clock cap for the token-exchange HTTP call. urllib's ``timeout`` +# only bounds socket operations AFTER DNS resolution succeeds; getaddrinfo +# blocks in C and ignores it entirely, so on a networkless Windows host the +# resolver can hang for many minutes (observed: a 17-minute event-loop stall +# on 2026-08-22 that took the whole backend down with it). +_DNS_GRACE_SECONDS = 5.0 + + +def _urlopen_bounded(req, timeout: float): + """urlopen() with a hard wall-clock cap of timeout + _DNS_GRACE_SECONDS. + + Runs the call on a daemon thread and abandons it if the cap fires, so a + DNS/getaddrinfo hang cannot block the caller indefinitely. Raises the + worker's exception, or TimeoutError when the cap fires. + """ + import urllib.request + + box: dict = {} + abandoned = threading.Event() + + def _worker() -> None: + try: + resp = urllib.request.urlopen(req, timeout=timeout) + except BaseException as exc: # re-raised on the caller's thread + box["exc"] = exc + return + if abandoned.is_set(): + # The caller already timed out; nobody will read this response, + # so release its socket instead of leaking it with the thread. + try: + resp.close() + except Exception: + pass + return + box["resp"] = resp + + t = threading.Thread( + target=_worker, name="copilot-token-exchange", daemon=True + ) + t.start() + t.join(timeout + _DNS_GRACE_SECONDS) + if t.is_alive(): + abandoned.set() + raise TimeoutError( + "copilot token exchange exceeded hard cap of " + f"{timeout + _DNS_GRACE_SECONDS:.0f}s (DNS/getaddrinfo hang?)" + ) + if "exc" in box: + raise box["exc"] + if "resp" not in box: + raise TimeoutError("copilot token exchange worker died without result") + return box["resp"] + + def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[str, float, Optional[str]]: """Exchange a raw GitHub token for a short-lived Copilot API token. @@ -536,11 +605,34 @@ def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[st Results are cached in-process and reused until close to expiry. Raises ``ValueError`` on failure. """ - import urllib.request - fp = _token_fingerprint(raw_token) - # Check in-process cache first + # Fast paths outside the lock: a valid in-process JWT needs no exchange, + # and a recent failure means queueing behind the in-flight holder (up to + # ~50 s) would only park an executor thread to learn the same answer. + cached = _jwt_cache.get(fp) + if cached and time.time() < cached[1] - _JWT_REFRESH_MARGIN_SECONDS: + return cached + _fail_until = _exchange_failure_cache.get(fp, 0.0) + if time.time() < _fail_until: + raise ValueError( + "Copilot token exchange recently failed; skipping re-attempt " + f"for another {int(_fail_until - time.time())}s" + ) + + # Note: a waiter's own ``timeout`` is not honoured across the lock wait — + # by design of single-flight, it observes the holder's outcome instead. + with _exchange_lock_for(fp): + return _exchange_copilot_token_locked(raw_token, fp, timeout=timeout) + + +def _exchange_copilot_token_locked( + raw_token: str, fp: str, *, timeout: float +) -> tuple[str, float, Optional[str]]: + import urllib.request + + # Re-check the caches under the lock: a concurrent caller may have just + # completed (or just failed) the exchange we were queued behind. cached = _jwt_cache.get(fp) if cached: api_token, expires_at, base_url = cached @@ -593,7 +685,7 @@ def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[st permanent_failure = False for attempt in range(_EXCHANGE_MAX_ATTEMPTS): try: - with urllib.request.urlopen(req, timeout=timeout) as resp: + with _urlopen_bounded(req, timeout) as resp: data = json.loads(resp.read().decode()) break except Exception as exc: # noqa: BLE001 — retry all, re-raise below diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 8db53a707b..cc19e94c6b 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -264,6 +264,12 @@ def cron_list(show_all: bool = False): last_run = job.get("last_run_at", "?") if last_status == "ok": status_display = color("ok", Colors.GREEN) + elif last_status == "delivery_failed": + # The agent succeeded but the result never reached the user — + # not green, and the detail lives in last_delivery_error + # (last_error is None for these runs). + detail = job.get("last_delivery_error") or "?" + status_display = color(f"delivery_failed: {detail}", Colors.YELLOW) else: status_display = color(f"{last_status}: {job.get('last_error', '?')}", Colors.RED) streak = int(job.get("failure_streak") or 0) @@ -286,6 +292,17 @@ def cron_list(show_all: bool = False): if delivery_err: print(f" {color('⚠ Delivery failed:', Colors.YELLOW)} {delivery_err}") + # A live adapter acked the last send but returned no message_id / + # raw_response (Slack/Matrix/Mattermost shape): accepted as delivered, + # but say so here rather than only in a WARNING log line. + unverified = job.get("last_delivery_unverified") + if unverified: + targets = ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) + print( + f" {color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} " + f"adapter acked {targets} without message_id/raw_response" + ) + fire_err = job.get("last_fire_error") if isinstance(fire_err, dict) and fire_err.get("detail"): print( @@ -688,7 +705,10 @@ def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: issues: List[str] = [] last_status = str(job.get("last_status") or "").strip().lower() - if last_status and last_status != "ok": + # "delivery_failed" means the agent run itself succeeded, so it is not a + # failed last run — the dedicated delivery issue below reports it (and + # last_error is None, which would render as "unknown error" here). + if last_status and last_status not in {"ok", "delivery_failed"}: err = str(job.get("last_error") or "unknown error").strip() issues.append(f"last run failed: {err}") @@ -696,6 +716,11 @@ def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: if delivery_err: issues.append(f"last delivery failed: {delivery_err}") + unverified = job.get("last_delivery_unverified") + if unverified: + targets = ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) + issues.append(f"last delivery unverified (adapter acked without evidence): {targets}") + if job.get("enabled", True) and job.get("state") not in {"paused", "completed"}: next_run = str(job.get("next_run_at") or "").strip() if not next_run: @@ -766,6 +791,7 @@ def cron_create(args): prompt=args.prompt, name=getattr(args, "name", None), deliver=getattr(args, "deliver", None), + failure_deliver=getattr(args, "failure_deliver", None), repeat=getattr(args, "repeat", None), skill=getattr(args, "skill", None), skills=_normalize_skills(getattr(args, "skill", None), getattr(args, "skills", None)), @@ -842,6 +868,7 @@ def cron_edit(args): prompt=getattr(args, "prompt", None), name=getattr(args, "name", None), deliver=getattr(args, "deliver", None), + failure_deliver=getattr(args, "failure_deliver", None), repeat=getattr(args, "repeat", None), skills=final_skills, script=getattr(args, "script", None), diff --git a/hermes_cli/dashboard_auth/__init__.py b/hermes_cli/dashboard_auth/__init__.py index c07b2ade6f..9a997fd224 100644 --- a/hermes_cli/dashboard_auth/__init__.py +++ b/hermes_cli/dashboard_auth/__init__.py @@ -19,6 +19,7 @@ from hermes_cli.dashboard_auth.base import ( ProviderError, RefreshExpiredError, assert_protocol_compliance, + classify_jwks_lookup_error, ) from hermes_cli.dashboard_auth.registry import ( register_provider, @@ -39,6 +40,7 @@ __all__ = [ "ProviderError", "RefreshExpiredError", "assert_protocol_compliance", + "classify_jwks_lookup_error", "register_provider", "get_provider", "list_providers", diff --git a/hermes_cli/dashboard_auth/base.py b/hermes_cli/dashboard_auth/base.py index 2d744c6cf3..02db55f65e 100644 --- a/hermes_cli/dashboard_auth/base.py +++ b/hermes_cli/dashboard_auth/base.py @@ -110,6 +110,45 @@ class RefreshExpiredError(Exception): """ +def classify_jwks_lookup_error(exc: BaseException) -> Exception: + """Map a ``PyJWKClient.get_signing_key_from_jwt`` failure to the protocol. + + Only a genuine transport failure (the IDP's JWKS endpoint could not be + fetched) is a :class:`ProviderError` — middleware turns that into 503 + "auth provider unreachable" so a flaky IDP never forces a logout. + + Everything else means the token itself cannot be verified by this + provider and is an :class:`InvalidCodeError` (``verify_session`` returns + ``None``, the middleware tries the next provider / refresh / 401): + + * ``jwt.DecodeError`` — the bearer is not a JWT at all (an opaque peer + key, a legacy session token, garbage). #94558: hosted agents answered + every non-JWT bearer with a fast 503 ``Auth provider 'nous' + unreachable`` even though Portal was healthy, because "cannot parse" + and "cannot reach" were folded into one branch. + * ``jwt.PyJWKSetError`` — the JWKS was fetched fine but holds no key for + this token's ``kid`` (rotated/foreign key). The provider was reached; + the token is simply not one of ours. + + ``PyJWKClientConnectionError`` is the only ``PyJWKClientError`` subclass + that denotes unreachability; a bare ``PyJWKClientError`` (unexpected + JWKS shape) is kept as a provider fault since the IDP misbehaved. + """ + try: + import jwt + except Exception: # pragma: no cover - jwt is a hard dep of these providers + return ProviderError(f"JWKS lookup failed: {exc!r}") + if isinstance(exc, jwt.PyJWKClientConnectionError): + return ProviderError(f"JWKS lookup failed: {exc}") + if isinstance(exc, (jwt.DecodeError, jwt.PyJWKSetError)): + return InvalidCodeError(f"token not verifiable by this provider: {exc}") + if isinstance(exc, jwt.PyJWKClientError): + return ProviderError(f"JWKS lookup failed: {exc}") + if isinstance(exc, jwt.InvalidTokenError): + return InvalidCodeError(f"token not verifiable by this provider: {exc}") + return ProviderError(f"JWKS lookup failed: {exc!r}") + + class DashboardAuthProvider(ABC): """Protocol every dashboard-auth provider plugin implements. diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 015465d083..4c59886375 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -35,6 +35,7 @@ from hermes_cli.colors import Colors, color from hermes_cli.models import _HERMES_USER_AGENT from hermes_cli.vercel_auth import describe_vercel_auth from hermes_constants import OPENROUTER_MODELS_URL +from hermes_state_common import FTS_STORAGE_VERSION from utils import base_url_host_matches @@ -429,6 +430,28 @@ def check_info(text: str): print(f" {color('→', Colors.CYAN)} {text}") +def _doctor_memory_config(hermes_home: Path | None = None) -> dict: + """Return the effective memory section used by doctor diagnostics.""" + home = hermes_home if hermes_home is not None else HERMES_HOME + try: + from hermes_cli.config import _expand_env_vars, read_user_config_raw + + config_path = home / "config.yaml" + if not config_path.exists(): + return {} + config = _expand_env_vars(read_user_config_raw(config_path)) + try: + from hermes_cli import managed_scope + + config = managed_scope.apply_managed_overlay(config) + except Exception: + pass + section = config.get("memory") if isinstance(config, dict) else None + return section if isinstance(section, dict) else {} + except Exception: + return {} + + # ── state.db health/stats thresholds (advisory only — module constants, # deliberately NOT config: doctor warnings are guidance, not policy) ── STATE_DB_SIZE_WARN_BYTES = 1 * 1024 * 1024 * 1024 # 1 GiB logical size @@ -498,21 +521,20 @@ def _render_state_db_stats(stats: dict, holders=None) -> list: "optimize-storage' with the gateway stopped)", )) - # Advisory: oversized database. Suggest auto_prune, and — when the v23 - # FTS rebuild is pending OR the DB still carries the legacy inline - # trigram layout (fts_storage_version marker absent) — the offline + # Advisory: oversized database. Suggest auto_prune, and — when the FTS + # rebuild is pending OR the DB predates the current trigram layout — the offline # optimize-storage pass that migrates/compacts the FTS indexes. if logical is not None and logical > STATE_DB_SIZE_WARN_BYTES: detail = ( "consider enabling sessions.auto_prune in config.yaml " "to bound growth" ) - legacy_trigram = ( + stale_trigram = ( fts is not None and fts.get("messages_fts_trigram") - and stats.get("fts_storage_version") is None + and (stats.get("fts_storage_version") or 0) < FTS_STORAGE_VERSION ) - if stats.get("fts_rebuild_pending") or legacy_trigram: + if stats.get("fts_rebuild_pending") or stale_trigram: detail += ( "; run 'hermes sessions optimize-storage' offline " "(with the gateway stopped) to compact FTS storage" @@ -1980,8 +2002,19 @@ def run_doctor(args): else: check_warn(f"{_DHH} not found", "(will be created on first use)") - # Check expected subdirectories - expected_subdirs = ["cron", "sessions", "logs", "skills", "memories"] + from tools.memory_tool import get_builtin_memory_store_flags + + _memory_config = _doctor_memory_config(hermes_home) + _memory_enabled, _user_profile_enabled = get_builtin_memory_store_flags( + {"memory": _memory_config} + ) + + # Check expected subdirectories. The built-in file store does not create or + # consume memories/ when both targets are disabled, so stale migration files + # are not an active diagnostic surface. + expected_subdirs = ["cron", "sessions", "logs", "skills"] + if _memory_enabled or _user_profile_enabled: + expected_subdirs.append("memories") for subdir_name in expected_subdirs: subdir_path = hermes_home / subdir_name if subdir_path.exists(): @@ -2016,22 +2049,28 @@ def run_doctor(args): check_ok(f"Created {_DHH}/SOUL.md with basic template") fixed_count += 1 - # Check memory directory + # Check only enabled built-in stores. External providers are additive, but + # users can explicitly disable either legacy file target; stale files left + # by a migration must not be presented as active memory usage. memories_dir = hermes_home / "memories" - if memories_dir.exists(): + if not (_memory_enabled or _user_profile_enabled): + check_info("Built-in memory files disabled by config") + elif memories_dir.exists(): check_ok(f"{_DHH}/memories/ directory exists") memory_file = memories_dir / "MEMORY.md" user_file = memories_dir / "USER.md" - if memory_file.exists(): - size = len(memory_file.read_text(encoding="utf-8-sig").strip()) - check_ok(f"MEMORY.md exists ({size} chars)") - else: - check_info("MEMORY.md not created yet (will be created when the agent first writes a memory)") - if user_file.exists(): - size = len(user_file.read_text(encoding="utf-8-sig").strip()) - check_ok(f"USER.md exists ({size} chars)") - else: - check_info("USER.md not created yet (will be created when the agent first writes a memory)") + if _memory_enabled: + if memory_file.exists(): + size = len(memory_file.read_text(encoding="utf-8-sig").strip()) + check_ok(f"MEMORY.md exists ({size} chars)") + else: + check_info("MEMORY.md not created yet (will be created when the agent first writes a memory)") + if _user_profile_enabled: + if user_file.exists(): + size = len(user_file.read_text(encoding="utf-8-sig").strip()) + check_ok(f"USER.md exists ({size} chars)") + else: + check_info("USER.md not created yet (will be created when the agent first writes a memory)") else: check_warn(f"{_DHH}/memories/ not found", "(will be created on first use)") if should_fix: @@ -3291,21 +3330,7 @@ def run_doctor(args): check_warn("No GITHUB_TOKEN", f"(60 req/hr rate limit — set in {_DHH}/.env for better rates)") _section("Memory Provider") - _active_memory_provider = "" - try: - from hermes_cli.config import read_user_config_raw as _read_raw_mem - _mem_cfg_path = HERMES_HOME / "config.yaml" - if _mem_cfg_path.exists(): - # Raw-file diagnostic (+ managed overlay below, unchanged). - _raw_cfg = _read_raw_mem(_mem_cfg_path) - try: - from hermes_cli import managed_scope - _raw_cfg = managed_scope.apply_managed_overlay(_raw_cfg) - except Exception: - pass - _active_memory_provider = (_raw_cfg.get("memory") or {}).get("provider", "") - except Exception: - pass + _active_memory_provider = _memory_config.get("provider", "") if not _active_memory_provider: check_ok("Built-in memory active", "(no external provider configured — this is fine)") diff --git a/hermes_cli/env_loader.py b/hermes_cli/env_loader.py index 434641b834..1976fcec80 100644 --- a/hermes_cli/env_loader.py +++ b/hermes_cli/env_loader.py @@ -51,6 +51,10 @@ _SECRET_SOURCE_VALUES_BY_HOME: dict[str, dict[str, str]] = {} _APPLIED_HOMES: set[str] = set() _SECRET_SOURCE_CACHE_LOCK = threading.RLock() +# Routed profile homes whose dotenv load was skipped under multiplex, so the +# skip is logged once per home rather than on every lazy import mid-turn. +_SCOPED_SKIP_LOGGED: set[str] = set() + def _known_hermes_env_keys() -> set[str]: """Return the combined set of known Hermes env-var keys. @@ -483,10 +487,42 @@ def load_hermes_dotenv( - callers that only maintain the installation can set ``load_external_secrets=False`` to avoid loading optional secret-manager dependencies into the process that replaces that same environment. + - routed multiplex profile loads hydrate external sources into the + profile's private secret snapshot without mutating the shared process + environment; unscoped startup loads retain the normal behavior above. """ - loaded: list[Path] = [] - home_path = Path(hermes_home or os.getenv("HERMES_HOME", Path.home() / ".hermes")) + + # A multiplex gateway hosts every profile in one process. While a routed + # profile-home override is active, copying that profile's .env into + # os.environ would expose its credentials to sibling turns and every + # subsequently spawned child. An unscoped startup load remains process + # configuration and must retain the normal loading path. + # External secret sources still need their normal refresh path, so resolve + # them against the existing profile-local mapping instead of simply + # returning before all hydration work. + from agent.secret_scope import is_multiplex_active + from hermes_constants import get_hermes_home_override + + if is_multiplex_active() and get_hermes_home_override() is not None: + home_key = str(home_path.resolve()) + if home_key not in _SCOPED_SKIP_LOGGED: + _SCOPED_SKIP_LOGGED.add(home_key) + import logging + + logging.getLogger(__name__).debug( + "multiplex: skipping process-global dotenv load for routed " + "profile home %s (credentials resolve via the profile scope)", + home_path, + ) + if load_external_secrets: + from hermes_cli import _early_recovery + + if not _early_recovery._should_skip_external_secret_sources(): + hydrate_profile_secret_sources(home_path) + return [] + + loaded: list[Path] = [] user_env = home_path / ".env" project_env_path = Path(project_env) if project_env else None diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 28fe339b47..4a4f9776ce 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -481,6 +481,46 @@ def _probe_loop_tick_socket( pass +def _probe_loop_tick_tcp( + port: int, + timeout: float = 1.0, +) -> bool | None: + """Ping the loop-scheduling witness via TCP loopback (Windows). + + Same protocol and semantics as the Unix socket variant: connect to + 127.0.0.1: and expect one byte "1" as proof the loop is + dispatching. Used on Windows / non-POSIX systems where AF_UNIX is not + available in asyncio. + + Returns: + True — the loop answered. + False — the port was reachable but did not answer, or refused. + None — invalid port / could not connect for unrelated reasons. + """ + try: + port_num = int(port) + if port_num <= 0 or port_num > 65535: + return None + except (TypeError, ValueError): + return None + sock = None + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(max(float(timeout), 0.0)) + sock.connect(("127.0.0.1", port_num)) + return sock.recv(1) == b"1" + except Exception: + # Connection refused, timeout, transient errors: witness exists + # but is silent (or the process is dead and the port is closed). + return False + finally: + if sock is not None: + try: + sock.close() + except Exception: + pass + + def _probe_loop_tick_socket_sustained( pid: int, home: Path | None, @@ -488,6 +528,7 @@ def _probe_loop_tick_socket_sustained( timeout: float = 1.0, strikes: int = 3, gap_s: float = 0.2, + tcp_port: int | None = None, ) -> bool | None: """Probe the tick socket until a reply or the sustained-miss budget. @@ -509,7 +550,10 @@ def _probe_loop_tick_socket_sustained( """ total = max(int(strikes), 0) for attempt in range(total): - result = _probe_loop_tick_socket(pid, home, timeout=timeout) + if tcp_port is not None: + result = _probe_loop_tick_tcp(tcp_port, timeout=timeout) + else: + result = _probe_loop_tick_socket(pid, home, timeout=timeout) if result is True: return True if result is None: @@ -579,14 +623,26 @@ def probe_gateway_loop_liveness( # up, or a stale file from a previous PID. Not evidence of a wedge. return GATEWAY_LOOP_UNKNOWN - witness = _probe_loop_tick_socket(pid, home, timeout=tick_timeout) + # Pick the right witness probe: TCP loopback (Windows / non-POSIX) + # takes priority if the producer published a port, otherwise fall back + # to the AF_UNIX socket (POSIX / legacy). + tcp_port = payload.get("loop_tick_tcp_port") + try: + tcp_port_int = int(tcp_port) if tcp_port is not None else None + except (TypeError, ValueError): + tcp_port_int = None + + if tcp_port_int is not None and tcp_port_int > 0: + witness = _probe_loop_tick_tcp(tcp_port_int, timeout=tick_timeout) + tick_armed = True + else: + witness = _probe_loop_tick_socket(pid, home, timeout=tick_timeout) + tick_armed = payload.get("loop_tick_socket", _LOOP_TICK_ABSENT) if witness is True: # The loop answered a ping — it is dispatching right now. A stale # heartbeat file is a stalled write or a saturated executor, not a # wedge (#90502). return GATEWAY_LOOP_ALIVE - - tick_armed = payload.get("loop_tick_socket", _LOOP_TICK_ABSENT) age = time.time() - mtime if age <= stale_budget: if witness is False: @@ -620,6 +676,7 @@ def probe_gateway_loop_liveness( timeout=tick_timeout, strikes=tick_strikes - 1, gap_s=tick_gap_s, + tcp_port=tcp_port_int, ) if sustained is False: # Both witnesses agree, sustained: the loop did not schedule for @@ -2199,14 +2256,17 @@ def _gateway_list() -> None: label += " (current)" parts = [f" {marker} {label:<24s}"] if prof.gateway_running: + pid = None try: from gateway.status import get_running_pid pid = get_running_pid(prof.path / "gateway.pid", cleanup_stale=False) - if pid: - parts.append(f"PID {pid}") except Exception: pass + if pid: + parts.append(f"PID {pid}") + elif named_profile_served_by_running_multiplexer(prof.name): + parts.append("served by the default multiplexer") else: parts.append("not running") print(" — ".join(parts)) @@ -6423,18 +6483,20 @@ def _running_under_gateway_supervisor() -> bool: return is_gateway_supervisor_process() -def named_profile_served_by_running_multiplexer() -> bool: +def named_profile_served_by_running_multiplexer(profile_name: str | None = None) -> bool: """True when a live default multiplexer already ticks this named profile. - Shared by the named-profile start guard and cron liveness: a satellite - profile has no gateway.pid of its own, but the default multiplexer's - ticker still fires its jobs (#97120). + Shared by the named-profile start guard, cron liveness, and the + ``gateway status`` / ``gateway list`` / ``profile list`` reports: a + satellite profile has no gateway.pid of its own, but the default + multiplexer's ticker still fires its jobs (#97120) and serves its + platforms. ``profile_name`` defaults to the current HERMES_HOME profile. """ try: - suffix = _profile_suffix() + suffix = profile_name if profile_name is not None else _profile_suffix() except Exception: return False - if not suffix: + if not suffix or suffix == "default": return False try: @@ -9208,7 +9270,12 @@ def _gateway_command_inner(args): from hermes_cli import gateway_windows _windows_service_installed = gateway_windows.is_installed() - if supports_systemd_services() and ( + if not snapshot.running and named_profile_served_by_running_multiplexer(): + # Satellite profile: no gateway.pid / service of its own, but the + # default multiplexer is the live inbound process for it. + print("✓ Gateway is running via the default-profile multiplexer") + print(" Manage it from the default profile: hermes gateway status") + elif supports_systemd_services() and ( get_systemd_unit_path(system=False).exists() or get_systemd_unit_path(system=True).exists() ): diff --git a/hermes_cli/goals.py b/hermes_cli/goals.py index df1b86df44..d159adbb65 100644 --- a/hermes_cli/goals.py +++ b/hermes_cli/goals.py @@ -42,6 +42,8 @@ from dataclasses import dataclass, field, asdict from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple +from hermes_cli._subprocess_compat import noninteractive_git_env + logger = logging.getLogger(__name__) @@ -153,12 +155,22 @@ JUDGE_SYSTEM_PROMPT = ( "You are a strict judge evaluating whether an autonomous agent has " "achieved a user's stated goal. You receive the goal text, the agent's " "most recent response, and — when present — a list of background " - "processes the agent has running. Decide one of three verdicts.\n\n" + "processes the agent has running. Decide one of four verdicts.\n\n" "DONE — the goal is fully satisfied:\n" "- The response explicitly confirms the goal was completed, OR\n" - "- The response clearly shows the final deliverable was produced, OR\n" - "- The response explains the goal is unachievable / blocked / needs " - "user input (treat this as DONE with reason describing the block).\n\n" + "- The response clearly shows the final deliverable was produced.\n" + "DONE requires the deliverable to actually exist. If the response only " + "explains why the goal cannot be reached, the verdict is BLOCKED, not " + "DONE.\n\n" + "BLOCKED — the goal cannot be satisfied as stated:\n" + "- The response explains the goal is genuinely unachievable (impossible, " + "out of scope, no valid path to the deliverable), or refuses to " + "fabricate a deliverable that cannot exist, OR\n" + "- The response explains progress is blocked and the next step needs " + "user input to proceed.\n" + "Return BLOCKED with the reason describing what is blocking. BLOCKED is " + "a refusal, not a completion — never return BLOCKED for a goal that " + "was achieved.\n\n" "WAIT — the goal is NOT done, but the next step is to wait for async " "work to finish rather than act again. Choose this ONLY when the agent's " "progress is genuinely gated on something running on its own:\n" @@ -180,6 +192,7 @@ JUDGE_SYSTEM_PROMPT = ( "take right now. This is the default when in doubt.\n\n" "Reply ONLY with a single JSON object on one line. Shapes:\n" '{"verdict": "done", "reason": ""}\n' + '{"verdict": "blocked", "reason": ""}\n' '{"verdict": "continue", "reason": ""}\n' '{"verdict": "wait", "wait_on_session": "", "reason": ""}\n' '{"verdict": "wait", "wait_on_pid": , "reason": ""}\n' @@ -203,7 +216,7 @@ JUDGE_USER_PROMPT_TEMPLATE = ( "Agent's most recent response:\n{response}\n\n" "{background_block}" "Current time: {current_time}\n\n" - "Is the goal satisfied — done, continue, or wait?" + "Is the goal satisfied — done, blocked, continue, or wait?" ) # Used when the user has added /subgoal criteria. The judge must @@ -247,11 +260,11 @@ JUDGE_USER_PROMPT_WITH_CONTRACT_TEMPLATE = ( "process to satisfy the Verification criterion (e.g. CI is the " "verification and it's still running), return WAIT on that process " "instead of re-poking — re-poking now would be pure busy-work.\n" - "- If the response explains the work is blocked / unachievable / needs " - "user input (e.g. the stated Stop condition was hit), treat it as DONE " - "with the reason describing the block.\n" + "- If the response explains the work is genuinely unachievable or hits " + "the stated Stop condition and needs user input, the goal is NOT done — " + "return BLOCKED with the reason describing the block.\n" "- Otherwise the goal is NOT done — CONTINUE.\n\n" - "Is the goal satisfied per its completion contract — done, continue, or wait?" + "Is the goal satisfied per its completion contract — done, blocked, continue, or wait?" ) @@ -483,6 +496,7 @@ def workspace_fingerprint(cwd: Optional[str] = None) -> str: ["git", "rev-parse", "HEAD"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, cwd=workdir, + stdin=subprocess.DEVNULL, env=noninteractive_git_env(), ) if head.returncode != 0: return "" @@ -490,6 +504,7 @@ def workspace_fingerprint(cwd: Optional[str] = None) -> str: ["git", "status", "--porcelain"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, cwd=workdir, + stdin=subprocess.DEVNULL, env=noninteractive_git_env(), ) if status.returncode != 0: return "" @@ -553,7 +568,7 @@ class GoalState: max_turns: int = DEFAULT_MAX_TURNS created_at: float = 0.0 last_turn_at: float = 0.0 - last_verdict: Optional[str] = None # "done" | "continue" | "skipped" + last_verdict: Optional[str] = None # "done" | "blocked" | "continue" | "wait" | "skipped" last_reason: Optional[str] = None paused_reason: Optional[str] = None # why we auto-paused (budget, etc.) consecutive_parse_failures: int = 0 # judge-output parse failures in a row @@ -1027,7 +1042,7 @@ def _parse_judge_response(raw: str) -> Tuple[str, str, bool, Optional[Dict[str, """Parse the judge's reply. Fail-open on unusable output. Returns ``(verdict, reason, parse_failed, wait_directive)`` where: - - ``verdict`` is ``"done"``, ``"continue"``, or ``"wait"``. + - ``verdict`` is ``"done"``, ``"blocked"``, ``"continue"``, or ``"wait"``. - ``parse_failed`` is True when the judge returned output that couldn't be interpreted as the expected JSON verdict (empty body, prose, malformed JSON). Callers use it to auto-pause after N consecutive @@ -1084,7 +1099,7 @@ def _parse_judge_response(raw: str) -> Tuple[str, str, bool, Optional[Dict[str, done = bool(done_val) verdict = "done" if done else "continue" - if verdict not in {"done", "continue", "wait"}: + if verdict not in {"done", "blocked", "continue", "wait"}: verdict = "continue" if verdict != "wait": @@ -1178,7 +1193,7 @@ def judge_goal( """Ask the auxiliary model whether the goal is satisfied. Returns ``(verdict, reason, parse_failed, wait_directive, transport_failed)`` where verdict - is ``"done"``, ``"continue"``, ``"wait"``, or ``"skipped"`` (when the + is ``"done"``, ``"blocked"``, ``"continue"``, ``"wait"``, or ``"skipped"`` (when the judge couldn't be reached). ``wait_directive`` is set only for ``"wait"`` (``{"pid": int}`` or ``{"seconds": int}``); ``None`` otherwise. @@ -1882,7 +1897,7 @@ class GoalManager: - ``status``: current goal status after update - ``should_continue``: bool — caller should fire another turn - ``continuation_prompt``: str or None - - ``verdict``: "done" | "continue" | "wait" | "skipped" | "inactive" + - ``verdict``: "done" | "blocked" | "continue" | "wait" | "skipped" | "inactive" - ``reason``: str - ``message``: user-visible one-liner to print/send """ @@ -1999,6 +2014,28 @@ class GoalManager: "message": f"⏳ Goal parked (judge) — waiting on {tgt}: {reason}", } + # BLOCKED verdict: the judge ruled the goal genuinely cannot be + # satisfied as stated (impossible, out of scope, needs user input). + # This is NOT done — don't keep burning turns on an unachievable goal + # and don't wave it through as complete (#100954). Pause so the user + # sees the judge's reason and can re-scope (/goal set) or override + # (/goal resume). + if verdict == "blocked": + state.status = "paused" + state.paused_reason = f"judged unachievable: {reason}" + save_goal(self.session_id, state) + return { + "status": "paused", + "should_continue": False, + "continuation_prompt": None, + "verdict": "blocked", + "reason": reason, + "message": ( + f"🚫 Goal judged unachievable — paused: {reason} " + "Re-scope with /goal set, or override with /goal resume." + ), + } + if verdict == "done": state.status = "done" save_goal(self.session_id, state) @@ -2202,7 +2239,7 @@ def run_kanban_goal_loop( Returns a decision dict: ``{"outcome", "turns_used", "reason"}`` where outcome is one of ``"completed_by_worker"``, ``"review_requested_by_worker"``, ``"changes_requested_by_reviewer"``, ``"blocked_budget"``, - ``"blocked_by_worker"``, or ``"stopped"``. + ``"blocked_unachievable"``, ``"blocked_by_worker"``, or ``"stopped"``. """ def _log(msg: str) -> None: @@ -2258,6 +2295,22 @@ def run_kanban_goal_loop( verdict = "continue" _log(f"kanban goal loop: turn {turns_used}/{max_turns} verdict={verdict} reason={_truncate(reason, 120)}") + if verdict == "blocked": + # The judge ruled the goal cannot be satisfied at all — this is + # NOT done (#100954). Block the card now with the judge's reason + # instead of spending the remaining turns re-poking an impossible + # goal, and never let it land in done. + _log(f"kanban goal loop: task {task_id} judged unachievable; blocking") + try: + block_fn(f"Goal-mode judge ruled the goal unachievable: {reason}") + except Exception as exc: + _log(f"kanban goal loop: block_fn failed ({exc})") + return { + "outcome": "blocked_unachievable", + "turns_used": turns_used, + "reason": f"judge verdict blocked: {reason}", + } + if verdict == "done": if nudged_to_finalize: # Already asked once to call kanban_complete and it still diff --git a/hermes_cli/gui_uninstall.py b/hermes_cli/gui_uninstall.py index 358f812a30..6e54d4e9dc 100644 --- a/hermes_cli/gui_uninstall.py +++ b/hermes_cli/gui_uninstall.py @@ -156,10 +156,9 @@ def packaged_gui_app_paths() -> "list[Path]": data_base / "applications" / "Hermes.desktop", data_base / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png", ] - # Fixed-size hicolor dirs: the icon is copied at its native size - # (read from the PNG header), so sweep the standard ones plus the - # 1024x1024 dir the shipped asset lands in. - for size in ("256x256", "512x512", "1024x1024"): + # Fixed-size hicolor dirs the installer may have written (resized + # panel sizes plus leftover native-size copies from older builds). + for size in ("24x24", "32x32", "48x48", "256x256", "512x512", "1024x1024"): paths.append(data_base / "icons" / "hicolor" / size / "apps" / "hermes.png") return paths diff --git a/hermes_cli/inventory.py b/hermes_cli/inventory.py index 320bbc0310..1b3f918c90 100644 --- a/hermes_cli/inventory.py +++ b/hermes_cli/inventory.py @@ -33,10 +33,16 @@ Substrate facts (verified May 2026): from __future__ import annotations +from contextvars import copy_context from dataclasses import dataclass, replace +from threading import Lock, Thread, current_thread from typing import Any, Optional +_pricing_prewarm_lock = Lock() +_pricing_prewarm_threads: dict[tuple[str, tuple[tuple[str, str], ...]], Thread] = {} + + # ─── Public types ─────────────────────────────────────────────────────── @@ -125,6 +131,7 @@ def build_models_payload( picker_hints: bool = False, canonical_order: bool = False, pricing: bool = False, + pricing_cache_only: bool = False, capabilities: bool = False, featured: bool = False, force_fresh_nous_tier: bool = False, @@ -155,6 +162,9 @@ def build_models_payload( show $/Mtok columns and gate paid models on free accounts — mirroring the ``hermes model`` CLI picker. Adds network calls (pricing fetch + Nous tier check); only set for interactive pickers. + - ``pricing_cache_only``: when pricing is enabled, use only values already + resident in process caches. Normal picker opens use this while a + background worker warms cold pricing endpoints. - ``capabilities``: add a per-row ``capabilities`` map ``{model: {fast, reasoning}}`` so pickers can gate the model-options controls (fast toggle / reasoning) to what each model actually @@ -304,7 +314,11 @@ def build_models_payload( if canonical_order: rows = _reorder_canonical(rows) if pricing: - _apply_pricing(rows, force_fresh_nous_tier=force_fresh_nous_tier) + _apply_pricing( + rows, + force_fresh_nous_tier=force_fresh_nous_tier, + cached_only=pricing_cache_only, + ) if capabilities: _apply_capabilities(rows) if featured: @@ -336,19 +350,27 @@ def build_model_options_payload( cache so live catalogs repopulate fully """ refresh = bool(refresh) - return build_models_payload( + payload = build_models_payload( ctx, explicit_only=bool(explicit_only), include_unconfigured=bool(include_unconfigured), picker_hints=True, canonical_order=True, pricing=True, + pricing_cache_only=not refresh, capabilities=True, featured=True, refresh=refresh, probe_custom_providers=refresh, probe_current_custom_provider=not refresh, ) + if not refresh: + _prewarm_pricing_async( + payload["providers"], + current_provider=ctx.current_provider, + current_base_url=ctx.current_base_url, + ) + return payload # ─── Public: auxiliary-task pickers ───────────────────────────────────── @@ -798,11 +820,35 @@ def _filter_explicit_provider_rows(rows: list[dict], ctx: ConfigContext) -> list # just accepted those same credentials when building it. kept.append(row) continue + if _external_process_signed_in(slug): + # External-process providers (copilot-acp) authenticate through + # their own CLI (`copilot login`), which — like the Anthropic + # OAuth case above — leaves no trace in active_provider, + # model.provider, or env vars. Verified CLI credentials are a + # deliberate sign-in; without this the desktop picker drops the + # row the picker-discovery side just accepted. + kept.append(row) + continue if is_provider_explicitly_configured(slug): kept.append(row) return kept +def _external_process_signed_in(slug: str) -> bool: + """True when an external-process provider has verified CLI credentials.""" + try: + from hermes_cli.auth import ( + PROVIDER_REGISTRY, + get_external_process_provider_status, + ) + pconfig = PROVIDER_REGISTRY.get(slug) + if not pconfig or pconfig.auth_type != "external_process": + return False + return bool(get_external_process_provider_status(slug).get("auth_verified")) + except Exception: + return False + + def _provider_is_keyless(slug: str) -> bool: """True when the provider's Hermes overlay declares it keyless.""" try: @@ -920,6 +966,7 @@ def _apply_pricing( rows: list[dict], *, force_fresh_nous_tier: bool = False, + cached_only: bool = False, ) -> None: """Enrich each provider row with per-model pricing + Nous tier gating. @@ -942,6 +989,7 @@ def _apply_pricing( _format_price_per_mtok, check_nous_free_tier, compute_sale_discount, + get_cached_nous_free_tier, get_pricing_for_provider, partition_nous_models_by_tier, ) @@ -955,10 +1003,36 @@ def _apply_pricing( if not models: continue try: - raw_pricing = get_pricing_for_provider(slug) or {} + pricing_kwargs = {"cached_only": True} if cached_only else {} + raw_pricing = get_pricing_for_provider(slug, **pricing_kwargs) or {} except Exception: raw_pricing = {} + cached_nous_tier: Optional[bool] = None + if slug == "nous" and cached_only: + cached_nous_tier = get_cached_nous_free_tier() + if cached_nous_tier is None: + # Entitlement is not yet known. Keep the response nonblocking, + # but fail closed until this profile's prewarm has populated + # both caches; otherwise a free account can briefly select + # paid models on its first picker open. + row["free_tier_pending"] = True + row["unavailable_models"] = list(models) + # Every model renders locked until the prewarm lands; say why + # on the existing per-provider warning surface instead of + # leaving the user staring at a greyed-out list. + if not row.get("warning"): + row["warning"] = ( + "Checking Nous plan entitlement… models unlock on the " + "next picker open or refresh." + ) + continue if not raw_pricing: + if slug == "nous": + row["free_tier"] = bool(cached_nous_tier) + row["pricing_pending"] = True + row["unavailable_models"] = ( + list(models) if cached_nous_tier else [] + ) continue formatted: dict[str, dict] = {} @@ -1008,9 +1082,12 @@ def _apply_pricing( if slug == "nous": try: if nous_free_tier is None: - nous_free_tier = check_nous_free_tier( - force_fresh=force_fresh_nous_tier - ) + if cached_only: + nous_free_tier = cached_nous_tier + else: + nous_free_tier = check_nous_free_tier( + force_fresh=force_fresh_nous_tier + ) row["free_tier"] = bool(nous_free_tier) if nous_free_tier: _selectable, unavailable = partition_nous_models_by_tier( @@ -1076,6 +1153,68 @@ def _local_runtime_row(ctx: "ConfigContext") -> dict | None: return None +def _prewarm_pricing_async( + rows: list[dict], + *, + current_provider: str = "", + current_base_url: str = "", +) -> Optional[Thread]: + """Warm picker pricing caches without delaying the current payload.""" + from hermes_constants import hermes_home_key + from hermes_cli.models import pricing_cache_scope + + profile_key = hermes_home_key() + endpoint_scope = tuple( + sorted( + ( + slug, + pricing_cache_scope( + slug, + current_provider=current_provider, + current_base_url=current_base_url, + ), + ) + for slug in { + str(row.get("slug") or "").lower() + for row in rows + if row.get("slug") + } + ) + ) + prewarm_key = (profile_key, endpoint_scope) + + with _pricing_prewarm_lock: + current = _pricing_prewarm_threads.get(prewarm_key) + if current is not None and current.is_alive(): + return current + + # The worker mutates only private copies while the pricing helpers + # populate their shared process caches. + worker_rows = [ + {**row, "models": list(row.get("models") or [])} + for row in rows + ] + + def _worker() -> None: + try: + _apply_pricing(worker_rows) + finally: + with _pricing_prewarm_lock: + if _pricing_prewarm_threads.get(prewarm_key) is current_thread(): + _pricing_prewarm_threads.pop(prewarm_key, None) + + worker_context = copy_context() + thread = Thread( + target=worker_context.run, + args=(_worker,), + name="hermes-picker-pricing-prewarm", + daemon=True, + ) + _pricing_prewarm_threads[prewarm_key] = thread + thread.start() + return thread + + def _moa_provider_row(current_provider: str = "") -> dict | None: """Build the virtual ``moa`` provider row for model pickers. diff --git a/hermes_cli/kanban.py b/hermes_cli/kanban.py index ffbcfad8ca..167db27f1b 100644 --- a/hermes_cli/kanban.py +++ b/hermes_cli/kanban.py @@ -2311,18 +2311,23 @@ def _worker_run_id_for(task_id: str) -> Optional[int]: return None -def _goal_mode_handoff_rejection(task: Optional[kb.Task], evidence: str) -> Optional[str]: - """Apply the goal judge to every terminal worker handoff, including review.""" +def _goal_mode_handoff_rejection(task: Optional[kb.Task], evidence: str): + """Apply the goal judge to every terminal worker handoff, including review. + + Returns ``(verdict, reason_or_None)`` — ``"done"`` allows the handoff; + ``"blocked"`` means the judge ruled the goal unachievable (#100954); + ``"continue"``/``"wait"`` reject with the judge's reason. + """ if task is None or not task.goal_mode: - return None + return ("done", None) try: from agent.auxiliary_client import get_text_auxiliary_client client, model = get_text_auxiliary_client("goal_judge") except Exception: - return None + return ("done", None) if client is None or not model: - return None + return ("done", None) from hermes_cli.goals import judge_goal @@ -2341,7 +2346,7 @@ def _goal_mode_handoff_rejection(task: Optional[kb.Task], evidence: str) -> Opti judge_exc, exc_info=True, ) - return reason if verdict != "done" else None + return (verdict, None if verdict == "done" else reason) def _cmd_complete(args: argparse.Namespace) -> int: @@ -2379,10 +2384,20 @@ def _cmd_complete(args: argparse.Namespace) -> int: # to every terminal handoff so request-review cannot bypass the # acceptance contract that protects complete. task = kb.get_task(conn, tid) - rejection = _goal_mode_handoff_rejection( + gate_verdict, rejection = _goal_mode_handoff_rejection( task, (summary or args.result or "").strip(), ) + if gate_verdict == "blocked": + print( + f"kanban: goal completion of {tid} rejected: judge ruled " + f"the goal unachievable — {rejection}. Re-scope with " + f"kanban edit, or record the block with kanban block " + f"instead of completing.", + file=sys.stderr, + ) + failed.append(tid) + continue if rejection is not None: print( f"kanban: goal completion of {tid} rejected by judge: {rejection}. " @@ -2532,10 +2547,18 @@ def _cmd_request_review(args: argparse.Namespace) -> int: return 2 reviewer = getattr(args, "reviewer", None) with kb.connect_closing() as conn: - rejection = _goal_mode_handoff_rejection( + gate_verdict, rejection = _goal_mode_handoff_rejection( kb.get_task(conn, tid), summary or "", ) + if gate_verdict == "blocked": + print( + f"kanban: goal review handoff of {tid} rejected: judge ruled " + f"the goal unachievable — {rejection}. Record the block with " + f"kanban block instead of requesting review.", + file=sys.stderr, + ) + return 1 if rejection is not None: print( f"kanban: goal review handoff of {tid} rejected by judge: " @@ -3428,8 +3451,10 @@ def _cmd_repair(args: argparse.Namespace) -> int: print(f" corrupt copy quarantined at: {report.backup_path}", file=sys.stderr) print( - " Recover manually (e.g. `sqlite3 kanban.db \".recover\"` into a " - "fresh file) or move the file aside to start a new board.", + " Recover manually (copy kanban.db aside FIRST, then run " + "`sqlite3 \".recover\"` into a fresh file — never against " + "the live path, a WAL-reset-vulnerable sqlite3 CLI can corrupt it " + "further) or move the file aside to start a new board.", file=sys.stderr, ) return 1 diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index 9f25bea6f1..6c4fd6179a 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -10717,6 +10717,32 @@ def _retag_legacy_worker_sessions(workspaces_root_path: str) -> None: _log.debug("kanban worker: legacy session retag skipped (%s)", exc) +def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]: + """Wrap a managed-gateway worker in the shared restart-safe scope.""" + if task.current_run_id is None: + # Outside managed systemd this is harmless, but a managed dispatch must + # never mint an untraceable scope. Check topology through the shared + # helper first, using a placeholder suffix that cannot be launched. + from tools.process_registry import restart_safe_gateway_child_argv + + scoped = restart_safe_gateway_child_argv( + command, unit_suffix=f"kanban-{task.id}-run-missing" + ) + if scoped is not command: + raise RuntimeError( + "cannot create restart-safe systemd scope for Kanban worker: " + "the claimed task has no current run id" + ) + return command + + from tools.process_registry import restart_safe_gateway_child_argv + + return restart_safe_gateway_child_argv( + command, + unit_suffix=f"kanban-{task.id}-run-{task.current_run_id}", + ) + + def _default_spawn( task: Task, workspace: str, @@ -10744,7 +10770,13 @@ def _default_spawn( profile_arg = normalize_profile_name(task.assignee) prompt = f"work kanban task {task.id}" - env = dict(os.environ) + from agent.secret_scope import is_multiplex_active + from tools.environments.local import build_subprocess_env + + env = build_subprocess_env( + scrub_secrets=is_multiplex_active(), + inherit_profile_home=True, + ) # The dispatcher is detached from every conversation. Its worker must never # inherit routing mirrored by a previous gateway turn, even before the first # session binds ContextVars in this process. @@ -10895,6 +10927,12 @@ def _default_spawn( # turn, prints text, exits rc=0, and the dispatcher records a # protocol violation (incident 2026-06-09 t_d9cbe312). cmd.append("-Q") + + # A worker spawned by a managed systemd gateway must leave the gateway's + # cgroup before startup; otherwise restarting the service kills the worker + # that is performing the handoff. + cmd = _restart_safe_worker_argv(task, cmd) + # Redirect output to a per-task log under /logs/. # Anchored at the board root (not the shared kanban root), so # `hermes kanban log` on a specific board reads its own file and @@ -11685,8 +11723,8 @@ def purge_stale_done_notify_subs( *, max_age_days: int = 30, ) -> int: - """Delete notify subscriptions whose task has sat in ``done`` untouched - for longer than ``max_age_days``. + """Delete notify subscriptions whose task has sat in ``done`` or + ``blocked`` untouched for longer than ``max_age_days``. The notifier keeps subscriptions alive through ``done`` because a completed task can be reopened (review corrections, continuation) and @@ -11695,7 +11733,10 @@ def purge_stale_done_notify_subs( subscription rows forever — each one scanned every notifier tick. This GC bounds that: a task that has been ``done`` with no new events for the retention window is treated as settled and its subscriptions - are purged. Age is measured from the task's most recent event + are purged. ``blocked`` tasks (circuit-breaker trips, dead workers) + are reaped on the same clock — they are abandoned, not idle, unlike a + ``backlog``/``ready`` card that is merely waiting for pickup (#100955). + Age is measured from the task's most recent event (falling back to ``completed_at`` then ``created_at``), so ANY activity — including a reopen, which also moves the task off ``done`` — resets or exempts it. @@ -11714,7 +11755,7 @@ def purge_stale_done_notify_subs( cur = conn.execute( "DELETE FROM kanban_notify_subs WHERE task_id IN (" " SELECT t.id FROM tasks t" - " WHERE t.status = 'done'" + " WHERE t.status IN ('done', 'blocked')" " AND COALESCE(" " (SELECT MAX(e.created_at) FROM task_events e" " WHERE e.task_id = t.id)," diff --git a/hermes_cli/linux_desktop_entry.py b/hermes_cli/linux_desktop_entry.py index 53bde536f3..00de49c3c7 100644 --- a/hermes_cli/linux_desktop_entry.py +++ b/hermes_cli/linux_desktop_entry.py @@ -15,9 +15,9 @@ Two values must be absolute for the entry to work: checkout. Do not copy the icon: ``Exec`` already depends on that tree. Cache refresh is best-effort and tool-gated: ``update-desktop-database`` -for the freedesktop menu cache, and ``kbuildsycoca6``/``kbuildsycoca5`` -for Plasma. Run each tool only when it exists. A missing tool is not an -error. +for the freedesktop menu cache, ``gtk-update-icon-cache`` for the user +hicolor tree, and ``kbuildsycoca6``/``kbuildsycoca5`` for Plasma. Run +each tool only when it exists. A missing tool is not an error. Import-light and side-effect-free at import time: the uninstaller and the Electron main process both use this without loading the full CLI. @@ -25,6 +25,7 @@ Electron main process both use this without loading the full CLI. from __future__ import annotations +import io import os import shutil import subprocess diff --git a/hermes_cli/loops.py b/hermes_cli/loops.py index 92cdefdfe3..04e8dc7c02 100644 --- a/hermes_cli/loops.py +++ b/hermes_cli/loops.py @@ -768,6 +768,18 @@ class LoopManager: "reason": s.last_stop_reason, "message": f"✓ Loop finished after {s.ticks_fired} tick{'s' if s.ticks_fired != 1 else ''} — {reason}", } + if verdict == "blocked": + # Judge ruled the stop condition unachievable — don't spin + # until the tick budget; pause so the user can re-scope. + s.status = "paused" + s.paused_reason = f"stop condition judged unachievable: {reason}" + save_loop(self.session_id, s) + return { + "status": "paused", + "stopped": True, + "reason": s.paused_reason, + "message": f"⏸ Loop paused — {s.paused_reason}. /loop resume to keep going, /loop stop to end it.", + } # 3. --times user cap. if s.times and s.ticks_fired >= s.times: diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 66159317c1..ae45055a33 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -446,6 +446,7 @@ import shutil import stat import subprocess import tempfile +import time as _time_mod from pathlib import Path from typing import Optional @@ -483,7 +484,7 @@ from hermes_cli.subcommands.skin import build_skin_parser from hermes_cli.subcommands.console import build_console_parser from hermes_cli.subcommands.update import build_update_parser from hermes_cli.subcommands.uninstall import build_uninstall_parser -from hermes_cli.subcommands.dashboard import build_dashboard_parser +from hermes_cli.subcommands.dashboard import build_dashboard_parser, build_serve_parser from hermes_cli.subcommands.gui import build_gui_parser from hermes_cli.subcommands.logs import build_logs_parser from hermes_cli.subcommands.prompt_size import build_prompt_size_parser @@ -971,8 +972,14 @@ def _relative_time(ts) -> str: return relative_time(ts) -def _has_any_provider_configured() -> bool: - """Check if at least one inference provider is usable.""" +def _has_any_provider_configured(*, strict_profile_scope: bool = False) -> bool: + """Check if at least one inference provider is usable. + + ``strict_profile_scope``: the caller has bound a NAMED profile's home and + secret scope and wants an answer for that profile only — launch-process + env and host-wide fallbacks (gh auth, Claude Code credentials) must not + make it appear ready. Unscoped callers keep the legacy behavior. + """ from hermes_cli.config import get_env_path, get_hermes_home, load_config from hermes_cli.auth import get_auth_status @@ -1015,7 +1022,13 @@ def _has_any_provider_configured() -> bool: for pconfig in PROVIDER_REGISTRY.values(): if pconfig.auth_type == "api_key": provider_env_vars.update(pconfig.api_key_env_vars) - if any(os.getenv(v) for v in provider_env_vars): + if strict_profile_scope: + from agent.secret_scope import current_secret_scope + + read_provider_env = (current_secret_scope() or {}).get + else: + read_provider_env = os.getenv + if any(read_provider_env(v) for v in provider_env_vars): return True # Check .env file for keys @@ -1047,7 +1060,10 @@ def _has_any_provider_configured() -> bool: auth = json.loads(auth_file.read_text(encoding="utf-8-sig")) active = auth.get("active_provider") - if active: + active_config = PROVIDER_REGISTRY.get(str(active or "").strip().lower()) + if active and not ( + strict_profile_scope and active_config and active_config.auth_type == "api_key" + ): status = get_auth_status(active) if status.get("logged_in"): return True @@ -1066,20 +1082,21 @@ def _has_any_provider_configured() -> bool: return True # Check provider-specific auth fallbacks (for example, Copilot via gh auth). - try: - for provider_id, pconfig in PROVIDER_REGISTRY.items(): - if pconfig.auth_type != "api_key": - continue - status = get_auth_status(provider_id) - if status.get("logged_in"): - return True - except Exception: - pass + if not strict_profile_scope: + try: + for provider_id, pconfig in PROVIDER_REGISTRY.items(): + if pconfig.auth_type != "api_key": + continue + status = get_auth_status(provider_id) + if status.get("logged_in"): + return True + except Exception: + pass # Check for Claude Code OAuth credentials (~/.claude/.credentials.json) # Only count these if Hermes has been explicitly configured — Claude Code # being installed doesn't mean the user wants Hermes to use their tokens. - if _has_hermes_config: + if _has_hermes_config and not strict_profile_scope: try: from agent.anthropic_adapter import ( read_claude_code_credentials, @@ -6720,7 +6737,15 @@ def _write_desktop_build_stamp(project_root: Path, *, source_mode: bool) -> None def _desktop_packaged_executable(desktop_dir: Path) -> Optional[Path]: """Return the current platform's unpacked Electron app executable.""" - release_dir = desktop_dir / "release" + return _desktop_packaged_executable_in(desktop_dir / "release") + + +def _desktop_packaged_executable_in(release_dir: Path) -> Optional[Path]: + """Return the unpacked Electron app executable under *release_dir*. + + *release_dir* is electron-builder's ``directories.output`` — the live + ``apps/desktop/release`` or a stage-and-swap staging dir (#86443). + """ if sys.platform == "darwin": candidates = list(release_dir.glob("mac*/Hermes.app/Contents/MacOS/Hermes")) elif sys.platform == "win32": @@ -6754,6 +6779,91 @@ def _desktop_packaged_executable(desktop_dir: Path) -> Optional[Path]: return max(existing, key=lambda p: p.stat().st_mtime) +# ─── Desktop stage-and-swap pack (#86443) ─────────────────────────────────── +# +# electron-builder packs IN PLACE: before-pack.mjs wipes ``release/- +# unpacked`` (or the mac ``Hermes.app``) and the Electron unpack + asar + rename +# then rebuild it. Any failure after that wipe — corrupt cached zip, blocked +# download, missing dep, disk full — leaves the user with NO app, and +# ``hermes update`` used to report "partially complete" over an empty +# release/. Fix the class, not the predicate: build into a STAGING output +# dir next to release/, verify the staged result, and only then swap it over +# the live tree with renames. On any failure the live app is untouched. + +_DESKTOP_STAGING_PREFIX = ".staging-" +_DESKTOP_PREVIOUS_SUFFIX = ".previous" + + +def _desktop_staging_dir(desktop_dir: Path) -> Path: + """Fresh, unique staging output dir: ``apps/desktop/.staging--``. + + A sibling of ``release/`` (same filesystem → the swap is a rename, not a + copy) but NOT inside it, so nothing globbing ``release/*-unpacked`` or + ``release/mac*`` can mistake the half-built tree for the live app. + Leftovers from a killed earlier build are swept first (best-effort). + """ + for stale in desktop_dir.glob(f"{_DESKTOP_STAGING_PREFIX}*"): + shutil.rmtree(stale, ignore_errors=True) + return desktop_dir / f"{_DESKTOP_STAGING_PREFIX}{os.getpid()}-{int(_time_mod.time())}" + + +def _desktop_unpacked_root(exe: Path, release_dir: Path) -> Path: + """The directory directly under *release_dir* that holds *exe* + (``linux-unpacked``, ``win-unpacked``, ``mac-arm64``…) — electron-builder's + ``appOutDir``, the unit that gets swapped as a whole.""" + unpacked = exe + while unpacked.parent != release_dir: + if unpacked.parent == unpacked: + raise ValueError(f"{exe} is not under {release_dir}") + unpacked = unpacked.parent + return unpacked + + +def _swap_staged_desktop_app(desktop_dir: Path, staging_dir: Path) -> Optional[Path]: + """Promote a VERIFIED staged pack over the live ``release/`` app. + + ``release/`` → ``release/.previous``, + ``/`` → ``release/``, then drop ``.previous``. + Two renames; the only window with no live app is between them, and a + failure there rolls ``.previous`` back. Returns the live executable, or + ``None`` (live app untouched or restored) when the swap could not happen. + Best-effort cleanup of the staging dir; never raises. + """ + staged_exe = _desktop_packaged_executable_in(staging_dir) + if staged_exe is None: + shutil.rmtree(staging_dir, ignore_errors=True) + return None + release_dir = desktop_dir / "release" + try: + staged_root = _desktop_unpacked_root(staged_exe, staging_dir) + live_root = release_dir / staged_root.name + previous = release_dir / (staged_root.name + _DESKTOP_PREVIOUS_SUFFIX) + release_dir.mkdir(parents=True, exist_ok=True) + shutil.rmtree(previous, ignore_errors=True) + moved_aside = False + if live_root.exists(): + os.rename(live_root, previous) + moved_aside = True + try: + os.rename(staged_root, live_root) + except OSError: + if moved_aside: + os.rename(previous, live_root) # restore; live app back as it was + raise + if moved_aside: + shutil.rmtree(previous, ignore_errors=True) + except (OSError, ValueError) as exc: + logger.warning("desktop stage-and-swap failed, live app kept: %s", exc) + return None + finally: + shutil.rmtree(staging_dir, ignore_errors=True) + return live_root / staged_exe.relative_to(staged_root) + + +def _discard_desktop_staging(staging_dir: Path) -> None: + shutil.rmtree(staging_dir, ignore_errors=True) + + # ─── Desktop exe integrity gate (#69179) ──────────────────────────────────── # # The desktop self-update chain (Desktop → hermes-setup --update → @@ -7070,8 +7180,10 @@ def _ensure_desktop_exe_launchable( # Self-heal setup for the retry: drop the (likely corrupt) cached Electron # zip and the content stamp so the next rebuild is a genuine re-download + - # re-stage rather than a replay of the same broken extraction. - _purge_electron_build_cache(desktop_dir) + # re-stage rather than a replay of the same broken extraction. Only the + # exe's OWN output dir is purged (a stage-and-swap staging dir, #86443), + # never the live release/ tree that still holds the last working app. + _purge_electron_build_cache(desktop_dir, release_dir=packaged_executable.parent.parent) try: _desktop_stamp_path().unlink() except OSError: @@ -7127,7 +7239,9 @@ def _electron_download_cache_dirs() -> list[Path]: return out -def _purge_electron_build_cache(desktop_dir: Path) -> list[Path]: +def _purge_electron_build_cache( + desktop_dir: Path, release_dir: Optional[Path] = None +) -> list[Path]: """Clear the cached Electron download + half-written unpacked dir so the next ``pack`` re-downloads and re-stages from scratch. @@ -7173,8 +7287,11 @@ def _purge_electron_build_cache(desktop_dir: Path) -> list[Path]: # Drop the half-written unpacked dir too: an interrupted prior pack leaves # a partial tree that poisons the rename even after the zip is fixed. # (before-pack.cjs also handles this, but clearing it here makes the retry - # robust even if the hook is somehow skipped.) - release_dir = desktop_dir / "release" + # robust even if the hook is somehow skipped.) ``release_dir`` lets a + # stage-and-swap caller point this at its STAGING output so a mid-retry + # purge never touches the live app under ``release/`` (#86443). + if release_dir is None: + release_dir = desktop_dir / "release" if release_dir.is_dir(): for unpacked in release_dir.glob("*-unpacked"): try: @@ -7553,6 +7670,7 @@ def _desktop_macos_relaunchable_fixup( desktop_dir: Path, *, publisher_signing_configured: Optional[bool] = None, + release_dir: Optional[Path] = None, ) -> bool: """Make a locally-built macOS desktop app survive in-place self-update without resetting the user's TCC permission grants. @@ -7584,7 +7702,9 @@ def _desktop_macos_relaunchable_fixup( ) if publisher_signing_configured: return True - exe = _desktop_packaged_executable(desktop_dir) + # ``release_dir`` (stage-and-swap, #86443): sign the STAGED bundle before + # it is promoted, so the live app is never touched mid-sign. + exe = _desktop_packaged_executable_in(release_dir or (desktop_dir / "release")) if exe is None: return True # exe = .../Hermes.app/Contents/MacOS/Hermes -> app bundle = .../Hermes.app @@ -8335,7 +8455,16 @@ def cmd_gui(args: argparse.Namespace): print(" → No Developer ID configured; ad-hoc signing this local rebuild " "(CSC_IDENTITY_AUTO_DISCOVERY=false)") npm_build_env = _npm_lifecycle_env(env) + # Stage-and-swap (#86443): electron-builder packs IN PLACE and + # before-pack.mjs wipes release/ first, so a pack that + # fails afterwards used to leave the user with NO app. Build into + # a fresh staging output dir instead; the live release/ tree is + # only replaced — by rename — after the staged result verifies. + staging_dir: Optional[Path] = None + build_cmd = [npm, "run", build_script] if not source_mode: + staging_dir = _desktop_staging_dir(desktop_dir) + build_cmd += ["--", f"-c.directories.output={staging_dir}"] # A running desktop instance launched from release/win-unpacked # holds Hermes.exe locked on Windows, so the pack can't replace # it ("Access is denied" / ERR_ELECTRON_BUILDER_CANNOT_EXECUTE). @@ -8344,13 +8473,17 @@ def cmd_gui(args: argparse.Namespace): stopped = _stop_desktop_processes_locking_build(desktop_dir) if stopped: print(f" ⚠ Stopped running desktop app to free the build output (pid {', '.join(map(str, stopped))})") + + def _staged_exe() -> Optional[Path]: + return _desktop_packaged_executable_in(staging_dir) if staging_dir else None + build_result = subprocess.run( - [npm, "run", build_script], cwd=desktop_dir, env=npm_build_env, check=False + build_cmd, cwd=desktop_dir, env=npm_build_env, check=False ) if ( build_result.returncode != 0 and not source_mode - and _desktop_packaged_executable(desktop_dir) is None + and _staged_exe() is None ): # Corrupt cached Electron zip → partial unpack → ENOENT on rename. # stdlib zipfile won't catch the common concat-junk case, so purge @@ -8364,7 +8497,7 @@ def cmd_gui(args: argparse.Namespace): purged: list[Path] = [] restored = False if not _electron_dist_ok(PROJECT_ROOT): - purged = _purge_electron_build_cache(desktop_dir) + purged = _purge_electron_build_cache(desktop_dir, release_dir=staging_dir) restored = _redownload_electron_dist(PROJECT_ROOT, env) if restored: print(" ⚠ Desktop build failed; refreshed the Electron download and retrying once...") @@ -8374,13 +8507,13 @@ def cmd_gui(args: argparse.Namespace): # is still locked by a running instance; stop it before retry. _stop_desktop_processes_locking_build(desktop_dir) build_result = subprocess.run( - [npm, "run", build_script], cwd=desktop_dir, env=npm_build_env, check=False + build_cmd, cwd=desktop_dir, env=npm_build_env, check=False ) if ( build_result.returncode != 0 and not source_mode and not env.get("ELECTRON_MIRROR") - and _desktop_packaged_executable(desktop_dir) is None + and _staged_exe() is None ): print(" ⚠ Desktop build still failing; the Electron download from " "GitHub looks blocked. Re-downloading via a public mirror " @@ -8391,9 +8524,13 @@ def cmd_gui(args: argparse.Namespace): if not _electron_dist_ok(PROJECT_ROOT): _redownload_electron_dist(PROJECT_ROOT, env, mirror=mirror) _stop_desktop_processes_locking_build(desktop_dir) - build_result = subprocess.run([npm, "run", build_script], cwd=desktop_dir, env=mirror_env, check=False) + build_result = subprocess.run(build_cmd, cwd=desktop_dir, env=mirror_env, check=False) if build_result.returncode != 0: print("✗ Desktop GUI build failed") + if staging_dir is not None: + _discard_desktop_staging(staging_dir) + if _desktop_packaged_executable(desktop_dir) is not None: + print(" ↩ The previous desktop app was left untouched and still works.") print(f" Run manually: cd apps/desktop && npm run {build_script}") if sys.platform == "win32": print(" If this says \"Access is denied\" on Hermes.exe, close any") @@ -8401,28 +8538,37 @@ def cmd_gui(args: argparse.Namespace): print(" If the log shows Electron download retries, rebuild via a mirror:") print(" ELECTRON_MIRROR= hermes desktop --force-build") sys.exit(build_result.returncode or 1) - packaged_executable = _desktop_packaged_executable(desktop_dir) if not source_mode: + assert staging_dir is not None + staged_executable = _staged_exe() # Locally-built apps are ad-hoc signed; make them relaunchable after # an in-place self-update (otherwise macOS reports "Hermes is # damaged"). No-op on non-macOS and on real-identity builds. - _desktop_macos_relaunchable_fixup(desktop_dir) + # Signs the STAGED bundle so the live app is never half-signed. + _desktop_macos_relaunchable_fixup(desktop_dir, release_dir=staging_dir) # Windows integrity gate (#69179): never declare the rebuild a # success on a Hermes.exe Windows cannot load (truncated PE from # a corrupt cached Electron zip, wrong-arch tree, interrupted - # rcedit rewrite). Roll back to the .bak tree preserved by - # before-pack.mjs when possible, then fail loudly so the - # updater's retry-once rebuilds from a fresh Electron download - # instead of silently shipping the broken exe. + # rcedit rewrite). Verified on the STAGED exe: a failure here + # simply discards the staging dir — the live app was never + # touched — and fails loudly so the updater's retry-once + # rebuilds from a fresh Electron download. verified_executable, rolled_back = _ensure_desktop_exe_launchable( - desktop_dir, packaged_executable + desktop_dir, staged_executable ) - if packaged_executable is not None and ( - rolled_back or verified_executable is None - ): + if staged_executable is None or rolled_back or verified_executable is None: + _discard_desktop_staging(staging_dir) + if staged_executable is None: + print(f"✗ Desktop build produced no launchable app in {staging_dir}") + print(" ↩ The previous desktop app was left untouched and still works.") + sys.exit(1) + # Verified: swap the staged tree over the live one (rename). + packaged_executable = _swap_staged_desktop_app(desktop_dir, staging_dir) + if packaged_executable is None: + print(f"✗ Could not install the rebuilt desktop app into {desktop_dir / 'release'}") + print(" ↩ The previous desktop app was left untouched and still works.") sys.exit(1) - packaged_executable = verified_executable # Build succeeded — write the stamp so next run can skip _write_desktop_build_stamp(PROJECT_ROOT, source_mode=source_mode) @@ -11043,6 +11189,7 @@ def cmd_profile(args): profile_exists, _read_config_model, _check_gateway_running, + _served_by_running_multiplexer, _count_skills, _read_distribution_meta, _get_wrapper_dir, @@ -11056,7 +11203,7 @@ def cmd_profile(args): sys.exit(1) profile_dir = get_profile_dir(name) model, provider = _read_config_model(profile_dir) - gw = _check_gateway_running(profile_dir) + gw = _check_gateway_running(profile_dir) or _served_by_running_multiplexer(name) skills = _count_skills(profile_dir) dist_name, dist_version, dist_source = _read_distribution_meta(profile_dir) alias_name = find_alias_for_profile(name) @@ -11988,18 +12135,29 @@ def cmd_dashboard(args): # this, a profile's configured MCP servers never connect, so desktop # sessions show no MCP tools. Spawn discovery in the background here so a # slow/dead server can't block dashboard startup. - try: - from hermes_cli.mcp_startup import start_background_mcp_discovery + # + # Desktop-spawned headless backends start it AFTER the socket binds + # instead (start_server's ready path): the thread's first act is the + # ~350ms `mcp` SDK import, which holds the GIL against the main thread's + # own web_server import and pushes the READY sentinel — and every + # renderer paint behind it — back by that much. The Desktop can't issue + # an agent turn until its WebSocket is up anyway, and _make_agent's + # bounded wait_for_mcp_discovery + the late-binding refresh cover a + # server that is still connecting when the first turn lands. + _mcp_discovery_after_bind = _headless_backend and os.environ.get("HERMES_DESKTOP") == "1" + if not _mcp_discovery_after_bind: + try: + from hermes_cli.mcp_startup import start_background_mcp_discovery - start_background_mcp_discovery( - logger=logger, - thread_name="dashboard-mcp-discovery", - ) - except Exception: - logger.debug( - "Background MCP tool discovery failed at dashboard startup", - exc_info=True, - ) + start_background_mcp_discovery( + logger=logger, + thread_name="dashboard-mcp-discovery", + ) + except Exception: + logger.debug( + "Background MCP tool discovery failed at dashboard startup", + exc_info=True, + ) from hermes_cli.web_server import start_server @@ -12022,6 +12180,7 @@ def cmd_dashboard(args): headless=_headless_backend, ssh_session_token=_ssh_session_token, ssh_owner_nonce=_ssh_owner_nonce, + start_mcp_discovery_after_bind=_mcp_discovery_after_bind, ) @@ -12235,7 +12394,19 @@ _AGENT_SUBCOMMANDS = { def _is_tui_chat_launch(args) -> bool: - return bool(getattr(args, "tui", False) or os.environ.get("HERMES_TUI") == "1") + if getattr(args, "tui", False) or os.environ.get("HERMES_TUI") == "1": + return True + # The chat path decides TUI-vs-classic via _resolve_use_tui (--cli/--tui + # flags, TTY gate, HERMES_TUI env, display.interface config). Bare + # `hermes`/`hermes chat` with a TUI display config was previously missed + # here, so the wrapper pre-warmed its own MCP discovery while the TUI + # gateway (spawned moments later) ran a second one — an idle stdio MCP + # server copy held dead for the whole session. Only chat commands can + # launch the TUI; other commands (mcp serve, gateway, acp, cron) keep + # their own discovery behavior untouched. + if getattr(args, "command", None) not in {None, "chat"}: + return False + return _resolve_use_tui(args) def _command_has_dedicated_mcp_startup(args) -> bool: @@ -12296,6 +12467,17 @@ def _prepare_agent_startup(args) -> None: "plugin discovery failed at CLI startup", exc_info=True, ) + # -t/--toolsets narrows which configured MCP servers get spawned, on + # every discovery path (inline below, background thread, TUI/desktop + # deferred start). Built-in toolset names never match a server key, so + # `-t terminal` simply spawns nothing; `-t all` keeps the full set. + try: + from hermes_cli.mcp_startup import set_mcp_server_filter + + set_mcp_server_filter(getattr(args, "toolsets", None)) + except Exception: + logger.debug("MCP server filter setup failed", exc_info=True) + _run_inline_mcp_discovery = True if _is_tui_chat_launch(args): # The TUI launcher hands off to a dedicated startup path that already @@ -12324,9 +12506,14 @@ def _prepare_agent_startup(args) -> None: try: # MCP tool discovery remains synchronous for entrypoints that do # not own a later bounded/executor startup path. + from hermes_cli.mcp_startup import get_mcp_server_filter from tools.mcp_tool import discover_mcp_tools - discover_mcp_tools() + _mcp_filter = get_mcp_server_filter() + if _mcp_filter is None: + discover_mcp_tools() + else: + discover_mcp_tools(allowed_mcp_names=_mcp_filter) except Exception: logger.debug( "MCP tool discovery failed at CLI startup", @@ -12411,6 +12598,47 @@ def _set_chat_arg_defaults(args) -> None: setattr(args, attr, default) +def _try_fast_serve_launch() -> bool: + """Dispatch an unambiguous built-in ``serve`` without the full CLI tree. + + Desktop launches this exact command on every cold start. Building parsers + for unrelated Hermes commands performs thousands of filesystem-backed + translation lookups on Windows even though none of those commands are + usable in this process. Unknown or globally-scoped arguments fall back to + normal parsing so compatibility and error reporting remain unchanged. + """ + if os.environ.get("HERMES_DISABLE_FAST_SERVE_LAUNCH") == "1": + return False + + argv = sys.argv[1:] + if not argv or argv[0] != "serve" or "-h" in argv or "--help" in argv: + return False + + # Container routing is top-level policy and must run before host dispatch. + try: + from hermes_cli.config import get_container_exec_info + + if get_container_exec_info(): + return False + except Exception: + return False + + parser = build_serve_parser( + cmd_dashboard=cmd_dashboard, + add_help=False, + exit_on_error=False, + ) + try: + args, unknown = parser.parse_known_args(argv[1:]) + except (argparse.ArgumentError, ValueError): + return False + if unknown: + return False + + cmd_dashboard(args) + return True + + def _try_fast_chat_launch() -> bool: """Fast path for unambiguous interactive chat launches (all hosts). diff --git a/hermes_cli/mcp_startup.py b/hermes_cli/mcp_startup.py index c368805405..c57b00eb43 100644 --- a/hermes_cli/mcp_startup.py +++ b/hermes_cli/mcp_startup.py @@ -9,6 +9,38 @@ from typing import Optional _mcp_discovery_lock = threading.Lock() _mcp_discovery_started = False _mcp_discovery_thread: Optional[threading.Thread] = None +_mcp_discovery_deferred: Optional[threading.Timer] = None +# Process-wide MCP server-name allowlist derived from ``-t/--toolsets``. +# ``None`` = no filter (spawn every configured server). Set once at CLI +# startup by ``set_mcp_server_filter`` and honored by every discovery path +# in this module (inline, background, deferred), so a ``-t terminal`` +# oneshot never cold-starts MCP subprocesses it cannot use. +_mcp_server_filter: Optional[list[str]] = None + + +def set_mcp_server_filter(toolsets: object) -> Optional[list[str]]: + """Derive the MCP spawn allowlist from a ``-t/--toolsets`` value. + + Built-in toolset names in the list are harmless (they never match a + configured ``mcp_servers`` key). ``all``/``*`` or an empty/absent value + clears the filter. Returns the stored list for logging/tests. + """ + global _mcp_server_filter + names: list[str] = [] + if isinstance(toolsets, str): + names = [t.strip() for t in toolsets.split(",") if t.strip()] + elif isinstance(toolsets, (list, tuple, set)): + for item in toolsets: + names.extend(t.strip() for t in str(item).split(",") if t.strip()) + if not names or "all" in names or "*" in names: + _mcp_server_filter = None + else: + _mcp_server_filter = names + return _mcp_server_filter + + +def get_mcp_server_filter() -> Optional[list[str]]: + return _mcp_server_filter def _has_configured_mcp_servers() -> bool: @@ -169,7 +201,52 @@ def _discover_mcp_tools_without_interactive_oauth() -> None: with suppress_interactive_oauth(): from tools.mcp_tool import discover_mcp_tools - discover_mcp_tools() + # Only pass the kwarg when a filter is set: many tests (and any + # out-of-tree caller) stub discover_mcp_tools with a zero-arg + # callable, and the unfiltered call shape is unchanged. + if _mcp_server_filter is None: + discover_mcp_tools() + else: + discover_mcp_tools(allowed_mcp_names=_mcp_server_filter) + + +def defer_background_mcp_discovery(*, logger, thread_name: str, delay: float) -> None: + """Arm ``start_background_mcp_discovery`` to run ``delay`` seconds from now. + + Used by the Desktop ``serve`` backend after its socket is announced: the + discovery thread's first act is the ~350ms ``mcp`` SDK import, which holds + the GIL against the renderer's connect + first hydration reads if it starts + at bind time, and against the web_server import if it starts before. Any + consumer that needs discovery sooner (``wait_for_mcp_discovery`` from an + agent build) fires the deferred start immediately, so the bounded join and + the late-binding refresh behave exactly as if it had been started eagerly. + """ + global _mcp_discovery_deferred + with _mcp_discovery_lock: + if _mcp_discovery_started or _mcp_discovery_deferred is not None: + return + + def _fire() -> None: + global _mcp_discovery_deferred + with _mcp_discovery_lock: + _mcp_discovery_deferred = None + start_background_mcp_discovery(logger=logger, thread_name=thread_name) + + timer = threading.Timer(delay, _fire) + timer.daemon = True + timer.name = f"{thread_name}-deferred" + _mcp_discovery_deferred = timer + timer.start() + + +def _start_deferred_mcp_discovery_now() -> None: + """Run an armed deferred start immediately (idempotent, thread-safe).""" + with _mcp_discovery_lock: + timer = _mcp_discovery_deferred + if timer is None: + return + timer.cancel() + timer.function() def wait_for_mcp_discovery( @@ -188,6 +265,7 @@ def wait_for_mcp_discovery( ``mcp_single_query_discovery_timeout`` instead (default 15s vs 1.5s interactive) because one-shot sessions have no second turn to recover. """ + _start_deferred_mcp_discovery_now() thread = _mcp_discovery_thread if thread is None or not thread.is_alive(): return diff --git a/hermes_cli/model_catalog.py b/hermes_cli/model_catalog.py index 761ab0d096..d54edfaea9 100644 --- a/hermes_cli/model_catalog.py +++ b/hermes_cli/model_catalog.py @@ -74,7 +74,10 @@ DEFAULT_CATALOG_URL = ( DEFAULT_CATALOG_FALLBACK_URLS: tuple[str, ...] = ( "https://raw.githubusercontent.com/NousResearch/hermes-agent/main/website/static/api/model-catalog.json", ) -DEFAULT_TTL_HOURS = 1 +DEFAULT_TTL_MINUTES = 20 +# Legacy key. ``ttl_hours`` is honoured only when the user set it explicitly; +# the shipped default is ``ttl_minutes`` above. +DEFAULT_TTL_HOURS = DEFAULT_TTL_MINUTES / 60.0 DEFAULT_FETCH_TIMEOUT = 8.0 SUPPORTED_SCHEMA_VERSION = 1 @@ -104,10 +107,28 @@ def _load_catalog_config() -> dict[str, Any]: if not isinstance(raw, dict): raw = {} + # ``ttl_minutes`` is the shipped default (20). ``ttl_hours`` is the legacy + # key: honoured when a user set it explicitly and ``ttl_minutes`` is still + # at its default (load_config() deep-merges the default in, so "present" + # alone doesn't mean "user-set"), so old customized configs keep their + # chosen window. + ttl_minutes = raw.get("ttl_minutes") + try: + ttl_minutes = float(ttl_minutes) if ttl_minutes not in (None, "") else DEFAULT_TTL_MINUTES + except (TypeError, ValueError): + ttl_minutes = DEFAULT_TTL_MINUTES + if ttl_minutes == DEFAULT_TTL_MINUTES and raw.get("ttl_hours"): + try: + ttl_minutes = float(raw["ttl_hours"]) * 60.0 + except (TypeError, ValueError): + pass + if ttl_minutes <= 0: + ttl_minutes = DEFAULT_TTL_MINUTES + return { "enabled": bool(raw.get("enabled", True)), "url": str(raw.get("url") or DEFAULT_CATALOG_URL), - "ttl_hours": float(raw.get("ttl_hours") or DEFAULT_TTL_HOURS), + "ttl_hours": ttl_minutes / 60.0, "providers": raw.get("providers") if isinstance(raw.get("providers"), dict) else {}, } @@ -330,6 +351,33 @@ def get_catalog(*, force_refresh: bool = False) -> dict[str, Any]: return {} +def refresh_interval_seconds() -> float: + """Return the configured catalog TTL in seconds (the gateway poll cadence).""" + return max(60.0, _load_catalog_config()["ttl_hours"] * 3600.0) + + +def refresh_catalogs() -> bool: + """Force-refresh every remote model catalog the picker reads from. + + Fetches the curated manifest, the OpenRouter live list (tool-support / + free-pricing filter) and the Nous Portal recommendations, writing each + to its disk cache so the next ``/model`` open in ANY process on this + machine sees the new lists. Blocking; run it off the event loop. + Returns True when the manifest refresh succeeded. + """ + if not _load_catalog_config()["enabled"]: + return False + catalog = get_catalog(force_refresh=True) + try: + from hermes_cli.models import fetch_nous_recommended_models, fetch_openrouter_models + + fetch_openrouter_models(force_refresh=True) + fetch_nous_recommended_models(force_refresh=True) + except Exception: + logger.debug("provider catalog refresh failed", exc_info=True) + return bool(catalog) + + def _fetch_provider_override(provider: str) -> dict[str, Any] | None: """If ``model_catalog.providers..url`` is set, fetch that instead.""" cfg = _load_catalog_config() diff --git a/hermes_cli/model_data_policy_guard.py b/hermes_cli/model_data_policy_guard.py index 6cb86991dc..3fb3fd281a 100644 --- a/hermes_cli/model_data_policy_guard.py +++ b/hermes_cli/model_data_policy_guard.py @@ -52,20 +52,18 @@ def _is_meta_contributor(model_lower: str, provider_lower: str) -> bool: _META_CONTRIBUTOR_MESSAGE = ( "!!! CONTRIBUTOR TIER — TRAINS ON YOUR DATA !!!\n" "\n" - "muse-spark-1.2-contributor is Meta's contributor tier: heavily discounted\n" - "token pricing in exchange for permission to use your prompts and completions\n" - "to train future Meta models.\n" + "This is Meta's contributor tier. Selecting it permits Meta to use your\n" + "prompts and completions to train future Meta models.\n" "\n" - " Price per 1M tokens: input $0.10 | output $0.20 | cached input $0.002\n" - " (vs. standard muse-spark-1.2: input $1.25 | output $4.25 | cached $0.15)\n" + "See current pricing and rate limits for the Meta Model API here:\n" + " https://dev.meta.ai/docs/pricing-rate-limits/\n" "\n" "It lowers the barrier to entry for prototyping, testing integrations, and\n" "scaling experiments where training on your data is acceptable. Do NOT use it\n" "for confidential, proprietary, personal, or otherwise sensitive data. For the\n" - "same model at standard pricing with no training on your data, select the\n" - "standard variant, muse-spark-1.2.\n" + "same model with no training on your data, select the standard variant\n" + "(without the -contributor suffix).\n" "\n" - "Source: https://dev.meta.ai/docs/pricing-rate-limits/\n" "Confirm only if training on your prompts and completions is acceptable." ) diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index 5f9c11d085..f70c71f30f 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -1016,11 +1016,18 @@ def resolve_persist_behavior( 1. ``--once`` explicitly opts out → ``False`` (next turn only). 2. ``--session`` explicitly opts out → ``False`` (this session only). 3. ``--global`` explicitly opts in → ``True``. - 4. ``--provider`` given without an explicit persist flag → ``False`` + 4. No default configured yet (neither ``model.default`` nor + ``model.provider`` set — a fresh install whose first-ever pick this + is) → ``True``. Without a persisted provider, ``resolve_provider`` + falls through to whatever ``*_API_KEY`` env var is lying around on + the next launch (#86414), so the first pick becomes the default + instead of evaporating. Applies to every surface (CLI, gateway, + Desktop picker) so no client has to hardcode ``--global``. + 5. ``--provider`` given without an explicit persist flag → ``False`` (session only). Provider switches are typically exploratory — the user is trying a different backend for this conversation, not reconfiguring the default. ``--global`` can still force persist. - 5. Otherwise defer to ``model.persist_switch_by_default`` in + 6. Otherwise defer to ``model.persist_switch_by_default`` in ``config.yaml`` (defaults to ``False``: a plain ``/model `` affects only the current session). Users who want the old persist-by-default behavior can set the key to ``true``; a one-off @@ -1036,17 +1043,20 @@ def resolve_persist_behavior( return False if is_global: return True - if explicit_provider: - return False try: from hermes_cli.config import load_config model_cfg = load_config().get("model") - if isinstance(model_cfg, dict): - return bool(model_cfg.get("persist_switch_by_default", False)) except Exception: - pass - return False + return False + if isinstance(model_cfg, dict): + if not (model_cfg.get("default") or model_cfg.get("provider")): + return True + if explicit_provider: + return False + return bool(model_cfg.get("persist_switch_by_default", False)) + # Flat-string form: a non-empty string IS a configured default. + return not model_cfg # --------------------------------------------------------------------------- @@ -3354,6 +3364,19 @@ def list_authenticated_providers( if any(os.environ.get(ev) for ev in pcfg.api_key_env_vars): has_creds = True break + # External-process providers (copilot-acp) hold no API key, OAuth + # token, or pool entry by design — the spawned ACP subprocess brings + # its own auth. "Configured" means the executable resolves, which is + # exactly what get_auth_status() reports for them; without this branch + # the has_creds filter below unconditionally hides the provider from + # every picker (#63662). + if not has_creds and overlay.auth_type == "external_process": + try: + from hermes_cli.auth import get_auth_status + _ext_status = get_auth_status(hermes_slug) or {} + has_creds = bool(_ext_status.get("logged_in") or _ext_status.get("configured")) + except Exception as exc: + logger.debug("External-process check failed for %s: %s", pid, exc) # Check auth store and credential pool for non-env-var credentials. # This applies to OAuth providers AND api_key providers that also # support OAuth (e.g. anthropic supports both API key and Claude Code @@ -3528,7 +3551,20 @@ def list_authenticated_providers( _cp_config = _auth_registry.get(_cp.slug) _cp_has_creds = False if _cp_config and _cp_config.api_key_env_vars: - _cp_has_creds = any(os.environ.get(ev) for ev in _cp_config.api_key_env_vars) + _cp_lit = {ev for ev in _cp_config.api_key_env_vars if os.environ.get(ev)} + _cp_has_creds = bool(_cp_lit) + # A regional "-cn" twin lit only by key vars it shares with its + # non-CN sibling (e.g. alibaba-coding-plan-cn off the intl + # ALIBABA_CODING_PLAN_API_KEY) is a phantom picker row (#101122). + # Hide it unless the user configured that CN provider -- and only + # when it has a dedicated var of its own the user could set instead. + _sib = _auth_registry.get(_cp.slug[:-3]) if _cp.slug.endswith("-cn") else None + _sib_vars = set(_sib.api_key_env_vars) if _sib else set() + if ( + _cp_lit and _cp_lit <= _sib_vars < set(_cp_config.api_key_env_vars) + and _cp.slug != current_provider + ): + continue # Also check auth store and credential pool if not _cp_has_creds: try: diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 0d561d1b6c..8334e4f15c 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -103,6 +103,7 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [ ("openai/gpt-5.4-mini", ""), # Google ("google/gemini-3.1-pro-preview", ""), + ("google/gemini-3.8-flash", ""), ("google/gemini-3.7-flash", ""), # xAI ("x-ai/grok-4.6", ""), @@ -133,6 +134,9 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [ ("nvidia/nemotron-3-super-120b-a12b", ""), # Meta ("meta/muse-spark-1.2", ""), + ("meta/muse-spark-1.2-contributor", ""), + ("meta/muse-spark-1.3", ""), + ("meta/muse-spark-1.3-contributor", ""), # Sakana ("sakana/fugu-ultra", ""), # OpenRouter routers @@ -286,6 +290,7 @@ _PROVIDER_MODELS: dict[str, list[str]] = { "openai/gpt-5.4-mini", # Google "google/gemini-3.1-pro-preview", + "google/gemini-3.8-flash", "google/gemini-3.7-flash", # xAI "x-ai/grok-4.6", @@ -575,6 +580,7 @@ _PROVIDER_MODELS: dict[str, list[str]] = { "nemotron-3-ultra-free", "nemotron-3.5-lightning-free", "muse-spark-1.2-contributor-free", + "muse-spark-1.3-contributor-free", ], # OpenCode free tier — keyless (no OpenCode account needed). This is the # OFFLINE FLOOR only: provider_model_ids("opencode-free") revalidates live @@ -595,6 +601,7 @@ _PROVIDER_MODELS: dict[str, list[str]] = { "nemotron-3-ultra-free", "nemotron-3.5-lightning-free", "muse-spark-1.2-contributor-free", + "muse-spark-1.3-contributor-free", ], # Synced against https://opencode.ai/docs/go/ + live GET /zen/go/v1/models # (2026-08-20). @@ -627,6 +634,7 @@ _PROVIDER_MODELS: dict[str, list[str]] = { "hy3", "hy3-preview", "muse-spark-1.2-contributor", + "muse-spark-1.3-contributor", # Go-subscription twin of the Zen keyless Ox Alpha (live go/v1 # catalog 2026-08-21; NOT keyless — Go relay requires a Go key). "ox-alpha-free", @@ -1030,35 +1038,61 @@ def union_with_portal_paid_recommendations( # session while still picking up upgrades quickly. # --------------------------------------------------------------------------- _FREE_TIER_CACHE_TTL: int = 180 # seconds (3 minutes) -_free_tier_cache: tuple[bool, float] | None = None # (result, timestamp) +_free_tier_cache: dict[str, tuple[bool, float]] = {} -def check_nous_free_tier(*, force_fresh: bool = False) -> bool: +def _pricing_profile_key() -> str: + """Return the stable profile identity for process-local pricing caches.""" + from hermes_constants import hermes_home_key + + return hermes_home_key() + + +def get_cached_nous_free_tier() -> Optional[bool]: + """Return this profile's live cached entitlement, or ``None`` if unknown.""" + cached = _free_tier_cache.get(_pricing_profile_key()) + if cached is None: + return None + result, cached_at = cached + if time.monotonic() - cached_at >= _FREE_TIER_CACHE_TTL: + return None + return result + + +def check_nous_free_tier( + *, force_fresh: bool = False, cached_only: bool = False +) -> bool: """Check if the current Nous Portal user is on a free (unpaid) tier. Results are cached for ``_FREE_TIER_CACHE_TTL`` seconds to avoid hitting the Portal API on every call. The cache is short-lived so that an account upgrade is reflected within a few minutes. + ``cached_only`` returns a live cached answer or the fail-open ``False`` + default without contacting Portal. + Returns True only when entitlement is known to be free. Unknown/error states return False so this compatibility wrapper does not block users. """ - global _free_tier_cache now = time.monotonic() - if not force_fresh and _free_tier_cache is not None: - cached_result, cached_at = _free_tier_cache - if now - cached_at < _FREE_TIER_CACHE_TTL: + profile_key = _pricing_profile_key() + if not force_fresh: + cached_result = get_cached_nous_free_tier() + if cached_result is not None: return cached_result + if cached_only: + return False + try: from hermes_cli.nous_account import get_nous_portal_account_info account_info = get_nous_portal_account_info(force_fresh=force_fresh) result = account_info.is_free_tier - _free_tier_cache = (result, now) + _free_tier_cache[profile_key] = (result, now) return result except Exception: - _free_tier_cache = (False, now) + _free_tier_cache[profile_key] = (False, now) return False # default to paid on error — don't block users @@ -2093,6 +2127,25 @@ def warm_nous_reasoning_caps_async() -> None: _warm_reasoning_caps_async(_refresh_nous_reasoning_caps) +def refresh_reasoning_caps_async(provider: Optional[str]) -> None: + """Force a background re-fetch of *provider*'s reasoning-capability catalog. + + The in-memory cache is otherwise held for the process lifetime, so a + route that flips to reasoning-mandatory mid-process (GLM-5.3-flash, Sep + 2026) keeps being sent disables it now rejects. Called from the + conversation loop's reasoning_mandatory recovery so the profile guard is + right again on the next request; no-op for providers without a catalog. + """ + refresh = { + "nous": _refresh_nous_reasoning_caps, + "nous-portal": _refresh_nous_reasoning_caps, + "nousresearch": _refresh_nous_reasoning_caps, + "openrouter": _refresh_openrouter_reasoning_caps, + }.get(str(provider or "").strip().lower()) + if refresh is not None: + _warm_reasoning_caps_async(refresh) + + # Canonical low→high ordering used for nearest-level clamping. Kept as an # alias of the single source of truth in ``agent.reasoning_effort``. from agent.reasoning_effort import EFFORT_LADDER as _REASONING_EFFORT_ORDER @@ -2320,6 +2373,7 @@ def ai_gateway_model_ids(*, force_refresh: bool = False) -> list[str]: # Cache: maps model_id → {"prompt": str, "completion": str} per endpoint _pricing_cache: dict[str, dict[str, dict[str, str]]] = {} +_pricing_provider_cache_keys: dict[tuple[str, str], str] = {} # A failed fetch caches its empty result too, so an unreachable endpoint isn't # re-dialed on every call — but only until this deadline. Cached forever, one @@ -2806,26 +2860,145 @@ def restrict_to_nous_policy( return kept -def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: - """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita).""" +def get_cached_nous_inference_base_url() -> str: + """Return the profile's persisted Nous endpoint without refreshing auth.""" + try: + from hermes_cli.auth import ( + _load_auth_store, + _load_provider_state, + _optional_base_url, + _validate_nous_inference_url_from_network, + ) + + state = _load_provider_state(_load_auth_store(), "nous") or {} + return ( + _validate_nous_inference_url_from_network( + _optional_base_url(state.get("inference_base_url")) + ) + or "" + ).rstrip("/").removesuffix("/v1") + except Exception: + return "" + + +def pricing_cache_scope( + provider: str, + *, + current_provider: str = "", + current_base_url: str = "", +) -> str: + """Return the current endpoint identity used by a provider's pricing cache. + + This only resolves local configuration; it never fetches a catalog. Picker + prewarm single-flight uses the result to let an endpoint rotation start a + new worker while the previous endpoint is still slow or unreachable. + """ normalized = normalize_provider(provider) if normalized == "openrouter": + return "https://openrouter.ai/api" + if normalized == "ai-gateway": + from hermes_constants import AI_GATEWAY_BASE_URL + + return AI_GATEWAY_BASE_URL.rstrip("/") + if normalized == "novita": + return ( + os.getenv("NOVITA_BASE_URL", "").strip() + or "https://api.novita.ai/openai/v1" + ).rstrip("/") + if normalized == "deepinfra": + cache_key, _url = _deepinfra_catalog_url() + return cache_key + if normalized == "fireworks": + return "models.dev/fireworks" + if normalized == "nous": + try: + from hermes_cli.auth import _nous_inference_env_override + + env_base = _nous_inference_env_override() + except Exception: + env_base = None + if env_base: + return env_base.rstrip("/").removesuffix("/v1") + if normalize_provider(current_provider) == "nous" and current_base_url: + return current_base_url.rstrip("/").removesuffix("/v1") + persisted_base = get_cached_nous_inference_base_url() + if persisted_base: + return persisted_base + return _pricing_provider_cache_keys.get( + (_pricing_profile_key(), normalized), _DEFAULT_NOUS_INFERENCE_BASE + ) + return "" + + +def get_pricing_for_provider( + provider: str, + *, + force_refresh: bool = False, + cached_only: bool = False, +) -> dict[str, dict[str, str]]: + """Return pricing for providers that publish it. + + ``cached_only`` never starts provider I/O. Normal picker opens use it so + cold endpoints cannot hold the response path; a background prewarm fills + the same caches for later opens. + """ + normalized = normalize_provider(provider) + if cached_only: + if normalized == "deepinfra": + cache_key, _url = _deepinfra_catalog_url() + if cache_key not in _deepinfra_catalog_cache: + return {} + return _fetch_deepinfra_pricing() + + cache_key = _pricing_provider_cache_keys.get( + (_pricing_profile_key(), normalized) + ) + if cache_key is None: + if normalized == "openrouter": + cache_key = "https://openrouter.ai/api" + elif normalized == "ai-gateway": + from hermes_constants import AI_GATEWAY_BASE_URL + + cache_key = AI_GATEWAY_BASE_URL.rstrip("/") + elif normalized == "fireworks": + cache_key = "models.dev/fireworks" + return (_cached_catalog(cache_key) or {}) if cache_key else {} + + if normalized == "openrouter": + _pricing_provider_cache_keys[ + (_pricing_profile_key(), normalized) + ] = "https://openrouter.ai/api" return fetch_models_with_pricing( api_key=_resolve_openrouter_api_key(), base_url="https://openrouter.ai/api", force_refresh=force_refresh, ) if normalized == "ai-gateway": + from hermes_constants import AI_GATEWAY_BASE_URL + + _pricing_provider_cache_keys[ + (_pricing_profile_key(), normalized) + ] = AI_GATEWAY_BASE_URL.rstrip("/") return fetch_ai_gateway_pricing(force_refresh=force_refresh) if normalized == "novita": + base_url = os.getenv("NOVITA_BASE_URL", "").strip() or "https://api.novita.ai/openai/v1" + _pricing_provider_cache_keys[ + (_pricing_profile_key(), normalized) + ] = base_url.rstrip("/") return _fetch_novita_pricing(force_refresh=force_refresh) if normalized == "deepinfra": return _fetch_deepinfra_pricing(force_refresh=force_refresh) if normalized == "fireworks": + _pricing_provider_cache_keys[ + (_pricing_profile_key(), normalized) + ] = "models.dev/fireworks" return _fireworks_pricing_from_models_dev(force_refresh=force_refresh) if normalized == "nous": api_key, base_url = _resolve_nous_pricing_credentials() if base_url: + _pricing_provider_cache_keys[ + (_pricing_profile_key(), normalized) + ] = base_url.rstrip("/") return fetch_models_with_pricing( api_key=api_key, base_url=base_url, @@ -3602,7 +3775,7 @@ _BORROWED_MODEL_PROVIDERS: frozenset[str] = frozenset() # Zen / Go re-expose dozens of upstream vendors and rotate them frequently, so # their stale curated entries must not pollute the top of the picker. (#49129) _LIVE_FIRST_PICKER_PROVIDERS: frozenset[str] = frozenset( - {"opencode-zen", "opencode-go"} + {"opencode-zen", "opencode-go", "meta-ai"} ) @@ -3970,23 +4143,60 @@ def model_supports_fast_mode(model_id: Optional[str]) -> bool: def _is_anthropic_fast_model(model_id: Optional[str]) -> bool: """Return True if the model accepts the Anthropic Fast Mode ``speed`` param. - This gates the *speed=fast request parameter*, which Anthropic supports on - Opus 4.6 only (Opus 4.7 explicitly 400s). It is deliberately NOT a general - "is this a fast model" check: for Opus 4.8 the fast offering is a SEPARATE - model id (``…-opus-4.8-fast``) selected via the model field, not the speed - parameter — see ``agent.anthropic_adapter._supports_fast_mode`` and its - test. Keep this in lock-step with that adapter gate so the UI never shows a - Fast toggle that the runtime would silently drop. + This gates the *speed=fast request parameter*, which Anthropic supports + on Opus 4.8 and Opus 5 (research preview, Claude API only). It is + deliberately NOT a general "is this a fast model" check: + + - Opus 4.6 had fast mode at launch and LOST it (2026-06-29) — the param + is silently ignored (standard speed, standard billing), so exposing a + toggle for it would show users a switch that does nothing. + - Opus 4.7 hard-400s on the parameter. + - Dedicated ``…-fast`` model ids (e.g. OpenRouter's + ``claude-opus-4.8-fast``) select fast inference via the model field + and must not also receive the speed parameter. + + Keep this in lock-step with ``agent.anthropic_adapter._supports_fast_mode`` + so the UI never shows a Fast toggle that the runtime would drop. """ raw = _strip_vendor_prefix(str(model_id or "")) base = raw.split(":")[0] if not base.startswith("claude-"): return False - # Only Opus 4.6 supports the speed=fast parameter at present. - return "opus-4-6" in base or "opus-4.6" in base + if "-fast" in base: + return False + return any(v in base for v in ("opus-4-8", "opus-4.8", "opus-5")) -def resolve_fast_mode_overrides(model_id: Optional[str]) -> dict[str, Any] | None: +def _fast_mode_route_supported( + model_id: Optional[str], provider: Optional[str], base_url: Optional[str] +) -> bool: + """Only the first-party endpoint that bills for fast mode may receive its params. + + OpenRouter, Nous, Copilot, Azure, Bedrock, and custom base_urls either + strip ``service_tier``/``speed`` (charging nothing) or 400 on them. + """ + from urllib.parse import urlparse + + from agent.model_metadata import is_grok_46_family + + if _is_anthropic_fast_model(model_id): + allowed = {"anthropic": "api.anthropic.com"} + elif is_grok_46_family(str(model_id or "")): + allowed = {"xai": "api.x.ai"} + else: + allowed = {"openai": "api.openai.com", "openai-codex": "chatgpt.com"} + if provider and normalize_provider(provider) not in allowed: + return False + host = (urlparse(str(base_url or "")).hostname or "").lower() + return not host or host in allowed.values() + + +def resolve_fast_mode_overrides( + model_id: Optional[str], + *, + provider: Optional[str] = None, + base_url: Optional[str] = None, +) -> dict[str, Any] | None: """Return request_overrides for fast/priority mode, or None if unsupported. Returns provider-appropriate overrides: @@ -3994,12 +4204,21 @@ def resolve_fast_mode_overrides(model_id: Optional[str]) -> dict[str, Any] | Non - Anthropic models: ``{"speed": "fast"}`` (Anthropic Fast Mode beta) - Grok 4.6: ``{"service_tier": "priority"}`` (xAI Priority Processing) + When ``provider``/``base_url`` are given the result is also gated on the + route (see ``_fast_mode_route_supported``) so proxies never see the + params. This is the single fast-mode gate for static ``/fast fast`` and + the bounded ``auto``/``cold`` windows in ``agent.fast_mode``. + The overrides are injected into the API request kwargs by - ``_build_api_kwargs`` in run_agent.py — each API path handles its own - keys (service_tier for OpenAI/Codex, speed for Anthropic Messages). + ``build_api_kwargs`` — each API path handles its own keys + (service_tier for OpenAI/Codex, speed for Anthropic Messages). """ if not model_supports_fast_mode(model_id): return None + if (provider or base_url) and not _fast_mode_route_supported( + model_id, provider, base_url + ): + return None if _is_anthropic_fast_model(model_id): return {"speed": "fast"} return {"service_tier": "priority"} @@ -4017,13 +4236,18 @@ def _resolve_copilot_catalog_api_key() -> str: ``auth.json`` under ``credential_pool.copilot[]``. The pool is populated by ``hermes auth add copilot`` and by ``_seed_from_env`` when the env var is set in ``~/.hermes/.env``. + 3. ``~/.copilot/config.json`` ``copilotTokens`` — the GitHub Copilot + CLI's own store, written by ``copilot login`` on hosts without an + OS keychain. Without it, a user whose ONLY credential is the ACP + CLI login sees the copilot-acp picker fall back to the stale + curated list instead of the models their subscription serves. - Without (2), users whose only Copilot credential is in the pool see - the ``/model`` picker fall back to a stale hardcoded list because the - live catalog fetch silently 401s. To avoid wedging on a malformed pool - entry, each candidate is exchanged via ``exchange_copilot_token`` — - only entries that actually exchange successfully are returned, so a - later valid entry is reachable when an earlier one is unsupported. + Without (2)/(3), users without env-var credentials see the ``/model`` + picker fall back to a stale hardcoded list because the live catalog + fetch silently 401s. To avoid wedging on a malformed entry, each + candidate is exchanged via ``exchange_copilot_token`` — only entries + that actually exchange successfully are returned, so a later valid + entry is reachable when an earlier one is unsupported. """ try: from hermes_cli.auth import resolve_api_key_provider_credentials @@ -4052,7 +4276,11 @@ def _resolve_copilot_catalog_api_key() -> str: if not valid: continue try: - api_token, _expires_at = exchange_copilot_token(raw) + # exchange_copilot_token returns (api_token, expires_at, + # base_url) — a 2-name unpack raises ValueError, which the + # except below silently swallowed, disabling this entire + # resolution path. + api_token = exchange_copilot_token(raw)[0] except Exception: continue if api_token: @@ -4060,6 +4288,41 @@ def _resolve_copilot_catalog_api_key() -> str: except Exception: pass + # 3. Copilot CLI plaintext token store (JSONC — strip //-comment lines). + try: + import json as _json + + from hermes_cli.copilot_auth import ( + exchange_copilot_token, + validate_copilot_token, + ) + + cli_config = os.path.expanduser("~/.copilot/config.json") + if os.path.isfile(cli_config): + with open(cli_config, "r", encoding="utf-8", errors="ignore") as fh: + raw_text = "\n".join( + line for line in fh.read().splitlines() + if not line.lstrip().startswith("//") + ) + data = _json.loads(raw_text) if raw_text.strip() else {} + tokens = data.get("copilotTokens") + if isinstance(tokens, dict): + for raw in tokens.values(): + raw = str(raw or "").strip() + if not raw: + continue + valid, _ = validate_copilot_token(raw) + if not valid: + continue + try: + api_token = exchange_copilot_token(raw)[0] + except Exception: + continue + if api_token: + return api_token + except Exception: + pass + return "" @@ -5017,12 +5280,14 @@ def copilot_default_headers(*, is_agent_turn: bool = True) -> dict[str, str]: } -def _copilot_catalog_item_is_text_model(item: dict[str, Any]) -> bool: +def _copilot_catalog_item_is_text_model( + item: dict[str, Any], *, ignore_picker_flag: bool = False +) -> bool: model_id = str(item.get("id") or "").strip() if not model_id: return False - if item.get("model_picker_enabled") is False: + if not ignore_picker_flag and item.get("model_picker_enabled") is False: return False capabilities = item.get("capabilities") @@ -5101,6 +5366,25 @@ def fetch_github_model_catalog( continue seen_ids.add(model_id) models.append(item) + if not models and items: + # GitHub has been observed returning + # ``model_picker_enabled: false`` for EVERY model on some + # accounts/token types, which would silently reject the + # whole live catalog and strand the picker on the stale + # curated fallback. The flag is a display hint, not an + # availability contract — when honoring it empties the + # catalog, retry without it (chat/endpoint checks still + # apply, so embeddings and non-chat rows stay excluded). + for item in items: + if not _copilot_catalog_item_is_text_model( + item, ignore_picker_flag=True + ): + continue + model_id = str(item.get("id") or "").strip() + if not model_id or model_id in seen_ids: + continue + seen_ids.add(model_id) + models.append(item) if models: _github_model_catalog_cache = copy.deepcopy(models) _github_model_catalog_cache_key = api_key diff --git a/hermes_cli/nous_auth_keepalive.py b/hermes_cli/nous_auth_keepalive.py index 947bbd1787..f86acc705c 100644 --- a/hermes_cli/nous_auth_keepalive.py +++ b/hermes_cli/nous_auth_keepalive.py @@ -19,8 +19,31 @@ from hermes_cli.auth import ( logger = logging.getLogger(__name__) -NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS = 6 * 60 * 60 +# Two things have to line up for the keepalive to actually keep anything alive. +# +# 1. The tick has to be frequent enough to see the credential before it dies. +# Nous credential lifetimes are not fixed: this varies by account, and +# installs have been observed at both ~3594s and ~899s. The tick therefore +# derives from the lifetime the server actually issued rather than assuming +# an hour, capped by the configured interval and floored so a pathological +# lifetime cannot spin the thread. +# +# 2. The refresh has to fire while the tick can still act on it. The refresh +# only triggers once a credential is within a skew window of expiry, so a +# tick spaced wider than that window steps straight over it. The keepalive +# widens the window to "will this credential outlive my next tick?" instead +# of the request-path default of 120s. Without this, ticking faster only +# narrows the gap; it never closes it. +# +# The original 6-hour tick against a one-hour credential failed both tests, so +# in practice every hour expired reactively into a 401 plus a re-auth retry. +NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS = 15 * 60 +NOUS_AUTH_KEEPALIVE_MIN_INTERVAL_SECONDS = 60 +# Ticks per credential lifetime. Four keeps the refresh comfortably ahead of +# expiry without making the thread chatty. +NOUS_AUTH_KEEPALIVE_TICKS_PER_LIFETIME = 4 NOUS_AUTH_KEEPALIVE_INITIAL_DELAY_SECONDS = 60 +NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY = "keepalive_interval_seconds" _keepalive_lock = threading.Lock() _keepalive_stop = threading.Event() @@ -36,6 +59,92 @@ def _timeout_seconds(value: Optional[float]) -> float: return 15.0 +def _nous_config() -> dict: + """Return the ``nous:`` section of config.yaml, or {} on any failure. + + Imported lazily: this module is loaded by the gateway and the web server + during startup, and the config loader pulls in a wider dependency graph + than the keepalive itself needs. + """ + try: + from hermes_cli.config import load_config + + section = load_config().get("nous") + return section if isinstance(section, dict) else {} + except Exception: + return {} + + +def _interval_seconds(value: Optional[int]) -> int: + """Resolve the keepalive tick interval. + + Explicit argument wins, then ``nous.keepalive_interval_seconds`` in + config.yaml, then the module default. This is a behavioural threshold + rather than a credential, so it lives in config.yaml and not in .env. + A non-positive result disables the keepalive thread entirely, which is + the documented way to turn it off. + """ + if value is not None: + try: + return int(value) + except (TypeError, ValueError): + return NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS + + raw = _nous_config().get(NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS + try: + return int(float(raw)) + except (TypeError, ValueError): + logger.warning( + "Ignoring invalid nous.%s=%r; using %ds", + NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY, + raw, + NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS, + ) + return NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS + + +def _observed_lifetime_seconds() -> Optional[int]: + """Lifetime the server issued for the current Nous credentials, in seconds. + + Both the access token and the invoke agent key carry their own lifetime and + they are not always equal, so the shorter one governs. Returns None when no + usable value is stored, in which case the caller keeps its configured tick. + """ + state = get_provider_auth_state("nous") or {} + lifetimes = [] + for key in ("expires_in", "agent_key_expires_in"): + try: + value = int(float(state.get(key))) + except (TypeError, ValueError): + continue + if value > 0: + lifetimes.append(value) + return min(lifetimes) if lifetimes else None + + +def _tick_seconds(configured_interval: int, lifetime: Optional[int]) -> int: + """Tick fast enough to refresh several times per credential lifetime.""" + if not lifetime or lifetime <= 0: + return configured_interval + derived = lifetime // NOUS_AUTH_KEEPALIVE_TICKS_PER_LIFETIME + return max( + NOUS_AUTH_KEEPALIVE_MIN_INTERVAL_SECONDS, + min(configured_interval, derived), + ) + + +def _refresh_horizon_seconds(tick_seconds: int, floor_seconds: int) -> int: + """How much life a credential needs to be left alone this tick. + + A credential that will not survive until the next tick has to be refreshed + now, because nothing will look at it again before it expires. Hence + tick + skew rather than the request path's bare skew. + """ + return max(floor_seconds, tick_seconds + ACCESS_TOKEN_REFRESH_SKEW_SECONDS) + + def _entry_state(entry: object) -> dict: return { "agent_key": getattr(entry, "agent_key", None), @@ -47,6 +156,7 @@ def _entry_state(entry: object) -> dict: def _refresh_selected_pool_entry( *, min_key_ttl_seconds: int, + min_access_ttl_seconds: Optional[int] = None, ) -> Optional[bool]: """Refresh the current Nous credential pool entry when it is stale. @@ -73,9 +183,11 @@ def _refresh_selected_pool_entry( if entry is None: return False + if min_access_ttl_seconds is None: + min_access_ttl_seconds = ACCESS_TOKEN_REFRESH_SKEW_SECONDS access_expiring = _is_expiring( getattr(entry, "expires_at", None), - ACCESS_TOKEN_REFRESH_SKEW_SECONDS, + min_access_ttl_seconds, ) key_usable = _agent_key_is_usable(_entry_state(entry), min_key_ttl_seconds) if access_expiring or not key_usable: @@ -91,6 +203,7 @@ def _refresh_selected_pool_entry( def refresh_nous_auth_keepalive_once( *, min_key_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, + min_access_ttl_seconds: Optional[int] = None, timeout_seconds: Optional[float] = None, ) -> bool: """Refresh Nous auth once if credentials are configured.""" @@ -98,6 +211,7 @@ def refresh_nous_auth_keepalive_once( pool_result = _refresh_selected_pool_entry( min_key_ttl_seconds=min_key_ttl_seconds, + min_access_ttl_seconds=min_access_ttl_seconds, ) if pool_result is not None: return pool_result @@ -135,21 +249,28 @@ def _keepalive_loop( return while not stop_event.is_set(): + # Re-read each pass: the lifetime can change when the account, plan, or + # server-side policy does, and a keepalive that caches it would go stale + # in exactly the case it exists to cover. + tick = _tick_seconds(interval_seconds, _observed_lifetime_seconds()) + horizon = _refresh_horizon_seconds(tick, min_key_ttl_seconds) refresh_nous_auth_keepalive_once( - min_key_ttl_seconds=min_key_ttl_seconds, + min_key_ttl_seconds=horizon, + min_access_ttl_seconds=horizon, timeout_seconds=timeout_seconds, ) - stop_event.wait(interval_seconds) + stop_event.wait(tick) def start_nous_auth_keepalive( *, - interval_seconds: int = NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS, + interval_seconds: Optional[int] = None, initial_delay_seconds: int = NOUS_AUTH_KEEPALIVE_INITIAL_DELAY_SECONDS, min_key_ttl_seconds: int = NOUS_INVOKE_JWT_MIN_TTL_SECONDS, timeout_seconds: Optional[float] = None, ) -> Optional[threading.Thread]: """Start the process-wide Nous auth keepalive thread.""" + interval_seconds = _interval_seconds(interval_seconds) if interval_seconds <= 0: return None diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index 5a97c8a18d..162b6e70ad 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -933,7 +933,7 @@ class _Runtime: task, self.relay.tools.call_end, tool_call.handle, - fields, + self.relay.ToolExecutionResult(fields), metadata=self._event_metadata(), ) except Exception: diff --git a/hermes_cli/platform_actions.py b/hermes_cli/platform_actions.py index 52e0b68fce..de52a591e7 100644 --- a/hermes_cli/platform_actions.py +++ b/hermes_cli/platform_actions.py @@ -99,7 +99,36 @@ class PlatformActions: platform_enum = Platform(str(platform).strip().lower()) except Exception: return None, _err("unknown_platform", f"unknown platform {platform!r}") - adapter = getattr(runner, "adapters", {}).get(platform_enum) + # Multiplex/Team-Gateway: a secondary profile's adapters live in + # runner._profile_adapters[profile], not runner.adapters (the default + # profile's registry) — every other adapter-resolution path in this + # codebase (_authorization_adapter, plugin message-injection) goes + # through this same profile-aware, fail-closed lookup so a plugin + # scoped to one profile can never act through another profile's bot + # identity. Falls back to the bare default-profile lookup only when + # the gateway runner predates this method (defensive, not expected). + resolve_fn = getattr(runner, "_authorization_adapter", None) + if callable(resolve_fn): + try: + from hermes_cli.profiles import get_active_profile_name + + profile_name = get_active_profile_name() + except Exception: + # Fail closed: an unresolvable profile must not degrade to the + # default profile's bot (the same rule _authorization_adapter + # applies to a stamped profile with no registry entry). + logger.debug( + "platform_actions: profile resolution failed for %s", + self._plugin_id, exc_info=True, + ) + return None, _err( + "adapter_not_registered", + f"no {platform_enum.value} adapter is registered " + "(active profile could not be resolved)", + ) + adapter = resolve_fn(platform_enum, profile_name) + else: + adapter = getattr(runner, "adapters", {}).get(platform_enum) if adapter is None: return None, _err( "adapter_not_registered", diff --git a/hermes_cli/plugins.py b/hermes_cli/plugins.py index 2d3eb9fca7..d950dac09f 100644 --- a/hermes_cli/plugins.py +++ b/hermes_cli/plugins.py @@ -4282,18 +4282,21 @@ class PluginManager: # first process sees plugin backends (tracking #64177). self._refresh_secret_sources_after_discovery() if force: - # config.yaml shell hooks live in ``_hooks`` but are - # config-owned, not plugin-owned — the ledger-driven - # unload() above wiped them and cannot restore them. - # Re-register so force-reload is symmetric (#60036; - # tracking #64178 — salvaged from PR #64188). - self._re_register_shell_hooks_after_force() + # config.yaml shell hooks and outbound webhooks live in + # ``_hooks`` but are config-owned, not plugin-owned — + # the ledger-driven unload() above wiped them and + # cannot restore them. Re-register so force-reload is + # symmetric (#60036; tracking #64178 — salvaged from + # PR #64188; outbound webhooks added per #92682 review). + self._re_register_config_hooks_after_force() except BaseException: self._discovered = False raise - def _re_register_shell_hooks_after_force(self) -> None: - """Restore config.yaml shell hooks wiped by force-clear of ``_hooks``.""" + def _re_register_config_hooks_after_force(self) -> None: + """Restore config.yaml shell hooks/outbound webhooks wiped by + force-clear of ``_hooks``. Each re-register call is independently + guarded so one failing does not skip the other.""" try: from agent.shell_hooks import re_register_config_hooks @@ -4301,6 +4304,14 @@ class PluginManager: except Exception as exc: # Import cycle / missing module must not abort force reload. logger.debug("force-reload shell-hook re-register skipped: %s", exc) + try: + from agent.outbound_webhooks import ( + re_register_config_hooks as re_register_outbound_webhooks, + ) + + re_register_outbound_webhooks() + except Exception as exc: + logger.debug("force-reload outbound-webhook re-register skipped: %s", exc) def _refresh_secret_sources_after_discovery(self) -> None: """If any plugin secret source is enabled, reset cache and re-apply. diff --git a/hermes_cli/profiles.py b/hermes_cli/profiles.py index 669b8fba32..dddd96be5c 100644 --- a/hermes_cli/profiles.py +++ b/hermes_cli/profiles.py @@ -849,6 +849,22 @@ def _check_gateway_running(profile_dir: Path) -> bool: return False +def _served_by_running_multiplexer(profile_name: str) -> bool: + """True when the live default gateway multiplexes ``profile_name``. + + A served named profile has no gateway.pid of its own, so + ``_check_gateway_running`` alone reports it stopped while the default + multiplexer is actually its inbound process. Single shared lookup with the + named-profile start guard and cron liveness (#97120). + """ + try: + from hermes_cli.gateway import named_profile_served_by_running_multiplexer + + return named_profile_served_by_running_multiplexer(profile_name) + except Exception: + return False + + # In-process cache for skill counts. Walking ``skills_dir.rglob("SKILL.md")`` # recurses the entire skill tree (each skill carries references/scripts/assets # sub-trees); the default profile alone has ~270 skills, and ``list_profiles`` @@ -1095,7 +1111,10 @@ def list_profiles() -> List[ProfileInfo]: name=name, path=entry, is_default=False, - gateway_running=_check_gateway_running(entry), + gateway_running=( + _check_gateway_running(entry) + or _served_by_running_multiplexer(name) + ), model=model, provider=provider, has_env=(entry / ".env").exists(), @@ -1272,6 +1291,18 @@ def create_profile( # Strip runtime files for stale in _CLONE_ALL_STRIP: (profile_dir / stale).unlink(missing_ok=True) + # A clone-all copies auth.json and .anthropic_oauth.json verbatim. + # Single-use OAuth grants (Anthropic / Codex / xAI) forked that way + # are one credential with two owners: the first profile to refresh + # revokes the pair for every sibling (#100339). Drop the copies; the + # clone reads the root grant through the credential-pool fallback. + from hermes_cli.auth import strip_cloned_single_use_oauth_grants + stripped = strip_cloned_single_use_oauth_grants(profile_dir) + if any(stripped.values()): + logger.info( + "profile %s: dropped cloned single-use OAuth grants %s " + "(inherits the root grant instead)", canon, stripped, + ) else: # Bootstrap directory structure profile_dir.mkdir(parents=True, exist_ok=True) @@ -2019,6 +2050,19 @@ def _stop_gateway_process(profile_dir: Path) -> None: raw = pid_file.read_text(encoding="utf-8-sig").strip() data = json.loads(raw) if raw.startswith("{") else {"pid": int(raw)} pid = int(data["pid"]) + # Cross-profile kill refusal (#89315): the record's hermes_home stamp + # names the gateway's TRUE owner. A contaminated/poisoned gateway.pid + # inside this profile dir can point at another profile's live gateway + # — killing it starts the mutual SIGTERM restart loop from the issue. + from gateway.status import recorded_gateway_home_conflicts + + if recorded_gateway_home_conflicts(data, expected_home=profile_dir): + print( + f"✗ Refusing to stop PID {pid}: its recorded HERMES_HOME " + f"belongs to a different profile than {profile_dir} " + "(stale/poisoned PID record, #89315)." + ) + return # Route through terminate_pid so Windows uses the appropriate # primitive (taskkill / TerminateProcess) — raw os.kill with # _signal.SIGKILL raises AttributeError at import time on Windows, diff --git a/hermes_cli/proxy/adapters/nous_portal.py b/hermes_cli/proxy/adapters/nous_portal.py index 18c0123a25..913f99dff2 100644 --- a/hermes_cli/proxy/adapters/nous_portal.py +++ b/hermes_cli/proxy/adapters/nous_portal.py @@ -82,18 +82,19 @@ class NousPortalAdapter(UpstreamAdapter): failed_credential: UpstreamCredential, status_code: int, ) -> Optional[UpstreamCredential]: - _ = failed_credential if status_code != 401: return None logger.info("proxy: Nous upstream rejected bearer; force-refreshing invoke JWT") return self._get_credential( force_refresh=True, + stale_access_token=failed_credential.bearer, ) def _get_credential( self, *, force_refresh: bool = False, + stale_access_token: Optional[str] = None, ) -> UpstreamCredential: with self._lock: state = self._read_state() @@ -105,6 +106,7 @@ class NousPortalAdapter(UpstreamAdapter): try: refreshed = resolve_nous_runtime_credentials( force_refresh=force_refresh, + stale_access_token=stale_access_token or None, ) except AuthError as exc: if _is_terminal_nous_refresh_error(exc): diff --git a/hermes_cli/proxy/server.py b/hermes_cli/proxy/server.py index 880ac4830a..e0734ce6be 100644 --- a/hermes_cli/proxy/server.py +++ b/hermes_cli/proxy/server.py @@ -112,11 +112,17 @@ def create_app(adapter: UpstreamAdapter) -> "web.Application": app[_adapter_key] = adapter async def handle_health(request: "web.Request") -> "web.Response": + # ``is_authenticated`` is documented as cheap (see UpstreamAdapter), + # but both shipped adapters read auth state off disk, and the Nous one + # does it under ``_auth_store_lock()`` — 15s cross-process. Offload it + # so a healthcheck poll can never freeze the loop behind a lock held by + # a concurrent ``hermes auth`` command. + authenticated = await asyncio.to_thread(adapter.is_authenticated) return web.json_response( { "status": "ok", "upstream": adapter.display_name, - "authenticated": adapter.is_authenticated(), + "authenticated": authenticated, } ) @@ -134,8 +140,14 @@ def create_app(adapter: UpstreamAdapter) -> "web.Application": code="path_not_allowed", ) + # ``UpstreamAdapter.get_credential`` is synchronous and hard-blocking: + # the Nous adapter takes ``_auth_store_lock()`` (a cross-process lock + # with a 15s timeout), reads auth.json, and may perform a token-refresh + # POST, taking the lock a second time to persist a terminal error. Run + # it on a worker thread so a refresh or a contended lock cannot freeze + # every other in-flight streaming completion on this single loop. try: - cred = adapter.get_credential() + cred = await asyncio.to_thread(adapter.get_credential) except Exception as exc: logger.warning("proxy: credential resolution failed: %s", exc) return _json_error(401, str(exc), code="upstream_auth_failed") @@ -211,8 +223,16 @@ def create_app(adapter: UpstreamAdapter) -> "web.Application": session = session_or_response if upstream_resp.status in {401, 429}: + # Third and last blocking method on the adapter contract, and the + # most expensive: the Nous adapter routes this straight into + # ``_get_credential(force_refresh=True)``, so the refresh POST that + # ``get_credential`` only performs near expiry is unconditional + # here — under the same 15s cross-process ``_auth_store_lock()``. + # The xAI adapter loads its key pool off disk and rotates it under + # ``self._lock``. Offload it for the same reason as the two above. try: - retry_cred = adapter.get_retry_credential( + retry_cred = await asyncio.to_thread( + adapter.get_retry_credential, failed_credential=cred, status_code=upstream_resp.status, ) diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index 605d2d7135..bee78f192e 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -1952,6 +1952,33 @@ def _resolve_explicit_runtime( return None +def _is_external_process_provider(provider: str) -> bool: + """Whether ``provider`` is declared as an external-process (CLI) provider. + + Reads the CLI provider registry first (which now absorbs registered + ProviderProfiles, in-tree and out), then falls back to the profile registry + directly so the check works before the CLI registry has been extended. + """ + name = (provider or "").strip().lower() + if not name: + return False + try: + from hermes_cli.auth import PROVIDER_REGISTRY + + pconfig = PROVIDER_REGISTRY.get(name) + if pconfig is not None: + return pconfig.auth_type == "external_process" + except Exception: + pass + try: + from providers import get_provider_profile + + profile = get_provider_profile(name) + except Exception: + return False + return profile is not None and getattr(profile, "auth_type", "") == "external_process" + + def resolve_runtime_provider( *, requested: Optional[str] = None, @@ -2337,10 +2364,13 @@ def resolve_runtime_provider( "requested_provider": requested_provider, } - if provider == "copilot-acp": + # External-process providers (an agent CLI driven over stdio, e.g. ACP). + # Keyed on the registered provider's auth_type rather than on one name, so a + # provider shipped outside this tree lands on the same credential path. + if _is_external_process_provider(provider): creds = resolve_external_process_provider_credentials(provider) return { - "provider": "copilot-acp", + "provider": provider, "api_mode": "chat_completions", "base_url": creds.get("base_url", "").rstrip("/"), "api_key": creds.get("api_key", ""), diff --git a/hermes_cli/session_lost_and_found.py b/hermes_cli/session_lost_and_found.py index 90d8acba9a..09a362076b 100644 --- a/hermes_cli/session_lost_and_found.py +++ b/hermes_cli/session_lost_and_found.py @@ -23,6 +23,7 @@ data, and derived FTS indexes are rebuilt from scratch. from __future__ import annotations +import logging import re import shutil import sqlite3 @@ -31,6 +32,8 @@ import tempfile from pathlib import Path from typing import Any, Optional +logger = logging.getLogger(__name__) + # Hermes session ids are timestamps: 20260812_135332_ab12cd. This is the # strongest sentinel available for classifying schema-less rows. SESSION_ID_PATTERN = re.compile(r"^\d{8}_\d{6}_") @@ -55,6 +58,11 @@ SESSION_MODEL_USAGE_NFIELD = 18 _EPOCH_LOW = 1_000_000_000.0 # 2001 _EPOCH_HIGH = 4_000_000_000.0 # 2096 +# Title prefix of every session row this lane synthesises (legacy-layout rows +# and stubbed parents). The recovery verifier keys on it to tell synthesised +# rows from positionally mapped ones. +STUB_TITLE_PREFIX = "[best-effort recovered" + SQLITE3_CLI_GUIDANCE = ( "A last-resort page-level salvage is available when a `.recover`-capable " "`sqlite3` command-line shell is installed: its `.recover` command can " @@ -67,25 +75,122 @@ SQLITE3_CLI_GUIDANCE = ( "with --allow-partial." ) +# SQLite's WAL-reset bug (https://sqlite.org/wal.html#walresetbug) lets a +# fresh opener unlink a live WAL/SHM sidecar pair and split the database into +# two concurrent generations whose acknowledged writes can silently vanish. +# It is real in CLI builds up to 3.51.2; fixed in 3.51.3+ with backports +# 3.50.7 and 3.44.6 — the same version gate hermes_state applies to the +# embedded library (#69784). The system `sqlite3` CLI on Debian/Ubuntu is +# routinely in the vulnerable band (e.g. 3.45.1), and #100368's forensics +# caught exactly this shell converting a live Hermes state.db into two +# generations. A salvage shell must therefore be version-gated, not just +# capability-gated, before it is pointed at (a copy of) a Hermes database. +# +# The predicate lives in hermes_cli.sqlite_runtime (stdlib-only, shared with +# the installer/update gates) so the embedded runtime and the salvage shell +# can never disagree about which versions are safe. +from hermes_cli.sqlite_runtime import is_sqlite_wal_reset_vulnerable as _wal_reset_vulnerable # noqa: E502 + +_WAL_RESET_VULNERABLE_GUIDANCE = ( + "salvage against a Hermes database with the WAL-reset bug " + "(https://sqlite.org/wal.html#walresetbug, fixed in 3.51.3+ / backports " + "3.50.7 / 3.44.6; the vulnerable fresh-opener can unlink a live WAL/SHM " + "pair and split the database into two generations, losing acknowledged " + "writes — #100368). Install a fixed sqlite3 CLI (3.51.3+, e.g. `brew " + "install sqlite` or the precompiled sqlite-tools from sqlite.org)" +) + class LostAndFoundError(RuntimeError): """Raised when the CLI .recover pass cannot produce a usable database.""" -def find_sqlite3_cli() -> Optional[str]: - """Return a ``.recover``-capable sqlite3 CLI path, or None. +def _parse_sqlite3_cli_version(binary: str) -> Optional[tuple[int, int, int]]: + """Parse the reporting version of the sqlite3 CLI at *binary*. - PATH presence is not enough: distro builds (e.g. Ubuntu's) can ship a - sqlite3 shell compiled without the ``sqlite_dbpage`` virtual table that - ``.recover`` requires — those fail every recovery with - ``no such table: sqlite_dbpage``. Probe capability on a scratch DB once - instead of discovering it mid-recovery. + Returns ``None`` when the CLI cannot be executed or its version line + cannot be understood (older shells print the version only in + interactive mode; the modern ``--version`` flag covers every build in + the supported range). """ + try: + probe = subprocess.run( + [binary, "--version"], + capture_output=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + return None + if probe.returncode != 0: + return None + match = re.search(rb"(\d+)\.(\d+)\.(\d+)", probe.stdout) + if match is None: + return None + return tuple(int(part) for part in match.groups()) + +_last_cli_refusal: dict[str, Any] = {} + + +def find_sqlite3_cli_refusal() -> dict[str, Any]: + """Why the last :func:`find_sqlite3_cli` call in this process refused. + + ``{"reason": ...}`` with ``reason`` in ``missing``, ``no_dbpage`` (the + shell cannot run ``.recover``), or ``wal_reset_vulnerable``; empty when + the last probe found a usable shell or never ran. + """ + return dict(_last_cli_refusal) + + +def find_sqlite3_cli() -> Optional[str]: + """Return a salvage-safe ``.recover``-capable sqlite3 CLI path, or None. + + PATH presence is not enough, and neither is `.recover` support alone: + + 1. Distro builds (e.g. Ubuntu's) can ship a sqlite3 shell compiled + without the ``sqlite_dbpage`` virtual table that ``.recover`` + requires — those fail every recovery with ``no such table: + sqlite_dbpage``. Capability is probed on a scratch DB once. + 2. A `.recover`-capable CLI can still carry the WAL-reset opener bug + (fixed 3.51.3+ / backports 3.50.7 / 3.44.6). The salvage lane runs + the CLI against a *snapshot copy* of the source, so it cannot hit + the live sidecars itself; but the same binary is what operators + reach for when following the old guidance, and refusing it here + keeps the vulnerable shells out of the documented workflow + entirely. Probe the version once. + + Refusals are recorded for :func:`find_sqlite3_cli_refusal` so callers + can explain exactly what to install instead of a generic "not found". + """ + global _last_cli_refusal + _last_cli_refusal = {} binary = shutil.which("sqlite3") if binary is None: + _last_cli_refusal = {"reason": "missing"} return None - return binary if _cli_supports_recover(binary) else None + if not _cli_supports_recover(binary): + _last_cli_refusal = {"reason": "no_dbpage", "binary": binary} + return None + version = _parse_sqlite3_cli_version(binary) + if version is not None and _wal_reset_vulnerable(version): + version_str = ".".join(str(part) for part in version) + logger.warning( + "sqlite3 CLI %s reports version %s, which still carries the " + "WAL-reset opener bug; refusing to use it for salvage", + binary, + version_str, + ) + _last_cli_refusal = { + "reason": "wal_reset_vulnerable", + "binary": binary, + "version": version_str, + "detail": ( + f"reports version {version_str}, which has " + + _WAL_RESET_VULNERABLE_GUIDANCE + ), + } + return None + return binary def _cli_supports_recover(binary: str) -> bool: @@ -325,20 +430,25 @@ def _copy_direct_tables( ) -> dict[str, int]: """Copy rows .recover managed to attribute to real canonical tables.""" + # Lazy import: session_recovery imports this module inside a function, so + # a module-level import here would be circular. + from hermes_cli.session_recovery import ( + _AUXILIARY_TABLE_SCHEMAS, + _AUXILIARY_TABLES, + _CANONICAL_TABLES, + ) + copied: dict[str, int] = {} - for table in ( - "system_prompts", - "sessions", - "messages", - "session_model_usage", - "compression_locks", - "gateway_routing", - "async_delegations", - ): + for table in (*_CANONICAL_TABLES, *_AUXILIARY_TABLES): source_columns = _table_columns(lf_conn, table) if not source_columns: continue dest_columns = _table_columns(dest, table) + if not dest_columns and table in _AUXILIARY_TABLE_SCHEMAS: + # Lazily-created gateway table: base SessionDB never made it on + # the fresh destination, so create it before copying. + _AUXILIARY_TABLE_SCHEMAS[table](dest) + dest_columns = _table_columns(dest, table) columns = [c for c in dest_columns if c in source_columns] if not columns: continue @@ -451,7 +561,7 @@ def map_lost_and_found_rows( cells[1] if _looks_like_source(cells[1]) else "recovered", _heuristic_started_at(cells), - "[best-effort recovered] legacy session " + f"{STUB_TITLE_PREFIX}] legacy session " "row (layout unknown)", ), ).rowcount @@ -520,7 +630,7 @@ def stub_missing_parent_sessions(dest: sqlite3.Connection) -> dict[str, Any]: for session_id, info in sorted(orphan_ids.items()): while True: title = ( - f"[best-effort recovered {sequence}] session metadata " + f"{STUB_TITLE_PREFIX} {sequence}] session metadata " "was unreadable" ) sequence += 1 diff --git a/hermes_cli/session_recovery.py b/hermes_cli/session_recovery.py index 6d5f5e8bf8..14ea0bfff9 100644 --- a/hermes_cli/session_recovery.py +++ b/hermes_cli/session_recovery.py @@ -45,6 +45,34 @@ _TOPIC_TABLES = ( "telegram_dm_topic_bindings", ) + + +def _init_delivery_ledger_schema(conn: sqlite3.Connection) -> None: + from gateway.delivery_ledger import _initialize_schema + + _initialize_schema(conn) + + +# Tables that live in state.db but are created lazily by a gateway module on +# first use, so base ``SessionDB`` never creates them on a fresh destination. +# Every entry maps the table to the initializer that owns its DDL; recovery +# creates the table on the destination before copying, so owed rows survive +# instead of silently vanishing from a "complete" salvage (#100313, #86236). +# Add new lazily-created state.db tables HERE, never as one-off ``if table ==`` +# branches. +_AUXILIARY_TABLE_SCHEMAS: dict[str, Callable[[sqlite3.Connection], None]] = { + "delivery_obligations": _init_delivery_ledger_schema, +} + +_AUXILIARY_TABLES = tuple(_AUXILIARY_TABLE_SCHEMAS) + +_INVENTORY_TABLES = ( + *_CANONICAL_TABLES, + "state_meta", + *_TOPIC_TABLES, + *_AUXILIARY_TABLES, +) + # These values describe derived indexes or the schema that owns an optional # table. A fresh destination must generate them from its own current schema. _GENERATED_META_KEYS = frozenset({ @@ -305,7 +333,7 @@ def _inspect_connection(conn: sqlite3.Connection) -> dict[str, Any]: # A damaged journal pragma must not block rows that are still readable. report["warnings"].append(f"journal mode: {exc}") - for table in (*_CANONICAL_TABLES, "state_meta", *_TOPIC_TABLES): + for table in _INVENTORY_TABLES: report["tables"][table] = _table_inventory(conn, table) for required in ("sessions", "messages"): @@ -385,6 +413,25 @@ def inspect_session_database( temp_dir.cleanup() +def _ensure_auxiliary_destination_schema( + destination: sqlite3.Connection, + table: str, +) -> None: + """Create a lazy auxiliary table on the recovered destination. + + Recovery initializes the destination through base ``SessionDB``, which + does not create gateway-owned tables. Copying into a missing dest table + would report ``missing`` / ``no compatible columns`` and drop the rows. + """ + + initialize = _AUXILIARY_TABLE_SCHEMAS.get(table) + if initialize is None: + raise SessionRecoverySafetyError( + f"no destination schema initializer registered for table {table!r}" + ) + initialize(destination) + + def _copy_table( source: sqlite3.Connection, destination: sqlite3.Connection, @@ -1243,7 +1290,7 @@ def _verify_recovered_database( ) counts: dict[str, int] = {} - for table in (*_CANONICAL_TABLES, "state_meta", *_TOPIC_TABLES): + for table in _INVENTORY_TABLES: columns = _table_columns(conn, table) if columns: counts[table] = int( @@ -1251,7 +1298,7 @@ def _verify_recovered_database( ) verification["table_counts"] = counts - for table in ("sessions", "messages"): + for table in ("sessions", "messages", *_AUXILIARY_TABLES): expected = expected_counts.get(table) if expected is not None and counts.get(table) != expected: message = ( @@ -1384,6 +1431,49 @@ def _finalize_derived_metadata(destination: sqlite3.Connection) -> dict[str, Any return result +def _lost_and_found_plausibility_errors( + conn: sqlite3.Connection, +) -> list[str]: + """Flag systematic timestamp mis-mapping in a salvaged database. + + Structural checks (integrity, FK, FTS, row counts) pass on mis-mapped + salvage because every row still inserts. Only semantics give it away: + the physical column order of a source upgraded via ALTER TABLE differs + from the destination template's declared order, so positional cell + mapping lands counters/strings where ``started_at``/``timestamp`` + belong — and the NOT NULL substitutes turn gaps into 0.0. When every + mapped row violates the epoch floor, the mapping was wrong. + + Stub rows written by ``stub_missing_parent_sessions`` legitimately carry + ``started_at = 0.0`` when no timestamped message survived, so they are + excluded from the denominator. + """ + from hermes_cli.session_lost_and_found import _EPOCH_LOW, STUB_TITLE_PREFIX + + errors: list[str] = [] + checks = ( + ("sessions", "started_at", f"WHERE COALESCE(title, '') NOT LIKE '{STUB_TITLE_PREFIX}%'"), + ("messages", "timestamp", ""), + ) + for table, column, mapped_filter in checks: + (total,) = conn.execute(f"SELECT COUNT(*) FROM {table} {mapped_filter}").fetchone() + if not total: + continue + (implausible,) = conn.execute( + f"SELECT COUNT(*) FROM {table} {mapped_filter} " + f"{'AND' if mapped_filter else 'WHERE'} ({column} IS NULL OR {column} < ?)", + (_EPOCH_LOW,), + ).fetchone() + if implausible == total: + errors.append( + f"{table}.{column} is implausible in all {total} salvaged row(s) " + "(NULL or before 2001-09): the source's physical column order " + "did not match the destination template, so cells were mapped " + "onto the wrong columns" + ) + return errors + + def _recover_via_lost_and_found( *, source: Path, @@ -1406,6 +1496,7 @@ def _recover_via_lost_and_found( SQLITE3_CLI_GUIDANCE, LostAndFoundError, find_sqlite3_cli, + find_sqlite3_cli_refusal, map_lost_and_found_rows, rebuild_fts_indexes, run_cli_lost_and_found_recover, @@ -1414,6 +1505,16 @@ def _recover_via_lost_and_found( sqlite3_bin = find_sqlite3_cli() if sqlite3_bin is None: + refusal = find_sqlite3_cli_refusal() + if refusal.get("reason") == "wal_reset_vulnerable": + raise SessionRecoverySourceError( + "Partial recovery requires a page-level salvage shell, but " + "the only sqlite3 CLI on PATH is not safe to use for it: it " + + refusal["detail"] + + ". The readable table schemas for: " + + ", ".join(missing_required) + + " are still required." + ) raise SessionRecoverySourceError( "Partial recovery still requires readable table schemas for: " + ", ".join(missing_required) @@ -1483,6 +1584,21 @@ def _recover_via_lost_and_found( ) verification["complete"] = False + # Structural checks cannot see a positional mis-mapping (#101409): + # every row still inserts, so integrity/FK/FTS stay green. A + # systematic timestamp violation is the semantic tell — surface it + # so a mis-mapped salvage is never reported as verified. + plausibility_conn = sqlite3.connect(str(output), isolation_level=None) + try: + plausibility_errors = _lost_and_found_plausibility_errors( + plausibility_conn + ) + finally: + plausibility_conn.close() + if plausibility_errors: + verification["errors"].extend(plausibility_errors) + verification["healthy"] = False + source_unchanged = ( _source_fingerprint(source) == inspection["source_fingerprint"] ) @@ -1662,6 +1778,27 @@ def recover_session_database( progress_cb=progress_cb, source_rows=table_inspection.get("rows"), ) + + for table in _AUXILIARY_TABLES: + table_inspection = inspection["tables"][table] + if not table_inspection.get("available"): + copy_report[table] = { + "status": "missing", + "copied_rows": 0, + } + continue + _ensure_auxiliary_destination_schema(destination_conn, table) + copy_function = ( + _copy_table_salvage if allow_partial else _copy_table + ) + copy_report[table] = copy_function( + source_conn, + destination_conn, + table, + chunk_size=chunk_size, + progress_cb=progress_cb, + source_rows=table_inspection.get("rows"), + ) orphan_cleanup = ( _cleanup_partial_orphans(destination_conn) if allow_partial @@ -1678,8 +1815,15 @@ def recover_session_database( verification = _verify_recovered_database( output, expected_counts={ - table: inspection["tables"][table].get("rows") - for table in _CANONICAL_TABLES + **{ + table: inspection["tables"][table].get("rows") + for table in _CANONICAL_TABLES + }, + **{ + table: inspection["tables"][table].get("rows") + for table in _AUXILIARY_TABLES + if inspection["tables"].get(table, {}).get("available") + }, }, copy_report=copy_report, allow_partial=allow_partial, diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 9a3d0b6f98..d528ddae8e 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -112,7 +112,7 @@ _DEFAULT_PROVIDER_MODELS = { "ai-gateway": ["anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "openai/gpt-5", "google/gemini-3-flash"], "kilocode": ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "openai/gpt-5.4", "google/gemini-3-pro-preview", "google/gemini-3-flash-preview"], "opencode-zen": ["x-preview-f-free", "gpt-5.6-sol", "gpt-5.4", "gpt-5.3-codex", "claude-opus-5", "claude-sonnet-5", "gemini-3.7-flash", "glm-5.2", "kimi-k3", "minimax-m3"], - "opencode-free": ["deepseek-v4-flash-free", "hy3-free", "mimo-v2.5-free", "laguna-s-2.1-free", "nemotron-3-ultra-free", "nemotron-3.5-lightning-free", "muse-spark-1.2-contributor-free"], + "opencode-free": ["deepseek-v4-flash-free", "hy3-free", "mimo-v2.5-free", "laguna-s-2.1-free", "nemotron-3-ultra-free", "nemotron-3.5-lightning-free", "muse-spark-1.2-contributor-free", "muse-spark-1.3-contributor-free"], "opencode-go": ["kimi-k3", "kimi-k2.7-code", "kimi-k2.6", "gpt-5.6-luna", "grok-4.5", "glm-5.3", "glm-5.3-flash", "glm-5.2", "mimo-v2.5-pro", "mimo-v2.5", "minimax-m3", "minimax-m2.7", "qwen3.8-max", "qwen3.7-max", "deepseek-v4-pro", "hy3"], "huggingface": [ "Qwen/Qwen3.5-397B-A17B", "Qwen/Qwen3-235B-A22B-Thinking-2507", diff --git a/hermes_cli/subcommands/cron.py b/hermes_cli/subcommands/cron.py index b9d7f08569..4501578b2c 100644 --- a/hermes_cli/subcommands/cron.py +++ b/hermes_cli/subcommands/cron.py @@ -42,6 +42,16 @@ def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None: "local profile's canonical Bot Chat as a message the bot responds to)" ), ) + cron_create.add_argument( + "--failure-deliver", + dest="failure_deliver", + help=( + "Override target for FAILURE notices only (same grammar as " + "--deliver). 'local' suppresses failure notices entirely; run " + "state stays visible in `hermes cron list`. Omit = failures " + "follow --deliver." + ), + ) cron_create.add_argument("--repeat", type=int, help="Optional repeat count") cron_create.add_argument( "--skill", @@ -142,6 +152,14 @@ def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None: cron_edit.add_argument("--prompt", help="New prompt/task instruction") cron_edit.add_argument("--name", help="New job name") cron_edit.add_argument("--deliver", help="New delivery target") + cron_edit.add_argument( + "--failure-deliver", + dest="failure_deliver", + help=( + "Override target for failure notices (same grammar as --deliver; " + "'local' suppresses; '' clears the override)" + ), + ) cron_edit.add_argument("--repeat", type=int, help="New repeat count") cron_edit.add_argument( "--skill", diff --git a/hermes_cli/subcommands/dashboard.py b/hermes_cli/subcommands/dashboard.py index 0b695e076a..8b2e6cc487 100644 --- a/hermes_cli/subcommands/dashboard.py +++ b/hermes_cli/subcommands/dashboard.py @@ -84,6 +84,60 @@ def _add_server_runtime_args(parser) -> None: ) +def _configure_serve_parser(parser, *, cmd_dashboard: Callable) -> None: + """Attach the canonical ``serve`` arguments to *parser*. + + Kept separate from the full subcommand tree so Desktop's hot path can parse + only the command it launches. Both callers use this exact function, keeping + the lean parser and normal CLI semantics in lockstep. + """ + _add_server_runtime_args(parser) + # Accepted but redundant: ``serve`` is always headless. Kept so callers + # using the legacy flag do not trip an argparse error. + parser.add_argument("--no-open", action="store_true", help=argparse.SUPPRESS) + parser.add_argument( + "--ssh-session-token-file", + dest="ssh_session_token_file", + metavar="PATH", + default=None, + help="Read a one-shot Desktop SSH session token from PATH", + ) + parser.add_argument( + "--ssh-owner-nonce", + dest="ssh_owner_nonce", + metavar="NONCE", + default=None, + help="Identify a Desktop-owned SSH backend process", + ) + parser.set_defaults( + func=cmd_dashboard, + no_open=True, + headless_backend=True, + command="serve", + ) + + +def build_serve_parser( + *, + cmd_dashboard: Callable, + add_help: bool = True, + exit_on_error: bool = True, +) -> argparse.ArgumentParser: + """Build the standalone parser used by the lean ``serve`` dispatch path.""" + parser = argparse.ArgumentParser( + prog="hermes serve", + description=( + "Run the Hermes backend server - the JSON-RPC/WebSocket gateway the " + "desktop app and remote clients connect to. Headless: it never opens " + "a browser UI." + ), + add_help=add_help, + exit_on_error=exit_on_error, + ) + _configure_serve_parser(parser, cmd_dashboard=cmd_dashboard) + return parser + + def build_dashboard_parser( subparsers, *, cmd_dashboard: Callable, cmd_dashboard_register: Callable ) -> None: @@ -142,32 +196,7 @@ def build_dashboard_parser( "a browser UI." ), ) - _add_server_runtime_args(serve_parser) - # Accepted but redundant: `serve` is always headless (see set_defaults - # below). Kept so callers that pass the legacy `--no-open` flag (e.g. the - # desktop backend spawn) don't trip "unrecognized arguments". - serve_parser.add_argument( - "--no-open", action="store_true", help=argparse.SUPPRESS - ) - serve_parser.add_argument( - "--ssh-session-token-file", - dest="ssh_session_token_file", - metavar="PATH", - default=None, - help="Read a one-shot Desktop SSH session token from PATH", - ) - serve_parser.add_argument( - "--ssh-owner-nonce", - dest="ssh_owner_nonce", - metavar="NONCE", - default=None, - help="Identify a Desktop-owned SSH backend process", - ) - # `headless_backend` marks the lean path: desktop/remote clients speak pure - # JSON-RPC/WS, so `serve` skips the web UI build AND never serves the SPA - # (cmd_dashboard exports HERMES_SERVE_HEADLESS=1). `dashboard` leaves it - # unset and serves the browser UI as before. - serve_parser.set_defaults(func=cmd_dashboard, no_open=True, headless_backend=True) + _configure_serve_parser(serve_parser, cmd_dashboard=cmd_dashboard) # `hermes dashboard register` — register a self-hosted dashboard OAuth # client with Nous Portal and write the client_id into ~/.hermes/.env. diff --git a/hermes_cli/terminal_notify.py b/hermes_cli/terminal_notify.py new file mode 100644 index 0000000000..6c1877cd8c --- /dev/null +++ b/hermes_cli/terminal_notify.py @@ -0,0 +1,99 @@ +"""Terminal-native desktop notifications: OSC 9 and Warp's OSC 777 CLI-agent protocol. + +Both emitters ride on the existing ``display.bell_on_prompt`` / +``display.bell_on_complete`` flags (see ``cli._ring_bell``) — no extra config. + +- **OSC 9** (``ESC ] 9 ; BEL``): Ghostty, iTerm2, Kitty and WezTerm + raise an OS notification; terminals that don't know the sequence drop it. +- **OSC 777** (``ESC ] 777 ; notify ; warp://cli-agent ; BEL``): Warp's + structured CLI-agent protocol (tab status + notification mailbox). Only sent + when Warp advertises support and the build is newer than the last release + that set the protocol var without being able to render the payload. + +Sequences are written to ``/dev/tty`` because prompt_toolkit's stdout wrapper +can buffer or strip raw escapes; when ``/dev/tty`` can't be opened (Windows, +no controlling terminal) they fall back to ``sys.stdout``. Never raises. +""" + +from __future__ import annotations + +import json +import os +import re +import sys + +_C0_AND_DEL = re.compile(r"[\x00-\x1f\x7f]") +_WARP_PROTOCOL_VERSION = 1 +# Last Warp release per channel that set WARP_CLI_AGENT_PROTOCOL_VERSION but +# could not render structured payloads (Warp's reference agent plugin, +# should-use-structured.sh). Bash compares these lexicographically; so do we. +_WARP_LAST_BROKEN = { + "stable": "v0.2026.03.25.08.24.stable_05", + "preview": "v0.2026.03.25.08.24.preview_05", +} + + +def _write_tty(seq: str) -> None: + """Write raw escapes to /dev/tty, falling back to sys.stdout. Never raises.""" + try: + with open("/dev/tty", "w", encoding="utf-8") as tty: + tty.write(seq) + return + except OSError: + pass + try: + sys.stdout.write(seq) + sys.stdout.flush() + except Exception: + pass + + +def osc9(body: str) -> str: + """OSC 9 sequence with C0 controls and DEL stripped from the body.""" + return f"\x1b]9;{_C0_AND_DEL.sub('', body)}\x07" + + +def warp_supported(env=None) -> bool: + """True when running in a Warp build that can render OSC 777 agent payloads.""" + env = os.environ if env is None else env + if env.get("TERM_PROGRAM") != "WarpTerminal" or not env.get("WARP_CLI_AGENT_PROTOCOL_VERSION"): + return False + client = env.get("WARP_CLIENT_VERSION", "") + if not client: + return False + for channel, last_broken in _WARP_LAST_BROKEN.items(): + if channel in client and client <= last_broken: + return False + return True + + +def warp_osc777(event: str, detail: str, session_id: str = "") -> str: + """OSC 777 ``warp://cli-agent`` notification; ``event`` is ``stop`` or ``permission_request``. + + Payload mirrors the reference plugin's build-payload.sh: common fields plus + ``summary`` (permission_request) or ``response`` (stop), truncated to 200. + """ + try: + advertised = int(os.environ.get("WARP_CLI_AGENT_PROTOCOL_VERSION", "1")) + except ValueError: + advertised = 1 + cwd = os.getcwd() + payload = { + "v": min(advertised, _WARP_PROTOCOL_VERSION), + "agent": "hermes", + "event": event, + "session_id": session_id, + "cwd": cwd, + "project": os.path.basename(cwd), + } + payload["summary" if event == "permission_request" else "response"] = detail[:200] + return f"\x1b]777;notify;warp://cli-agent;{json.dumps(payload, separators=(',', ':'))}\x07" + + +def notify(context: str, *, prompt: bool, session_id: str = "", detail: str = "") -> None: + """Emit OSC 9 (plus Warp OSC 777 when supported) for a blocking prompt or turn end.""" + seq = osc9(f"Hermes: {context}") + if warp_supported(): + event = "permission_request" if prompt else "stop" + seq += warp_osc777(event, detail or context, session_id) + _write_tty(seq) diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index c25a4433e5..01a6ee5031 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -111,7 +111,7 @@ CONFIGURABLE_TOOLSETS = [ ("tts", "🔊 Text-to-Speech", "text_to_speech"), ("stt", "🎙️ Speech-to-Text", "voice transcription (gateway voice messages + voice mode)"), ("skills", "📚 Skills", "list, view, manage"), - ("todo", "📋 Task Planning", "todo"), + ("todo", "📋 Task Planning", "todo_list"), ("memory", "💾 Memory", "persistent memory across sessions"), ("context_engine", "🧩 Context Engine", "runtime tools from the active context engine"), ("session_search", "🔎 Session Search", "search past conversations"), diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index d382908481..579cc8e269 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -56,6 +56,22 @@ def _m(): return main +def _no_prompt_git_kwargs() -> dict: + """``subprocess.run`` kwargs for the updater's network git calls. + + GitHub answers anonymous fetches with HTTP 401 during outages (and for + unreachable repos); git then prompts ``Username for 'https://github.com':`` + on the inherited terminal and the update sits there forever. Disable the + prompt so the fetch fails fast into ``_classify_fetch_failure``. Only the + *prompt* is disabled — a configured credential helper / askpass still + runs, so a private-fork origin keeps authenticating non-interactively. + """ + env = dict(os.environ) + env["GIT_TERMINAL_PROMPT"] = "0" + env["GCM_INTERACTIVE"] = "Never" + return {"stdin": subprocess.DEVNULL, "env": env} + + _UPDATE_RUNTIME_RELOAD_MODULES = ( "hermes_constants", "tools.environments.local", @@ -212,7 +228,7 @@ def _run_config_check_fresh() -> tuple: _reload_config_modules() from hermes_cli.config import check_config_version - return check_config_version() + return check_config_version(raise_on_parse_error=True) def _run_migrate_config_fresh(*, interactive: bool = False, quiet: bool = False) -> dict: @@ -1033,9 +1049,9 @@ def _print_curator_first_run_notice() -> None: def _print_fts_optimize_available_notice() -> None: """Advertise the opt-in v23 search-index optimization after `hermes update`. - Only fires when the current profile's state.db is still on the legacy - (pre-v23) inline FTS layout. Leads with the reclaimable-space figure and - points at the exact command. Honors ``sessions.fts_optimize_notice``: + Only fires when the current profile's state.db still needs an FTS storage + rebuild. Leads with the reclaimable-space figure and points at the exact + command. Honors ``sessions.fts_optimize_notice``: ``advise`` (default) prints an advisory notice, ``require`` prints a firmer required-upgrade notice, ``off`` suppresses it. Silent for fresh/already-optimized installs. @@ -1071,13 +1087,17 @@ def _print_fts_optimize_available_notice() -> None: return db = None interrupted = False + needs_upgrade = False try: db = SessionDB(db_path=db_path, read_only=True) - # read_only opens skip schema init, so probe the layout directly. + # read_only opens skip schema init, so probe the stored layout directly. row = db._conn.execute( "SELECT sql FROM sqlite_master " "WHERE type = 'table' AND name = 'messages_fts'" ).fetchone() + needs_upgrade = bool(row) and getattr( + db, "_db_needs_fts_storage_upgrade" + )(db._conn) # An interrupted `optimize-storage` run: the table is already the # v23 shape, but backfill markers / demoted trash tables remain. # Offer the command again — re-running resumes and finishes it. @@ -1103,9 +1123,8 @@ def _print_fts_optimize_available_notice() -> None: db.close() except Exception: pass - sql = (row[0] if row else "") or "" - if not sql or ("tool_name" in sql and not interrupted): - # v23 layout already present (fresh/optimized) — nothing to offer. + if not needs_upgrade and not interrupted: + # Current layout already present (fresh/optimized) — nothing to offer. return if interrupted: @@ -3393,6 +3412,7 @@ def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool: cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", + **_no_prompt_git_kwargs(), ) return result.returncode == 0 except Exception: @@ -3488,6 +3508,7 @@ def _sync_with_upstream_if_needed( cwd=cwd, capture_output=True, check=True, + **_no_prompt_git_kwargs(), ) except subprocess.CalledProcessError: print(" ✗ Failed to fetch upstream. Skipping upstream sync.") @@ -3527,6 +3548,7 @@ def _sync_with_upstream_if_needed( git_cmd + ["pull", "--ff-only", "upstream", "main"], cwd=cwd, check=True, + **_no_prompt_git_kwargs(), ) except subprocess.CalledProcessError: print( @@ -4486,7 +4508,17 @@ def _classify_fetch_failure(stderr: str) -> str: ) if "Could not resolve host" in stderr or "unable to access" in stderr: return "✗ Network error — cannot reach the remote repository." - if "Authentication failed" in stderr or "could not read Username" in stderr: + if "could not read Username" in stderr or "terminal prompts disabled" in stderr: + # Anonymous fetch of a public repo got HTTP 401. GitHub does this + # during outages (and for renamed/private repos) — it is not a + # credentials problem on the user's side. + return ( + "✗ GitHub rejected the anonymous fetch (asked for a login) — this" + " usually means a GitHub outage; try again in a few minutes" + " (https://www.githubstatus.com). If it persists, check" + " `git remote -v` points at a public repo." + ) + if "Authentication failed" in stderr: return "✗ Authentication failed — check your git credentials or SSH key." return "✗ Failed to fetch updates from origin." @@ -4627,6 +4659,7 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + **_no_prompt_git_kwargs(), ) if fetch_result is not None and fetch_result.returncode == 0: upstream_exists = True @@ -4639,6 +4672,7 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + **_no_prompt_git_kwargs(), ) upstream_exists = False compare_branch = f"origin/{branch}" @@ -4650,6 +4684,7 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + **_no_prompt_git_kwargs(), ) upstream_exists = False compare_branch = f"origin/{branch}" @@ -8594,6 +8629,7 @@ def _cmd_update_impl(args, gateway_mode: bool): cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + **_no_prompt_git_kwargs(), ) if fetch_result.returncode != 0: _print_fetch_failure(fetch_result.stderr) @@ -10721,6 +10757,26 @@ def _cmd_update_impl(args, gateway_mode: bool): node_failures, already_restarted_units=set(restarted_services) ) + # Check if any pre-update serve/dashboard runtimes survived on + # pre-update code generations (#100479). This is the SUCCESS-path + # twin of the abort-recovery probe above: the restart phase only + # restarts units, so an sshd-spawned `serve --isolated` or a manual + # `hermes serve` (no unit) is left running its pre-update + # sys.modules graph — and its cron ticker keeps firing agent jobs + # that ImportError on every symbol added in the pulled range. Runs + # AFTER the dashboard cleanup so a manual dashboard that cleanup + # killed and respawned is (correctly) not a survivor. The rows also + # feed the plan-vs-execution reconciliation below, so a survivor is + # escalated (exit 1) instead of merely printed. ``None`` means the + # probe itself failed; the reconciliation then stays fail-closed. + _stale_serve_rows: "list | None" = None + try: + _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) + if _stale_serve_rows: + _warn_stale_serve_runtimes(_stale_serve_rows) + except Exception as _serve_warn_exc: + logger.debug("Failed to check for surviving serve runtimes: %s", _serve_warn_exc) + print() print("Tip: You can now select a provider and model:") print(" hermes model # Select provider and model") @@ -10830,6 +10886,13 @@ def _cmd_update_impl(args, gateway_mode: bool): externally_supervised_profiles=externally_supervised_profiles, killed_pids=killed_pids, failed_units=failed_or_stale_units, + # Serve/dashboard runtimes reconcile by incarnation + # liveness, not by the gateway's unit names (#100479). + stale_serve_pids=( + {row.get("pid") for row in _stale_serve_rows} + if _stale_serve_rows is not None + else None + ), ) if report_unaccounted_runtimes(_runtime_outcomes): gateway_fleet_restart_incomplete = True diff --git a/hermes_cli/update_inventory.py b/hermes_cli/update_inventory.py index 2434f4c7a3..4fc392e5d6 100644 --- a/hermes_cli/update_inventory.py +++ b/hermes_cli/update_inventory.py @@ -425,6 +425,49 @@ def print_update_plan(plan: UpdatePlan) -> None: ) +_SERVE_KINDS = ("serve", "dashboard") + + +def _serve_unit_matches_profile(profile: str, unit: object) -> bool: + """Does *unit* name a ``hermes-serve*``/``hermes-dashboard*`` unit for *profile*? + + Serve/dashboard runtimes have their OWN unit vocabulary; the gateway's + ``hermes-gateway*`` names never cover them (#100479). Exact names only — + ``work`` must not claim ``hermes-serve-workbench`` — and a scope prefix + (``user/hermes-serve``) is tolerated because the restart phase records + scope-qualified identities in some lists. + """ + name = str(unit).removesuffix(".service") + if "/" in name: + name = name.rsplit("/", 1)[-1] + if profile == "default": + return name in {"hermes-serve", "hermes-dashboard"} + return name in {f"hermes-serve-{profile}", f"hermes-dashboard-{profile}"} + + +def _serve_runtime_outcome( + r: RuntimeRecord, + *, + killed: set, + failed_set: set, + restarted_set: set, + stale_serves: "set | None", +) -> str: + """Outcome for one serve/dashboard runtime — never the gateway's.""" + if r.pid is not None and r.pid in killed: + return "stopped" + if any(_serve_unit_matches_profile(r.profile, u) for u in failed_set): + return "failed" + if stale_serves is not None: + # Incarnation-verified: the pre-update process is gone (replaced by + # its unit / the dashboard cleanup respawn / the Desktop app) or it + # is still alive on pre-update code. + return "unaccounted" if r.pid in stale_serves else "restarted" + if any(_serve_unit_matches_profile(r.profile, s) for s in restarted_set): + return "restarted" + return "unaccounted" + + def match_runtime_outcomes( plan: "UpdatePlan", *, @@ -433,6 +476,7 @@ def match_runtime_outcomes( externally_supervised_profiles: list, killed_pids: set, failed_units: list, + stale_serve_pids: "set | None" = None, ) -> list[dict[str, Any]]: """Reconcile the plan's runtimes against what the restart phase DID. @@ -450,6 +494,18 @@ def match_runtime_outcomes( ``unaccounted`` — the plan saw it and NO bookkeeping mentions it: the blind-spot tripwire (same philosophy as the fleet matrix's DOWN row). Never raises; on any probe error returns what it has. + + Serve/dashboard runtimes are reconciled in their OWN vocabulary + (#100479): a ``hermes-serve*``/``hermes-dashboard*`` unit, a killed + PID, or — when the caller passes ``stale_serve_pids`` (the + ``(pid, create_time)``-verified survivor probe, + :func:`hermes_cli.update_abort_recovery._surviving_pre_update_serve_runtimes`) + — liveness: a pre-update serve whose incarnation is gone was replaced + (unit restart, dashboard cleanup respawn, Desktop respawn) and counts as + ``restarted``; one still alive is ``unaccounted``. They never borrow the + gateway's outcome: ``relaunched_profiles`` and ``hermes-gateway*`` name a + different process that shares the profile, nothing more. Without the + probe result, an untouched serve stays ``unaccounted`` (fail closed). """ outcomes: list[dict[str, Any]] = [] try: @@ -458,23 +514,57 @@ def match_runtime_outcomes( relaunched = set(relaunched_profiles or []) external = set(externally_supervised_profiles or []) killed = {int(p) for p in (killed_pids or set())} + stale_serves = ( + {int(p) for p in stale_serve_pids} if stale_serve_pids is not None else None + ) for runtime in plan.runtimes: r = runtime if isinstance(runtime, RuntimeRecord) else None if r is None: continue + if r.kind in _SERVE_KINDS: + outcomes.append( + { + "kind": r.kind, + "profile": r.profile, + "pid": r.pid, + "mechanism": r.restart_via, + "outcome": _serve_runtime_outcome( + r, + killed=killed, + failed_set=failed_set, + restarted_set=restarted_set, + stale_serves=stale_serves, + ), + } + ) + continue outcome = "unaccounted" + # The bare "hermes-gateway" unit name is gateway-specific: a + # serve/dashboard runtime that merely shares the default + # profile is a different process the gateway restart never + # touched, and must not borrow its outcome (#100479). if r.profile in relaunched or r.profile in external: outcome = "restarted" elif r.pid is not None and r.pid in killed: outcome = "stopped" elif any( - r.profile in unit or (r.profile == "default" and "hermes-gateway" in unit) + r.profile in unit + or ( + r.kind == "gateway" + and r.profile == "default" + and "hermes-gateway" in unit + ) for unit in failed_set ): outcome = "failed" elif any( - r.profile in svc or (r.profile == "default" and "hermes-gateway" in svc) + r.profile in svc + or ( + r.kind == "gateway" + and r.profile == "default" + and "hermes-gateway" in svc + ) for svc in restarted_set ): outcome = "restarted" @@ -511,8 +601,14 @@ def report_unaccounted_runtimes(outcomes: list[dict[str, Any]]) -> bool: f" — planned mechanism: {o['mechanism']}" ) print(" Restart them manually, then verify:") - print(" hermes gateway restart # active profile") - print(" hermes -p gateway restart # named profile") + if any(o.get("kind") not in _SERVE_KINDS for o in missed): + print(" hermes gateway restart # active profile") + print(" hermes -p gateway restart # named profile") + if any(o.get("kind") in _SERVE_KINDS for o in missed): + # A serve/dashboard is not reachable by any `gateway restart` + # command (#100479): name the process, not the wrong verb. + print(" systemctl --user restart hermes-serve.service # unit-managed serve") + print(" relaunch `hermes serve` / `hermes dashboard` / the Desktop app") return True diff --git a/hermes_cli/web_git.py b/hermes_cli/web_git.py index 3ea3c77c02..f06f736d49 100644 --- a/hermes_cli/web_git.py +++ b/hermes_cli/web_git.py @@ -19,7 +19,7 @@ import shutil import subprocess from pathlib import Path -from hermes_cli._subprocess_compat import noninteractive_git_env +from hermes_cli._subprocess_compat import harden_git_argv, noninteractive_git_env _GIT_TIMEOUT = 30 _GH_TIMEOUT = 30 @@ -42,7 +42,7 @@ def _git(cwd: str, args: list[str], *, timeout: int = _GIT_TIMEOUT) -> tuple[int the real auth error in the toast instead.""" try: proc = subprocess.run( - ["git", *args], + ["git", *harden_git_argv(args)], cwd=cwd, capture_output=True, text=True, encoding='utf-8', errors='replace', diff --git a/hermes_cli/web_models.py b/hermes_cli/web_models.py index fa5dd37243..b03f649417 100644 --- a/hermes_cli/web_models.py +++ b/hermes_cli/web_models.py @@ -306,6 +306,17 @@ class TTSSpeakRequest(BaseModel): text: str +class TTSLeaseRequest(BaseModel): + """Body for ``POST /api/audio/tts-lease``. + + ``lease`` names the toggle/surface holding the lease (``desktop:read-aloud``, + ``desktop:conversation``); ``active`` True acquires + warms, False releases. + """ + + lease: str + active: bool = True + + # --- from web_server.py (originally lines 11549-11551) --- class OAuthSubmitBody(BaseModel): diff --git a/hermes_cli/web_routers/profiles.py b/hermes_cli/web_routers/profiles.py index 843bf0649f..8cc359636c 100644 --- a/hermes_cli/web_routers/profiles.py +++ b/hermes_cli/web_routers/profiles.py @@ -12,7 +12,6 @@ late-binding seam in :mod:`hermes_cli.web_deps` so tests that ``monkeypatch.setattr(web_server, "_helper", ...)`` keep working. """ -import asyncio # noqa: F401 — used by handlers import copy import functools import inspect @@ -84,6 +83,12 @@ _strip_session_list_rows = late("_strip_session_list_rows") _write_profile_mcp_servers = late("_write_profile_mcp_servers") _write_profile_model = late("_write_profile_model") +# Returned by the offloaded file readers below to mean "the file is not there", +# which a plain ``None`` cannot express: ``desktop.json`` may legitimately hold +# the document ``null``, and that is an existing-but-empty overlay rather than +# an absent one. +_MISSING = object() + # Bounded cache lifetime for the expensive sidebar scan. Short enough that the # UI never shows meaningfully stale data, long enough to coalesce the desktop's @@ -197,6 +202,11 @@ def _sidebar_singleflight_cache(func): if cached is not miss: return cached result = func(*args, **kwargs) + # A 200 carrying errors[] is a FAILED profile scan, not a + # successful empty page. Caching it holds the empty recents in + # front of a store that has already recovered, for the whole TTL. + if isinstance(result, dict) and result.get("errors"): + return result try: snapshot = copy.deepcopy(result) except Exception: @@ -908,15 +918,23 @@ async def get_active_profile_endpoint(): the running dashboard/gateway is scoped to (derived from HERMES_HOME). """ from hermes_cli import profiles as profiles_mod - try: - active = profiles_mod.get_active_profile() or "default" - except Exception: - active = "default" - try: - current = profiles_mod.get_active_profile_name() or "default" - except Exception: - current = "default" - return {"active": active, "current": current} + + def _run(): + # Both reads touch the filesystem: get_active_profile() reads the + # active_profile state file and get_active_profile_name() resolves + # HERMES_HOME against the profiles root. Batched into one hop so the + # sidebar's polling costs a single executor round-trip, not two. + try: + active = profiles_mod.get_active_profile() or "default" + except Exception: + active = "default" + try: + current = profiles_mod.get_active_profile_name() or "default" + except Exception: + current = "default" + return {"active": active, "current": current} + + return await run_in_threadpool(_run) @router.post("/api/profiles/active") @@ -927,8 +945,14 @@ async def set_active_profile_endpoint(body: ProfileActiveUpdate): it changes which profile subsequent CLI commands and gateways use. """ from hermes_cli import profiles as profiles_mod + + def _run(): + return profiles_mod.set_active_profile(body.name) + try: - profiles_mod.set_active_profile(body.name) + # set_active_profile() stats the target profile, creates the state + # directory and writes active_profile through a temp file + replace. + await run_in_threadpool(_run) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) except ValueError as e: @@ -1001,8 +1025,16 @@ async def open_profile_terminal_endpoint(name: str): @router.patch("/api/profiles/{name}") async def rename_profile_endpoint(name: str, body: ProfileRename): from hermes_cli import profiles as profiles_mod + + def _run(): + return profiles_mod.rename_profile(name, body.new_name) + try: - path = profiles_mod.rename_profile(name, body.new_name) + # rename_profile() stops a running gateway through the same 10-second + # _stop_gateway_process() poll that delete does, then renames the + # profile directory, rewrites the Honcho host blocks and regenerates + # the wrapper script. + path = await run_in_threadpool(_run) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) except (ValueError, FileExistsError) as e: @@ -1037,8 +1069,18 @@ async def delete_profile_endpoint(name: str): its own dialog before this request, so we always pass ``yes=True`` to skip the CLI's interactive prompt.""" from hermes_cli import profiles as profiles_mod + + def _run(): + return profiles_mod.delete_profile(name, yes=True) + try: - path = profiles_mod.delete_profile(name, yes=True) + # delete_profile() stops a running gateway by polling its PID once + # every 500 ms for up to 10 s (profiles._stop_gateway_process) and + # then rmtree()s the profile directory. Deleting a profile whose + # gateway is up — which this path announces as "⚠ Gateway is running + # — it will be stopped" — therefore parks the loop for a full ten + # seconds, and the desktop's WebSocket ready-probe gives up at ten. + path = await run_in_threadpool(_run) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) except ValueError as e: @@ -1052,18 +1094,27 @@ async def delete_profile_endpoint(name: str): @router.get("/api/profiles/{name}/soul") async def get_profile_soul(name: str): soul_path = _resolve_profile_dir(name) / "SOUL.md" - if soul_path.exists(): - try: - return {"content": soul_path.read_text(encoding="utf-8-sig"), "exists": True} - except OSError as e: - raise HTTPException(status_code=500, detail=f"Could not read SOUL.md: {e}") - return {"content": "", "exists": False} + def _run(): + # Probe and read in the same hop: two round-trips would also widen the + # window between the existence check and the read. + if not soul_path.exists(): + return _MISSING + return soul_path.read_text(encoding="utf-8-sig") + + try: + content = await run_in_threadpool(_run) + except OSError as e: + raise HTTPException(status_code=500, detail=f"Could not read SOUL.md: {e}") + if content is _MISSING: + return {"content": "", "exists": False} + return {"content": content, "exists": True} @router.put("/api/profiles/{name}/soul") async def update_profile_soul(name: str, body: ProfileSoulUpdate): soul_path = _resolve_profile_dir(name) / "SOUL.md" - try: + + def _run(): from utils import atomic_write_text # PUT replaces the whole persona document from the dashboard editor. @@ -1083,6 +1134,12 @@ async def update_profile_soul(name: str, body: ProfileSoulUpdate): atomic_write_text( soul_path, body.content, preserve_mode=True, create_mode=0o644 ) + + try: + # atomic_write_text() writes a temp file, fsyncs it and replaces the + # original — three syscalls that block for as long as the filesystem + # takes to durably commit the persona document. + await run_in_threadpool(_run) except OSError as e: _log.exception("PUT /api/profiles/%s/soul failed", name) raise HTTPException(status_code=500, detail=f"Could not write SOUL.md: {e}") @@ -1100,12 +1157,18 @@ async def update_profile_description_endpoint(name: str, body: ProfileDescriptio from hermes_cli import profiles as profiles_mod profile_dir = _resolve_profile_dir(name) text = (body.description or "").strip() - try: + + def _run(): profiles_mod.write_profile_meta( profile_dir, description=text, description_auto=False, ) + + try: + # write_profile_meta() reads profile.yaml, merges the new keys and + # writes the document back out. + await run_in_threadpool(_run) except Exception as e: _log.exception("PUT /api/profiles/%s/description failed", name) raise HTTPException(status_code=500, detail=str(e)) @@ -1125,7 +1188,8 @@ async def update_profile_model_endpoint(name: str, body: ProfileModelUpdate): if not provider or not model: raise HTTPException(status_code=400, detail="provider and model are required") try: - _write_profile_model(profile_dir, provider, model) + # _write_profile_model() reads and rewrites the profile's config.yaml. + await run_in_threadpool(_write_profile_model, profile_dir, provider, model) except Exception as e: _log.exception("PUT /api/profiles/%s/model failed", name) raise HTTPException(status_code=500, detail=str(e)) @@ -1142,10 +1206,21 @@ async def describe_profile_auto_endpoint(name: str, body: ProfileDescribeAuto): ``ok: false`` with a reason rather than an HTTP error so the UI can surface it inline and let the operator fix config and retry. """ + # Resolution stays on the loop: it is a name check plus one stat, and it + # owns the 400/404 mapping that the ``except Exception`` below would + # otherwise flatten into a 500. _resolve_profile_dir(name) - try: + + def _run(): from hermes_cli import profile_describer - outcome = profile_describer.describe_profile(name, overwrite=bool(body.overwrite)) + return profile_describer.describe_profile(name, overwrite=bool(body.overwrite)) + + try: + # describe_profile() is a plain def that reaches auxiliary_client's + # call_llm() — a synchronous provider round-trip with a 60 s ceiling, + # six times the desktop's WebSocket disconnect threshold. Held on the + # loop it stalls every other dashboard request for that whole window. + outcome = await run_in_threadpool(_run) except Exception as e: _log.exception("POST /api/profiles/%s/describe-auto failed", name) raise HTTPException(status_code=500, detail=str(e)) @@ -1180,11 +1255,9 @@ async def export_profile_endpoint(name: str, body: ProfileExport): except OSError as exc: raise HTTPException(status_code=500, detail=f"Could not create export directory: {exc}") - loop = asyncio.get_running_loop() try: - result = await loop.run_in_executor( - None, - lambda: profiles_mod.export_profile(name, output, extra_files=body.extra_files or None), + result = await run_in_threadpool( + profiles_mod.export_profile, name, output, extra_files=body.extra_files or None ) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) @@ -1204,11 +1277,9 @@ async def import_profile_endpoint(body: ProfileImport): if not archive: raise HTTPException(status_code=400, detail="archive path is required") - loop = asyncio.get_running_loop() try: - profile_dir = await loop.run_in_executor( - None, - lambda: profiles_mod.import_profile(archive, name=(body.name or "").strip() or None), + profile_dir = await run_in_threadpool( + profiles_mod.import_profile, archive, name=(body.name or "").strip() or None ) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) @@ -1251,10 +1322,19 @@ async def get_profile_desktop_overlay(name: str): """The desktop appearance/interface overlay bundled with an imported profile (``desktop.json`` at the profile root), or ``exists: false``.""" overlay_path = _resolve_profile_dir(name) / "desktop.json" - if not overlay_path.is_file(): - return {"exists": False, "desktop": None} - try: + + def _run(): + if not overlay_path.is_file(): + return _MISSING import json as _json - return {"exists": True, "desktop": _json.loads(overlay_path.read_text(encoding="utf-8-sig"))} + return _json.loads(overlay_path.read_text(encoding="utf-8-sig")) + + try: + overlay = await run_in_threadpool(_run) except Exception as e: raise HTTPException(status_code=500, detail=f"Could not read desktop.json: {e}") + # _MISSING rather than None: an overlay file holding the document ``null`` + # exists, and must not be reported as absent. + if overlay is _MISSING: + return {"exists": False, "desktop": None} + return {"exists": True, "desktop": overlay} diff --git a/hermes_cli/web_routers/sessions.py b/hermes_cli/web_routers/sessions.py index a4da40c3a1..657840ed62 100644 --- a/hermes_cli/web_routers/sessions.py +++ b/hermes_cli/web_routers/sessions.py @@ -31,7 +31,7 @@ from hermes_cli.web_models import ( SessionPrune, SessionRename, ) -from hermes_state import is_malformed_db_error +from hermes_state import is_malformed_db_error, is_transient_sqlite_error # Same logger the handlers used before extraction (identical logger object). _log = logging.getLogger("hermes_cli.web_server") @@ -197,6 +197,22 @@ def get_sessions( db.close() except HTTPException: raise + except sqlite3.OperationalError as exc: + _log.exception("GET /api/sessions failed") + # 503, not 500: the store is busy, not gone. The desktop keeps the + # sidebar it already has instead of reading a 500 as an authoritative + # empty list. Retrying the OPEN here is deliberately not done — the + # bounded retry lives in SessionDB's read-only constructor, so every + # read-only opener gets it, not just this route. + transient = is_transient_sqlite_error(exc) + raise HTTPException( + status_code=503 if transient else 500, + detail=( + "Session store is busy (disk I/O or lock). Retry; the list was not cleared." + if transient + else "Internal server error" + ), + ) from exc except Exception: _log.exception("GET /api/sessions failed") raise HTTPException(status_code=500, detail="Internal server error") diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 77fe99e715..27609dfff0 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -303,6 +303,17 @@ def _start_desktop_cron_ticker(stop_event: "threading.Event", interval: int = 60 profile_homes = list(profiles_to_serve(multiplex=True)) if len(profile_homes) > 1: start_kwargs["profile_homes"] = profile_homes + # Stand down, per tick, for any profile whose OWN gateway is + # running: that gateway ticks it with live adapters, and the + # tick-lock race otherwise lets this adapter-less ticker win + # and deliver the job through the standalone path (#100489). + # Evaluated every cycle so a gateway starting/stopping later + # is picked up without a dashboard restart. + from hermes_cli.profiles import _check_gateway_running + + start_kwargs["profile_gate"] = ( + lambda _name, home: not _check_gateway_running(Path(home)) + ) from hermes_logging import enable_profile_log_routing enable_profile_log_routing(profile_homes) @@ -320,6 +331,11 @@ def _start_desktop_cron_ticker(stop_event: "threading.Event", interval: int = 60 provider.start(stop_event, **start_kwargs) +# Desktop `serve` only (start_server(start_mcp_discovery_after_bind=True)): +# seconds after the READY sentinel before the MCP discovery thread starts. +_DESKTOP_MCP_DISCOVERY_DELAY_S = 1.0 + + def _warm_gateway_module() -> None: """Pre-import heavy modules so the event loop is not stalled on first use. @@ -1429,8 +1445,8 @@ _SCHEMA_OVERRIDES: Dict[str, Dict[str, Any]] = { }, "agent.service_tier": { "type": "select", - "description": "API service tier (OpenAI/Anthropic)", - "options": ["", "auto", "default", "flex"], + "description": "Fast mode: fast = always, auto = first N seconds of each turn, cold = first turn only", + "options": ["", "normal", "fast", "auto", "cold"], }, "delegation.reasoning_effort": { "type": "select", @@ -1527,6 +1543,9 @@ _CATEGORY_MERGE: Dict[str, str] = { # `session.terminal_continue` is the only schema-surfaced session field — # fold it into general rather than spawning a one-field orphan category. "session": "general", + # `nous.keepalive_interval_seconds` is the only schema-surfaced nous field + # (Portal tokens live in auth.json) — fold it into the agent tab. + "nous": "agent", } # Display order for tabs — unlisted categories sort alphabetically after these. @@ -1836,6 +1855,7 @@ from hermes_cli.web_models import ( # noqa: F401 LearningNodeEdit, DebugShareRequest, TTSSpeakRequest, + TTSLeaseRequest, OAuthSubmitBody, BulkDeleteSessions, SessionImport, @@ -3397,6 +3417,7 @@ _PORT_BINDING_PLATFORM_PORTS: Dict[str, Tuple[str, int]] = { "sms": ("webhook_port", 8080), "whatsapp_cloud": ("webhook_port", 8090), "line": ("port", 8646), + "teams": ("port", 3978), } # Platform states that mean the adapter is NOT serving its port right now. @@ -5665,6 +5686,43 @@ async def speak_text(payload: TTSSpeakRequest, profile: Optional[str] = None): } +@app.post("/api/audio/tts-lease") +async def tts_lease(payload: TTSLeaseRequest, profile: Optional[str] = None): + """Desktop TTS-output toggles as warm-up / release signals. + + "Read replies aloud" and voice-conversation mode are explicit "speech is + about to be needed" gestures. ``active: true`` registers the toggle as a + lease on the TTS engine and pre-loads the configured provider (local + piper/kittentts model, lazily-installed SDK) so the first spoken reply + doesn't pay the load as dead air; ``active: false`` drops the lease and, + once no surface holds one, unloads resident local models. + + Blocking work (model load, voice download) runs off the event loop. + Warm-up failures are reported in the body, never as an HTTP error — the + toggle must succeed even when the engine can't preload. + """ + lease = (payload.lease or "").strip() + if not lease: + raise HTTPException(status_code=400, detail="lease is required") + + def _apply(): + from tools.tts_tool import acquire_tts_lease, release_tts_lease + + if payload.active: + with _config_profile_scope(profile): + return acquire_tts_lease(lease) + return release_tts_lease(lease) + + try: + result = await asyncio.get_running_loop().run_in_executor(None, _apply) + except HTTPException: + raise + except Exception as exc: + _log.warning("TTS lease %s (%s) failed: %s", lease, payload.active, exc) + result = {"leases": None, "action": "error", "error": str(exc)} + return {"ok": True, "lease": lease, "active": payload.active, **result} + + def _split_text_for_speak_stream(text: str, cap: int) -> list: """Split *text* into provider-cap-sized pieces on sentence boundaries. @@ -11046,20 +11104,63 @@ def _claude_code_only_status() -> Dict[str, Any]: def _copilot_acp_status() -> Dict[str, Any]: """Status for copilot-acp — credentials are owned by the Copilot CLI. - There is no cheap programmatic credential probe for the ACP subprocess, so - this is a read-only "managed by the Copilot CLI" card (like claude-code): - Hermes never claims a login state it can't verify. + ``logged_in`` is claimed only on positive evidence (a supported env token + or a known on-disk GitHub Copilot credential store, via + ``auth.get_external_process_provider_status``). The Copilot CLI may also + hold its session in an OS keychain Hermes can't read, so the unverified + state is presented as "managed by the Copilot CLI" — never as signed out. """ + try: + from hermes_cli.auth import get_external_process_provider_status + status = get_external_process_provider_status("copilot-acp") or {} + except Exception: + status = {} + verified = bool(status.get("auth_verified")) + configured = bool(status.get("configured")) + if verified: + source_label = status.get("auth_source") or "Copilot credentials detected" + elif configured: + found = status.get("resolved_command") or status.get("command") or "copilot" + source_label = f"Managed by the GitHub Copilot CLI ({found})" + else: + source_label = "GitHub Copilot CLI not found on PATH" return { - "logged_in": False, + "logged_in": verified, "source": "copilot_cli", - "source_label": "Managed by the GitHub Copilot CLI", + "source_label": source_label, "token_preview": None, "expires_at": None, "has_refresh_token": False, + "configured": configured, } +def _external_process_cli_command(provider_id: str, default: str) -> str: + """Render an external-process provider's sign-in command with the CLI the + user actually has configured. + + The static catalog assumes the default executable name; users who point + Hermes at a custom binary (``HERMES_COPILOT_ACP_COMMAND`` / + ``COPILOT_CLI_PATH``) would otherwise be told to run a command that isn't + the one Hermes spawns. Non-external-process providers get ``default`` back + untouched. + """ + try: + from hermes_cli.auth import PROVIDER_REGISTRY, get_external_process_provider_status + pconfig = PROVIDER_REGISTRY.get(provider_id) + if not pconfig or pconfig.auth_type != "external_process": + return default + status = get_external_process_provider_status(provider_id) or {} + command = str(status.get("command") or "").strip() + if command: + parts = default.split(" ", 1) + tail = f" {parts[1]}" if len(parts) > 1 else "" + return f"{command}{tail}" + except Exception: + pass + return default + + # Explicit, hand-tuned OAuth/account provider cards. These carry the bits that # can't be derived from the unified provider catalog: the OAuth ``flow`` shape, # the per-provider ``status_fn``, the ``cli_command`` fallback, and curated @@ -11125,7 +11226,11 @@ _OAUTH_PROVIDER_CATALOG: tuple[Dict[str, Any], ...] = ( "id": "copilot-acp", "name": "GitHub Copilot (ACP)", "flow": "external", - "cli_command": "copilot /login", + # `copilot login` is the CLI's non-interactive device-code login + # subcommand; the previous `copilot /login` form is not a valid + # invocation (slash-commands only exist inside an interactive + # session, reachable as `copilot -i /login`). + "cli_command": "copilot login", "docs_url": "https://docs.github.com/en/copilot", "status_fn": _copilot_acp_status, }, @@ -11384,7 +11489,7 @@ async def list_oauth_providers(profile: Optional[str] = None): "id": p["id"], "name": p["name"], "flow": p["flow"], - "cli_command": p["cli_command"], + "cli_command": _external_process_cli_command(p["id"], p["cli_command"]), "docs_url": p["docs_url"], "disconnect_hint": disconnect_hint, "disconnect_command": _oauth_provider_disconnect_command(p), @@ -12893,6 +12998,13 @@ def _normalize_dashboard_cron_updates( ) if "deliver" in normalized: normalized["deliver"] = _cron_optional_text(normalized["deliver"]) or "local" + if "failure_deliver" in normalized: + # Same text normalization as deliver, but empty CLEARS the override + # (failures fall back to deliver) rather than coalescing to a target + # — the field is optional by design (NS-788). + normalized["failure_deliver"] = _cron_optional_text( + normalized["failure_deliver"] + ) if "context_from" in normalized: normalized["context_from"] = _cron_string_list(normalized["context_from"]) if "enabled_toolsets" in normalized: @@ -13447,35 +13559,62 @@ def _gateway_fire_endpoint(profile: str, home: Path) -> str: """Resolve the loopback URL of the gateway api_server's cron-fire route. Port resolution mirrors gateway/config.py's api_server load order for the - TARGET profile: ``platforms.api_server.extra.port`` in the profile's - config.yaml, then ``API_SERVER_PORT`` (process env for the active profile, - the profile's own .env otherwise), then the adapter default 8642. The bind - host is the adapter's loopback default — the dashboard and gateway share a - network namespace in every supported deployment (same host process tree, - or the same container under s6). + LISTENER-OWNER profile: ``platforms.api_server.extra.port`` in that + profile's config.yaml, then ``API_SERVER_PORT`` (process env for the + active profile, the profile's own .env otherwise), then the adapter + default 8642. The bind host is the adapter's loopback default — the + dashboard and gateway share a network namespace in every supported + deployment (same host process tree, or the same container under s6). Multiplex mode (one gateway serving several profiles) exposes per-profile mirrors under ``/p//…``, so a non-default profile routes through - the default gateway's port with that prefix; per-profile-gateway mode - (each profile its own process/port) uses the bare path on the profile's - own port. + the default gateway's port with that prefix — only the DEFAULT profile's + api_server is bound in that mode, so the port must be read from the + default home, never the target profile's (a secondary's own + ``API_SERVER_PORT`` is a port nothing listens on). Per-profile-gateway + mode (each profile its own process/port) uses the bare path on the + profile's own port. """ import os as _os + multiplex = False + try: + from gateway.config import _env_multiplex_profiles_override + + cfg = load_config() + multiplex = bool(cfg_get(cfg, "gateway", "multiplex_profiles", default=False)) + env_flag = _env_multiplex_profiles_override() + if env_flag is not None: + multiplex = env_flag + except Exception: + _log.debug("cron fire: multiplex detection failed; assuming single-profile", exc_info=True) + + listener_profile, listener_home = profile, home + if multiplex and profile != "default": + from hermes_constants import get_default_hermes_root + + listener_profile, listener_home = "default", get_default_hermes_root() + _log.info( + "cron fire: multiplex gateway — resolving api_server port for %s " + "from the default profile's listener (%s)", + profile, + listener_home, + ) + port = 0 try: # Profile-scoped read through the CANONICAL loader (managed-scope # overlay, ${ENV_VAR} expansion, profile pathing) — never a raw # yaml.safe_load of config.yaml (tests/hermes_cli/ # test_config_read_guard.py). The HERMES_HOME override scopes - # get_config_path() to the TARGET profile, same pattern the + # get_config_path() to the LISTENER-OWNER profile, same pattern the # deprecated _fire_cron_job_for_profile used for its store scope. from hermes_constants import ( reset_hermes_home_override, set_hermes_home_override, ) - token = set_hermes_home_override(str(home)) + token = set_hermes_home_override(str(listener_home)) try: profile_cfg = load_config() finally: @@ -13490,8 +13629,8 @@ def _gateway_fire_endpoint(profile: str, home: Path) -> str: if not port: raw = ( _os.getenv("API_SERVER_PORT", "") - if profile == _cron_default_profile() - else _profile_env_value(home, "API_SERVER_PORT") + if listener_profile == _cron_default_profile() + else _profile_env_value(listener_home, "API_SERVER_PORT") ) try: port = int(raw) if raw else 0 @@ -13500,18 +13639,6 @@ def _gateway_fire_endpoint(profile: str, home: Path) -> str: if not port: port = 8642 - multiplex = False - try: - cfg = load_config() - multiplex = bool(cfg_get(cfg, "gateway", "multiplex_profiles", default=False)) - env_flag = _os.getenv("GATEWAY_MULTIPLEX_PROFILES", "").strip().lower() - if env_flag in {"1", "true", "yes", "on"}: - multiplex = True - elif env_flag in {"0", "false", "no", "off"}: - multiplex = False - except Exception: - pass - if multiplex and profile != "default": return f"http://127.0.0.1:{port}/p/{profile}/api/cron/fire" return f"http://127.0.0.1:{port}/api/cron/fire" @@ -14230,26 +14357,35 @@ async def list_credential_pool(): from agent.credential_pool import load_pool from hermes_cli.auth import read_credential_pool - providers = [] - # read_credential_pool(None) lists every provider that has pooled entries; - # load_pool() then gives us the rich PooledCredential objects per provider. - raw_pool = read_credential_pool() - for provider_id in sorted(raw_pool.keys()): - try: - pool = load_pool(provider_id) - except Exception: - _log.exception("load_pool(%s) failed", provider_id) - continue - entries = pool.entries() - if not entries: - continue - providers.append({ - "provider": provider_id, - "entries": [ - _pool_entry_summary(e, i) for i, e in enumerate(entries, start=1) - ], - }) - return {"providers": providers} + # load_pool() may hit the network synchronously (Copilot token exchange + # over raw urllib). urllib's timeout does NOT bound DNS resolution + # (getaddrinfo blocks in C), so on a networkless Windows host this froze + # the uvicorn event loop for 17 minutes (2026-08-22 00:03-00:20 stall). + # Keep every provider load off the loop - same pattern as + # get_memory_status below. + def _run(): + providers = [] + # read_credential_pool(None) lists every provider that has pooled entries; + # load_pool() then gives us the rich PooledCredential objects per provider. + raw_pool = read_credential_pool() + for provider_id in sorted(raw_pool.keys()): + try: + pool = load_pool(provider_id) + except Exception: + _log.exception("load_pool(%s) failed", provider_id) + continue + entries = pool.entries() + if not entries: + continue + providers.append({ + "provider": provider_id, + "entries": [ + _pool_entry_summary(e, i) for i, e in enumerate(entries, start=1) + ], + }) + return {"providers": providers} + + return await asyncio.to_thread(_run) @app.post("/api/credentials/pool") @@ -14268,40 +14404,46 @@ async def add_credential_pool_entry(body: CredentialPoolAdd): if not provider or not api_key: raise HTTPException(status_code=400, detail="provider and api_key are required") - try: - pool = load_pool(provider) - label = (body.label or "").strip() or f"key #{len(pool.entries()) + 1}" - entry = PooledCredential( - provider=provider, - id=_uuid.uuid4().hex[:6], - label=label, - auth_type=AUTH_TYPE_API_KEY, - priority=0, - source=SOURCE_MANUAL, - access_token=api_key, - ) - pool.add_entry(entry) - # Re-adding a credential is an explicit re-engagement signal: lift - # every suppression for this provider so a source deleted earlier - # (via DELETE below or `hermes auth remove`) can seed again. - # Mirrors the `hermes auth add` behaviour in auth_commands.py. - if not provider.startswith(CUSTOM_POOL_PREFIX): - try: - from hermes_cli.auth import ( - _load_auth_store, - unsuppress_credential_source, - ) - suppressed = _load_auth_store().get("suppressed_sources", {}) - for src in list(suppressed.get(provider, []) or []): - unsuppress_credential_source(provider, src) - except Exception: - _log.exception("unsuppress after pool add failed (non-fatal)") - except HTTPException: - raise - except Exception as exc: - _log.exception("POST /api/credentials/pool failed") - raise HTTPException(status_code=400, detail=str(exc)) from exc - return {"ok": True, "provider": provider, "count": len(pool.entries())} + # load_pool() may run synchronous OAuth token exchanges (network I/O); + # keep it off the event loop - see list_credential_pool (2026-08-22 + # 17-minute stall fix). + def _run(): + try: + pool = load_pool(provider) + label = (body.label or "").strip() or f"key #{len(pool.entries()) + 1}" + entry = PooledCredential( + provider=provider, + id=_uuid.uuid4().hex[:6], + label=label, + auth_type=AUTH_TYPE_API_KEY, + priority=0, + source=SOURCE_MANUAL, + access_token=api_key, + ) + pool.add_entry(entry) + # Re-adding a credential is an explicit re-engagement signal: lift + # every suppression for this provider so a source deleted earlier + # (via DELETE below or `hermes auth add`) can seed again. + # Mirrors the `hermes auth add` behaviour in auth_commands.py. + if not provider.startswith(CUSTOM_POOL_PREFIX): + try: + from hermes_cli.auth import ( + _load_auth_store, + unsuppress_credential_source, + ) + suppressed = _load_auth_store().get("suppressed_sources", {}) + for src in list(suppressed.get(provider, []) or []): + unsuppress_credential_source(provider, src) + except Exception: + _log.exception("unsuppress after pool add failed (non-fatal)") + return {"ok": True, "provider": provider, "count": len(pool.entries())} + except HTTPException: + raise + except Exception as exc: + _log.exception("POST /api/credentials/pool failed") + raise HTTPException(status_code=400, detail=str(exc)) from exc + + return await asyncio.to_thread(_run) @app.delete("/api/credentials/pool/{provider}/{index}") @@ -14322,44 +14464,50 @@ async def remove_credential_pool_entry(provider: str, index: int): from hermes_cli.auth import suppress_credential_source provider = (provider or "").strip().lower() - try: - pool = load_pool(provider) - removed = pool.remove_index(index) - except Exception as exc: - _log.exception("DELETE /api/credentials/pool failed") - raise HTTPException(status_code=400, detail=str(exc)) from exc - if removed is None: - raise HTTPException(status_code=404, detail="No pool entry at that index") - - cleaned: List[str] = [] - hints: List[str] = [] - step = find_removal_step(provider, removed.source or "") - if step is not None: + # load_pool() may run synchronous token exchanges and the removal steps do + # blocking disk writes - keep them off the event loop (see + # list_credential_pool; 2026-08-22 17-minute stall fix). + def _run(): try: - result = step.remove_fn(provider, removed) - cleaned = list(result.cleaned) - hints = list(result.hints) - if result.suppress: - suppress_credential_source(provider, removed.source) - except Exception: - # Cleanup is best-effort, but suppression is the actual bug fix — - # without it the entry resurrects on the next load_pool(). Apply - # it even when source-specific cleanup blew up. - _log.exception( - "credential source cleanup failed for %s/%s; suppressing anyway", - provider, removed.source, - ) + pool = load_pool(provider) + removed = pool.remove_index(index) + except Exception as exc: + _log.exception("DELETE /api/credentials/pool failed") + raise HTTPException(status_code=400, detail=str(exc)) from exc + if removed is None: + raise HTTPException(status_code=404, detail="No pool entry at that index") + + cleaned: List[str] = [] + hints: List[str] = [] + step = find_removal_step(provider, removed.source or "") + if step is not None: try: - suppress_credential_source(provider, removed.source) + result = step.remove_fn(provider, removed) + cleaned = list(result.cleaned) + hints = list(result.hints) + if result.suppress: + suppress_credential_source(provider, removed.source) except Exception: - _log.exception("suppress_credential_source failed") - return { - "ok": True, - "provider": provider, - "count": len(pool.entries()), - "cleaned": cleaned, - "hints": hints, - } + # Cleanup is best-effort, but suppression is the actual bug fix - + # without it the entry resurrects on the next load_pool(). Apply + # it even when source-specific cleanup blew up. + _log.exception( + "credential source cleanup failed for %s/%s; suppressing anyway", + provider, removed.source, + ) + try: + suppress_credential_source(provider, removed.source) + except Exception: + _log.exception("suppress_credential_source failed") + return { + "ok": True, + "provider": provider, + "count": len(pool.entries()), + "cleaned": cleaned, + "hints": hints, + } + + return await asyncio.to_thread(_run) # --------------------------------------------------------------------------- @@ -15044,7 +15192,13 @@ def _fallback_profile_dicts(profiles_mod) -> List[Dict[str, Any]]: "provider": provider, "has_env": _safe(lambda entry=entry_path: (entry / ".env").exists(), False), "skill_count": _safe(lambda entry=entry_path: profiles_mod._count_skills(entry), 0), - "gateway_running": _safe(lambda entry=entry_path: profiles_mod._check_gateway_running(entry), False), + "gateway_running": _safe( + lambda entry=entry_path, name=entry.name: ( + profiles_mod._check_gateway_running(entry) + or profiles_mod._served_by_running_multiplexer(name) + ), + False, + ), "description": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description", ""), ""), "description_auto": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description_auto", False), False), "distribution_name": None, @@ -19601,6 +19755,7 @@ def start_server( headless: bool = False, ssh_session_token: Optional[str] = None, ssh_owner_nonce: Optional[str] = None, + start_mcp_discovery_after_bind: bool = False, ): """Start the web UI server. @@ -19615,6 +19770,10 @@ def start_server( ``ssh_session_token`` and ``ssh_owner_nonce`` are process-local Desktop SSH bootstrap state. Neither is persisted or exported to child processes. + + ``start_mcp_discovery_after_bind`` (Desktop ``serve``) defers the + background MCP discovery thread until the ready sentinel has been written, + so its SDK import cannot hold the GIL against the pre-bind import path. """ _apply_ssh_session_token(ssh_session_token or "") _apply_ssh_owner_nonce(ssh_owner_nonce) @@ -19992,6 +20151,27 @@ def start_server( print(f" Hermes Web UI → http://{host}:{actual_port}") _maybe_open_browser(host, actual_port, open_browser, initial_profile) + if start_mcp_discovery_after_bind: + # Deferred from cmd_dashboard for Desktop `serve` (see there). + # Not started at the bind itself either: the ~350ms `mcp` SDK + # import holds the GIL, and at bind time the renderer is doing + # its WebSocket handshake + first hydration reads against this + # loop (measured: starting it here gave back most of the + # READY gain as a slower connect). One second later the shell + # is painted and idle. An agent build inside that second fires + # the deferred start itself (wait_for_mcp_discovery), so its + # bounded join and the late-binding refresh are unchanged. + try: + from hermes_cli.mcp_startup import defer_background_mcp_discovery + + defer_background_mcp_discovery( + logger=_log, + thread_name="dashboard-mcp-discovery", + delay=_DESKTOP_MCP_DISCOVERY_DELAY_S, + ) + except Exception: + _log.debug("Deferred MCP discovery arm failed", exc_info=True) + # Collapse the peer-hangup teardown flood (#50005). When the Desktop # forcibly closes its WebSocket mid-write, asyncio logs a full # traceback per pending connection-lost callback — 50+ identical diff --git a/hermes_constants.py b/hermes_constants.py index b2d25110c7..1d426df9d5 100644 --- a/hermes_constants.py +++ b/hermes_constants.py @@ -139,6 +139,14 @@ def get_hermes_home() -> Path: return _hermes_home_from_env() +# Resolved keys, keyed by the path string that was handed in. Path.resolve() +# is a filesystem call, and this function sits under every ToolRegistry +# lookup through current_scope_key(), so without this the registry pays a +# syscall per lookup. A process only ever sees a handful of home paths, so +# the dict stays tiny. Only paths that really exist are stored, see below. +_HOME_KEY_CACHE: dict[str, str] = {} + + def hermes_home_key(path: str | Path | None = None) -> str: """Return a stable key for a Hermes home/profile directory. @@ -146,10 +154,37 @@ def hermes_home_key(path: str | Path | None = None) -> str: Runtime registries use this key to isolate plugin-owned entries while keeping built-in registrations process-global. ``strict=False`` preserves useful behavior for profiles whose directories have not been created yet. + + The resolved value is remembered per input path. A directory that does + not exist yet is resolved without touching the cache, because the answer + can change once it is created, for example when part of the path turns + out to be a symlink. """ - candidate = Path(path) if path else get_hermes_home() - resolved = candidate.expanduser().resolve(strict=False) - return os.path.normcase(str(resolved)) + candidate = Path(path) if path is not None else get_hermes_home() + raw = str(candidate) + cached = _HOME_KEY_CACHE.get(raw) + if cached is not None: + return cached + expanded = candidate.expanduser() + try: + resolved = expanded.resolve(strict=True) + except OSError: + # Not on disk yet. Fall back to the lenient resolve and do not store + # it, so the real answer is picked up once the directory appears. + return os.path.normcase(str(expanded.resolve(strict=False))) + key = os.path.normcase(str(resolved)) + _HOME_KEY_CACHE[raw] = key + return key + + +def reset_hermes_home_key_cache() -> None: + """Forget every remembered home key. + + For tests that move a home directory around on disk under one path. + Normal callers never need this: a different home path is a different + cache key already. + """ + _HOME_KEY_CACHE.clear() def normalize_scope(scope: str | Path | None) -> str | None: diff --git a/hermes_state.py b/hermes_state.py index 244e2faf3d..19745bde79 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -60,6 +60,7 @@ from hermes_cli.sqlite_runtime import ( ) from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, TypeVar, cast +import hermes_state_holders as _state_holders from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) _BRANCH_CHILD_SQL, _COMPRESSION_CHILD_SQL, @@ -100,6 +101,7 @@ from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) _clear_lock_holder_record, _describe_lock_holder, _read_lock_holder_record, + is_advisory_lock_contention, ) from hermes_state_portability import SessionPortabilityMixin from hermes_state_schema import SessionSchemaMixin @@ -115,6 +117,13 @@ logger = logging.getLogger(__name__) MAX_SAFE_RESUME_MESSAGES = 20_000 MAX_SAFE_EXPORT_MESSAGES = 20_000 +# Auto-maintenance only VACUUMs when at least this fraction of the database +# file is reclaimable (``PRAGMA freelist_count / PRAGMA page_count``). Below +# it a full rewrite costs more I/O than it returns — pruning a handful of small +# sessions on a dense multi-GB state.db should never rewrite the whole file to +# reclaim a few MB (#54189). Composes with ``min_vacuum_interval_days``. +AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25 + def _configured_transcript_limit(key: str, fallback: int) -> int: """Resolve a transcript safety limit from config at call time. @@ -374,6 +383,20 @@ DEFAULT_DB_PATH = get_hermes_home() / "state.db" # query; short enough that transient fd pressure doesn't strand the read pool. _READ_OPEN_RETRY_SECONDS = 60.0 +# Transient SQLITE_IOERR retry budget for READ-ONLY opens (#100436). A WAL +# database being actively written (checkpoint, WAL reset/truncate, frame +# flush) can surface "disk I/O error" to a concurrent ``mode=ro`` reader in +# a millisecond-wide transition window: the read-only connection cannot +# perform the WAL recovery a read through a stale or mid-update -shm file +# needs, because recovery requires writing the -shm index, which mode=ro +# refuses. The window closes on its own (the writer finishes the transition), +# so a bounded number of short retries makes the open succeed instead of +# 500-ing the whole /api/sessions poll (or any other read-only opener). +# Deliberately NOT attempted on writable opens: a writer owns the +# transition, so an IOERR there means a real storage/fd problem. +_READ_ONLY_IOERR_RETRY_ATTEMPTS = 3 +_READ_ONLY_IOERR_RETRY_BACKOFF_S = 0.05 + # Hard ceiling on read-only connections ALIVE at once against one database # FILE — pooled idle ones and checked-out ones together, summed over every # SessionDB in this process that points at that file. See _PathReadBudget. @@ -1119,6 +1142,17 @@ def _strip_stale_tool_call_markers( return messages +def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str: + """Normalize profile namespace for Telegram topic-mode tables. + + Empty / missing values map to ``\"default\"`` so non-multiplexed gateways + keep a single namespace. Multiplexed callers must pass the *routed* + profile (``source.profile``), never the process-global active profile. + """ + name = str(profile_name or "").strip() + return name if name else "default" + + def format_session_db_unavailable(prefix: str = "Session database not available") -> str: """Format a user-facing 'session DB unavailable' message with cause. @@ -1764,13 +1798,14 @@ def _log_wal_reset_bug_once( # for git/pip/system Python installs (#75153). repair_hint = _wal_reset_repair_hint() logger.warning( - "%s: linked SQLite %s is vulnerable to the WAL-reset corruption " - "bug (https://sqlite.org/wal.html#walresetbug) — %s. " + "%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset " + "corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. " "Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); " "%s. See `hermes doctor`. This warning fires once per " "process per database.", db_label, sqlite3.sqlite_version, + sys.executable, action, repair_hint, ) @@ -2084,6 +2119,52 @@ def is_malformed_db_error(exc: BaseException) -> bool: return any(marker in str(exc).lower() for marker in _MALFORMED_DB_MARKERS) +# SQLITE_IOERR, matched as a plain substring so wrapped error strings still +# classify. Shared by the read-only open retry and the write-path BEGIN retry. +_DISK_IO_ERROR_MARKER = "disk i/o error" + +# Broader set for HTTP classification: a read that failed for one of these +# reasons found the store BUSY, not gone. Callers map it to 503 (retry, the +# list was not cleared) instead of 500. Corruption is deliberately absent — +# a malformed store must surface, not be retried into a timeout. +_TRANSIENT_SQLITE_MARKERS = ( + _DISK_IO_ERROR_MARKER, + "database is locked", + "database table is locked", + "busy", +) + + +def is_transient_sqlite_error(exc: BaseException) -> bool: + """True when a SQLite failure means "busy right now", not "damaged". + + One predicate so the read paths cannot drift apart on what counts as + recoverable: the read-only open retry, and the HTTP 503-vs-500 split on + the session-list endpoints, classify the same way. + """ + if not isinstance(exc, sqlite3.OperationalError): + return False + message = str(exc).lower() + return any(marker in message for marker in _TRANSIENT_SQLITE_MARKERS) + + +def _is_transient_read_only_ioerr(exc: sqlite3.OperationalError, *, attempt: int) -> bool: + """True when a read-only open should be retried rather than raised. + + A ``mode=ro`` connection cannot perform WAL recovery (recovery needs to + write the -shm index, which read-only mode refuses), so a concurrent WAL + checkpoint / reset / frame-flush can surface ``SQLITE_IOERR`` ("disk I/O + error") to a reader on an otherwise healthy database (#100436). The + transition is millisecond-scale, so a bounded number of short retries + clears it without changing classification for genuine storage failures — + a persistent IOERR still exhausts the budget and propagates. + """ + return ( + attempt < _READ_ONLY_IOERR_RETRY_ATTEMPTS + and _DISK_IO_ERROR_MARKER in str(exc).lower() + ) + + def is_malformed_schema_error(exc: BaseException) -> bool: """True only when SQLite explicitly reports malformed schema text. @@ -2204,7 +2285,10 @@ def classify_persistence_error(exc_or_str) -> str: if isinstance(exc_or_str, CompressionSessionBusyError): return "compression" if isinstance(exc_or_str, StateDbReplacedError): + # Includes DeletedWalGenerationError (subclass). return "replaced" + if isinstance(exc_or_str, StateDbCorruptError): + return "corrupt" text = str(exc_or_str).lower() if "turn lease" in text: return "turn_lease" @@ -2214,6 +2298,8 @@ def classify_persistence_error(exc_or_str) -> str: return "compression" if "was replaced underneath" in text: return "replaced" + if "deleted state.db-wal" in text or "deleted state.db-shm" in text: + return "replaced" # Structural corruption BEFORE the lock and disk buckets: "database disk # image is malformed" contains "disk" (and some wrapped corruption # strings mention "locked" recovery attempts), so later buckets would @@ -2278,10 +2364,11 @@ def _cross_process_repair_lock(db_path: Path): """Serialize state.db schema surgery across processes. Yields True when this process holds the repair lock for *db_path*, False - when the bounded acquire timed out. Unlike the kanban init lock — whose - critical section is idempotent, so proceeding without the lock is merely - redundant work — proceeding here would be exactly the unsafe interleaving - we are trying to prevent, so a caller that gets False must NOT do surgery. + when the bounded acquire timed out or the lock file could not be opened at + all. Unlike the kanban init lock — whose critical section is idempotent, + so proceeding without the lock is merely redundant work — proceeding here + would be exactly the unsafe interleaving we are trying to prevent, so a + caller that gets False must NOT do surgery. ``flock`` is the right primitive for this: the kernel drops the lock when the holding process dies, so a crashed repairer cannot leave a stale lock @@ -2299,14 +2386,22 @@ def _cross_process_repair_lock(db_path: Path): lock_path.parent.mkdir(parents=True, exist_ok=True) handle = lock_path.open("a+b") except OSError as exc: - # Read-only dir, exhausted fds, exotic filesystem: fall back to the - # in-process behaviour that shipped before this lock existed rather - # than refusing to repair a DB we could otherwise heal. + # Fail closed, exactly as a timed-out acquire does. A lock file we + # cannot even open means the filesystem is out of space, inodes or + # descriptors — and a sibling that opened ITS handle before the disk + # filled is still inside writable_schema surgery or VACUUM. Yielding + # True here let two processes run schema surgery on the same live + # state.db concurrently, which is itself the corruption source this + # lock exists to remove (#100368: the disk-full trigger, then a fresh + # corruption on every boot with other writers alive). Callers already + # handle False by re-probing and reporting, and on a read-only + # directory no repair strategy could have written anyway. logger.warning( - "Could not open state.db repair lock %s (%s) — proceeding with " - "in-process serialisation only.", lock_path, exc, + "Could not open state.db repair lock %s (%s) — skipping schema " + "surgery rather than running it without cross-process authority.", + lock_path, exc, ) - yield True + yield False return acquired = False @@ -2321,7 +2416,15 @@ def _cross_process_repair_lock(db_path: Path): msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) acquired = True break - except (BlockingIOError, OSError): + except (BlockingIOError, OSError) as exc: + if not is_advisory_lock_contention(exc): + logger.warning( + "Could not acquire state.db repair lock %s (%s) — " + "skipping schema surgery on a non-contention error.", + lock_path, exc, + ) + acquired = None + break if time.monotonic() >= deadline: break time.sleep(_REPAIR_LOCK_POLL_SECONDS) @@ -2333,7 +2436,10 @@ def _cross_process_repair_lock(db_path: Path): _REPAIR_LOCK_POLL_SECONDS, "state.db repair lock", ) - if not acquired: + if acquired is None: + # Non-contention failure already logged with its errno. + acquired = False + elif not acquired: record = None if _IS_WINDOWS else _read_lock_holder_record(handle) logger.warning( "state.db repair lock %s held by another process for more " @@ -2362,6 +2468,65 @@ def _cross_process_repair_lock(db_path: Path): handle.close() +def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]: + """Non-blocking cross-process lock for one auto-maintenance pass. + + The kernel releases this advisory lock if the holder exits, unlike a + durable pid/meta marker. A caller that cannot acquire it must skip the + pass: otherwise two startups can both pass the interval check and the + second can prune a row the first has only just closed recoverably. + """ + lock_path = db_path.with_name(db_path.name + ".auto-maintenance.lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + except OSError as exc: + logger.warning( + "Could not open state.db auto-maintenance lock %s (%s) — skipping " + "automatic maintenance.", + lock_path, + exc, + ) + return None + + try: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking( # type: ignore[attr-defined] + handle.fileno(), msvcrt.LK_NBLCK, 1 # type: ignore[attr-defined] + ) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (BlockingIOError, OSError): + handle.close() + return None + return handle + + +def _release_auto_maintenance_lock(handle: Any) -> None: + """Release a handle returned by :func:`_try_acquire_auto_maintenance_lock`.""" + try: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking( # type: ignore[attr-defined] + handle.fileno(), msvcrt.LK_UNLCK, 1 # type: ignore[attr-defined] + ) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: # pragma: no cover - best effort release + pass + finally: + handle.close() + + def _bump_schema_cookie(conn: sqlite3.Connection) -> None: """Increment the schema cookie after direct ``sqlite_master`` surgery. @@ -2743,7 +2908,12 @@ def _persistent_repair_exhausted_error(db_path: Path) -> str: f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — " "the corruption is beyond the schema/FTS repair strategies " "(likely b-tree page damage). Manual recovery required: restore " - f"a backup, or salvage with `sqlite3 {db_path} \".recover\"`. " + "a backup, or salvage with `hermes sessions recover --source " + f"{db_path} --inspect-only`, then (if it reports recoverable) " + f"`hermes sessions recover --source {db_path} --output " + "recovered-state.db` (recovery snapshots the damaged file first, " + "then runs the page-level `.recover` lane on the copy; do NOT " + "point a raw `sqlite3` shell at the live database). " f"Delete {_repair_ledger_path(db_path).name} to force another " "automatic attempt." ) @@ -2942,8 +3112,9 @@ def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]": f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; " f"copying the damaged DB needs {need / 1e9:.2f}GB and must " f"leave {headroom / 1e9:.2f}GB headroom. Free disk space, " - f"then retry (or recover manually with `sqlite3 {db_path} " - '".recover"`).' + "then retry (or recover manually with " + f"`hermes sessions recover --source {db_path} " + "--inspect-only` first)." ) logger.error("Refusing forensic backup of %s: %s", db_path, reason) return None, reason @@ -2957,7 +3128,8 @@ def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]": f"could not determine free space on {db_path.parent} ({exc}); " "refusing the forensic copy rather than risk filling the " f"volume. Free disk space, then retry (or recover manually " - f'with `sqlite3 {db_path} ".recover"`).' + f"with `hermes sessions recover --source {db_path} " + "--inspect-only` first)." ) logger.error("Refusing forensic backup of %s: %s", db_path, reason) return None, reason @@ -3433,58 +3605,17 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]: conn.close() +def _foreign_state_db_holders(db_path: Path) -> List[Tuple[int, str]]: + """Compatibility delegate to the state-holder authority.""" + return _state_holders.foreign_state_db_holders(db_path) + + def _live_writer_holds_db(db_path: Path) -> bool: - """True when a connection outside this call still holds ``db_path`` open. - - Detection works by asking SQLite for the thing a repair actually needs and - a live writer cannot grant: ``PRAGMA locking_mode=EXCLUSIVE`` followed by - ``BEGIN IMMEDIATE``. In WAL mode, entering exclusive locking mode - requires exclusive locks on the WAL index, so any other open connection — - reader or writer — makes it fail with SQLITE_BUSY. Neither statement - parses the schema, so this works on the malformed databases repair exists - to handle. - - Fails **open** (returns False) on anything other than a positive - busy/locked signal: refusing to repair a database that nobody is actually - holding would strand the very self-heal path this guard protects. - - Scope: the WAL-index exclusive lock is what makes this detect a holder, so - the guard is effective in WAL mode. On SQLite builds carrying the WAL-reset - bug and on NFS/SMB, Hermes deliberately runs ``state.db`` in - ``journal_mode=DELETE`` (see :func:`apply_wal_with_fallback`); there a held - reader takes only a SHARED lock, ``BEGIN IMMEDIATE`` still acquires - RESERVED, and this probe returns False. In that mode repair is serialised - only by the cross-process repairer lock rather than by this holder probe. - The 2026-08 incident that motivated the guard was in WAL mode, which this - covers; broadening detection to DELETE mode is left to a follow-up. - """ - probe = None - try: - probe = _connect_repair_durable(db_path, timeout=0.0) - probe.execute("PRAGMA locking_mode=EXCLUSIVE") - probe.execute("BEGIN IMMEDIATE") - probe.execute("ROLLBACK") - return False - except sqlite3.OperationalError as exc: - lowered = str(exc).lower() - return "locked" in lowered or "busy" in lowered - except sqlite3.DatabaseError: - # Malformed/unreadable: no evidence of a live holder either way. - return False - except Exception: - return False - finally: - if probe is not None: - try: - # Drop exclusive locking mode before closing so the probe - # itself never leaves the file pinned. - probe.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - try: - probe.close() - except Exception: - pass + """Compatibility delegate to the repair-admission authority.""" + return _state_holders.live_writer_holds_db( + db_path, + connect_repair_durable=_connect_repair_durable, + ) def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]: @@ -3556,16 +3687,19 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A result = report with _cross_process_repair_lock(db_path) as holding_lock: if not holding_lock: - # Another process is still inside its critical section. It may - # nonetheless have healed the file already (long VACUUM after a - # successful strategy), so re-probe before reporting failure. + # Another process is still inside its critical section, or the + # lock file itself could not be opened (full disk / no fds). It + # may nonetheless have healed the file already (long VACUUM after + # a successful strategy), so re-probe before reporting failure. if _db_opens_cleanly(db_path) is None: report["repaired"] = True report["strategy"] = "repaired_by_other_process" else: report["error"] = ( - "another process holds the state.db repair lock; skipped " - "schema surgery to avoid racing it" + "could not obtain the state.db repair lock (held by " + "another process, or the lock file was unopenable); " + "skipped schema surgery to avoid racing a concurrent " + "repairer" ) else: # The fast check above avoids taking the lock for a known-exhausted @@ -3601,11 +3735,13 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A # database.journal_mode setting is the restore target. before_mode = _probe_journal_mode_for_repair(db_path) result = _repair_state_db_schema_locked( - db_path, backup=backup, report=report + db_path, + backup=backup, + report=report, + journal_mode_before=before_mode, ) if result.get("repaired"): result["journal_mode_before"] = before_mode - _restore_journal_mode_after_repair(db_path, before_mode) # Environmental aborts happen before a strategy gets to mutate the # isolated snapshot. They are retriable operating conditions, not # proof that the damaged database exhausted a repair strategy. @@ -3641,7 +3777,9 @@ def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]: return None -def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str]) -> None: +def _restore_journal_mode_after_repair( + db_path: Path, before_mode: Optional[str], *, conn=None +) -> None: """Re-apply the journal mode after schema surgery (#89674). A repaired/rebuilt SQLite file comes back in the default journal mode @@ -3651,6 +3789,14 @@ def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str] inside the repair path, not at open (the open-time flip #89393 warns about is a different door). + ``conn`` must be the exclusive repair guard connection when called from + the repair path (#101064): opening a fresh connection AFTER the guard + released let a writer still holding the unlinked old ``-wal`` inode + coexist with a brand-new ``state.db-wal`` this connection created — two + generations of one store. The transactional promotion already leaves the + destination in its pre-repair mode, so on that path this is mostly the + WAL-companion re-assertion; the reopen is the hazard, not the mode. + The restore runs through :func:`apply_wal_with_fallback` — the canonical journal-mode path — rather than issuing a switch pragma directly, so it inherits the vulnerable-SQLite WAL-reset gate (a rebuilt file IS a new @@ -3666,12 +3812,15 @@ def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str] Best-effort by design: the repair itself already succeeded, so failures to re-apply are logged at WARNING, never raised. """ + owned_conn = conn is None try: - conn = _connect_repair_durable(db_path) + if owned_conn: + conn = _connect_repair_durable(db_path) try: after = apply_wal_with_fallback(conn, db_label=db_path.name) finally: - conn.close() + if owned_conn: + conn.close() if before_mode and after != before_mode: logger.warning( "state.db repair changed journal_mode %r -> %r " @@ -3689,7 +3838,11 @@ def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str] def _repair_state_db_schema_locked( - db_path: Path, *, backup: bool, report: Dict[str, Any] + db_path: Path, + *, + backup: bool, + report: Dict[str, Any], + journal_mode_before: Optional[str] = None, ) -> Dict[str, Any]: """Repair strategies for :func:`repair_state_db_schema`. @@ -3831,6 +3984,11 @@ def _repair_state_db_schema_locked( report.get("strategy"), db_path, ) + _restore_journal_mode_after_repair( + db_path, + journal_mode_before, + conn=live_guard, + ) if not report.get("repaired"): # Logged HERE, not inside the strategies: they run against the # scratch copy, and naming that throwaway path in the one @@ -4207,6 +4365,23 @@ class StateDbReplacedError(RuntimeError): """ +class DeletedWalGenerationError(StateDbReplacedError): + """A live process holds a deleted state.db-wal / -shm generation. + + Opening or writing through this handle would mint a second WAL inode + (or keep committing on the orphan) — the split-brain that produces + intermittent SQLITE_CORRUPT / SQLITE_IOERR. Stop the writers; do not + unlink the WAL yourself. ``database.journal_mode: delete`` is operator + containment, not a default change. + + Subclasses :class:`StateDbReplacedError` so every downstream consumer + that already stops SQLite writes and diverts pending transcripts on a + replaced store (gateway retry queue, run_agent flush) handles the split + WAL generation identically — the correct response is the same: stop + writing, preserve the transcript tail on disk. + """ + + # SQLite header: 4-byte big-endian application_id at offset 68. Distinct from # inode: ``cp`` onto the same path keeps st_ino and truncates+rewrites. _STATE_DB_APPLICATION_ID_OFFSET = 68 @@ -4217,6 +4392,53 @@ _STATE_DB_REPLACED_MSG = ( "gateway pending_messages spool) and restore or reopen after operator " "intervention." ) +_DELETED_WAL_GENERATION_MSG = ( + "FATAL: a live process holds a deleted state.db-wal or state.db-shm " + "inode while the path names a different (or missing) generation. " + "Refusing to open or write so a second WAL cannot be minted. " + "Stop the gateway, dashboard, and cron writers that hold the deleted " + "sidecar, then reopen. Do not delete the WAL yourself. " + "database.journal_mode: delete is operator containment, not a new default." +) + + +class StateDbCorruptError(sqlite3.DatabaseError): + """A live SessionDB observed structural (non-FTS) corruption and is quarantined. + + Raised once a write on this handle reports bare ``SQLITE_CORRUPT`` / + ``SQLITE_NOTADB`` that is neither FTS-scoped (``_is_fts_write_corruption_error``) + nor a replaced-file case (``StateDbReplacedError``). Subclasses + ``sqlite3.DatabaseError`` so every existing ``except sqlite3.Error`` + degrade path keeps working; ``sqlite_errorcode``/``sqlite_errorname`` + are copied from the originating error. + + The quarantine is sticky for the life of the handle: later writes fail + fast, the handle never reopens after ``close()``, and ``close()`` skips + its own WAL checkpoint. Field evidence (the #90837 lost/reordered-page + signature, the #90950 page-1 clobber): a handle that kept writing for ~50 + minutes after the first structural error checkpointed 15 pages under the + wrong page numbers on shutdown, turning a still-readable file into + ``file is not a database``. Stopping the writes is what prevents that; + skipping the explicit checkpoint is the second line of defence. SQLite + still runs its own last-connection checkpoint inside ``close()`` (and + deletes the ``-wal`` sidecar) unless ``SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE`` + is set — Python exposes it via ``Connection.setconfig()`` on 3.12+, so + quarantine disables the close-time checkpoint there and the WAL survives + on disk for forensics; on 3.11 the internal checkpoint is unavoidable + (post-quarantine it can only carry pre-corruption committed frames, since + no further writes are accepted). The + recovery boundary is a process restart on a repaired or restored file. + """ + + +_STATE_DB_CORRUPT_MSG = ( + "FATAL: state.db reported structural corruption (database disk image is " + "malformed outside the FTS shadow tables) on a live handle; refusing further " + "writes, automatic reopen, and the close-time WAL checkpoint on this file. " + "Stop the gateway, then run `hermes sessions recover --source " + "--inspect-only` or restore a snapshot. Unwritten transcripts are diverted to " + "sessions/.jsonl (and the gateway pending_messages spool)." +) def divert_session_transcript_jsonl(session_id: str, messages) -> "Optional[Path]": @@ -4341,6 +4563,84 @@ def _stat_db_file_identity(path: Path) -> "Optional[tuple]": return (st.st_dev, st.st_ino) +def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]: + """Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars.""" + identities: Dict[str, tuple] = {} + base = os.fspath(db_path) + for suffix in ("-wal", "-shm"): + ident = _stat_db_file_identity(Path(base + suffix)) + if ident is not None: + identities[suffix] = ident + return identities + + +_canonical_sqlite_path = _state_holders.canonical_sqlite_path + + +def _watched_sqlite_sidecar_paths(db_path) -> Set[str]: + base = os.path.abspath(os.fspath(db_path)) + return { + _canonical_sqlite_path(base + "-wal"), + _canonical_sqlite_path(base + "-shm"), + } + + +def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]: + """Return processes holding an unlinked ``state.db-wal`` / ``-shm``. + + Linux-only (``/proc//fd`` readlink). Windows and other hosts + return ``[]`` — Windows cannot unlink a sidecar another process still + holds, and macOS does not use the `` (deleted)`` suffix. + + The scan includes this process: on the SessionDB open/write refuse + path, the in-process writer that still holds the orphan inode is the + one that must not mint a replacement WAL (and must stop committing). + ``_foreign_state_db_holders`` keeps skipping this PID for FTS + maintenance so a process does not block its own optional repair. + """ + if not sys.platform.startswith("linux"): + return [] + + holders: List[Tuple[int, str]] = [] + watched = _watched_sqlite_sidecar_paths(db_path) + try: + for pid_str in os.listdir("/proc"): + if not pid_str.isdigit(): + continue + pid = int(pid_str) + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue + for fd in fds: + try: + target = os.readlink(f"{fd_dir}/{fd}") + except OSError: + continue + if " (deleted)" not in target: + continue + if _canonical_sqlite_path(target) in watched: + holders.append((pid, target)) + except Exception as exc: + logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc) + return holders + return holders + + +def refuse_deleted_wal_generation(db_path) -> None: + """Raise if any process holds a deleted WAL/SHM generation for *db_path*. + + Called *before* ``sqlite3.connect`` so a second opener cannot mint a + replacement WAL inode while a live writer still holds the orphan. + """ + holders = iter_deleted_sqlite_sidecar_holders(db_path) + if not holders: + return + logger.error(_DELETED_WAL_GENERATION_MSG) + raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) + + # ── Process-wide shared SessionDB registry (#90837) ── # # The registry itself lives in hermes_state_registry.py — a bounded @@ -4805,37 +5105,6 @@ def _concrete_state_db_holder_pids( return pids -def _read_proc_cmdline(pid: int) -> Optional[str]: - """Read /proc//cmdline, world-readable even when fd table is not. - - Returns the cmdline as a space-joined string, or None when unreadable - (process exited, or hidepid mount). - """ - try: - with open(f"/proc/{pid}/cmdline", "rb") as f: - raw = f.read() - if not raw: - return None - return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() - except OSError: - return None - - -_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve", - "hermes-agent", "hermes gateway", "hermes chat") - - -def _looks_like_hermes(cmdline: str) -> bool: - """Heuristic: does this cmdline look like a Hermes process? - - Used to decide whether an uninspectable process (fd table unreadable - due to different user) should be treated as a potential state.db holder. - We only flag processes that look like Hermes, not every system daemon. - """ - lower = cmdline.lower() - return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS) - - # Lifecycle statuses surfaced by session pickers. Classification looks ONLY at # a session's final message row — role, whether it carries tool_calls, and its # finish_reason — so it stays O(1) per session (see @@ -4878,6 +5147,18 @@ def classify_session_status( return SESSION_STATUS_COMPLETE +# Parent→child ``profile_name`` inheritance fence (#88381). ``agent::...`` +# gateway keys encode the profile namespace; a keyless row (CLI / subagent +# lineage) carries none and inherits freely. Two keyed rows must agree on +# ``agent::`` — a default child (``agent:main:``) forked from a sibling +# profile's row must not be durably mislabelled as that profile's. +_SAME_KEY_NAMESPACE_SQL = ( + "p.session_key IS NULL OR sessions.session_key IS NULL" + " OR substr(p.session_key, 1, instr(substr(p.session_key, 7), ':') + 6)" + " = substr(sessions.session_key, 1, instr(substr(sessions.session_key, 7), ':') + 6)" +) + + class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin): """ SQLite-backed session storage with FTS5 search. @@ -4886,6 +5167,19 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) single writer via WAL mode). Each method opens its own cursor. """ + # Only these state-owned producers participate in automatic stale-open + # reconciliation. Messaging-platform and UI/desktop sources have separate + # lifecycle owners; unknown/future sources fail closed (#60609). + _AUTO_PRUNE_STALE_OPEN_SOURCES: Tuple[str, ...] = ( + "cli", + "cron", + "kanban", + "acp", + "api_server", + "subagent", + "tool", + ) + # ── Write-contention tuning ── # With multiple hermes processes (gateway + CLI sessions + worktree agents) # all sharing one state.db, WAL write-lock contention causes visible TUI @@ -5087,6 +5381,14 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._db_file_application_id: int = 0 self._db_file_generation_token: str = "" self._db_replaced = False + # Sticky: set once a write on THIS handle reports bare SQLITE_CORRUPT / + # NOTADB that is not FTS-scoped and not a replaced-file case. Never + # cleared; the recovery boundary is a process restart on a repaired or + # restored file (see StateDbCorruptError). + self._db_corrupt = False + self._db_corrupt_reason = "" + self._db_sidecar_identity: Dict[str, tuple] = {} + self._db_wal_generation_lost = False # One-shot guard for the usermerge-floor config write on the # incremental FTS merge cadence (see _merge_fts_incrementally). self._fts_usermerge_floor_applied = False @@ -5125,46 +5427,67 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # must already exist + be initialised (callers guard on # db_path.exists()); a SELECT against an empty file raises and # the caller degrades per-profile. - self._conn = _connect_tracked_db( - f"file:{self.db_path}?mode=ro", - tracking_path=self.db_path, - uri=True, - check_same_thread=False, - timeout=1.0, - isolation_level=None, - ) - self._conn.row_factory = sqlite3.Row - # FTS capability flags normally come from writable schema - # initialisation. Probe existing virtual tables with SELECTs - # only so read-only search keeps its FTS and trigram paths. - # Close the connection on ANY probe failure (e.g. malformed - # schema raises DatabaseError, not the OperationalError the - # probe handles). The constructor's outer finally also covers - # failures before this probe and BaseException paths, so a - # leaked tracked connection cannot block _backup_db_file's - # raw-copy for the rest of the process — the writable heal - # that follows would then repair WITHOUT its forensic backup. - try: - apply_database_pragmas(self._conn, db_label="state.db") - cursor = self._conn.cursor() - self._fts_enabled = ( - self._fts_table_probe(cursor, "messages_fts") is True - ) - if self._fts_enabled: - self._trigram_available = ( - self._fts_table_probe( - cursor, - "messages_fts_trigram", - ) - is True - ) - except BaseException: - conn, self._conn = self._conn, None + open_attempt = 0 + while True: try: - conn.close() - except Exception: - pass - raise + self._conn = _connect_tracked_db( + f"file:{self.db_path}?mode=ro", + tracking_path=self.db_path, + uri=True, + check_same_thread=False, + timeout=1.0, + isolation_level=None, + ) + self._conn.row_factory = sqlite3.Row + # FTS capability flags normally come from writable schema + # initialisation. Probe existing virtual tables with + # SELECTs only so read-only search keeps its FTS and + # trigram paths. Close the connection on ANY probe + # failure (e.g. malformed schema raises DatabaseError, + # not the OperationalError the probe handles). The + # constructor's outer finally also covers failures + # before this probe and BaseException paths, so a + # leaked tracked connection cannot block + # _backup_db_file's raw-copy for the rest of the + # process — the writable heal that follows would then + # repair WITHOUT its forensic backup. + try: + apply_database_pragmas(self._conn, db_label="state.db") + cursor = self._conn.cursor() + self._fts_enabled = ( + self._fts_table_probe(cursor, "messages_fts") + is True + ) + if self._fts_enabled: + self._trigram_available = ( + self._fts_table_probe( + cursor, + "messages_fts_trigram", + ) + is True + ) + except BaseException: + conn, self._conn = self._conn, None + try: + conn.close() + except Exception: + pass + raise + break + except sqlite3.OperationalError as ioerr: + # A WAL checkpoint / reset / frame-flush in flight on + # the writer side can surface SQLITE_IOERR to a + # concurrent mode=ro reader (it cannot perform the + # recovery the read needs — recovery writes the -shm + # index, which mode=ro refuses). The transition closes + # in milliseconds, so retry a bounded number of times + # before classifying the store as failed (#100436). + if not _is_transient_read_only_ioerr( + ioerr, attempt=open_attempt + ): + raise + open_attempt += 1 + time.sleep(_READ_ONLY_IOERR_RETRY_BACKOFF_S) self._record_db_file_identity() initialization_complete = True return @@ -5217,6 +5540,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) raise sqlite3.DatabaseError(msg) def _connect_and_init(): + # Refuse before sqlite3.connect (under the startup lock) so we + # cannot mint a replacement WAL while a live writer still + # holds a deleted sidecar inode. + refuse_deleted_wal_generation(self.db_path) self._conn = _connect_tracked_db( str(self.db_path), check_same_thread=False, @@ -5607,6 +5934,17 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # through stale WAL/shm assumptions (#89332). Refuse instead. if self._db_replaced or self._db_file_was_replaced(): self._halt_db_replaced() + # A quarantined handle must never come back: reopening would hand a + # fresh connection (and its own close-time checkpoint) to a file we + # already know is structurally damaged. + if self._db_corrupt: + raise self._corrupt_error( + f"state.db connection for {self.db_path} is quarantined after " + f"structural corruption; refusing to reopen for a {context} " + "after close(). " + ) + if self._db_wal_generation_lost or self._wal_generation_was_lost(): + self._halt_deleted_wal_generation() logger.warning( "state.db connection for %s was closed while a %s was still in " "flight — reopening (teardown/worker race, #94736)", @@ -5704,6 +6042,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # means a legacy shape that doesn't index tool metadata → optimize. return "tool_name" not in sql + @staticmethod + def _db_has_trigram_tool_calls_projection(cursor: sqlite3.Cursor) -> bool: + """True when the trigram vtable still includes tool_calls payload.""" + row = cursor.execute( + "SELECT sql FROM sqlite_master " + "WHERE type = 'table' AND name = 'messages_fts_trigram'" + ).fetchone() + if row is None: + return False + sql = (row[0] if not isinstance(row, sqlite3.Row) else row["sql"]) or "" + return "tool_calls" in sql.lower() + + @classmethod + def _db_needs_fts_storage_upgrade( + cls, cursor: sqlite3.Cursor + ) -> bool: + """True when the current FTS storage layout should be treated as stale.""" + return ( + cls._db_has_legacy_inline_fts(cursor) + or cls._db_has_trigram_tool_calls_projection(cursor) + ) + def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None: """Log once that the trigram tokenizer is missing; base FTS5 stays enabled.""" if getattr(self, "_trigram_unavailable_warned", False): @@ -5932,6 +6292,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Set on the first compression-busy collision so the short wait is # measured from then, not from the start of the write. compression_deadline: Optional[float] = None + # One retry for SQLITE_IOERR raised by BEGIN IMMEDIATE itself. The + # callback has not run at that point, so there is no durable effect + # to replay and the retry is exactly-once safe (#99502's contract). + # Once the callback starts, an IOERR leaves the write's settlement + # unknown and must propagate — this helper owns non-idempotent + # transcript/counter mutations, not just idempotent UPSERTs. + ioerr_begin_retried = False # Transient engine-level error observed on contended WAL appends # (dual gateway/agent writers; FTS5 trigram sync holds the write @@ -5944,7 +6311,9 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return "no more rows available" in str(exc).lower() while True: + self._raise_if_db_corrupt() self._raise_if_db_replaced() + fn_started = False try: with self._lock: if self._conn is None: @@ -5953,6 +6322,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._reopen_after_close_locked(context="write") self._conn.execute("BEGIN IMMEDIATE") try: + fn_started = True result = fn(self._conn) self._conn.commit() except BaseException: @@ -6005,7 +6375,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) from exc if _is_no_more_rows(exc) and self._sleep_before_write_retry(deadline, patience_s): continue - # Non-lock error or patience exhausted — propagate. + if ( + _DISK_IO_ERROR_MARKER in err_msg + and not fn_started + and not ioerr_begin_retried + and self._sleep_before_write_retry(deadline, patience_s) + ): + # BEGIN IMMEDIATE itself hit a transient WAL-transition + # IOERR. Nothing has been mutated, so retrying on the SAME + # connection replays nothing. Never close()+reopen to + # "heal" it: close() cancels this process's POSIX locks on + # the file for every sibling connection (howtocorrupt §2.2). + ioerr_begin_retried = True + continue + # Non-lock error, the callback already ran (settlement is + # unknown — do not replay), or patience exhausted. raise except sqlite3.DatabaseError as exc: if _is_no_more_rows(exc) and self._sleep_before_write_retry(deadline, patience_s): @@ -6030,6 +6414,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # explicit repair paths retain rebuild ownership. if self._enter_fts_fail_open(exc): continue + # Bare SQLITE_CORRUPT / NOTADB that survived the replaced-file + # check and the FTS-scoped fail-open is structural damage: + # quarantine the handle (see StateDbCorruptError). + if self._is_structural_corruption_error(exc): + self._halt_db_corrupt(exc) raise except sqlite3.Error as exc: # Catch-all for builds that surface 'no more rows available' @@ -6085,6 +6474,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) def _record_db_file_identity(self) -> None: """Snapshot inode plus the on-disk generation header when present.""" self._db_file_identity = _stat_db_file_identity(self.db_path) + self._db_sidecar_identity = _stat_sqlite_sidecar_identity(self.db_path) disk_id = _read_sqlite_application_id(self.db_path) if disk_id: self._db_file_application_id = disk_id @@ -6119,11 +6509,151 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) logger.error(_STATE_DB_REPLACED_MSG) raise StateDbReplacedError(_STATE_DB_REPLACED_MSG) + def _wal_generation_was_lost(self) -> bool: + """True when the WAL/SHM generation this instance opened is gone. + + Steady state (a sidecar generation is recorded): pure stat — a + recorded inode that is missing or replaced by a new file at the same + path means the generation split. No /proc walk on healthy writes. + + Empty-identity state (fresh DB whose WAL appears only after open, or + identity cleared by a clean ``close()``): fall back to a + ``/proc/self/fd`` deleted-fd probe, and adopt the current sidecars as + this handle's generation once the probe comes back clean. The full + ``/proc/*/fd`` walk is reserved for + :func:`refuse_deleted_wal_generation` on open, where we must see + *foreign* deleted holders before ``sqlite3.connect`` mints a new WAL. + """ + recorded = self._db_sidecar_identity or {} + base = os.fspath(self.db_path) + if recorded: + for suffix, recorded_ident in recorded.items(): + current = _stat_db_file_identity(Path(base + suffix)) + if current is None or current != recorded_ident: + return True + return False + if not self._wal_active: + # No WAL on this handle (journal_mode=delete/truncate fallback): + # there is no sidecar generation to lose, and probing every write + # would put a /proc walk on the hot path of exactly the + # delete-mode deployments the field report used as containment. + return False + if sys.platform.startswith("linux"): + watched = _watched_sqlite_sidecar_paths(self.db_path) + fd_dir = f"/proc/{os.getpid()}/fd" + try: + for fd in os.listdir(fd_dir): + try: + target = os.readlink(f"{fd_dir}/{fd}") + except OSError: + continue + if " (deleted)" in target and _canonical_sqlite_path(target) in watched: + return True + except OSError: + return False + # Probe clean (or unavailable on this platform): adopt whatever + # sidecar generation exists now so subsequent writes use the cheap + # stat check. + current_identity = _stat_sqlite_sidecar_identity(self.db_path) + if current_identity: + self._db_sidecar_identity = current_identity + return False + + def _halt_deleted_wal_generation(self) -> None: + """Stop writes; do not mint or keep committing on a split WAL.""" + self._db_wal_generation_lost = True + logger.error(_DELETED_WAL_GENERATION_MSG) + raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) + def _raise_if_db_replaced(self) -> None: if self._db_replaced: raise StateDbReplacedError(_STATE_DB_REPLACED_MSG) + if self._db_wal_generation_lost: + raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) if self._db_file_was_replaced(): self._halt_db_replaced() + if self._wal_generation_was_lost(): + self._halt_deleted_wal_generation() + + @classmethod + def _is_structural_corruption_error(cls, exc: BaseException) -> bool: + """Bare SQLITE_CORRUPT/NOTADB with no FTS provenance. + + ``_is_fts_write_corruption_error`` is the positive FTS classifier; + everything else in the ``corrupt`` bucket of + ``classify_persistence_error`` is damage to a canonical B-tree, the + schema, or the freelist — never repairable from the live write path. + """ + if not isinstance(exc, sqlite3.DatabaseError): + return False + if isinstance(exc, StateDbCorruptError): + return False + if cls._is_fts_write_corruption_error(exc): + return False + return classify_persistence_error(exc) == "corrupt" + + def _corrupt_error(self, prefix: str = "") -> "StateDbCorruptError": + """Build the quarantine error for this handle (message assembled once).""" + return StateDbCorruptError( + f"{prefix}{_STATE_DB_CORRUPT_MSG} (cause: {self._db_corrupt_reason})" + ) + + def _halt_db_corrupt(self, exc: BaseException) -> None: + """Quarantine this handle and raise; never run in-file repair here.""" + self._db_corrupt = True + self._db_corrupt_reason = str(exc) + self._disable_close_time_checkpoint() + logger.error( + "state.db %s reported structural corruption outside the FTS " + "indexes (%s); quarantining this handle: no further writes, no " + "automatic reopen, no explicit WAL checkpoint at close. Stop the " + "gateway and run `hermes sessions recover --source %s " + "--inspect-only`.", + self.db_path, + exc, + self.db_path, + ) + err = self._corrupt_error() + for attr in ("sqlite_errorcode", "sqlite_errorname"): + value = getattr(exc, attr, None) + if value is not None: + setattr(err, attr, value) + raise err from exc + + def _disable_close_time_checkpoint(self) -> None: + """Best-effort: stop SQLite's own last-connection checkpoint on close. + + Skipping our explicit ``PRAGMA wal_checkpoint(PASSIVE)`` in + ``close()`` is not enough on its own: ``sqlite3.Connection.close()`` + still runs SQLite's internal last-connection PASSIVE checkpoint and + unlinks the ``-wal``/``-shm`` sidecars. On the field incident's file + that close-time checkpoint is exactly what wrote 15 pages under the + wrong page numbers. Python 3.12+ exposes the switch as + ``Connection.setconfig(SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE)``; on 3.11 + neither the constant nor ``setconfig`` exists, so the internal + checkpoint remains (it can only carry pre-quarantine committed + frames — no further writes are accepted on this handle). + """ + flag = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None) + if flag is None: + return + conn = self._conn + setconfig = getattr(conn, "setconfig", None) + if conn is None or setconfig is None: + return + try: + setconfig(flag, True) + except Exception: + logger.debug( + "Could not disable SQLite's close-time checkpoint on the " + "quarantined handle for %s", + self.db_path, + exc_info=True, + ) + + def _raise_if_db_corrupt(self) -> None: + if self._db_corrupt: + raise self._corrupt_error() def _sleep_before_write_retry( self, deadline: float, patience_s: float @@ -6172,108 +6702,8 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return msg.startswith("fts5:") and "corrupt structure" in msg def _foreign_state_db_holders(self) -> List[Tuple[int, str]]: - """Return foreign processes holding this DB or its WAL sidecars. - - Automatic FTS repair is structural maintenance, not an ordinary WAL - write. It must not run while another process remains attached: a - sidecar reset under that holder can leave the two processes writing - through different WAL inodes. - - A scan failure is represented as an unknown holder. Skipping optional - automatic maintenance is safer than assuming quiescence; canonical - writes continue through the stale-FTS fail-open path. - """ - # The split-brain mechanism requires POSIX unlink semantics: Windows - # refuses to replace SQLite sidecars while another process has them - # open. Avoid psutil.open_files() there; querying arbitrary Windows - # processes can block for minutes on device-backed handles. - if _IS_WINDOWS: - return [] - if psutil is None: - return [(-1, "open-file scan unavailable")] - - def _canonical(path: str) -> str: - clean = path.removesuffix(" (deleted)") - return os.path.normcase(os.path.abspath(clean)) - - db_path = os.path.abspath(os.fspath(self.db_path)) - watched = { - _canonical(db_path), - _canonical(db_path + "-wal"), - _canonical(db_path + "-shm"), - } - holders: List[Tuple[int, str]] = [] - - # On Linux, read /proc//fd symlinks directly. psutil's - # open_files() filters through isfile_strict(), which stats the - # literal path — for an unlinked WAL sidecar the kernel returns - # "/path/state.db-wal (deleted)" and stat fails, so the entry is - # silently dropped and the split-brain holder is never seen. - # /proc readlinks preserve the "(deleted)" suffix so _canonical can - # strip it and match. - if sys.platform.startswith("linux"): - try: - own_pid = os.getpid() - for pid_str in os.listdir("/proc"): - if not pid_str.isdigit(): - continue - pid = int(pid_str) - if pid == own_pid: - continue - fd_dir = f"/proc/{pid}/fd" - try: - fds = os.listdir(fd_dir) - except OSError: - # Cannot read this process's fd table (different - # user, e.g. root gateway vs user desktop). - # /proc//cmdline is world-readable by default, - # so check whether this is a Hermes process — - # only flag uninspectable holders that look like - # another Hermes instance, not every system daemon. - cmdline = _read_proc_cmdline(pid) - if cmdline is not None and _looks_like_hermes(cmdline): - holders.append((pid, f"uninspectable holder: {cmdline[:80]}")) - continue - for fd in fds: - try: - target = os.readlink(f"{fd_dir}/{fd}") - except OSError: - continue - if _canonical(target) in watched: - holders.append((pid, target)) - except Exception as exc: - logger.warning( - "Could not prove state.db has no foreign holders; " - "deferring automatic FTS maintenance: %s", - exc, - ) - return holders or [(-1, f"open-file scan failed: {exc}")] - return holders - - # macOS / BSD: use psutil.open_files(). macOS does not use the - # "(deleted)" suffix convention, so psutil's filtering is safe here. - try: - for process in psutil.process_iter(["pid", "open_files"]): - info = process.info - pid = int(info["pid"]) - if pid == os.getpid(): - continue - # psutil's as_dict() converts AccessDenied to None, which - # or-() turns into an empty iteration. On macOS this is - # acceptable: the gateway/desktop topology from the issue is - # Linux-specific (systemd units running as root). - for opened in info.get("open_files") or (): - path = getattr(opened, "path", "") - if path and _canonical(path) in watched: - holders.append((pid, path)) - except Exception as exc: - logger.warning( - "Could not prove state.db has no foreign holders; " - "deferring automatic FTS maintenance: %s", - exc, - ) - return holders or [(-1, f"open-file scan failed: {exc}")] - return holders + """Return foreign processes holding this DB or its WAL sidecars.""" + return _foreign_state_db_holders(self.db_path) def _reap_inactive_orphan_desktop_holders( self, holders: List[Tuple[int, str]], *, min_age_seconds: float @@ -6346,8 +6776,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) """ if not self._fts_enabled or not self._is_fts_write_corruption_error(exc): return False + self._raise_if_db_corrupt() if self._db_replaced or self._db_file_was_replaced(): self._halt_db_replaced() + if self._db_wal_generation_lost or self._wal_generation_was_lost(): + self._halt_deleted_wal_generation() try: with self._lock: @@ -6413,6 +6846,8 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) databases (65K+ pages) due to the exclusive-lock I/O pressure from checkpointing thousands of frames at once (issue #45383). """ + if self._db_corrupt: + return # quarantined: never checkpoint over a damaged image try: with self._lock: result = self._conn.execute( @@ -6504,7 +6939,20 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._close_read_conn(conn) with self._lock: if self._conn: - if not self.read_only: + if self._db_corrupt: + # Quarantined handle (see StateDbCorruptError): no explicit + # checkpoint over a damaged page image. + logger.warning( + "Skipping the close-time WAL checkpoint for %s: this " + "handle observed structural corruption (%s). Take a " + "snapshot of state.db, -wal and -shm before restarting, " + "then run `hermes sessions recover --source %s " + "--inspect-only`.", + self.db_path, + self._db_corrupt_reason, + self.db_path, + ) + elif not self.read_only: # PASSIVE, not TRUNCATE. Every cron run_agent opens+closes a # transient SessionDB, so a TRUNCATE here fires a full WAL # reset many times/hour, racing the gateway's long-lived @@ -6521,6 +6969,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) conn, self._conn = self._conn, None self._close_connection_quietly(conn) + # A clean close of the last connection lets SQLite unlink the + # WAL/SHM sidecars — a legitimate end of this handle's sidecar + # generation, not a split (#94736 late writes must still + # self-heal). Drop the recorded generation so a teardown-race + # reopen re-adopts whatever exists then instead of halting. + self._db_sidecar_identity = {} def __del__(self) -> None: """Safety net: close the connection if the caller forgot. @@ -6789,7 +7243,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._delete_unreferenced_system_prompts(conn) if parent_session_id: conn.execute( - """UPDATE sessions + f"""UPDATE sessions SET cwd = COALESCE(sessions.cwd, (SELECT p.cwd FROM sessions p WHERE p.id = sessions.parent_session_id)), @@ -6801,7 +7255,8 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) WHERE p.id = sessions.parent_session_id)), profile_name = COALESCE(sessions.profile_name, (SELECT p.profile_name FROM sessions p - WHERE p.id = sessions.parent_session_id)) + WHERE p.id = sessions.parent_session_id + AND ({_SAME_KEY_NAMESPACE_SQL}))) WHERE id = ? AND parent_session_id IS NOT NULL""", (session_id,), ) @@ -7373,6 +7828,15 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # tuple so we never cross chats/threads/users. if chat_id is None or chat_type is None: return None + # Profile fence (#74285): a Telegram DM's peer tuple is identical + # for every bot (chat_id == user_id, no thread), so a sibling + # profile's row written into this store before the per-profile + # partition (legacy data) would otherwise be adopted here. Every + # profile-tree store has one owner; a row is ours when its + # profile_name is the owner or NULL (legacy rows this store + # minted). Stores outside the tree derive no owner and keep the + # historical unfenced behavior. + owner = self._own_profile_name() row = conn.execute( f""" SELECT s.*, @@ -7388,6 +7852,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) AND COALESCE(s.chat_id, '') = COALESCE(?, '') AND COALESCE(s.chat_type, '') = COALESCE(?, '') AND COALESCE(s.thread_id, '') = COALESCE(?, '') + AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?) AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) AND (COALESCE(s.message_count, 0) > 0 OR EXISTS ( SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 @@ -7407,7 +7872,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC LIMIT 1 """, - (source, user_id, chat_id, chat_type, thread_id), + (source, user_id, chat_id, chat_type, thread_id, owner, owner, owner), ).fetchone() return self._session_row_dict(row) if row else None @@ -8560,6 +9025,54 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._execute_write(_do) + def get_compression_recovery_deadline(self, session_id: str) -> float: + """Return the persisted anti-thrash recovery deadline (wall-clock epoch). + + ``0.0`` means "not armed". The deadline is the durable half of the + #14694 recovery clock: the gateway rebuilds the compressor on every + turn / cache eviction, so a process-local deadline restarted the + wait on each rebuild and a tripped session never earned its probe + (#100185). + """ + if not session_id: + return 0.0 + with self._read_ctx() as conn: + if conn is None: + return 0.0 + row = conn.execute( + "SELECT compression_recovery_deadline FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if row is None: + return 0.0 + value = ( + row["compression_recovery_deadline"] + if isinstance(row, sqlite3.Row) + else row[0] + ) + try: + return max(0.0, float(value or 0.0)) + except (TypeError, ValueError): + return 0.0 + + def set_compression_recovery_deadline(self, session_id: str, deadline: float) -> None: + """Persist the anti-thrash recovery deadline; ``0`` / ``None`` disarms it.""" + if not session_id: + return + try: + normalized = max(0.0, float(deadline or 0.0)) + except (TypeError, ValueError): + normalized = 0.0 + stored = normalized if normalized > 0.0 else None + + def _do(conn): + conn.execute( + "UPDATE sessions SET compression_recovery_deadline = ? WHERE id = ?", + (stored, session_id), + ) + + self._execute_write(_do) + # ────────────────────────────────────────────────────────────────────── # Compression locks # ────────────────────────────────────────────────────────────────────── @@ -9121,6 +9634,25 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._delete_unreferenced_system_prompts(conn) self._execute_write(_do) + def update_session_tool_names( + self, session_id: str, tool_names: Optional[List[str]] + ) -> None: + """Persist the session's resolved ``tools[]`` name order (JSON array). + + Read back by ``tools.mcp_tool.restore_agent_tool_prefix`` when a fresh + ``AIAgent`` is rebuilt for an existing session (gateway agent-cache + eviction) so a flipped ``check_fn`` verdict can't fork the cached tool + prefix. ``None`` clears the pin. + """ + payload = json.dumps(list(tool_names)) if tool_names is not None else None + + def _do(conn): + conn.execute( + "UPDATE sessions SET tool_names = ? WHERE id = ?", + (payload, session_id), + ) + self._execute_write(_do) + def update_session_model( self, session_id: str, model: str, provider: Optional[str] = None ) -> None: @@ -10138,57 +10670,54 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) max_idle_seconds: float, sources: Tuple[str, ...] = ("tui", "desktop", "subagent"), exclude_ids: Tuple[str, ...] = (), + exclude_pinned: bool = False, heartbeat_staleness_seconds: Optional[float] = None, heartbeat_ownership_grace_seconds: Optional[float] = None, + respect_gateway_heartbeats: bool = True, ) -> List[str]: """Close session rows orphaned by a dead gateway process (#65194, #94895). The TUI/desktop gateway reaps disconnected websocket sessions with an in-process ``threading.Timer`` grace timer; a gateway restart destroys - the timer and leaves the row ``ended_at IS NULL`` forever. This is - the startup-time complement: it closes rows for the given ``sources`` - whose ``started_at`` AND newest ``messages.timestamp`` are both older - than ``max_idle_seconds``, with a distinct + the timer and leaves the row ``ended_at IS NULL`` forever. This is the + startup-time complement: it closes rows for the given ``sources`` whose + ``started_at`` and canonical last-activity time are both older than + ``max_idle_seconds``, with a distinct ``end_reason='startup_orphan_reap'`` for traceability. - Both timestamps must be stale on purpose: message recency alone would - sweep a freshly created compression/branch child carrying old copied - message timestamps, while ``started_at`` alone would sweep a - long-lived session that is still actively producing messages. - Message-less rows fall back to ``started_at`` via COALESCE. + Canonical activity is the newest of ``last_activity_at`` (the in-turn + heartbeat) and the newest durable message timestamp, falling back to + ``started_at``. The separate ``started_at`` predicate protects freshly + created compression/branch children whose copied activity is old. - Only pass sources owned by the local UI stack (never messaging-gateway + Only pass sources whose lifecycle the caller owns (never messaging-gateway platforms like ``telegram`` — ending those triggers the #60609 routing - loop). ``exclude_ids`` spares rows this process still holds in - memory (a ``session.resume`` that landed during the startup grace - window). Non-destructive: messages are preserved and the row remains - resumable. First-reason-wins is preserved via ``ended_at IS NULL``. + loop). ``exclude_ids`` spares rows this process still holds in memory + (a ``session.resume`` that landed during the startup grace window). + ``exclude_pinned`` is intended for broad automatic sweeps; pinned rows + remain explicitly recoverable. Non-destructive: messages are preserved + and the row remains resumable. First-reason-wins is preserved via + ``ended_at IS NULL``. - Cross-backend liveness (#94895): when one ``state.db`` is shared by - N serve / gateway processes (isolated backends, fixed-port launchd - ``hermes serve``, desktop WS sidecar), each backend registers a row - in ``gateway_heartbeats`` refreshed every few seconds. A row is - only reaped when ``started_at``/message staleness hold AND no live - backend (heartbeat refreshed within ``heartbeat_staleness_seconds``, - default ``2 * max_idle_seconds``) could plausibly own it. + Cross-backend liveness (#94895): when one ``state.db`` is shared by N + serve / gateway processes, each backend refreshes a row in + ``gateway_heartbeats``. With ``respect_gateway_heartbeats`` enabled, a + row is only reaped when activity staleness holds AND no live backend + (heartbeat refreshed within ``heartbeat_staleness_seconds``, default + ``2 * max_idle_seconds``) could plausibly own it. Disable that gate only + for sources whose lifecycle is explicitly owned by state.db itself. Ownership inference: a live backend B ``owns`` a session S if ``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds`` - (default ``heartbeat_staleness_seconds``). The grace window - accommodates the deploy-time migration case where a backend just - wrote its first heartbeat row while its existing open sessions - predate the schema. The grace is bounded by the staleness window - so a fresh PID-reuse respawn cannot indefinitely protect sessions - inherited from a dead predecessor. + (default ``heartbeat_staleness_seconds``). The grace window covers a + migrating backend whose existing sessions predate its first heartbeat, + but is bounded so a fresh PID-reuse respawn cannot protect rows forever. + With no fresh heartbeat the predicate falls back to the legacy sweep. - When NO backend has ever written a heartbeat (legacy deployment - mid-upgrade before any process has registered) the predicate falls - back to the original behavior so we never silently strand a row - that pre-dates the schema. - - The SELECT + UPDATE run in one ``BEGIN IMMEDIATE`` write, so a sibling - process cannot sneak a new message or end-reason between the - staleness check and the close. Returns the swept session ids. + The SELECT, live-lease validation, and UPDATE run in one + ``BEGIN IMMEDIATE`` transaction. Active turn leases or compression + locks spare the row; expired/reclaimed guards are removed so their + former owner is fenced. Returns the swept session ids. """ srcs = tuple(s for s in sources if s) if max_idle_seconds <= 0 or not srcs: @@ -10200,56 +10729,76 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) hb_grace = ( heartbeat_ownership_grace_seconds - if heartbeat_ownership_grace_seconds and heartbeat_ownership_grace_seconds >= 0 + if heartbeat_ownership_grace_seconds is not None + and heartbeat_ownership_grace_seconds >= 0 else hb_staleness ) - cutoff = time.time() - max_idle_seconds - hb_cutoff = time.time() - hb_staleness + now = time.time() + cutoff = now - max_idle_seconds + hb_cutoff = now - hb_staleness placeholders = ",".join("?" for _ in srcs) staleness = ( - "started_at < ? AND COALESCE((SELECT MAX(m.timestamp) FROM messages m" - " WHERE m.session_id = sessions.id), started_at) < ?" + f"started_at < ? AND {_sql_session_last_active('sessions')} < ?" ) - - def _do(conn): - # Cross-process liveness gate (#94895). A session is "owned by - # a live backend" if any row in gateway_heartbeats is fresh - # (last_heartbeat >= hb_cutoff) AND was alive no later than - # ``sessions.started_at + hb_grace`` (heartbeats.started_at <= - # sessions.started_at + hb_grace). If at least one live backend - # matches, the row is not orphaned. - # - # ``hb_cutoff`` and ``hb_grace`` are computed above so all - # backends running concurrent sweep queries agree on the same - # boundaries. We do NOT clear heartbeats here — that's each - # backend's atexit responsibility via ``clear_backend_heartbeat``. - orphan_predicate = ( - f"{staleness} AND NOT EXISTS (" + pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else "" + heartbeat_params: Tuple[float, ...] = () + orphan_predicate = staleness + if respect_gateway_heartbeats: + orphan_predicate += ( + " AND NOT EXISTS (" "SELECT 1 FROM gateway_heartbeats h" " WHERE h.last_heartbeat >= ?" - f" AND h.started_at <= sessions.started_at + ?" + " AND h.started_at <= sessions.started_at + ?" ")" ) + heartbeat_params = (hb_cutoff, hb_grace) + + def _do(conn): rows = conn.execute( f"SELECT id FROM sessions WHERE ended_at IS NULL" - f" AND source IN ({placeholders}) AND {orphan_predicate}", - (*srcs, cutoff, cutoff, hb_cutoff, hb_grace), + f" AND source IN ({placeholders}){pin_scope}" + f" AND {orphan_predicate}", + (*srcs, cutoff, cutoff, *heartbeat_params), ).fetchall() excluded = {str(x) for x in exclude_ids if x} - victims = [str(r["id"]) for r in rows if str(r["id"]) not in excluded] + victims = [] + for row in rows: + sid = str(row["id"]) + if sid in excluded: + continue + try: + self._check_transcript_write_guards( + conn, + sid, + compression_lock_holder=None, + turn_lease_holder=None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + except ( + SessionCompressionInProgressError, + SessionTurnLeaseLostError, + ): + continue + victims.append(sid) if not victims: return [] - now = time.time() + closed_at = time.time() marks = ",".join("?" for _ in victims) - # Re-apply the same predicates under the write lock so a - # row that raced to activity between SELECT and UPDATE is - # spared (and so a freshly registered heartbeat from a sibling - # that started during this transaction can still save the row). + # Re-apply every scope/liveness predicate under the write lock. conn.execute( f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'" f" WHERE id IN ({marks}) AND ended_at IS NULL" + f" AND source IN ({placeholders}){pin_scope}" f" AND {orphan_predicate}", - (now, *victims, cutoff, cutoff, hb_cutoff, hb_grace), + ( + closed_at, + *victims, + *srcs, + cutoff, + cutoff, + *heartbeat_params, + ), ) return victims @@ -10444,8 +10993,8 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Bot Mode's forever-chat registry: the session titled exactly this, on a # bot's profile, IS the bot's canonical chat — resolved by exact-title # lookup on every open (no session-id pointer exists). The title is the - # identity, which is why _set_session_title refuses user renames of a - # hidden row holding it (#92473). + # identity, which is why _set_session_title refuses renames of a hidden + # row holding it (#92473). CANONICAL_BOT_CHAT_TITLE = "Bot Chat" @classmethod @@ -10562,7 +11111,8 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) stored title has strictly lower authority, so the instant ``derived`` title upgrades to ``llm`` exactly once and neither can ever overwrite a name the user typed. Re-running the titler on an already-``llm`` row is - a no-op, which is what stops a session renaming itself. + a no-op, which is what stops a session renaming itself. The one thing + no writer may do is move a hidden canonical Bot Chat off its title. The read and the write are one compare-and-swap inside a single transaction, so a manual ``/title`` racing an in-flight generation @@ -10587,18 +11137,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # surface funnels through (gateway session.title, /title, CLI # rename, REST). Hidden is the discriminator: canonical chats are # born hidden; an ordinary visible session a user happens to call - # "Bot Chat" stays freely renameable. + # "Bot Chat" stays freely renameable. Provenance-blind: an + # automatic llm write outranks a derived title, so the auto-titler + # would otherwise rename the row too (#99517) — it no-ops instead. if ( - is_user - and (current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE + (current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE and bool(current["hidden"]) and title != self.CANONICAL_BOT_CHAT_TITLE ): - raise ValueError( - "This is the bot's canonical Bot Chat — its name is its " - "identity, and renaming it would orphan the conversation. " - "To start fresh, create a new bot instead." - ) + if is_user: + raise ValueError( + "This is the bot's canonical Bot Chat — its name is its " + "identity, and renaming it would orphan the conversation. " + "To start fresh, create a new bot instead." + ) + return 0 if not is_user and current["title"] is not None: if self._title_rank(current["title_source"]) >= new_rank: return 0 @@ -11134,8 +11687,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return f"{base} #{max_num + 1}" - def get_compression_tip(self, session_id: str) -> Optional[str]: - """Walk the compression-continuation chain forward and return the tip. + def get_compression_chain(self, session_id: str) -> List[str]: + """Walk the compression-continuation chain forward and return every id. + + Root-first order, ending at the tip; ``[session_id]`` when no + continuation exists. ``get_compression_tip`` is this walk's last + element — kept as the single implementation so the two can never + disagree about what the chain is. A compression continuation is a child of a session whose ``end_reason = 'compression'``. Older builds tried to distinguish @@ -11156,6 +11714,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) continuation exists. """ current = session_id + chain = [current] if current else [] seen = {current} if current else set() # Bound the walk defensively — compression chains this deep are # pathological and shouldn't happen in practice. 100 = plenty. @@ -11186,13 +11745,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) row = cursor.fetchone() if row is None: - return current + return chain child_id = row["id"] if not child_id or child_id in seen: - return current + return chain seen.add(child_id) current = child_id - return current + chain.append(child_id) + return chain + + def get_compression_tip(self, session_id: str) -> Optional[str]: + """The live tip of a compression-continuation chain (see + ``get_compression_chain`` for the walk's semantics). Returns the input + id when no continuation exists.""" + chain = self.get_compression_chain(session_id) + return chain[-1] if chain else session_id # Columns excluded from compact_rows projections: only the payload-heavy # blob no list consumer renders. Everything else — including gateway @@ -11234,6 +11801,213 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)} + def list_recent_sessions_bounded( + self, + *, + limit: int = 20, + exclude_sources: List[str] = None, + timeout_seconds: float = 3.0, + candidate_limit: int = None, + lineage_limit: int = None, + ) -> List[Dict[str, Any]]: + """List recent user conversations without an unbounded message scan. + + This is the latency-bounded browse path used by ``session_search()``. + It deliberately separates cheap candidate selection from expensive + hydration: + + 1. preselect a small set of session ids from the indexed durable + activity timestamp (falling back to ``started_at``); + 2. resolve only those candidates across compression ancestry/chains; + 3. calculate message-derived activity and previews only for that + bounded set. + + Compression ancestry and descendant traversal use ``UNION`` so a + corrupt cycle cannot revisit the same session for one logical root, + plus a total-row ceiling so a deep or highly branching lineage cannot + defeat the candidate bound. If that ceiling is reached before a + candidate resolves to a terminal root/tip, that incomplete lineage is + omitted from the browse result rather than expanded without bound. + + The query has a cooperative SQLite VM progress deadline. Expensive + statements that remain active beyond ``timeout_seconds`` are + interrupted at the next progress callback and this method raises + ``TimeoutError`` instead of holding a gateway callback indefinitely. + Cheap statements may finish between callbacks; the deadline is a + fail-safe for sustained work, not a real-time scheduler guarantee. + + This method intentionally supports only the filters needed by the + agent-tool browse shape. Rich dashboard/search callers keep using + :meth:`list_sessions_rich`. + """ + limit = max(1, int(limit)) + timeout_seconds = max(0.0, float(timeout_seconds)) + if candidate_limit is None: + candidate_limit = max(128, limit * 8) + candidate_limit = max(limit, min(int(candidate_limit), 2048)) + if lineage_limit is None: + lineage_limit = min(8192, candidate_limit * 8) + lineage_limit = max(candidate_limit, min(int(lineage_limit), 8192)) + + candidate_clauses = [ + "s.archived = 0", + "s.hidden = 0", + f"{_delegate_from_json('s.model_config')} IS NULL", + ] + candidate_params: List[Any] = [] + if exclude_sources: + placeholders = ",".join("?" for _ in exclude_sources) + candidate_clauses.append(f"s.source NOT IN ({placeholders})") + candidate_params.extend(exclude_sources) + candidate_where = " AND ".join(candidate_clauses) + + # A compression continuation is an implementation edge, unlike /new + # reset and /branch children which are independent user-visible + # conversations. The same predicate is used in both directions so a + # candidate tip maps to its logical root and the root maps back to the + # freshest live tip. + compression_parent_edge = f""" + parent.end_reason = 'compression' + AND child.parent_session_id = parent.id + AND json_extract( + COALESCE(child.model_config, '{{}}'), '$._branched_from' + ) IS NULL + AND {_delegate_from_json('child.model_config')} IS NULL + AND COALESCE(child.source, '') != 'tool' + """ + + query = f""" + WITH RECURSIVE + recent_candidates(id) AS ( + SELECT s.id + FROM sessions s + WHERE {candidate_where} + ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC, + s.started_at DESC, s.id DESC + LIMIT ? + ), + ancestors(candidate_id, cur_id) AS ( + SELECT id, id FROM recent_candidates + UNION + SELECT a.candidate_id, parent.id + FROM ancestors a + JOIN sessions child ON child.id = a.cur_id + JOIN sessions parent ON {compression_parent_edge} + LIMIT ? + ), + candidate_roots(root_id) AS ( + SELECT DISTINCT a.cur_id + FROM ancestors a + JOIN sessions child ON child.id = a.cur_id + WHERE NOT EXISTS ( + SELECT 1 + FROM sessions parent + WHERE {compression_parent_edge} + ) + ), + chain(root_id, cur_id) AS ( + SELECT root_id, root_id FROM candidate_roots + UNION + SELECT c.root_id, child.id + FROM chain c + JOIN sessions parent ON parent.id = c.cur_id + JOIN sessions child ON {compression_parent_edge} + LIMIT ? + ), + chain_rows AS ( + SELECT + c.root_id, + c.cur_id, + {_sql_session_last_active_by_id('c.cur_id')} AS activity, + CASE WHEN EXISTS ( + SELECT 1 + FROM sessions parent + JOIN sessions child ON {compression_parent_edge} + WHERE parent.id = c.cur_id + ) THEN 0 ELSE 1 END AS is_tip + FROM chain c + ), + ranked_tips AS ( + SELECT root_id, cur_id, activity, + ROW_NUMBER() OVER ( + PARTITION BY root_id + ORDER BY activity DESC, cur_id DESC + ) AS rank_in_root + FROM chain_rows + WHERE is_tip = 1 + ) + SELECT + tip.id, + tip.source, + tip.model, + tip.title, + s.started_at AS started_at, + tip.ended_at, + tip.end_reason, + tip.message_count, + tip.tool_call_count, + rt.activity AS last_active, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = tip.id + AND m.role = 'user' + AND m.content IS NOT NULL + AND {_PREVIEW_ELIGIBLE_SQL} + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + CASE WHEN s.id != tip.id THEN s.id ELSE NULL END + AS _lineage_root_id + FROM ranked_tips rt + JOIN sessions s ON s.id = rt.root_id + JOIN sessions tip ON tip.id = rt.cur_id + WHERE rt.rank_in_root = 1 + AND s.archived = 0 + AND s.hidden = 0 + AND {_LISTABLE_CHILD_SQL} + AND {_delegate_from_json('s.model_config')} IS NULL + ORDER BY rt.activity DESC, s.started_at DESC, tip.id DESC + LIMIT ? + """ + params = candidate_params + [ + candidate_limit, + lineage_limit, + lineage_limit, + limit, + ] + deadline = time.monotonic() + timeout_seconds + interrupted_by_deadline = False + + def _deadline_progress_handler() -> int: + nonlocal interrupted_by_deadline + if time.monotonic() >= deadline: + interrupted_by_deadline = True + return 1 + return 0 + + try: + with self._read_ctx() as conn: + conn.set_progress_handler(_deadline_progress_handler, 1000) + try: + rows = conn.execute(query, params).fetchall() + finally: + conn.set_progress_handler(None, 0) + except sqlite3.OperationalError as exc: + if interrupted_by_deadline and "interrupt" in str(exc).lower(): + raise TimeoutError( + f"recent-session browse exceeded {timeout_seconds:g}s deadline" + ) from exc + raise + + sessions = [] + for row in rows: + session = self._session_row_dict(row) + session["preview"] = _shape_preview(session.pop("_preview_raw", "")) + session["unread"] = self.session_unread(session) + sessions.append(session) + return sessions + def list_sessions_rich( self, source: str = None, @@ -11570,12 +12344,15 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # call per compression root. Batch that half instead: resolve # every tip id first, then fetch all tip rows in a single query. tip_ids_by_root: Dict[str, str] = {} + chain_by_root: Dict[str, List[str]] = {} for s in sessions: if s.get("end_reason") != "compression": continue - tip_id = self.get_compression_tip(s["id"]) + chain = self.get_compression_chain(s["id"]) + tip_id = chain[-1] if chain else s["id"] if tip_id != s["id"]: tip_ids_by_root[s["id"]] = tip_id + chain_by_root[s["id"]] = chain tip_rows = ( self._get_session_rich_rows_batch( @@ -11603,6 +12380,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if key in tip_row: merged[key] = tip_row[key] merged["_lineage_root_id"] = s["id"] + # Every id on the chain, intermediates included. Root and tip + # alone are not enough client-side: a persisted tile or route + # can hold a MIDDLE segment's id (it was the tip when opened, + # then rotated again), and with only the root/tip pair such a + # surface can no longer prove it names this conversation — + # which is how one chat ends up open twice after compaction. + merged["_lineage_ids"] = chain_by_root.get(s["id"]) or None projected.append(merged) sessions = projected @@ -11759,6 +12543,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) turn_lease_ttl_seconds: float = 300.0, reject_active_turn_lease: bool = False, reject_active_compression_lock: bool = False, + allow_closed_compression_parent: bool = False, ) -> None: """Transcript-write admission checks, run INSIDE the write txn. @@ -11858,6 +12643,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) session is not None and session["ended_at"] is not None and session["end_reason"] == "compression" + and not allow_closed_compression_parent ): raise CompressionSessionClosedError(session_id) @@ -12868,6 +13654,56 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return self._execute_write(_do) + def _dedupe_display_generations(self, rows): + """Collapse compaction generations so each message appears once. + + Compaction epochs copy the protected tail into each new generation, so + one logical message can exist as several rows (identical + role/content/timestamp) with different ``active`` flags and ids. A + display read must surface each exactly once: prefer the live row, then + the newest generation. + + This is the ONE definition shared by every display projection — + :meth:`get_messages` (REST), :meth:`get_resume_conversations` and + :meth:`get_ancestor_display_prefix` (gateway resume), and + :meth:`get_messages_as_conversation` (warm-session payload) — so the + surfaces cannot disagree about the same transcript. *rows* must already + be ordered by ``id``; the returned list keeps that order. + """ + seen: Dict[Tuple[Any, ...], Any] = {} + for row in rows: + dedupe_content = row["content"] + if row["role"] == "user": + from agent.context_compressor import split_user_originated_turn + + candidate = { + "role": "user", + "content": self._decode_content(row["content"]), + "display_kind": row["display_kind"], + "display_metadata": self._decode_display_metadata( + row["display_metadata"] + ), + } + handoff, live_view = split_user_originated_turn(candidate) + if handoff is not None and live_view is not None: + dedupe_content = self._encode_content(live_view.get("content")) + # Tool fields participate in the dedupe key: compaction copies them + # verbatim, so identical tool messages across generations still + # collapse, while distinct tool calls that happen to share + # role/content/timestamp are never merged. + key = ( + row["role"], + dedupe_content, + row["timestamp"], + row["tool_call_id"], + row["tool_calls"], + row["tool_name"], + ) + cur = seen.get(key) + if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]): + seen[key] = row + return sorted(seen.values(), key=lambda r: r["id"]) + def get_messages( self, session_id: str, @@ -12932,14 +13768,9 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if after_id is not None: params.append(after_id) if include_compacted: - # Compaction epochs copy the protected tail into each new - # generation, so the same logical message can exist as several - # rows (identical role/content/timestamp) with different active - # flags and ids. A display read must surface each message exactly - # once: prefer the live row, then the newest generation. Read the - # full display set (a session's rows are bounded; the UI-level - # 500-row cap lives in the endpoint, not here), dedupe in Python, - # then apply paging. + # Read the full display set (a session's rows are bounded; the + # UI-level 500-row cap lives in the endpoint, not here), dedupe + # generations, then apply paging. with self._read_ctx() as conn: cursor = conn.execute( "SELECT * FROM messages WHERE session_id = ?" + active_clause @@ -12947,41 +13778,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) [session_id], ) all_rows = cursor.fetchall() - seen: dict = {} - for row in all_rows: - dedupe_content = row["content"] - if row["role"] == "user": - from agent.context_compressor import split_user_originated_turn - - candidate = { - "role": "user", - "content": self._decode_content(row["content"]), - "display_kind": row["display_kind"], - "display_metadata": self._decode_display_metadata( - row["display_metadata"] - ), - } - handoff, live_view = split_user_originated_turn(candidate) - if handoff is not None and live_view is not None: - dedupe_content = self._encode_content( - live_view.get("content") - ) - # Tool fields participate in the dedupe key: compaction copies - # them verbatim, so identical tool messages across generations - # still collapse, while distinct tool calls that happen to - # share role/content/timestamp are never merged. - key = ( - row["role"], - dedupe_content, - row["timestamp"], - row["tool_call_id"], - row["tool_calls"], - row["tool_name"], - ) - cur = seen.get(key) - if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]): - seen[key] = row - rows = sorted(seen.values(), key=lambda r: r["id"]) + rows = self._dedupe_display_generations(all_rows) if latest: rows = rows[::-1] rows = rows[offset:] @@ -13223,6 +14020,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) include_inactive: bool = False, repair_alternation: bool = False, include_row_ids: bool = False, + include_compacted: bool = False, ) -> List[Dict[str, Any]]: """ Load messages in the OpenAI conversation format (role + content dicts). @@ -13232,6 +14030,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ``include_inactive=True`` to load soft-deleted (rewound) rows as well. See :meth:`rewind_to_message`. + ``include_compacted=True`` additionally loads rows preserved by + in-place compaction (``active=0, compacted=1``), deduped by + :meth:`_dedupe_display_generations`. DISPLAY reads want this; the + model-fed restore must NOT pass it, or a resumed session regrows the + very history compaction just summarized away. + ``repair_alternation=True`` runs ``repair_message_sequence`` over the loaded list before returning it. Callers that restore a session for LIVE REPLAY should pass it: a durable alternation violation (e.g. a @@ -13246,7 +14050,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if include_ancestors and not self._is_explicit_branch_session(session_id): session_ids = self._session_lineage_root_to_tip(session_id) - active_clause = "" if include_inactive else " AND active = 1" + if include_inactive: + active_clause = "" + elif include_compacted: + active_clause = " AND (active = 1 OR compacted = 1)" + else: + active_clause = " AND active = 1" with self._read_ctx() as conn: placeholders = ",".join("?" for _ in session_ids) rows = conn.execute( @@ -13264,6 +14073,9 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) tuple(session_ids), ).fetchall() + if include_compacted: + rows = self._dedupe_display_generations(rows) + return self._rows_to_conversation( rows, session_id=session_id, @@ -13274,12 +14086,14 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Columns every conversation projection decodes. Shared by # get_messages_as_conversation and get_resume_conversations so a single - # SELECT can feed both the model-fed and display views. + # SELECT can feed both the model-fed and display views. ``active`` rides + # along so a display read can split the compaction-archived rows from the + # live set (and feed _dedupe_display_generations) without a second query. _CONVERSATION_ROW_COLUMNS = ( "id, role, content, tool_call_id, tool_calls, tool_name, effect_disposition, " "finish_reason, reasoning, reasoning_content, reasoning_details, " "codex_reasoning_items, codex_message_items, platform_message_id, observed, " - "_compressed_summary, timestamp, " + "_compressed_summary, timestamp, active, " "api_content, display_kind, display_metadata" ) @@ -13490,21 +14304,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) copied transcript; including the live parent's rows would let messages written to the original after the fork leak into the branch. + The display projection also includes rows preserved by IN-PLACE + compaction (``active=0, compacted=1``), deduped by + :meth:`_dedupe_display_generations`. Without them a compacted + conversation resumes showing only its summary plus the carried-forward + tail — the user's own turns read as deleted even though every row is + still on disk, and the REST transcript read (which has always included + them) disagreed with this one about the same session (#92080). + The display fetch already reads a superset of the model fetch (the tip rows are part of the lineage), so serving both from one lineage SELECT halves the resume's DB work versus two separate calls, with byte-identical output (see test_get_resume_conversations_matches_separate_reads). """ - session_ids = ( - [session_id] - if self._is_explicit_branch_session(session_id) - else self._session_lineage_root_to_tip(session_id) - ) + session_ids = self._resume_lineage_ids(session_id) with self._read_ctx() as conn: placeholders = ",".join("?" for _ in session_ids) rows = conn.execute( f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders}) AND active = 1 " + f"FROM messages WHERE session_id IN ({placeholders}) " + # Compaction-archived rows (active=0, compacted=1) are display + # history; Undo/Rewind rows (active=0, compacted=0) are not. + "AND (active = 1 OR compacted = 1) " # ORDER BY id (insertion order) — see get_messages_as_conversation # for why timestamp ordering is unsafe. "ORDER BY id", @@ -13513,8 +14334,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Tip rows are exactly the model-fed set (get_messages_as_conversation # with session_ids=[session_id]); filtering the lineage fetch preserves - # their relative id order. - tip_rows = [r for r in rows if r["session_id"] == session_id] + # their relative id order. The model projection stays active-only — it + # is the compressed working context and must not regrow the history + # compaction just summarized away. + tip_rows = [r for r in rows if r["session_id"] == session_id and r["active"]] model_history = self._rows_to_conversation( tip_rows, session_id=session_id, @@ -13527,7 +14350,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) include_summary_markers=True, ) display_history = self._rows_to_conversation( - rows, + self._dedupe_display_generations(rows), session_id=session_id, include_ancestors=True, repair_alternation=False, @@ -13535,14 +14358,45 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) return model_history, display_history - def get_resume_message_count(self, session_id: str) -> int: - """Count active rows that a full resume would materialize.""" - session_ids = self._session_lineage_root_to_tip(session_id) + def _resume_lineage_ids(self, session_id: str) -> List[str]: + """Session ids a full (display) resume materializes for *session_id*. + + Compression continuations need their ended ancestors' rows for the + display transcript; an explicit ``/branch`` copy already owns its + transcript, so its lineage is itself alone. This is the ONE definition + shared by the resume readers (``get_resume_conversations``, + ``get_ancestor_display_prefix``) and the resume guard + (``assert_resume_safe`` / ``get_resume_message_count``) — the guard must + count exactly the rows a resume would load, never a superset. + """ + if self._is_explicit_branch_session(session_id): + return [session_id] + return self._session_lineage_root_to_tip(session_id) + + def get_resume_message_count( + self, session_id: str, *, tip_only: bool = False + ) -> int: + """Count the rows that a resume would materialize. + + ``tip_only=True`` counts the tip segment's ACTIVE rows — the set a + model-history restore loads (``get_messages_as_conversation`` without + ancestors, or the deferred Desktop resume that pages the display + transcript over REST and never materializes the ancestor prefix in + memory). + + Otherwise this counts the full-lineage DISPLAY set — active rows plus + the compaction-archived rows ``get_resume_conversations`` now loads + for the transcript. Counting only active rows here would let a + heavily-compacted conversation pass a limit sized for a handful of + live rows and then materialize tens of thousands. + """ + session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) + active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" placeholders = ",".join("?" for _ in session_ids) with self._read_ctx() as conn: row = conn.execute( f"SELECT COUNT(*) FROM messages " - f"WHERE session_id IN ({placeholders}) AND active = 1", + f"WHERE session_id IN ({placeholders}) AND {active_clause}", tuple(session_ids), ).fetchone() return int(row[0] if row else 0) @@ -13551,12 +14405,29 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self, session_id: str, max_messages: Optional[int] = None, + *, + tip_only: bool = False, ) -> int: """Return resume row count or reject a transcript too large to load. ``max_messages=None`` resolves the limit from config (``sessions.max_resume_messages``); 0 disables the guard and returns the (bounded) count without raising. + + ``tip_only=True`` bounds only the tip segment's ACTIVE rows, for + callers that never materialize the ancestor lineage or the + compaction archive in memory (tip-only model restore, deferred + Desktop resume whose display history is REST-paginated). A + heavily-compressed conversation — 85 compaction segments and ~29k + lineage rows behind a ~700-row tip — is exactly the shape compression + is supposed to produce; counting its whole lineage against a limit + sized for in-memory materialization rejected the healthiest sessions + (Desktop Bot Chat stuck on "Waking up…" with code 4130) while the + process would only ever have held the tip. + + The full (non-``tip_only``) bound counts the DISPLAY set — active plus + compaction-archived rows — because that is what + ``get_resume_conversations`` materializes for the transcript. """ if max_messages is None: max_messages = resolved_max_resume_messages() @@ -13568,19 +14439,24 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # return value, and an unbounded lineage COUNT here would do the # exact pathological work the disable exists to avoid. return 0 - session_ids = self._session_lineage_root_to_tip(session_id) + session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) + active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" placeholders = ",".join("?" for _ in session_ids) with self._read_ctx() as conn: row = conn.execute( "SELECT COUNT(*) FROM (" f"SELECT 1 FROM messages WHERE session_id IN ({placeholders}) " - "AND active = 1 LIMIT ?" + f"AND {active_clause} LIMIT ?" ")", (*session_ids, max_messages + 1), ).fetchone() message_count = int(row[0] if row else 0) if message_count > max_messages: - raise SessionResumeTooLargeError(message_count, max_messages) + raise SessionResumeTooLargeError( + message_count, + max_messages, + scope="in its tip segment" if tip_only else "across its lineage", + ) return message_count def assert_export_safe( @@ -13639,20 +14515,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) returns ONLY the genuine ancestor messages, identified by ``session_id != tip_session_id``. (#65919) """ - if self._is_explicit_branch_session(session_id): - return [] - - session_ids = self._session_lineage_root_to_tip(session_id) + session_ids = self._resume_lineage_ids(session_id) if len(session_ids) <= 1: return [] with self._read_ctx() as conn: placeholders = ",".join("?" for _ in session_ids) rows = conn.execute( f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders}) AND active = 1 " + f"FROM messages WHERE session_id IN ({placeholders}) " + # Display read: compaction-archived rows included, Undo/Rewind + # rows excluded (see get_resume_conversations). + "AND (active = 1 OR compacted = 1) " "ORDER BY id", tuple(session_ids), ).fetchall() + rows = self._dedupe_display_generations(rows) ancestor_ids = { int(row["id"]) for row in rows @@ -14852,6 +15729,14 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) < ?""" ) params.append(last_active_before) + # An automatic orphan sweep closes a stale open row so the user can + # still recover it. Age those rows from the sweep, not from their old + # activity, or the next prune pass can delete them immediately. + clauses.append( + "(COALESCE(s.end_reason, '') != 'startup_orphan_reap' " + "OR s.ended_at < ?)" + ) + params.append(last_active_before) if last_active_after is not None: clauses.append( """COALESCE( @@ -15113,6 +15998,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) older_than_days: Optional[float] = 90, source: str = None, sessions_dir: Optional[Path] = None, + exclude_active_write_guards: bool = False, **filters, ) -> int: """Delete sessions matching the filters. Returns count deleted. @@ -15150,6 +16036,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) on-disk transcript files (``.json`` / ``.jsonl`` / ``request_dump_*``) for every pruned session, outside the DB transaction. + + ``exclude_active_write_guards`` is for destructive automatic + maintenance: rows protected by a live turn lease or compression lock + are skipped, while expired or provably dead holders are reclaimed and + fenced in the same write transaction. """ self._apply_prune_age_filter(older_than_days, filters) where, where_params = self._prune_filter_where(source=source, **filters) @@ -15161,6 +16052,26 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) session_ids = {row["id"] for row in cursor.fetchall()} + if exclude_active_write_guards: + protected = set() + for sid in session_ids: + try: + self._check_transcript_write_guards( + conn, + sid, + compression_lock_holder=None, + turn_lease_holder=None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + allow_closed_compression_parent=True, + ) + except ( + SessionCompressionInProgressError, + SessionTurnLeaseLostError, + ): + protected.add(sid) + session_ids.difference_update(protected) + if not session_ids: return 0 @@ -15399,12 +16310,22 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) v1 — initial shape (no ON DELETE CASCADE on session_id FK) v2 — session_id FK gets ON DELETE CASCADE so session pruning automatically clears bindings. + v3 — ``profile_name`` dimension on both tables so multiplexed + gateways (shared ``state.db``) isolate topic mode/bindings + per Hermes profile (issue #76423). """ - def _do(conn): - conn.executescript( + # (table, column list, DDL body). ``profile_name`` leads the primary + # key so multiplexed profiles sharing one state.db never collide on a + # private chat_id (which is the user id, identical across bots). + tables = ( + ( + "telegram_dm_topic_mode", + "profile_name, chat_id, user_id, enabled, activated_at, updated_at, " + "has_topics_enabled, allows_users_to_create_topics, " + "capability_checked_at, intro_message_id, pinned_message_id", """ - CREATE TABLE IF NOT EXISTS telegram_dm_topic_mode ( - chat_id TEXT PRIMARY KEY, + profile_name TEXT NOT NULL DEFAULT 'default', + chat_id TEXT NOT NULL, user_id TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1, activated_at REAL NOT NULL, @@ -15413,10 +16334,16 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) allows_users_to_create_topics INTEGER, capability_checked_at REAL, intro_message_id TEXT, - pinned_message_id TEXT - ); - - CREATE TABLE IF NOT EXISTS telegram_dm_topic_bindings ( + pinned_message_id TEXT, + PRIMARY KEY (profile_name, chat_id) + """, + ), + ( + "telegram_dm_topic_bindings", + "profile_name, chat_id, thread_id, user_id, session_key, " + "session_id, managed_mode, linked_at, updated_at", + """ + profile_name TEXT NOT NULL DEFAULT 'default', chat_id TEXT NOT NULL, thread_id TEXT NOT NULL, user_id TEXT NOT NULL, @@ -15425,65 +16352,50 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) managed_mode TEXT NOT NULL DEFAULT 'auto', linked_at REAL NOT NULL, updated_at REAL NOT NULL, - PRIMARY KEY (chat_id, thread_id) - ); + PRIMARY KEY (profile_name, chat_id, thread_id) + """, + ), + ) + def _do(conn): + for table, columns, ddl in tables: + # Fresh installs get the v3 shape immediately. + conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})") + have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")} + if "profile_name" in have: + continue + # Pre-profile shape (v1 or v2) → v3. SQLite can't ALTER a + # primary key (or a foreign key), so rebuild; this also + # supplies the v2 ON DELETE CASCADE for v1 bindings tables. + # Legacy rows land in the "default" namespace only — never + # replicated across profiles. + legacy_columns = columns.replace("profile_name, ", "", 1) + conn.executescript( + f""" + CREATE TABLE {table}_new ({ddl}); + INSERT INTO {table}_new ({columns}) + SELECT 'default', {legacy_columns} FROM {table}; + DROP TABLE {table}; + ALTER TABLE {table}_new RENAME TO {table}; + """ + ) + + # Indexes after any rebuild so they always target the v3 shape + # (a legacy table lacking profile_name can't take the user index). + conn.executescript( + """ CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session ON telegram_dm_topic_bindings(session_id); CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user - ON telegram_dm_topic_bindings(user_id, chat_id); + ON telegram_dm_topic_bindings(profile_name, user_id, chat_id); """ ) - # v1 → v2: rebuild telegram_dm_topic_bindings if its session_id FK - # lacks ON DELETE CASCADE. SQLite can't ALTER a foreign key, so we - # rebuild the table. Only runs once per DB (version gate). - current = conn.execute( - "SELECT value FROM state_meta WHERE key = ?", - ("telegram_dm_topic_schema_version",), - ).fetchone() - current_version = int(current[0]) if current and str(current[0]).isdigit() else 0 - if current_version < 2: - fk_rows = conn.execute( - "PRAGMA foreign_key_list('telegram_dm_topic_bindings')" - ).fetchall() - needs_rebuild = any( - row[2] == "sessions" and (row[6] or "") != "CASCADE" - for row in fk_rows - ) - if needs_rebuild: - conn.executescript( - """ - CREATE TABLE telegram_dm_topic_bindings_new ( - chat_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - user_id TEXT NOT NULL, - session_key TEXT NOT NULL, - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - managed_mode TEXT NOT NULL DEFAULT 'auto', - linked_at REAL NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (chat_id, thread_id) - ); - INSERT INTO telegram_dm_topic_bindings_new - SELECT chat_id, thread_id, user_id, session_key, - session_id, managed_mode, linked_at, updated_at - FROM telegram_dm_topic_bindings; - DROP TABLE telegram_dm_topic_bindings; - ALTER TABLE telegram_dm_topic_bindings_new - RENAME TO telegram_dm_topic_bindings; - CREATE UNIQUE INDEX idx_telegram_dm_topic_bindings_session - ON telegram_dm_topic_bindings(session_id); - CREATE INDEX idx_telegram_dm_topic_bindings_user - ON telegram_dm_topic_bindings(user_id, chat_id); - """ - ) - conn.execute( "INSERT INTO state_meta (key, value) VALUES (?, ?) " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - ("telegram_dm_topic_schema_version", "2"), + ("telegram_dm_topic_schema_version", "3"), ) self._execute_write(_do) @@ -15492,6 +16404,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) *, chat_id: str, user_id: str, + profile_name: str = "default", has_topics_enabled: Optional[bool] = None, allows_users_to_create_topics: Optional[bool] = None, ) -> None: @@ -15499,9 +16412,15 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) This method intentionally owns the explicit topic migration. Ordinary SessionDB startup must not create these side tables. + + ``profile_name`` namespaces rows under a shared multiplex ``state.db`` + (issue #76423). Callers handling a multiplexed event must pass the + routed profile from ``source.profile``, not the process-global active + profile. """ self.apply_telegram_topic_migration() now = time.time() + profile_name = _normalize_telegram_topic_profile_name(profile_name) def _to_int(value: Optional[bool]) -> Optional[int]: if value is None: @@ -15512,11 +16431,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) conn.execute( """ INSERT INTO telegram_dm_topic_mode ( - chat_id, user_id, enabled, activated_at, updated_at, + profile_name, chat_id, user_id, enabled, activated_at, updated_at, has_topics_enabled, allows_users_to_create_topics, capability_checked_at - ) VALUES (?, ?, 1, ?, ?, ?, ?, ?) - ON CONFLICT(chat_id) DO UPDATE SET + ) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?) + ON CONFLICT(profile_name, chat_id) DO UPDATE SET user_id = excluded.user_id, enabled = 1, updated_at = excluded.updated_at, @@ -15525,6 +16444,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) capability_checked_at = excluded.capability_checked_at """, ( + profile_name, str(chat_id), str(user_id), now, @@ -15540,6 +16460,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self, *, chat_id: str, + profile_name: str = "default", clear_bindings: bool = True, ) -> None: """Disable Telegram DM topic mode for one private chat. @@ -15552,33 +16473,43 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) Never creates the topic-mode tables from scratch; if they don't exist there is nothing to disable and the call is a no-op. """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) + def _do(conn): try: conn.execute( "UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? " - "WHERE chat_id = ?", - (time.time(), str(chat_id)), + "WHERE profile_name = ? AND chat_id = ?", + (time.time(), profile_name, str(chat_id)), ) if clear_bindings: conn.execute( - "DELETE FROM telegram_dm_topic_bindings WHERE chat_id = ?", - (str(chat_id),), + "DELETE FROM telegram_dm_topic_bindings " + "WHERE profile_name = ? AND chat_id = ?", + (profile_name, str(chat_id)), ) except sqlite3.OperationalError: # Tables don't exist yet — nothing to disable. return self._execute_write(_do) - def is_telegram_topic_mode_enabled(self, *, chat_id: str, user_id: str) -> bool: + def is_telegram_topic_mode_enabled( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + ) -> bool: """Return whether Telegram DM topic mode is enabled for this chat/user.""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) with self._read_ctx() as conn: try: row = conn.execute( """ SELECT enabled FROM telegram_dm_topic_mode - WHERE chat_id = ? AND user_id = ? + WHERE profile_name = ? AND chat_id = ? AND user_id = ? """, - (str(chat_id), str(user_id)), + (profile_name, str(chat_id), str(user_id)), ).fetchone() except sqlite3.OperationalError: return False @@ -15592,16 +16523,18 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) *, chat_id: str, thread_id: str, + profile_name: str = "default", ) -> Optional[Dict[str, Any]]: """Return the session binding for a Telegram DM topic, if present.""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) with self._read_ctx() as conn: try: row = conn.execute( """ SELECT * FROM telegram_dm_topic_bindings - WHERE chat_id = ? AND thread_id = ? + WHERE profile_name = ? AND chat_id = ? AND thread_id = ? """, - (str(chat_id), str(thread_id)), + (profile_name, str(chat_id), str(thread_id)), ).fetchone() except sqlite3.OperationalError: return None @@ -15611,18 +16544,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self, *, chat_id: str, + profile_name: str = "default", ) -> List[Dict[str, Any]]: """All Telegram DM topic bindings for one chat, newest first. Read-only; returns [] if the bindings table doesn't exist yet (does not trigger the topic-mode migration). """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) with self._read_ctx() as conn: try: rows = conn.execute( "SELECT * FROM telegram_dm_topic_bindings " - "WHERE chat_id = ? ORDER BY updated_at DESC", - (str(chat_id),), + "WHERE profile_name = ? AND chat_id = ? " + "ORDER BY updated_at DESC", + (profile_name, str(chat_id)), ).fetchall() except sqlite3.OperationalError: return [] @@ -15657,6 +16593,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) *, chat_id: str, thread_id: str, + profile_name: str = "default", ) -> int: """Remove the binding row for a single (chat, thread) pair. @@ -15687,6 +16624,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) """ chat_id = str(chat_id) thread_id = str(thread_id) + profile_name = _normalize_telegram_topic_profile_name(profile_name) deleted = {"count": 0} def _do(conn): @@ -15694,9 +16632,9 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) cursor = conn.execute( """ DELETE FROM telegram_dm_topic_bindings - WHERE chat_id = ? AND thread_id = ? + WHERE profile_name = ? AND chat_id = ? AND thread_id = ? """, - (chat_id, thread_id), + (profile_name, chat_id, thread_id), ) deleted["count"] = cursor.rowcount or 0 except sqlite3.OperationalError: @@ -15712,15 +16650,16 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) remaining = conn.execute( """ SELECT 1 FROM telegram_dm_topic_bindings - WHERE chat_id = ? LIMIT 1 + WHERE profile_name = ? AND chat_id = ? LIMIT 1 """, - (chat_id,), + (profile_name, chat_id), ).fetchone() if remaining is None: conn.execute( "UPDATE telegram_dm_topic_mode " - "SET enabled = 0, updated_at = ? WHERE chat_id = ?", - (time.time(), chat_id), + "SET enabled = 0, updated_at = ? " + "WHERE profile_name = ? AND chat_id = ?", + (time.time(), profile_name, chat_id), ) except sqlite3.OperationalError: # telegram_dm_topic_mode absent — binding prune still stands. @@ -15738,6 +16677,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) session_key: str, session_id: str, managed_mode: str = "auto", + profile_name: str = "default", ) -> None: """Bind one Telegram DM topic thread to one Hermes session. @@ -15752,28 +16692,38 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) user_id = str(user_id) session_key = str(session_key) session_id = str(session_id) + profile_name = _normalize_telegram_topic_profile_name(profile_name) def _do(conn): existing_session = conn.execute( """ - SELECT chat_id, thread_id FROM telegram_dm_topic_bindings + SELECT profile_name, chat_id, thread_id + FROM telegram_dm_topic_bindings WHERE session_id = ? """, (session_id,), ).fetchone() if existing_session is not None: - linked_chat = existing_session["chat_id"] if isinstance(existing_session, sqlite3.Row) else existing_session[0] - linked_thread = existing_session["thread_id"] if isinstance(existing_session, sqlite3.Row) else existing_session[1] - if str(linked_chat) != chat_id or str(linked_thread) != thread_id: + if isinstance(existing_session, sqlite3.Row): + linked_profile = existing_session["profile_name"] + linked_chat = existing_session["chat_id"] + linked_thread = existing_session["thread_id"] + else: + linked_profile, linked_chat, linked_thread = existing_session + if ( + str(linked_profile) != profile_name + or str(linked_chat) != chat_id + or str(linked_thread) != thread_id + ): raise ValueError("session is already linked to another Telegram topic") conn.execute( """ INSERT INTO telegram_dm_topic_bindings ( - chat_id, thread_id, user_id, session_key, session_id, + profile_name, chat_id, thread_id, user_id, session_key, session_id, managed_mode, linked_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(chat_id, thread_id) DO UPDATE SET + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET user_id = excluded.user_id, session_key = excluded.session_key, session_id = excluded.session_id, @@ -15781,6 +16731,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) updated_at = excluded.updated_at """, ( + profile_name, chat_id, thread_id, user_id, @@ -15820,6 +16771,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) *, chat_id: str, user_id: str, + profile_name: str = "default", limit: int = 10, ) -> List[Dict[str, Any]]: """List previous Telegram sessions for this user that are not bound to a topic. @@ -15828,7 +16780,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) topic-mode tables are absent, fall back to a simpler query that just returns this user's Telegram sessions — there can't be any bindings yet. + + Scoped by ``profile_name`` so multiplexed profiles do not surface + each other's unlinked sessions (issue #76423). """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) + # sessions.profile_name is NULL/empty for legacy rows → treat as default. + profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?" with self._read_ctx() as conn: try: rows = conn.execute( @@ -15850,6 +16808,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ON sp.hash = s.system_prompt_hash WHERE s.source = 'telegram' AND s.user_id = ? + {profile_clause} AND NOT EXISTS ( SELECT 1 FROM telegram_dm_topic_bindings b WHERE b.session_id = s.id @@ -15857,7 +16816,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ORDER BY last_active DESC, s.started_at DESC LIMIT ? """, - (str(user_id), int(limit)), + (str(user_id), profile_name, int(limit)), ).fetchall() except sqlite3.OperationalError: # telegram_dm_topic_bindings doesn't exist yet — no bindings @@ -15930,6 +16889,31 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) logger.debug("Could not read logical DB size: %s", exc) return None + def _freelist_ratio(self) -> Optional[float]: + """Fraction of database pages that are on the freelist (reclaimable). + + ``PRAGMA freelist_count / PRAGMA page_count`` read over the existing + connection (never a byte-level probe of the live file — see + ``sqlite_safe_read``). This is what VACUUM would actually give back; + it is the gate :meth:`maybe_auto_prune_and_vacuum` uses to decide + whether a full rewrite pays off (#54189). + + Returns None if the pragmas cannot be read (callers treat that as + "unknown" and fall back to the time throttle alone). + """ + try: + with self._read_ctx() as conn: + if self._conn is None: + return None + page_count = int(conn.execute("PRAGMA page_count").fetchone()[0]) + freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0]) + if page_count <= 0: + return 0.0 + return freelist / page_count + except Exception as exc: + logger.debug("Could not read freelist ratio: %s", exc) + return None + def vacuum(self) -> int: """Run VACUUM to reclaim disk space after large deletes. @@ -15981,6 +16965,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") except Exception as exc: logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc) + # TRUNCATE may replace the WAL inode; adopt the post-VACUUM + # sidecars so the write-path generation guard does not halt a + # healthy exclusive maintenance connection. + self._record_db_file_identity() return optimized def maybe_auto_prune_and_vacuum( @@ -15990,30 +16978,59 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) vacuum: bool = True, sessions_dir: Optional[Path] = None, min_vacuum_interval_days: int = 30, + min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO, ) -> Dict[str, Any]: """Idempotent auto-maintenance: prune inactive sessions + optional VACUUM. Records the last run timestamp in state_meta so subsequent calls within ``min_interval_hours`` no-op. VACUUM has its own, typically longer, throttle controlled by ``min_vacuum_interval_days`` so routine - pruning does not repeatedly rewrite the database. Designed to be - called once at startup from long-lived entrypoints (CLI, gateway, cron - scheduler). + pruning does not repeatedly rewrite the database, and is additionally + gated on the reclaimable fraction of the file: it only runs when + ``PRAGMA freelist_count / PRAGMA page_count`` exceeds + ``min_vacuum_freelist_ratio`` (default + :data:`AUTO_VACUUM_MIN_FREELIST_RATIO`, 25%), so pruning a few small + sessions on a dense multi-GB database never triggers a full rewrite + (#54189). Designed to be called once at startup from long-lived + entrypoints (CLI, gateway, cron scheduler). When *sessions_dir* is provided, on-disk transcript files (``.json`` / ``.jsonl`` / ``request_dump_*``) for pruned sessions are removed as part of the same sweep (issue #3015). + Stale-open reconciliation (#54189): several state-owned producers + (cron, kanban workers, subagents, one-shot CLI runs) never set + ``ended_at`` when their process dies, and ``prune_sessions`` only + deletes ended rows — so retention was a no-op exactly where growth + concentrates. After pruning, this pass closes open rows from + :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES` whose activity is older than + ``retention_days`` (``end_reason='startup_orphan_reap'``). Closed rows + stay resumable and are aged from their close, so they get one more + full retention window before a later pass deletes them. Messaging + and UI sources are never touched here. + Never raises. On any failure, logs a warning and returns a dict with ``"error"`` set. Returns a dict with keys: - ``"skipped"`` (bool) — true if within min_interval_hours of last run - ``"pruned"`` (int) — number of sessions deleted + - ``"closed"`` (int) — stale open state-owned sessions marked ended - ``"vacuumed"`` (bool) — true if VACUUM ran + - ``"freelist_ratio"`` (float|None) — reclaimable fraction measured + when a VACUUM was considered (absent when it was not) - ``"error"`` (str, optional) — present only on failure """ - result: Dict[str, Any] = {"skipped": False, "pruned": 0, "vacuumed": False} + result: Dict[str, Any] = { + "skipped": False, + "pruned": 0, + "closed": 0, + "vacuumed": False, + } + maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path) + if maintenance_lock is None: + result["skipped"] = True + return result try: # Skip if another process/call did maintenance recently. last_raw = self.get_meta("last_auto_prune") @@ -16027,19 +17044,40 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) except (TypeError, ValueError): pass # corrupt meta; treat as no prior run + # Delete only sessions that were already explicitly closed. A + # startup orphan discovered by this pass is closed *after* pruning, + # preserving a full retention window in which it can be resumed. pruned = self.prune_sessions( older_than_days=retention_days, sessions_dir=sessions_dir, + exclude_active_write_guards=True, ) result["pruned"] = pruned - # Only VACUUM if we actually freed rows, and no more often than - # once every min_vacuum_interval_days -- a large prune (e.g. the - # first one to cross retention_days on a DB with tens of - # thousands of rows) can free enough pages that pruned > 0 fires - # on every subsequent startup even though a VACUUM already ran - # recently. VACUUM on this DB's size (FTS5 shadow tables) is not - # cheap -- it holds an exclusive lock for the full rewrite. + # Reap stale state-owned rows only. Runtime-owned messaging sources + # are intentionally outside this automatic destructive scope. + closed = self.sweep_orphaned_sessions( + max_idle_seconds=float(retention_days) * 86400.0, + sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES, + exclude_pinned=True, + # These sources are owned by state.db lifecycles, not by the + # dashboard/TUI gateway heartbeats used by startup recovery. + respect_gateway_heartbeats=False, + ) + result["closed"] = len(closed) + # Only VACUUM if we actually freed rows, no more often than once + # every min_vacuum_interval_days, AND only when the rewrite pays + # off: the reclaimable fraction of the file (freelist_count / + # page_count) must exceed AUTO_VACUUM_MIN_FREELIST_RATIO (#54189). + # A large prune (e.g. the first one to cross retention_days on a + # DB with tens of thousands of rows) can free enough pages that + # pruned > 0 fires on every subsequent startup even though a + # VACUUM already ran recently; and pruning one tiny session on a + # dense multi-GB DB would otherwise rewrite the whole file to + # reclaim a few MB. VACUUM on this DB's size (FTS5 shadow tables) + # is not cheap -- it holds an exclusive lock for the full rewrite. + # The time throttle says "not too often"; the ratio gate says + # "only when it pays off". Both must pass. last_vacuum_raw = self.get_meta("last_vacuum") vacuum_due = True if last_vacuum_raw: @@ -16048,20 +17086,32 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) except (TypeError, ValueError): vacuum_due = True if vacuum and pruned > 0 and vacuum_due: - try: - self.vacuum() - result["vacuumed"] = True - self.set_meta("last_vacuum", str(now)) - except Exception as exc: - logger.warning("state.db VACUUM failed: %s", exc) + ratio = self._freelist_ratio() + result["freelist_ratio"] = ratio + if ratio is None or ratio > min_vacuum_freelist_ratio: + try: + self.vacuum() + result["vacuumed"] = True + self.set_meta("last_vacuum", str(now)) + except Exception as exc: + logger.warning("state.db VACUUM failed: %s", exc) + else: + logger.debug( + "state.db auto-maintenance: skipping VACUUM, only " + "%.1f%% of pages reclaimable (threshold %.0f%%)", + ratio * 100.0, + min_vacuum_freelist_ratio * 100.0, + ) # Record the attempt even if pruned == 0, so we don't retry # every startup within the min_interval_hours window. self.set_meta("last_auto_prune", str(now)) - if pruned > 0: + if closed or pruned > 0: logger.info( - "state.db auto-maintenance: pruned %d session(s) inactive for %d days%s", + "state.db auto-maintenance: closed %d stale open session(s), " + "pruned %d session(s) inactive for %d days%s", + len(closed), pruned, retention_days, " + VACUUM" if result["vacuumed"] else "", @@ -16070,6 +17120,8 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Maintenance must never block startup. Log and return error marker. logger.warning("state.db auto-maintenance failed: %s", exc) result["error"] = str(exc) + finally: + _release_auto_maintenance_lock(maintenance_lock) return result diff --git a/hermes_state_common.py b/hermes_state_common.py index c35b6135a1..eec87c62bf 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -7,6 +7,7 @@ hermes_state re-imports every name here for backward compatibility. """ import contextlib +import errno import json import logging import os @@ -353,7 +354,7 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str: ) -SCHEMA_VERSION = 26 +SCHEMA_VERSION = 30 # FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the @@ -362,9 +363,31 @@ SCHEMA_VERSION = 26 # reaches the current version when a DB is either born fresh or explicitly # optimized via ``hermes sessions optimize-storage``. A legacy DB sits at # layout 0 (marker absent) with a working inline index until the user opts in. -# 1 = v23 external-content layout (content/tool_name/tool_calls, -# tool-row-excluded trigram) -FTS_STORAGE_VERSION = 1 +# 1 = v23 external-content layout with a tool-row-excluded trigram +# 2 = trigram also excludes structured tool_calls JSON +FTS_STORAGE_VERSION = 2 + +# Tool results are often multi-megabyte machine payloads. Index a useful +# prefix for new tool rows instead of tokenizing the entire body while the +# canonical message write holds SQLite's single writer lock. The high-water +# marker lets upgraded databases retain the exact token stream already stored +# for historical rows, so external-content delete/update commands stay valid +# without an eager full-index rebuild. +FTS_TOOL_CONTENT_PREFIX_CHARS = 8_192 +FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY = "fts_tool_full_content_high_water" + + +def _fts_indexed_content_sql(alias: str) -> str: + return f"""CASE WHEN {alias}.role = 'tool' + AND {alias}.id > COALESCE((SELECT CAST(value AS INTEGER) + FROM state_meta + WHERE key = '{FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY}'), -1) + THEN substr(COALESCE({alias}.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS}) + ELSE {alias}.content END""" + + +_FTS_NEW_INDEXED_CONTENT_SQL = _fts_indexed_content_sql("new") +_FTS_OLD_INDEXED_CONTENT_SQL = _fts_indexed_content_sql("old") # Cap on user-controlled FTS5 query input before regex/sanitizer processing. @@ -444,12 +467,14 @@ CREATE TABLE IF NOT EXISTS sessions ( compression_failure_error TEXT, compression_fallback_streak INTEGER NOT NULL DEFAULT 0, compression_ineffective_count INTEGER NOT NULL DEFAULT 0, + compression_recovery_deadline REAL, profile_name TEXT, rewind_count INTEGER NOT NULL DEFAULT 0, archived INTEGER NOT NULL DEFAULT 0, pinned INTEGER NOT NULL DEFAULT 0, hidden INTEGER NOT NULL DEFAULT 0, last_read_at REAL, + tool_names TEXT, FOREIGN KEY (parent_session_id) REFERENCES sessions(id), FOREIGN KEY (system_prompt_hash) REFERENCES system_prompts(hash) ); @@ -642,6 +667,11 @@ CREATE INDEX IF NOT EXISTS idx_sessions_handoff_state ON sessions(handoff_state, started_at); CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash ON sessions(system_prompt_hash); +-- Recent-session browsing must never derive recency by scanning messages. +-- This expression is the durable, indexable approximation used to preselect +-- a small candidate set before compression-chain and preview hydration. +CREATE INDEX IF NOT EXISTS idx_sessions_effective_activity + ON sessions(COALESCE(last_activity_at, started_at) DESC, started_at DESC); """ @@ -664,7 +694,7 @@ CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash # predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation. # The two state_meta PK probes per write are negligible next to the FTS # insert itself. -FTS_SQL = """ +FTS_SQL = f""" CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content, tool_name, @@ -680,7 +710,12 @@ WHEN (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) - VALUES (new.id, new.content, new.tool_name, new.tool_calls); + VALUES ( + new.id, + {_FTS_NEW_INDEXED_CONTENT_SQL}, + new.tool_name, + new.tool_calls + ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages @@ -690,82 +725,19 @@ WHEN (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) - VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); + VALUES ( + 'delete', + old.id, + {_FTS_OLD_INDEXED_CONTENT_SQL}, + old.tool_name, + old.tool_calls + ); END; -- UPDATE OF skips the trigger entirely for non-content column writes -- (status/compacted/observed/etc.), which is stronger than the WHEN gate -- alone and avoids FTS I/O saturation on large state.db (#68858 / #73639). CREATE TRIGGER IF NOT EXISTS messages_fts_update -AFTER UPDATE OF content, tool_name, tool_calls ON messages -WHEN (old.content IS NOT new.content - OR old.tool_name IS NOT new.tool_name - OR old.tool_calls IS NOT new.tool_calls) - AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) - VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); - INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) - VALUES (new.id, new.content, new.tool_name, new.tool_calls); -END; -""" - - -# Trigram FTS5 table for CJK substring search. The default unicode61 -# tokenizer splits CJK characters into individual tokens, breaking phrase -# matching. The trigram tokenizer creates overlapping 3-byte sequences so -# substring queries work natively for any script (CJK, Thai, etc.). -# -# The trigram index is the most expensive index in state.db (~2.6x the size -# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes -# while being almost entirely machine noise (base64 payloads, file dumps, -# delegation transcripts). The index therefore reads through -# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay -# fully stored in ``messages`` and fully searchable via the standard -# ``messages_fts`` index; they just don't get trigram (CJK substring) -# treatment. ``search_messages`` routes CJK queries that filter on -# ``role='tool'`` to the LIKE fallback for the same reason. -FTS_TRIGRAM_SQL = """ -CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS - SELECT id, role, content, tool_name, tool_calls - FROM messages - WHERE role <> 'tool'; - -CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( - content, - tool_name, - tool_calls, - content='messages_fts_trigram_src', - content_rowid='id', - tokenize='trigram' -); - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages -WHEN new.role <> 'tool' - AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) - VALUES (new.id, new.content, new.tool_name, new.tool_calls); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages -WHEN old.role <> 'tool' - AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) - VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE OF content, tool_name, tool_calls, role ON messages WHEN (old.content IS NOT new.content OR old.tool_name IS NOT new.tool_name @@ -776,12 +748,129 @@ WHEN (old.content IS NOT new.content OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_progress'), -1)) BEGIN - INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) - SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls - WHERE old.role <> 'tool'; - INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) - SELECT new.id, new.content, new.tool_name, new.tool_calls - WHERE new.role <> 'tool'; + INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) + VALUES ( + 'delete', + old.id, + {_FTS_OLD_INDEXED_CONTENT_SQL}, + old.tool_name, + old.tool_calls + ); + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) + VALUES ( + new.id, + {_FTS_NEW_INDEXED_CONTENT_SQL}, + new.tool_name, + new.tool_calls + ); +END; +""" + + +# Trigram FTS5 table for CJK substring search. The default unicode61 +# tokenizer splits CJK characters into individual tokens, breaking phrase +# matching. The trigram tokenizer creates overlapping 3-byte sequences so +# substring queries work natively for any script (CJK, Thai, etc.). +# +# The trigram index is the most expensive index in state.db (~2.6x the size +# of the text it covers). Tool output (~90% of message bytes, machine noise) +# and cron transcripts are excluded: the index reads through +# ``messages_fts_trigram_src``, a view that skips both classes. They stay +# fully stored in ``messages`` and searchable via the standard +# ``messages_fts`` index; they just don't get trigram (CJK substring) +# treatment. ``search_messages`` routes explicit tool/cron CJK searches to +# LIKE for the same reason. Structured ``tool_calls`` JSON likewise stays +# searchable through ``messages_fts``; excluding it here avoids indexing +# repetitive JSON syntax as trigrams (FTS_STORAGE_VERSION 2). +# +# Delegate-child (subagent) transcripts are excluded the same way (v30): +# on a fan-out-heavy install they were ~70% of all message bytes and +# ``session_search`` hides ``source='subagent'`` sessions anyway. A child +# is recognised by its source OR by the ``_delegate_from`` creation marker +# (children spawned under a gateway turn inherit the gateway's source). +# Compression/branch continuations of interactive sessions also carry +# ``parent_session_id`` but NOT the marker, so they stay trigram-indexed. +FTS_TRIGRAM_EXCLUDED_SOURCES = ("cron", "subagent") + +# Predicate over a ``sessions`` row (unqualified column names) selecting +# sessions whose rows belong in the trigram index. Shared by the view, the +# sync triggers, and the deferred-backfill INSERT ... SELECTs so they can +# never disagree about the index boundary. +FTS_TRIGRAM_SESSION_SQL = ( + "source NOT IN (" + + ", ".join(f"'{src}'" for src in FTS_TRIGRAM_EXCLUDED_SOURCES) + + ") AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL" +) + + +def fts_trigram_session_sql(alias: str) -> str: + """``FTS_TRIGRAM_SESSION_SQL`` with every column qualified by ``alias``.""" + return FTS_TRIGRAM_SESSION_SQL.replace("source ", f"{alias}.source ").replace( + "COALESCE(model_config", f"COALESCE({alias}.model_config" + ) + + +FTS_TRIGRAM_SQL = f""" +CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS + SELECT m.id, m.role, m.content, m.tool_name + FROM messages AS m + JOIN sessions AS s ON s.id = m.session_id + WHERE m.role <> 'tool' AND {fts_trigram_session_sql('s')}; + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( + content, + tool_name, + content='messages_fts_trigram_src', + content_rowid='id', + tokenize='trigram' +); + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages +WHEN new.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = new.session_id AND {FTS_TRIGRAM_SESSION_SQL}) + AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts_trigram(rowid, content, tool_name) + VALUES (new.id, new.content, new.tool_name); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages +WHEN old.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = old.session_id AND {FTS_TRIGRAM_SESSION_SQL}) + AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name) + VALUES ('delete', old.id, old.content, old.tool_name); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update +AFTER UPDATE OF content, tool_name, role ON messages +WHEN (old.content IS NOT new.content + OR old.tool_name IS NOT new.tool_name + OR old.role IS NOT new.role) + AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name) + SELECT 'delete', old.id, old.content, old.tool_name + WHERE old.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = old.session_id AND {FTS_TRIGRAM_SESSION_SQL}); + INSERT INTO messages_fts_trigram(rowid, content, tool_name) + SELECT new.id, new.content, new.tool_name + WHERE new.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = new.session_id AND {FTS_TRIGRAM_SESSION_SQL}); END; """ @@ -822,7 +911,7 @@ FTS_REBUILD_DEFERRAL_KEY = "fts_rebuild_deferral" # (which would create the external-content trigram source VIEW and leave the # DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a # legacy DB to the v23 shape. -LEGACY_FTS_SQL = """ +LEGACY_FTS_SQL = f""" CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content ); @@ -830,7 +919,8 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(rowid, content) VALUES ( new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '') + || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') ); END; @@ -839,17 +929,18 @@ CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN END; CREATE TRIGGER IF NOT EXISTS messages_fts_update -AFTER UPDATE OF content, tool_name, tool_calls ON messages BEGIN +AFTER UPDATE OF content, tool_name, tool_calls, role ON messages BEGIN DELETE FROM messages_fts WHERE rowid = old.id; INSERT INTO messages_fts(rowid, content) VALUES ( new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '') + || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') ); END; """ -LEGACY_FTS_TRIGRAM_SQL = """ +LEGACY_FTS_TRIGRAM_SQL = f""" CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( content, tokenize='trigram' @@ -858,7 +949,8 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN INSERT INTO messages_fts_trigram(rowid, content) VALUES ( new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '') + || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') ); END; @@ -867,11 +959,12 @@ CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON message END; CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update -AFTER UPDATE OF content, tool_name, tool_calls ON messages BEGIN +AFTER UPDATE OF content, tool_name, tool_calls, role ON messages BEGIN DELETE FROM messages_fts_trigram WHERE rowid = old.id; INSERT INTO messages_fts_trigram(rowid, content) VALUES ( new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '') + || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') ); END; """ @@ -924,6 +1017,32 @@ _IS_WINDOWS = sys.platform == "win32" # short bounded wait suffices — never re-enter the full timeout. _LOCK_BREAK_REACQUIRE_SECONDS = 5.0 +# errno set for "another process holds this advisory lock". flock() reports +# contention as EWOULDBLOCK/EAGAIN; msvcrt.locking() as EACCES (and EDEADLK +# when its internal retry gives up). Anything else — ESTALE on a dropped NFS +# handle, ENOTSUP/ENOLCK on a filesystem without advisory locks, EIO — is a +# persistent environment failure that no amount of polling turns into an +# acquire. Treating every OSError as contention made such a failure look +# like a live holder and burned the full 120s admission timeout on every +# attempt (#100108, PR #100130). +_LOCK_CONTENTION_ERRNOS = {errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK} +if hasattr(errno, "EDEADLK"): + _LOCK_CONTENTION_ERRNOS.add(errno.EDEADLK) + + +def is_advisory_lock_contention(exc: BaseException) -> bool: + """True when *exc* means another process holds the advisory lock. + + False for every other ``OSError`` (ESTALE, ENOTSUP, ENOLCK, EIO, ...): + callers must fail closed IMMEDIATELY rather than poll to the deadline, + because retrying cannot succeed and the wait only stalls the caller. + """ + if isinstance(exc, BlockingIOError): + return True + if not isinstance(exc, OSError): + return False + return exc.errno in _LOCK_CONTENTION_ERRNOS + def _proc_start_ticks(pid: int): """Kernel start time of *pid* in clock ticks, or None when unknowable. @@ -1032,7 +1151,11 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript """Bounded POSIX flock acquire with orphaned-holder staleness break. Returns ``(acquired, handle)``; *handle* may have been re-opened (the - caller owns closing whichever handle comes back). + caller owns closing whichever handle comes back). *acquired* is True on + success, False when a holder kept the lock past the deadline, and None + when a non-contention ``OSError`` (ESTALE/ENOTSUP/EIO) made acquisition + impossible — already logged here; callers treat None as "not acquired" + without emitting the held-by-another-process warning. Why breaking exists at all (issue #100108): ``flock`` belongs to the open file DESCRIPTION, which ``fork()`` duplicates into every child. A holder @@ -1055,7 +1178,21 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript while True: try: fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except (BlockingIOError, OSError): + except (BlockingIOError, OSError) as exc: + if not is_advisory_lock_contention(exc): + # ESTALE / ENOTSUP / EIO: not a holder, and polling cannot + # fix it. Defer NOW instead of pretending a live process + # held the lock for the whole timeout (#100108). + logger.warning( + "Could not acquire %s %s (%s) — deferring rather than " + "waiting out the %.0fs holder timeout on a " + "non-contention error.", + description, + lock_path, + exc, + timeout_seconds, + ) + return None, handle if time.monotonic() < deadline: time.sleep(poll_seconds) continue @@ -1128,39 +1265,59 @@ def _describe_lock_holder(record) -> str: @contextlib.contextmanager -def fts_rebuild_admission(db_path): +def fts_rebuild_admission(db_path, *, timeout_seconds=None): """Serialize full structural FTS rebuilds on *db_path* across processes. Yields True when this process holds the rebuild authority, False when the - bounded acquire timed out. A caller that gets False must NOT perform a - full rebuild — proceeding is exactly the concurrent-rebuild interleaving - this lock exists to prevent (fail closed). The deferred/stale breadcrumb - machinery already guarantees a skipped rebuild is retried later. + bounded acquire timed out or the lock file could not be opened at all. A + caller that gets False must NOT perform a full rebuild — proceeding is + exactly the concurrent-rebuild interleaving this lock exists to prevent + (fail closed). The deferred/stale breadcrumb machinery already guarantees + a skipped rebuild is retried later. ``db_path`` may be a str or Path; None (in-memory DB / tests without a file path) yields True — a private in-memory DB has no cross-process surface. + + *timeout_seconds* defaults to ``_FTS_REBUILD_LOCK_TIMEOUT_SECONDS``. + Opportunistic in-process retries (``retry_deferred_fts_recovery``) pass + ``0`` so a live holder never stalls a long-lived writer for two minutes; + the orphaned-holder break still applies on the single attempt. """ if db_path is None: yield True return + timeout = ( + _FTS_REBUILD_LOCK_TIMEOUT_SECONDS + if timeout_seconds is None + else max(float(timeout_seconds), 0.0) + ) lock_path = f"{db_path}.fts_rebuild.lock" try: handle = open(lock_path, "a+b") except OSError as exc: - # Read-only dir, exhausted fds, exotic filesystem: fall back to the - # pre-lock behaviour rather than refusing a rebuild we could run. + # Fail closed, exactly as a timed-out acquire does. A lock file we + # cannot even open means the filesystem is out of space, inodes or + # descriptors — and a sibling process that opened ITS handle before + # the disk filled is still holding the authority and rebuilding. + # Yielding True here handed every process on a full disk a concurrent + # structural rebuild of the same live state.db with no cross-process + # authority at all: the disk-full trigger and the re-corruption on + # every multi-writer boot in #100368. Deferring costs nothing that + # was reachable anyway — the breadcrumb retries, and on a read-only + # directory the rebuild's own writes could not have committed either. logger.warning( - "Could not open FTS rebuild lock %s (%s) — proceeding with " - "in-process serialisation only.", lock_path, exc, + "Could not open FTS rebuild lock %s (%s) — deferring this rebuild " + "rather than running it without cross-process authority.", + lock_path, exc, ) - yield True + yield False return acquired = False try: if _IS_WINDOWS: - deadline = time.monotonic() + _FTS_REBUILD_LOCK_TIMEOUT_SECONDS + deadline = time.monotonic() + timeout while True: try: import msvcrt @@ -1169,7 +1326,15 @@ def fts_rebuild_admission(db_path): msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) acquired = True break - except (BlockingIOError, OSError): + except (BlockingIOError, OSError) as exc: + if not is_advisory_lock_contention(exc): + logger.warning( + "Could not acquire FTS rebuild lock %s (%s) — " + "deferring on a non-contention error.", + lock_path, exc, + ) + acquired = None + break if time.monotonic() >= deadline: break time.sleep(_FTS_REBUILD_LOCK_POLL_SECONDS) @@ -1177,20 +1342,35 @@ def fts_rebuild_admission(db_path): acquired, handle = _acquire_db_flock( lock_path, handle, - _FTS_REBUILD_LOCK_TIMEOUT_SECONDS, + timeout, _FTS_REBUILD_LOCK_POLL_SECONDS, "FTS rebuild lock", ) - if not acquired: + if acquired is None: + # Non-contention failure: already logged with the real errno; + # a "held by another process" line here would be a lie. + acquired = False + elif not acquired: record = None if _IS_WINDOWS else _read_lock_holder_record(handle) - logger.warning( - "FTS rebuild lock %s held by another process for more than " - "%.0fs — deferring this rebuild to avoid racing the holder " - "(the stale-FTS breadcrumb keeps it retryable). " - "Recorded holder: %s.", - lock_path, _FTS_REBUILD_LOCK_TIMEOUT_SECONDS, - _describe_lock_holder(record), - ) + if timeout <= 0: + # Non-blocking probe from an in-process retry: a busy lock + # is expected and will be tried again, so keep it quiet. + logger.info( + "FTS rebuild lock %s is busy — deferring this retry " + "(the stale-FTS breadcrumb keeps it retryable). " + "Recorded holder: %s.", + lock_path, + _describe_lock_holder(record), + ) + else: + logger.warning( + "FTS rebuild lock %s held by another process for more than " + "%.0fs — deferring this rebuild to avoid racing the holder " + "(the stale-FTS breadcrumb keeps it retryable). " + "Recorded holder: %s.", + lock_path, timeout, + _describe_lock_holder(record), + ) yield acquired finally: try: diff --git a/hermes_state_holders.py b/hermes_state_holders.py new file mode 100644 index 0000000000..9f4176d40a --- /dev/null +++ b/hermes_state_holders.py @@ -0,0 +1,292 @@ +"""Process and descriptor authority for state.db structural maintenance. + +This module owns the proof that no foreign process still holds the active or +an unlinked SQLite DB/WAL/SHM generation. ``hermes_state`` supplies only the +SQLite connection factory needed by the final lock probe. +""" + +from __future__ import annotations + +import errno +import logging +import os +import sqlite3 +import sys +from pathlib import Path +from typing import Callable, List, Optional, Sequence, Set, Tuple + +try: # Hard dependency, but tolerate scaffold-phase imports before pip install. + import psutil +except ImportError: # pragma: no cover - stripped/scaffold installs only + psutil = None # type: ignore[assignment] + + +logger = logging.getLogger(__name__) + +_IS_WINDOWS = sys.platform == "win32" +_HERMES_EXECUTABLES = frozenset({"hermes", "hermes-agent", "hermes-acp"}) +_HERMES_PYTHON_MODULES = frozenset({"acp_adapter", "hermes_cli.main"}) +_HERMES_PYTHON_SCRIPTS = frozenset({"hermes_cli/main.py", "run_agent.py"}) +_PYTHON_SHORT_OPTIONS_WITH_OPERANDS = frozenset({"Q", "W", "X"}) +_PYTHON_LONG_OPTIONS_WITH_OPERANDS = frozenset( + {"--check-hash-based-pycs", "--jit"} +) + + +def _read_proc_argv(pid: int) -> Optional[List[str]]: + """Read /proc//cmdline without losing argv boundaries.""" + try: + with open(f"/proc/{pid}/cmdline", "rb") as handle: + raw = handle.read() + if not raw: + return None + argv = raw.decode("utf-8", "replace").split("\x00") + if argv[-1] == "": + argv.pop() + return argv or None + except OSError: + return None + + +def _looks_like_python_executable(program: str) -> bool: + name = os.path.basename(program).lower().removesuffix(".exe") + for prefix in ("python", "pypy"): + if name.startswith(prefix): + suffix = name[len(prefix) :] + return not suffix or all(char.isdigit() or char == "." for char in suffix) + return False + + +def _python_execution_target(argv: Sequence[str]) -> Optional[Tuple[str, str]]: + """Return the Python module or script selected by interpreter options.""" + index = 1 + while index < len(argv): + arg = argv[index] + if arg == "--": + index += 1 + return ("script", argv[index]) if index < len(argv) else None + if arg in _PYTHON_LONG_OPTIONS_WITH_OPERANDS: + index += 2 + continue + if arg.startswith("--check-hash-based-pycs=") or arg.startswith("--jit="): + index += 1 + continue + if arg.startswith("--"): + index += 1 + continue + if arg.startswith("-") and arg != "-": + options = arg[1:] + option_index = 0 + consumed_next = False + while option_index < len(options): + option = options[option_index] + attached = options[option_index + 1 :] + if option == "c": + return None + if option == "m": + if attached: + return "module", attached + index += 1 + return ("module", argv[index]) if index < len(argv) else None + if option in _PYTHON_SHORT_OPTIONS_WITH_OPERANDS: + consumed_next = not attached + break + option_index += 1 + index += 2 if consumed_next else 1 + continue + return "script", arg + return None + + +def _looks_like_hermes(argv: Sequence[str]) -> bool: + """Return whether argv identifies a supported Hermes execution target.""" + if not argv: + return False + program = os.path.basename(argv[0]).lower().removesuffix(".exe") + if program in _HERMES_EXECUTABLES: + return True + if not _looks_like_python_executable(program): + return False + target = _python_execution_target(argv) + if target is None: + return False + kind, value = target + normalized = value.lower().replace("\\", "/") + if kind == "module": + return normalized in _HERMES_PYTHON_MODULES + return any( + normalized == script or normalized.endswith(f"/{script}") + for script in _HERMES_PYTHON_SCRIPTS + ) + + +def canonical_sqlite_path(path: str) -> str: + """Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix.""" + return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)"))) + + +def foreign_state_db_holders(db_path: Path) -> List[Tuple[int, str]]: + """Return foreign holders of the DB or one of its WAL sidecars. + + A scan failure is represented as an unknown holder. Structural maintenance + must not assume quiescence when an old, unlinked SQLite generation may + still be open by another process. + """ + if _IS_WINDOWS: + return [] + + db_path_str = os.path.abspath(os.fspath(db_path)) + watched = { + canonical_sqlite_path(db_path_str), + canonical_sqlite_path(db_path_str + "-wal"), + canonical_sqlite_path(db_path_str + "-shm"), + } + holders: List[Tuple[int, str]] = [] + watched_ids: Set[Tuple[int, int]] = set() + db_dev: Optional[int] = None + for candidate in (db_path_str, db_path_str + "-wal", db_path_str + "-shm"): + try: + stat_result = os.stat(candidate) + except OSError as exc: + if exc.errno not in (errno.ENOENT, errno.ESRCH): + holders.append( + (-1, f"watched-file stat failed: {candidate}: {exc}") + ) + continue + watched_ids.add((stat_result.st_dev, stat_result.st_ino)) + if candidate == db_path_str: + db_dev = stat_result.st_dev + + if sys.platform.startswith("linux"): + try: + own_pid = os.getpid() + for pid_str in os.listdir("/proc"): + if not pid_str.isdigit(): + continue + pid = int(pid_str) + if pid == own_pid: + continue + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + argv = _read_proc_argv(pid) + if argv is not None and _looks_like_hermes(argv): + cmdline = " ".join(argv) + holders.append((pid, f"uninspectable holder: {cmdline[:80]}")) + continue + for fd in fds: + fd_path = f"{fd_dir}/{fd}" + try: + target = os.readlink(fd_path) + except OSError as exc: + if exc.errno in (errno.ENOENT, errno.ESRCH): + continue + argv = _read_proc_argv(pid) + if argv is not None and _looks_like_hermes(argv): + holders.append( + ( + pid, + f"uninspectable descriptor: {fd_path}: {exc}", + ) + ) + continue + target_is_watched = canonical_sqlite_path(target) in watched + try: + fd_stat = os.stat(fd_path) + except OSError as exc: + if exc.errno in (errno.ENOENT, errno.ESRCH): + continue + if target_is_watched: + holders.append( + (pid, f"uninspectable descriptor: {target}: {exc}") + ) + else: + argv = _read_proc_argv(pid) + if argv is not None and _looks_like_hermes(argv): + holders.append( + ( + pid, + "uninspectable descriptor: " + f"{target}: {exc}", + ) + ) + continue + if (fd_stat.st_dev, fd_stat.st_ino) in watched_ids or ( + target_is_watched + and target.endswith(" (deleted)") + and db_dev is not None + and fd_stat.st_dev == db_dev + ): + holders.append((pid, target)) + except Exception as exc: + logger.warning( + "Could not prove state.db has no foreign holders; " + "deferring structural maintenance: %s", + exc, + ) + holders.append((-1, f"open-file scan failed: {exc}")) + return holders + + if psutil is None: + return [(-1, "open-file scan unavailable")] + try: + for process in psutil.process_iter(["pid", "open_files"]): + info = process.info + pid = int(info["pid"]) + if pid == os.getpid(): + continue + for opened in info.get("open_files") or (): + path = getattr(opened, "path", "") + if path and canonical_sqlite_path(path) in watched: + holders.append((pid, path)) + except Exception as exc: + logger.warning( + "Could not prove state.db has no foreign holders; " + "deferring structural maintenance: %s", + exc, + ) + holders.append((-1, f"open-file scan failed: {exc}")) + return holders + + +def live_writer_holds_db( + db_path: Path, + *, + connect_repair_durable: Callable[..., sqlite3.Connection], +) -> bool: + """Return whether repair lacks proven exclusive ownership of ``db_path``.""" + foreign_holders = foreign_state_db_holders(db_path) + if any( + pid < 0 + or path.startswith("uninspectable holder:") + or path.startswith("uninspectable descriptor:") + or path.endswith(" (deleted)") + for pid, path in foreign_holders + ): + return True + + probe = None + try: + probe = connect_repair_durable(db_path, timeout=0.0) + probe.execute("PRAGMA locking_mode=EXCLUSIVE") + probe.execute("BEGIN IMMEDIATE") + probe.execute("ROLLBACK") + return False + except sqlite3.OperationalError as exc: + lowered = str(exc).lower() + return "locked" in lowered or "busy" in lowered + except sqlite3.DatabaseError: + return False + except Exception: + return False + finally: + if probe is not None: + try: + probe.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + try: + probe.close() + except Exception: + pass diff --git a/hermes_state_portability.py b/hermes_state_portability.py index ca8df96809..57a80cc493 100644 --- a/hermes_state_portability.py +++ b/hermes_state_portability.py @@ -54,8 +54,8 @@ class SessionPortabilityMixin: where = "cwd IS NOT NULL AND TRIM(cwd) != ''" if not include_archived: where += " AND archived = 0" - with self._lock: - rows = self._conn.execute( + with self._read_ctx() as conn: + rows = conn.execute( "SELECT cwd AS cwd, COUNT(*) AS sessions, " "MAX(COALESCE(ended_at, started_at, 0)) AS last_active " f"FROM sessions WHERE {where} GROUP BY cwd" @@ -119,8 +119,8 @@ class SessionPortabilityMixin: ORDER BY s.started_at DESC, s.id DESC LIMIT ? OFFSET ? """ - with self._lock: - cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset)) + with self._read_ctx() as conn: + cursor = conn.execute(query, (prefix, prefix_hi, limit, offset)) rows = cursor.fetchall() runs: List[Dict[str, Any]] = [] @@ -202,8 +202,8 @@ class SessionPortabilityMixin: {prompt_join} WHERE s.id IN ({placeholders}) """ - with self._lock: - cursor = self._conn.execute(query, ids) + with self._read_ctx() as conn: + cursor = conn.execute(query, ids) rows = cursor.fetchall() result: Dict[str, Dict[str, Any]] = {} for row in rows: @@ -229,8 +229,8 @@ class SessionPortabilityMixin: Returns ``id``, ``title``, and the full first-turn ``content`` so a caller can re-derive what the user typed. Newest first. """ - with self._lock: - rows = self._conn.execute( + with self._read_ctx() as conn: + rows = conn.execute( """ SELECT s.id, s.title, m.content FROM sessions s @@ -254,8 +254,8 @@ class SessionPortabilityMixin: Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed the titler the same (request, reply) shape the live path uses. """ - with self._lock: - row = self._conn.execute( + with self._read_ctx() as conn: + row = conn.execute( "SELECT content FROM messages " "WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL " "ORDER BY timestamp, id LIMIT 1", diff --git a/hermes_state_registry.py b/hermes_state_registry.py index 3c5b6ec24e..0f3bcedc20 100644 --- a/hermes_state_registry.py +++ b/hermes_state_registry.py @@ -42,7 +42,7 @@ from __future__ import annotations import logging import threading from pathlib import Path -from typing import TYPE_CHECKING, Dict, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple if TYPE_CHECKING: # pragma: no cover - import cycle guard, typed only from hermes_state import SessionDB @@ -91,6 +91,11 @@ _lock = threading.Lock() _generations: Dict[Path, _Generation] = {} # Object-keyed retired generations still draining holders. _retired: Dict[int, _Generation] = {} # id(db) → generation +# Paths whose next generation is currently being constructed. Construction +# stays outside _lock because schema reconciliation can take seconds, but peers +# for the SAME file must wait: otherwise every cold caller opens a writable +# SQLite connection before the registry chooses one winner. +_opening: Dict[Path, threading.Event] = {} def _open_session_db(path: Path) -> "SessionDB": @@ -132,42 +137,67 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": """ from hermes_state import _default_db_path - path = Path(db_path) if db_path is not None else Path(_default_db_path()) + raw_path = Path(db_path) if db_path is not None else Path(_default_db_path()) + try: + path = raw_path.resolve() + except OSError: + path = raw_path - with _lock: - generation = _generations.get(path) - if generation is not None: - current = _stat_db_file_identity(path) - if ( - current is not None - and generation.identity is not None - and current != generation.identity - ): - # File replaced: retire the live generation (its - # holders keep it until they release) and fall - # through to opening a fresh one below. - _retire_generation_locked(path, generation) - else: - generation.refcount += 1 - return generation.db + while True: + with _lock: + generation = _generations.get(path) + if generation is not None: + current = _stat_db_file_identity(path) + if ( + current is not None + and generation.identity is not None + and current != generation.identity + ): + # File replaced: retire the live generation (its + # holders keep it until they release) and elect one + # caller to construct the replacement below. + _retire_generation_locked(path, generation) + else: + generation.refcount += 1 + return generation.db + + opening = _opening.get(path) + if opening is None: + opening = threading.Event() + _opening[path] = opening + break + + # Another caller is constructing this path. Do not hold the global + # registry lock while waiting: unrelated databases continue opening. + # A failed opener signals too, so one waiter can retry as the successor. + opening.wait() + + # Open a fresh generation OUTSIDE the lock. The per-path opening marker + # prevents redundant writer connections without serialising other files. + try: + db = _open_session_db(path) + db._shared_registry_owned = True + identity = _stat_db_file_identity(path) + except BaseException: + with _lock: + if _opening.get(path) is opening: + _opening.pop(path, None) + opening.set() + raise - # Open a fresh generation OUTSIDE the lock: construction can - # take seconds (write-lock patience) and must not block every - # other state.db acquisition in the process. - db = _open_session_db(path) - db._shared_registry_owned = True - identity = _stat_db_file_identity(path) with _lock: existing = _generations.get(path) if existing is not None: - # Someone else opened a generation while we were - # constructing (or retired ours and installed a new one). - # Ours loses — close it (outside the lock) and use theirs. + # Defensive: a generation may have been installed by explicit + # registry manipulation while this open was in flight. existing.refcount += 1 winner = existing.db else: _generations[path] = _Generation(db, identity) winner = db + if _opening.get(path) is opening: + _opening.pop(path, None) + opening.set() if winner is not db: _teardown(db) return winner @@ -259,6 +289,18 @@ def close_all() -> int: return closed +def live_shared_session_dbs() -> List["SessionDB"]: + """Snapshot of every live (non-retired) shared SessionDB in this process. + + For periodic in-process maintenance (the gateway housekeeping tick's + deferred-FTS retry). Refcounts are NOT touched: the caller only invokes + a method on an instance that some holder already keeps alive; a + concurrent final release closes it and the callee sees ``_conn is None``. + """ + with _lock: + return [g.db for g in _generations.values() if not g.retired] + + def stats() -> Dict[str, int]: """Registry census for tests and diagnostics (no locks held long).""" with _lock: diff --git a/hermes_state_schema.py b/hermes_state_schema.py index 9813b12785..7896134dc2 100644 --- a/hermes_state_schema.py +++ b/hermes_state_schema.py @@ -26,6 +26,7 @@ from hermes_state_common import ( FTS_STALE_KEY, FTS_SQL, FTS_STORAGE_VERSION, + FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, FTS_TRIGRAM_SQL, LEGACY_FTS_SQL, LEGACY_FTS_TRIGRAM_SQL, @@ -43,6 +44,15 @@ logger = logging.getLogger("hermes_state") _FTS_HOLDER_ESCALATE_ATTEMPTS = 3 _FTS_HOLDER_ESCALATE_SECONDS = 60.0 +# Minimum spacing between in-process retries of a deferred stale-FTS rebuild +# (``retry_deferred_fts_recovery``). The startup open already paid the full +# admission wait once; later retries are non-blocking probes on this cadence +# so a live holder never stalls a long-lived writer. +_FTS_STALE_RETRY_SECONDS = 60.0 +# Each failed retry doubles the spacing up to this cap, so a holder that never +# goes away (a second long-lived writer) costs one deferral warning per hour, +# not one per minute. A successful rebuild clears the stale state entirely. +_FTS_STALE_RETRY_MAX_SECONDS = 3600.0 # Cache for schema_read_probe_statements() — parsing SCHEMA_SQL spins up an # in-memory SQLite database, so derive the statements once per process. @@ -290,6 +300,83 @@ class SessionSchemaMixin: ) return len(to_drop) + @staticmethod + def _execute_ddl_script_transactional( + cursor: sqlite3.Cursor, ddl: str + ) -> None: + """Execute a DDL script without ``executescript``'s implicit commit.""" + statement = "" + for line in ddl.splitlines(): + statement += line + "\n" + if sqlite3.complete_statement(statement): + cursor.execute(statement) + statement = "" + if statement.strip(): + raise sqlite3.OperationalError("incomplete FTS DDL statement") + + def _migrate_bounded_tool_fts_triggers( + self, cursor: sqlite3.Cursor, *, legacy: bool + ) -> None: + """Replace FTS triggers without rebuilding historical indexes. + + Existing rows keep their original full-content token stream. The + durable high-water id makes new tool rows use the bounded prefix in + both INSERT and matching external-content delete/update operations. + Trigger replacement is one savepoint so no concurrent writer can land + in a trigger gap. + """ + marker = cursor.execute( + "SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY,), + ).fetchone() + if marker is not None: + return + + trigram_present = cursor.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name = 'messages_fts_trigram'" + ).fetchone() is not None + names = _FTS_BASE_TRIGGERS + if legacy and trigram_present: + names += _FTS_TRIGRAM_TRIGGERS + existing = self._fts_trigger_count(cursor, names) + has_messages = cursor.execute( + "SELECT 1 FROM messages LIMIT 1" + ).fetchone() is not None + table_present = cursor.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name = 'messages_fts'" + ).fetchone() is not None + self._fts_tool_prefix_migration_requires_rebuild = bool( + table_present and has_messages and existing < len(names) + ) + + cursor.execute("SAVEPOINT bounded_tool_fts") + try: + high_water = cursor.execute( + "SELECT COALESCE(MAX(id), 0) FROM messages" + ).fetchone()[0] + cursor.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(high_water)), + ) + for name in names: + cursor.execute(f"DROP TRIGGER IF EXISTS {name}") + if legacy: + self._execute_ddl_script_transactional(cursor, LEGACY_FTS_SQL) + if trigram_present: + self._execute_ddl_script_transactional( + cursor, LEGACY_FTS_TRIGRAM_SQL + ) + else: + self._execute_ddl_script_transactional(cursor, FTS_SQL) + cursor.execute("RELEASE SAVEPOINT bounded_tool_fts") + except BaseException: + cursor.execute("ROLLBACK TO SAVEPOINT bounded_tool_fts") + cursor.execute("RELEASE SAVEPOINT bounded_tool_fts") + raise + def _cjk_update_trigger_is_narrowed(self, cursor: sqlite3.Cursor) -> bool: """True when messages_fts_cjk_update exists with AFTER UPDATE OF.""" row = cursor.execute( @@ -327,6 +414,56 @@ class SessionSchemaMixin: exc_info=True, ) + def _migrate_trigram_cron_exclusion(self, cursor: sqlite3.Cursor) -> bool: + """Install the source-filtered trigram view and purge historical rows. + + Covers the v29 cron exclusion and the v30 subagent exclusion — both + only change the view/trigger predicate and rebuild from it. + + Legacy inline indexes remain opt-in: their content is private to the + virtual table and cannot adopt this external-content view. For an + external layout, replacing the view and triggers is cheap, but the + existing inverted index still contains cron rows until FTS5 rebuilds + from the new view. Run that rebuild under the shared cross-process + admission gate used by every startup FTS repair. + """ + if self._db_has_legacy_inline_fts(cursor): + return True + if self._db_has_trigram_tool_calls_projection(cursor): + # The existing vtable still declares the FTS_STORAGE_VERSION 1 + # ``tool_calls`` column. Replacing the view underneath it would + # make the 'rebuild' read ``T.tool_calls`` from a view that no + # longer has it. Changing vtable columns is the opt-in + # ``hermes sessions optimize-storage`` path (it recreates the + # vtable from FTS_TRIGRAM_SQL, cron-filtered view included), so + # leave this install to that path instead of half-migrating it. + return True + trigram_exists = self._fts_table_probe(cursor, "messages_fts_trigram") + if trigram_exists is not True: + # Let the normal ensure path create/backfill a missing optional + # trigram table. ``None`` means this runtime cannot safely inspect + # an existing one, so leave the schema version behind for retry. + return trigram_exists is False + for name in _FTS_TRIGRAM_TRIGGERS: + cursor.execute(f"DROP TRIGGER IF EXISTS {name}") + cursor.execute("DROP VIEW IF EXISTS messages_fts_trigram_src") + if not self._ensure_fts_schema( + cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL + ): + return False + # Always rebuild while schema_version is behind, even if the view + # already has the new predicate. A process can die after replacing the + # view but before rebuilding/stamping; view text alone cannot prove the + # old cron postings were purged. + self._run_admitted_startup_rebuild( + cursor, + lambda: cursor.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) " + "VALUES('rebuild')" + ), + ) + return True + @staticmethod def _rebuild_fts_indexes( @@ -334,6 +471,14 @@ class SessionSchemaMixin: *, include_trigram: bool = True, ) -> None: + high_water = cursor.execute( + "SELECT COALESCE(MAX(id), 0) FROM messages" + ).fetchone()[0] + cursor.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(high_water)), + ) # Both FTS tables are external-content (v23+): the special 'rebuild' # command wipes the inverted index and repopulates it from the # content source (messages for the standard index, the tool-row- @@ -364,6 +509,14 @@ class SessionSchemaMixin: 'rebuild' source, so we DELETE + reinsert the concatenated content the legacy triggers produced. Never touches the v23 shape. """ + high_water = cursor.execute( + "SELECT COALESCE(MAX(id), 0) FROM messages" + ).fetchone()[0] + cursor.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(high_water)), + ) cursor.execute("DELETE FROM messages_fts") cursor.execute( "INSERT INTO messages_fts(rowid, content) " @@ -422,8 +575,14 @@ class SessionSchemaMixin: ) return None - def _recover_stale_fts(self, cursor: sqlite3.Cursor, *, legacy: bool) -> bool: - """Atomically rebuild stale base/trigram indexes and resume syncing.""" + def _recover_stale_fts( + self, cursor: sqlite3.Cursor, *, legacy: bool, timeout_seconds=None + ) -> bool: + """Atomically rebuild stale base/trigram indexes and resume syncing. + + *timeout_seconds* bounds the cross-process admission wait; None uses + the full startup budget, ``0`` is the non-blocking in-process retry. + """ foreign_holders = self._foreign_state_db_holders() if foreign_holders: now = time.time() @@ -502,7 +661,9 @@ class SessionSchemaMixin: # authority (fail closed). Losing the race means another process is # already performing this exact recovery; the stale breadcrumb stays # set, so this process simply keeps FTS detached and retries later. - with fts_rebuild_admission(getattr(self, "db_path", None)) as admitted: + with fts_rebuild_admission( + getattr(self, "db_path", None), timeout_seconds=timeout_seconds + ) as admitted: if not admitted: logger.warning( "Deferred stale state.db FTS rebuild: another process " @@ -512,6 +673,76 @@ class SessionSchemaMixin: return False return self._recover_stale_fts_locked(cursor, legacy=legacy) + def retry_deferred_fts_recovery(self) -> bool: + """Retry a deferred stale-FTS rebuild on this open SessionDB. + + ``_recover_stale_fts`` runs at open and fails closed when foreign + holders or the rebuild lock are busy, leaving ``_fts_stale`` set and + search on the LIKE fallback. Live write/search paths must never start + a full rebuild (#97940), so on a short-lived CLI that deferral is + cleared by the next process open — but a gateway opens state.db + once and stays up for days, so "next open" never came (#100108). + This is the in-process retry: bounded backoff from + ``_FTS_STALE_RETRY_SECONDS`` doubling to ``_FTS_STALE_RETRY_MAX_SECONDS``, + non-blocking admission (``timeout=0``) so a live holder is skipped and + tried again later, no new thread — the caller is an existing periodic + tick (gateway housekeeping). + + Returns True only when the index was rebuilt and sync triggers + restored. Never raises. + """ + if not getattr(self, "_fts_stale", False): + return False + if getattr(self, "_db_corrupt", False): + # Quarantined: never run FTS DDL/DML against a damaged image + # (mirrors _try_wal_checkpoint / close). This runs every + # housekeeping tick for the life of a gateway process, so a stale + # flag on a corrupt handle would otherwise retry the rebuild + # forever. Reset the backoff so any future un-quarantine path + # starts from the default interval, not a doubled stale one. + self._fts_stale_retry_after = 0.0 + self._fts_stale_retry_interval = 0.0 + return False + if getattr(self, "read_only", False) or getattr(self, "_conn", None) is None: + return False + now = time.monotonic() + if now < getattr(self, "_fts_stale_retry_after", 0.0): + return False + interval = float(getattr(self, "_fts_stale_retry_interval", 0.0)) + if interval <= 0.0: + interval = _FTS_STALE_RETRY_SECONDS + self._fts_stale_retry_after = now + interval + self._fts_stale_retry_interval = min( + max(interval, _FTS_STALE_RETRY_SECONDS, 1.0) * 2.0, + _FTS_STALE_RETRY_MAX_SECONDS, + ) + try: + with self._lock: + if self._conn is None or not self._fts_stale: + return False + cursor = self._conn.cursor() + legacy = self._db_has_legacy_inline_fts(cursor) + recovered = self._recover_stale_fts( + cursor, legacy=legacy, timeout_seconds=0.0 + ) + if recovered: + # CJK was detached alongside the base indexes; its own + # ensure path decides when it comes back online. + self._ensure_fts_cjk_schema(cursor) + self._fts_stale_retry_interval = 0.0 + try: + self._conn.commit() + except sqlite3.Error: + pass + return recovered + except Exception: # noqa: BLE001 - background retry must never raise + logger.warning( + "In-process retry of the deferred stale state.db FTS rebuild " + "failed; will retry later.", + exc_info=True, + ) + return False + def _recover_stale_fts_locked( self, cursor: sqlite3.Cursor, *, legacy: bool ) -> bool: @@ -1309,7 +1540,10 @@ class SessionSchemaMixin: # advances to SCHEMA_VERSION here like every other migration — # future v24+ migrations land automatically for legacy-FTS # users too. Only the FTS *layout* waits for opt-in. - if fts5_available and self._db_has_legacy_inline_fts(cursor): + if ( + fts5_available + and self._db_needs_fts_storage_upgrade(cursor) + ): self.set_meta("fts_optimize_available", "1", cursor=cursor) if current_version < 25: @@ -1319,6 +1553,15 @@ class SessionSchemaMixin: # rows, but clear migrated rows so future writes do not keep # one large prompt copy per session. self._dedupe_legacy_system_prompts(cursor) + if current_version < 30 and fts5_available: + # v29: cron sessions remain canonical and stay in the standard + # word index, but no longer inflate the trigram substring index. + # v30: delegate-child (subagent) transcripts get the same + # treatment (FTS_TRIGRAM_EXCLUDED_SOURCES + _delegate_from). + # Rebuild once so rows indexed by older trigger/view definitions + # do not survive indefinitely as stale matches and disk usage. + if not self._migrate_trigram_cron_exclusion(cursor): + fts_migrations_complete = False # The FTS storage layout is versioned independently of the main # schema (see the v23 note above). Stamp the current layout so the @@ -1333,7 +1576,7 @@ class SessionSchemaMixin: # transition actually completes. if ( fts5_available - and not self._db_has_legacy_inline_fts(cursor) + and not self._db_needs_fts_storage_upgrade(cursor) and cursor.execute( "SELECT 1 FROM state_meta " "WHERE key = 'fts_rebuild_high_water' LIMIT 1" @@ -1411,6 +1654,10 @@ class SessionSchemaMixin: # v23 view/external tables entirely. Fresh installs and opted-in # DBs have no legacy inline FTS, so they get the v23 DDL. legacy_fts = self._db_has_legacy_inline_fts(cursor) + if not self._fts_stale: + self._migrate_bounded_tool_fts_triggers( + cursor, legacy=legacy_fts + ) if self._fts_stale: if self._recover_stale_fts(cursor, legacy=legacy_fts): # CJK was detached alongside the corrupt base indexes and @@ -1429,6 +1676,8 @@ class SessionSchemaMixin: base_triggers_missing = ( self._fts_trigger_count(cursor, _FTS_BASE_TRIGGERS) < len(_FTS_BASE_TRIGGERS) + ) or getattr( + self, "_fts_tool_prefix_migration_requires_rebuild", False ) trigram_triggers_missing = ( self._fts_trigger_count(cursor, _FTS_TRIGRAM_TRIGGERS) @@ -1456,6 +1705,8 @@ class SessionSchemaMixin: base_triggers_missing = ( self._fts_trigger_count(cursor, _FTS_BASE_TRIGGERS) < len(_FTS_BASE_TRIGGERS) + ) or getattr( + self, "_fts_tool_prefix_migration_requires_rebuild", False ) trigram_triggers_missing = ( self._fts_trigger_count(cursor, _FTS_TRIGRAM_TRIGGERS) @@ -1510,7 +1761,8 @@ class SessionSchemaMixin: breadcrumb is persisted, mirroring ``_enter_fts_fail_open``'s ordering contract: triggers must never be live over an index with an unrebuilt gap. FTS stays detached for this instance; the winner's - rebuild — or ``_recover_stale_fts`` at the next startup — restores + rebuild — or ``retry_deferred_fts_recovery`` from the gateway + housekeeping tick, or ``_recover_stale_fts`` at the next startup — restores the index and triggers atomically. """ with fts_rebuild_admission(getattr(self, "db_path", None)) as admitted: diff --git a/hermes_state_search.py b/hermes_state_search.py index 40fddb70fd..91e95045e7 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -22,7 +22,11 @@ from hermes_state_common import ( FTS_SQL, FTS_STALE_KEY, FTS_STORAGE_VERSION, + FTS_TOOL_CONTENT_PREFIX_CHARS, + FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, + FTS_TRIGRAM_EXCLUDED_SOURCES, FTS_TRIGRAM_SQL, + fts_trigram_session_sql, MAX_FTS5_QUERY_CHARS, SCHEMA_VERSION, _FTS_CJK_TRIGGERS, @@ -152,18 +156,22 @@ class SessionSearchMixin: lo, hi = hw - 1000, hw + 1000 conn.execute( "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "SELECT m.id, " + "CASE WHEN m.role = 'tool' AND m.id > ? " + "THEN substr(COALESCE(m.content, ''), 1, ?) " + "ELSE m.content END, m.tool_name, m.tool_calls " "FROM messages m " "WHERE m.id > ? AND m.id <= ? " "AND NOT EXISTS (SELECT 1 FROM messages_fts_docsize d WHERE d.id = m.id)", - (lo, hi), + (hw, FTS_TOOL_CONTENT_PREFIX_CHARS, lo, hi), ) if include_trigram: conn.execute( - "INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " + "INSERT INTO messages_fts_trigram(rowid, content, tool_name) " + "SELECT m.id, m.content, m.tool_name " + "FROM messages m JOIN sessions s ON s.id = m.session_id " "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " + f"AND {fts_trigram_session_sql('s')} " "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", (lo, hi), ) @@ -191,9 +199,9 @@ class SessionSearchMixin: comparison, so they keep the legacy chunked ``LIMIT`` delete — those shadow tables are small by construction. """ - with self._lock: + with self._read_ctx() as conn: trash = [ - r[0] for r in self._conn.execute( + r[0] for r in conn.execute( "SELECT name FROM sqlite_master WHERE type = 'table' " "AND name LIKE ? ESCAPE '\\'", (self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",), @@ -319,9 +327,11 @@ class SessionSearchMixin: if include_trigram: conn.execute( "INSERT INTO messages_fts_trigram" - "(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", + "(rowid, content, tool_name) " + "SELECT m.id, m.content, m.tool_name " + "FROM messages m JOIN sessions s ON s.id = m.session_id " + "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " + f"AND {fts_trigram_session_sql('s')}", (progress, upper), ) # Publish progress in the same transaction as the rows it @@ -340,7 +350,9 @@ class SessionSearchMixin: return True # transient (lock contention) — caller retries if more is False: status = self.fts_rebuild_status() - if status is not None and status["indexed"] >= status["total"]: + if high_water <= 0 or ( + status is not None and status["indexed"] >= status["total"] + ): self._fts_rebuild_finish() return False return bool(more) @@ -568,6 +580,11 @@ class SessionSearchMixin: "('fts_rebuild_progress', '0') " "ON CONFLICT(key) DO UPDATE SET value = excluded.value" ) + conn.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(hw)), + ) return hw hw = conn.execute( @@ -576,6 +593,7 @@ class SessionSearchMixin: for k, v in ( ("fts_rebuild_high_water", str(hw)), ("fts_rebuild_progress", "0"), + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(hw)), ): conn.execute( "INSERT INTO state_meta (key, value) VALUES (?, ?) " @@ -643,23 +661,26 @@ class SessionSearchMixin: is a legacy inline-FTS install that can be optimized to the v23 external-content schema, or a previous optimize run was interrupted (legacy vtables already demoted, but backfill markers and/or trash - tables remain) and re-running would resume it, or the CJK-bigram - index needs a backfill/rebuild on this tokenizer-capable host, or - a prior demote left an empty external-content index without markers - (healable on re-run). + tables remain) and re-running would resume it, or this DB is v23 with the + old tool-calls-inclusive trigram projection (repairable via this same + migration flow), or the CJK-bigram index needs a backfill/rebuild on this + tokenizer-capable host, or a prior demote left an empty external-content + index without markers (healable on re-run). False for fresh and fully-optimized installs (and when FTS5 is unavailable).""" if not self._fts_enabled or self.read_only: return False - with self._lock: - if self._db_has_legacy_inline_fts(self._conn): + with self._read_ctx() as conn: + if self._db_has_legacy_inline_fts(conn): + return True + if self._db_has_trigram_tool_calls_projection(self._conn): return True # Interrupted optimize: demotion already removed the legacy # vtables (so the check above is False), but the transition is # unfinished until the backfill markers are cleared and the # demoted trash tables are torn down. Search stays complete # through the gap supplement meanwhile; re-running resumes. - if self._conn.execute( + if conn.execute( "SELECT 1 FROM state_meta " "WHERE key = 'fts_rebuild_high_water' LIMIT 1" ).fetchone(): @@ -667,20 +688,20 @@ class SessionSearchMixin: # CJK-bigram index work — only offerable when THIS process can # tokenize: a pending backfill (markers set at creation on a # populated DB) or a stale index awaiting a from-scratch rebuild. - if self._fts_cjk_loaded and self._conn.execute( + if self._fts_cjk_loaded and conn.execute( "SELECT 1 FROM state_meta WHERE key IN " f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1" ).fetchone(): return True - if self._has_fts_trash(self._conn): + if self._has_fts_trash(conn): return True # Pre-fix crash window: empty external-content index with # messages still present, no markers, no trash (teardown already # finished or never needed). Re-run seeds markers and backfills. - return self._fts_external_index_empty_with_messages(self._conn) + return self._fts_external_index_empty_with_messages(conn) def _demote_legacy_fts_to_trash(self) -> int: - """Demote the legacy inline FTS vtables and stage their shadow tables + """Demote upgrade-eligible FTS vtables and stage their shadow tables for chunked teardown. Returns MAX(messages.id) as the rebuild high water. O(1) schema surgery — the heavy delete is deferred to the chunked teardown, exactly as the validated auto path did. @@ -753,9 +774,16 @@ class SessionSearchMixin: progress_cb: Optional[Callable[[Dict[str, Any]], None]] = None, vacuum: bool = True, ) -> Dict[str, Any]: - """Migrate a legacy v22 inline-FTS DB to the v23 external-content - schema, foreground and to completion. Safe to re-run: if a previous - attempt was interrupted it resumes from the progress marker. + """Repair an older FTS layout into the current v23-compatible shape, + foreground and to completion. + + Supports two paths: + - legacy-v22 inline -> demote to v23 external-content + - v23 installs where ``messages_fts_trigram`` still stores + ``tool_calls`` payloads + + Safe to re-run: if a previous attempt was interrupted it resumes from + the progress marker. ``progress_cb`` receives {"phase", "percent", "indexed", "total"} dicts for a CLI progress bar. Returns a summary dict. @@ -779,11 +807,13 @@ class SessionSearchMixin: # finishing the backfill + teardown — this is what makes re-running # after an interruption safe. with self._lock: - legacy = self._db_has_legacy_inline_fts(self._conn) + needs_storage_upgrade = self._db_needs_fts_storage_upgrade( + self._conn + ) pending = self.get_meta("fts_rebuild_high_water") is not None - if legacy and not pending: + if needs_storage_upgrade and not pending: self._demote_legacy_fts_to_trash() - elif pending and not legacy: + elif pending and not needs_storage_upgrade: # Resume mid-demote: markers exist, empty v23 tables may still be # missing if the process died between the staged demote commit and # schema ensure. Re-ensure is IF NOT EXISTS and cheap. @@ -876,13 +906,13 @@ class SessionSearchMixin: # still empty against a non-empty messages table. Pre-fix code could # tear down trash and settle after a no-op backfill when markers were # missing — permanent search-index loss for historical rows. - with self._lock: - still_pending = self._conn.execute( + with self._read_ctx() as conn: + still_pending = conn.execute( "SELECT 1 FROM state_meta " "WHERE key = 'fts_rebuild_high_water' LIMIT 1" ).fetchone() is not None - still_trash = self._has_fts_trash(self._conn) - empty_index = self._fts_external_index_empty_with_messages(self._conn) + still_trash = self._has_fts_trash(conn) + empty_index = self._fts_external_index_empty_with_messages(conn) if still_pending or still_trash or empty_index: reason = ( "backfill_incomplete" if still_pending or empty_index @@ -1131,8 +1161,8 @@ class SessionSearchMixin: # excludes handoffs with a DB pick that includes them, soft-deleting # the wrong turn. fetch_limit = int(limit) * 2 + 5 - with self._lock: - cursor = self._conn.execute( + with self._read_ctx() as conn: + cursor = conn.execute( "SELECT id, timestamp, content FROM messages " "WHERE session_id = ? AND role = 'user'" f"{active_clause}{display_clause} " @@ -1760,6 +1790,24 @@ class SessionSearchMixin: if not query: return [] + # New oversized tool results only index a bounded prefix to keep the + # foreground write transaction short. An explicit tool-role search is + # the opt-in full-body path and scans canonical rows via LIKE. + if role_filter and "tool" in role_filter: + matches = self._search_messages_like_fallback( + query, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + limit=limit, + offset=offset, + sort=sort, + include_inactive=include_inactive, + ) + return self._finalize_search_matches( + matches, result_fields=result_fields + ) + self._refresh_fts_stale_state() if self._fts_stale: matches = self._search_messages_like_fallback( @@ -1876,6 +1924,12 @@ class SessionSearchMixin: # query explicitly filtering on role='tool' must therefore use # the LIKE fallback, which scans the base table directly. _wants_tool_rows = bool(role_filter) and "tool" in role_filter + # Cron and subagent transcripts are excluded too (see + # FTS_TRIGRAM_EXCLUDED_SOURCES); an explicit filter for them + # must likewise scan the base table. + _wants_cron_rows = bool(source_filter) and any( + src in FTS_TRIGRAM_EXCLUDED_SOURCES for src in source_filter + ) # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── # When the bigram index is available it serves EVERY CJK query @@ -1890,6 +1944,7 @@ class SessionSearchMixin: if ( self._fts_cjk_available and not _wants_tool_rows + and not _wants_cron_rows and not self._has_lone_cjk_run(raw_query) ): tokens = raw_query.split() @@ -1963,6 +2018,7 @@ class SessionSearchMixin: and not _any_short_cjk and self._trigram_available and not _wants_tool_rows + and not _wants_cron_rows ): # Trigram FTS5 path — quote each non-operator token to handle # FTS5 special chars (%, *, etc.) while preserving boolean @@ -2382,7 +2438,9 @@ class SessionSearchMixin: FAILS CLOSED: if another process holds the rebuild lock beyond the bounded wait, this call defers (returns 0) rather than racing it. Callers already treat 0 as "rebuild made no progress" and fall back - to the stale-FTS breadcrumb path, which retries at next startup. + to the stale-FTS breadcrumb path, which retries in-process from the + gateway housekeeping tick (``retry_deferred_fts_recovery``) and at + next startup. Safe to call when FTS tables don't exist (skips them). Returns the number of FTS indexes that were rebuilt. @@ -2396,6 +2454,14 @@ class SessionSearchMixin: ) return 0 with self._lock: + high_water = self._conn.execute( + "SELECT COALESCE(MAX(id), 0) FROM messages" + ).fetchone()[0] + self._conn.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(high_water)), + ) for tbl in self._FTS_TABLES: if not self._fts_table_exists(tbl): continue diff --git a/locales/af.yaml b/locales/af.yaml index 21806156ab..363563e9a4 100644 --- a/locales/af.yaml +++ b/locales/af.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nHuidige modus: `{mode}`\\n\\nKies \\'n opsie:" choice_fast: "fast — Priority Processing aan" choice_normal: "normal — standaardverwerking" + choice_auto: "auto — vinnig vir die eerste sekondes van elke beurt" + choice_cold: "cold — vinnig slegs vir die eerste beurt van 'n sessie" footer: status: "📎 Looptyd-voetstuk: **{state}**\nVelde: `{fields}`\nPlatform: `{platform}`" diff --git a/locales/ar.yaml b/locales/ar.yaml index 1f68628a3b..abdc371ead 100644 --- a/locales/ar.yaml +++ b/locales/ar.yaml @@ -160,6 +160,8 @@ gateway: picker_title: "⚡ **المعالجة ذات الأولوية**\n\nالوضع الحالي: `{mode}`\n\nاختر خيارًا:" choice_fast: "fast — المعالجة ذات الأولوية مُفعّلة" choice_normal: "normal — المعالجة القياسية" + choice_auto: "auto — سريع في الثواني الأولى من كل دور" + choice_cold: "cold — سريع في الدور الأول من الجلسة فقط" footer: status: "📎 تذييل التشغيل: **{state}**\nالحقول: `{fields}`\nالمنصّة: `{platform}`" diff --git a/locales/de.yaml b/locales/de.yaml index bc00bfe32f..d6e1528088 100644 --- a/locales/de.yaml +++ b/locales/de.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nAktueller Modus: `{mode}`\\n\\nOption wählen:" choice_fast: "fast — Priority Processing an" choice_normal: "normal — Standardverarbeitung" + choice_auto: "auto — schnell in den ersten Sekunden jedes Zugs" + choice_cold: "cold — schnell nur im ersten Zug einer Sitzung" footer: status: "📎 Laufzeit-Fußzeile: **{state}**\nFelder: `{fields}`\nPlattform: `{platform}`" diff --git a/locales/en.yaml b/locales/en.yaml index 2adac023f2..9b06ae1e96 100644 --- a/locales/en.yaml +++ b/locales/en.yaml @@ -141,8 +141,8 @@ gateway: fast: not_supported: "⚡ /fast is only available for OpenAI models that support Priority Processing." - status: "⚡ Priority Processing\n\nCurrent mode: `{mode}`\n\n_Usage:_ `/fast `" - unknown_arg: "⚠️ Unknown argument: `{arg}`\n\n**Valid options:** normal, fast, status" + status: "⚡ Priority Processing\n\nCurrent mode: `{mode}`\n\n_Usage:_ `/fast `" + unknown_arg: "⚠️ Unknown argument: `{arg}`\n\n**Valid options:** normal, fast, auto, cold, status" saved: "⚡ ✓ Priority Processing: **{label}** (saved to config)\n_(takes effect on next message)_" session_only: "⚡ ✓ Priority Processing: **{label}** (this session only)" label_fast: "FAST" @@ -152,6 +152,8 @@ gateway: picker_title: "⚡ **Priority Processing**\n\nCurrent mode: `{mode}`\n\nPick an option:" choice_fast: "fast — Priority Processing on" choice_normal: "normal — standard processing" + choice_auto: "auto — fast for the first seconds of every turn" + choice_cold: "cold — fast for the first turn of a session only" footer: status: "📎 Runtime footer: **{state}**\nFields: `{fields}`\nPlatform: `{platform}`" diff --git a/locales/es.yaml b/locales/es.yaml index 6b06a52afb..06cd2e9e23 100644 --- a/locales/es.yaml +++ b/locales/es.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nModo actual: `{mode}`\\n\\nElige una opción:" choice_fast: "fast — Priority Processing activado" choice_normal: "normal — procesamiento estándar" + choice_auto: "auto — rápido en los primeros segundos de cada turno" + choice_cold: "cold — rápido solo en el primer turno de una sesión" footer: status: "📎 Pie de ejecución: **{state}**\nCampos: `{fields}`\nPlataforma: `{platform}`" diff --git a/locales/fr.yaml b/locales/fr.yaml index 4ce9760969..4f1faa6cbf 100644 --- a/locales/fr.yaml +++ b/locales/fr.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nMode actuel : `{mode}`\\n\\nChoisissez une option :" choice_fast: "fast — Priority Processing activé" choice_normal: "normal — traitement standard" + choice_auto: "auto — rapide pendant les premières secondes de chaque tour" + choice_cold: "cold — rapide uniquement au premier tour d'une session" footer: status: "📎 Pied de page d'exécution : **{state}**\nChamps : `{fields}`\nPlateforme : `{platform}`" diff --git a/locales/ga.yaml b/locales/ga.yaml index 92ef5363ea..843dc5a1c2 100644 --- a/locales/ga.yaml +++ b/locales/ga.yaml @@ -141,6 +141,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nMód reatha: `{mode}`\\n\\nRoghnaigh rogha:" choice_fast: "fast — Priority Processing ar siúl" choice_normal: "normal — gnáthphróiseáil" + choice_auto: "auto — tapa do na chéad soicindí de gach seal" + choice_cold: "cold — tapa don chéad seal de sheisiún amháin" footer: status: "📎 Buntásc rite: **{state}**\nRéimsí: `{fields}`\nArdán: `{platform}`" diff --git a/locales/hu.yaml b/locales/hu.yaml index b8feb1b994..d134d4372f 100644 --- a/locales/hu.yaml +++ b/locales/hu.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nJelenlegi mód: `{mode}`\\n\\nVálassz egy opciót:" choice_fast: "fast — Priority Processing bekapcsolva" choice_normal: "normal — normál feldolgozás" + choice_auto: "auto — gyors minden kör első másodperceiben" + choice_cold: "cold — gyors csak a munkamenet első körében" footer: status: "📎 Futási idejű lábléc: **{state}**\nMezők: `{fields}`\nPlatform: `{platform}`" diff --git a/locales/it.yaml b/locales/it.yaml index 758be5d8a7..a3480541bc 100644 --- a/locales/it.yaml +++ b/locales/it.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nModalità attuale: `{mode}`\\n\\nScegli un\\'opzione:" choice_fast: "fast — Priority Processing attivo" choice_normal: "normal — elaborazione standard" + choice_auto: "auto — veloce nei primi secondi di ogni turno" + choice_cold: "cold — veloce solo nel primo turno di una sessione" footer: status: "📎 Footer di runtime: **{state}**\nCampi: `{fields}`\nPiattaforma: `{platform}`" diff --git a/locales/ja.yaml b/locales/ja.yaml index 28b41682aa..b691daf93d 100644 --- a/locales/ja.yaml +++ b/locales/ja.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\n現在のモード: `{mode}`\\n\\nオプションを選択:" choice_fast: "fast — Priority Processing オン" choice_normal: "normal — 標準処理" + choice_auto: "auto — 各ターンの最初の数秒間だけ高速" + choice_cold: "cold — セッションの最初のターンのみ高速" footer: status: "📎 ランタイムフッター: **{state}**\nフィールド: `{fields}`\nプラットフォーム: `{platform}`" diff --git a/locales/ko.yaml b/locales/ko.yaml index ecf58bbc68..f7f4f25a06 100644 --- a/locales/ko.yaml +++ b/locales/ko.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\n현재 모드: `{mode}`\\n\\n옵션을 선택하세요:" choice_fast: "fast — Priority Processing 켜기" choice_normal: "normal — 표준 처리" + choice_auto: "auto — 매 턴의 처음 몇 초 동안 빠름" + choice_cold: "cold — 세션의 첫 턴에만 빠름" footer: status: "📎 런타임 푸터: **{state}**\n필드: `{fields}`\n플랫폼: `{platform}`" diff --git a/locales/pt.yaml b/locales/pt.yaml index 1ac1fd4b00..f18100340e 100644 --- a/locales/pt.yaml +++ b/locales/pt.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nModo atual: `{mode}`\\n\\nEscolha uma opção:" choice_fast: "fast — Priority Processing ativado" choice_normal: "normal — processamento padrão" + choice_auto: "auto — rápido nos primeiros segundos de cada turno" + choice_cold: "cold — rápido apenas no primeiro turno de uma sessão" footer: status: "📎 Rodapé de execução: **{state}**\nCampos: `{fields}`\nPlataforma: `{platform}`" diff --git a/locales/ru.yaml b/locales/ru.yaml index 51c892e02e..7fd2c285f2 100644 --- a/locales/ru.yaml +++ b/locales/ru.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nТекущий режим: `{mode}`\\n\\nВыберите вариант:" choice_fast: "fast — Priority Processing включён" choice_normal: "normal — стандартная обработка" + choice_auto: "auto — быстро в первые секунды каждого хода" + choice_cold: "cold — быстро только на первом ходе сессии" footer: status: "📎 Нижний колонтитул среды выполнения: **{state}**\nПоля: `{fields}`\nПлатформа: `{platform}`" diff --git a/locales/tr.yaml b/locales/tr.yaml index a88b1d0586..7af791fb74 100644 --- a/locales/tr.yaml +++ b/locales/tr.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nMevcut mod: `{mode}`\\n\\nBir seçenek seçin:" choice_fast: "fast — Priority Processing açık" choice_normal: "normal — standart işleme" + choice_auto: "auto — her turun ilk saniyelerinde hızlı" + choice_cold: "cold — yalnızca oturumun ilk turunda hızlı" footer: status: "📎 Çalışma zamanı altbilgisi: **{state}**\nAlanlar: `{fields}`\nPlatform: `{platform}`" diff --git a/locales/uk.yaml b/locales/uk.yaml index 730a052cd5..5e63f0bdad 100644 --- a/locales/uk.yaml +++ b/locales/uk.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\nПоточний режим: `{mode}`\\n\\nОберіть варіант:" choice_fast: "fast — Priority Processing увімкнено" choice_normal: "normal — стандартна обробка" + choice_auto: "auto — швидко в перші секунди кожного ходу" + choice_cold: "cold — швидко лише на першому ході сесії" footer: status: "📎 Нижній колонтитул середовища: **{state}**\nПоля: `{fields}`\nПлатформа: `{platform}`" diff --git a/locales/zh-hant.yaml b/locales/zh-hant.yaml index 9468fbba1c..b9091c2aa2 100644 --- a/locales/zh-hant.yaml +++ b/locales/zh-hant.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **Priority Processing**\\n\\n目前模式：`{mode}`\\n\\n請選擇：" choice_fast: "fast — 開啟 Priority Processing" choice_normal: "normal — 標準處理" + choice_auto: "auto — 每輪的前幾秒快速" + choice_cold: "cold — 僅會話的第一輪快速" footer: status: "📎 執行階段頁尾：**{state}**\n欄位：`{fields}`\n平台：`{platform}`" diff --git a/locales/zh.yaml b/locales/zh.yaml index f659de9a24..dde25d0e82 100644 --- a/locales/zh.yaml +++ b/locales/zh.yaml @@ -137,6 +137,8 @@ gateway: picker_title: "⚡ **优先处理**\\n\\n当前模式：`{mode}`\\n\\n请选择：" choice_fast: "fast — 开启优先处理" choice_normal: "normal — 标准处理" + choice_auto: "auto — 每轮的前几秒快速" + choice_cold: "cold — 仅会话的第一轮快速" footer: status: "📎 运行时页脚：**{state}**\n字段：`{fields}`\n平台：`{platform}`" diff --git a/model_tools.py b/model_tools.py index 20ce327a2a..e5e4e657e8 100644 --- a/model_tools.py +++ b/model_tools.py @@ -279,7 +279,7 @@ _LEGACY_TOOLSET_MAP = { "browser_press", "browser_get_images", "browser_vision", "browser_console" ], - "cronjob_tools": ["cronjob"], + "cronjob_tools": ["cronjob_manage"], "file_tools": ["read_file", "write_file", "patch", "search_files"], "tts_tools": ["text_to_speech"], } @@ -602,7 +602,7 @@ def _compute_tool_definitions( # Same session-level seam as the browser_exec gate above. if "delegate_task" in available_tool_names: blocked_present = [ - t for t in ("clarify", "memory", "cronjob") if t in available_tool_names + t for t in ("clarify", "memory", "cronjob_manage") if t in available_tool_names ] if len(blocked_present) < 3: full_offvariant = "delegate_task, clarify, memory, or cronjob" @@ -788,7 +788,18 @@ def _resolve_active_context_length() -> int: # because they need agent-level state (TodoStore, MemoryStore, etc.). # The registry still holds their schemas; dispatch just returns a stub error # so if something slips through, the LLM sees a sensible message. -_AGENT_LOOP_TOOLS = {"todo", "memory", "session_search", "delegate_task"} +_AGENT_LOOP_TOOLS = {"todo_list", "memory", "session_search", "delegate_task"} + +# Legacy tool-name aliases (2026-08 renames): accepted at every dispatch seam +# (handle_function_call + both executors) so old sessions and saved prompts +# keep working; schemas only advertise the new names. +_LEGACY_TOOL_ALIASES = { + "todo": "todo_list", + "cronjob": "cronjob_manage", + "process": "process_manage", + "tour": "gui_tour", + "tip": "show_tip", +} _READ_SEARCH_TOOLS = {"read_file", "search_files"} @@ -1284,6 +1295,13 @@ def handle_function_call( function_args = {} _tool_middleware_trace = list(tool_request_middleware_trace or []) + # ── Legacy tool-name aliases (2026-08 renames) ──────────────────── + # Old sessions resuming mid-conversation (and users' muscle memory in + # saved skills/cron prompts) still emit the pre-rename names. Alias at + # the dispatch seam so every replay keeps working; new schemas only + # advertise the new names, so fresh sessions never see the old ones. + function_name = _LEGACY_TOOL_ALIASES.get(function_name, function_name) + # ── Tool Search bridge dispatch ────────────────────────────────── # tool_search and tool_describe are pure catalog reads — handle them # inline. tool_call is unwrapped to the underlying tool so that every @@ -1368,9 +1386,9 @@ def handle_function_call( "Use tool_search to find tools you can call." ) ) - # Probe-validate against the deferred tool's schema (ironclaw#5149): - # a blind call missing required arguments returns the parameter - # schema instead of dispatching into an opaque downstream failure. + # Validate against the deferred tool's concrete schema before + # dispatch. This covers constraints the provider cannot enforce + # through the generic tool_call ``arguments: object`` bridge. _probe_err = _ts_mod.validate_deferred_call_args(underlying_name, underlying_args) if _probe_err is not None: return _return_bridge_result(_probe_err) diff --git a/plugins/dashboard_auth/nous/__init__.py b/plugins/dashboard_auth/nous/__init__.py index 69acd18e36..fdb28173ee 100644 --- a/plugins/dashboard_auth/nous/__init__.py +++ b/plugins/dashboard_auth/nous/__init__.py @@ -85,6 +85,7 @@ from hermes_cli.dashboard_auth import ( LoginStart, ProviderError, RefreshExpiredError, + classify_jwks_lookup_error, Session, ) @@ -436,10 +437,11 @@ class NousDashboardAuthProvider(DashboardAuthProvider): signing_key = self._get_jwks_client().get_signing_key_from_jwt( access_token ) - except jwt.PyJWKClientError as exc: - raise ProviderError(f"JWKS lookup failed: {exc}") from exc - except Exception as exc: # pragma: no cover - defensive - raise ProviderError(f"JWKS lookup failed: {exc!r}") from exc + except Exception as exc: + # Unreachable JWKS -> ProviderError (503); a bearer that is not + # one of our JWTs (opaque peer key, foreign kid) -> InvalidCodeError + # (None / next provider). Folding both into 503 produced #94558. + raise classify_jwks_lookup_error(exc) from exc try: claims = jwt.decode( diff --git a/plugins/dashboard_auth/self_hosted/__init__.py b/plugins/dashboard_auth/self_hosted/__init__.py index 2672006571..7b7283b2df 100644 --- a/plugins/dashboard_auth/self_hosted/__init__.py +++ b/plugins/dashboard_auth/self_hosted/__init__.py @@ -93,6 +93,7 @@ from hermes_cli.dashboard_auth import ( LoginStart, ProviderError, RefreshExpiredError, + classify_jwks_lookup_error, Session, ) @@ -617,10 +618,11 @@ class SelfHostedOIDCProvider(DashboardAuthProvider): signing_key = self._get_jwks_client().get_signing_key_from_jwt( id_token ) - except jwt.PyJWKClientError as exc: - raise ProviderError(f"JWKS lookup failed: {exc}") from exc - except Exception as exc: # pragma: no cover - defensive - raise ProviderError(f"JWKS lookup failed: {exc!r}") from exc + except Exception as exc: + # Unreachable JWKS -> ProviderError (503); a bearer that is not + # one of our JWTs (opaque peer key, foreign kid) -> InvalidCodeError + # (None / next provider). Folding both into 503 produced #94558. + raise classify_jwks_lookup_error(exc) from exc try: claims = jwt.decode( diff --git a/plugins/image_gen/meta-ai/__init__.py b/plugins/image_gen/meta-ai/__init__.py new file mode 100644 index 0000000000..e5c2d43993 --- /dev/null +++ b/plugins/image_gen/meta-ai/__init__.py @@ -0,0 +1,297 @@ +"""Meta Model API image generation backend. + +Exposes Meta's ``muse-image`` model(s) as an :class:`ImageGenProvider`. +The Meta Model API (https://api.meta.ai/v1) is OpenAI-compatible, so we reuse +the OpenAI Python SDK pointed at Meta's base URL and authenticate with +``META_MODEL_API_KEY``. + +Output is base64 JSON (WebP) -> saved under ``$HERMES_HOME/cache/images/``. + +Selection precedence (first hit wins): + 1. ``model`` kwarg forwarded by the dispatcher (the ``hermes tools`` pick) + 2. ``META_IMAGE_MODEL`` env var (escape hatch for scripts / tests) + 3. ``image_gen.meta-ai.model`` in ``config.yaml`` + 4. ``image_gen.model`` in ``config.yaml`` (when it's one of our IDs) + 5. :data:`DEFAULT_MODEL` +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Dict, List, Optional, Tuple + +from agent.secret_scope import get_secret +from agent.image_gen_provider import ( + DEFAULT_ASPECT_RATIO, + ImageGenProvider, + error_response, + normalize_reference_images, + resolve_aspect_ratio, + save_b64_image, + save_url_image, + success_response, +) + +logger = logging.getLogger(__name__) + +DEFAULT_BASE_URL = "https://api.meta.ai/v1" +# Auth env vars, in priority order. Mirrors the bundled ``meta-ai`` chat +# provider (plugins/model-providers/meta-ai): MODEL_API_KEY is Meta's +# documented var; the rest are accepted aliases. +API_KEY_ENVS = ("MODEL_API_KEY", "META_API_KEY", "META_MODEL_API_KEY") +# Primary key shown in setup prompts / error messages. +API_KEY_ENV = "META_MODEL_API_KEY" +# Optional base-url override (same var the chat provider honors). +BASE_URL_ENV = "META_BASE_URL" + + +def _resolve_api_key() -> Optional[str]: + """First non-empty auth env var, checked in priority order.""" + for env in API_KEY_ENVS: + val = get_secret(env) + if val: + return val + return None + + +def _resolve_base_url() -> str: + return (os.environ.get(BASE_URL_ENV) or "").strip() or DEFAULT_BASE_URL + + +# --------------------------------------------------------------------------- +# Model catalog +# --------------------------------------------------------------------------- +# Catalog shown in `hermes tools` and matched against `image_gen.model`. +# The model id is sent verbatim to the Meta Model API (`/v1/images/generations`). +_MODELS: Dict[str, Dict[str, Any]] = { + "muse-image-1.0": { + "display": "Muse Image 1.0", + "speed": "~10s", + "strengths": "Meta Model API image generation", + "price": "$0.01/image", + }, +} +DEFAULT_MODEL = "muse-image-1.0" + +# aspect_ratio -> OpenAI-style size string +_SIZES: Dict[str, str] = { + "square": "1024x1024", + "landscape": "1536x1024", + "portrait": "1024x1536", +} + + +def _resolve_model(caller_model: Optional[str] = None) -> Tuple[str, Dict[str, Any]]: + """Return (model_id, metadata) using the documented precedence chain. + + ``caller_model`` is the ``model`` kwarg the dispatcher forwards from the + top-level ``image_gen.model`` config key (what ``hermes tools`` writes). + It wins when it names one of our models, mirroring the xai/krea/openrouter + providers, so a user's picker choice is never silently dropped. + """ + if caller_model and caller_model in _MODELS: + return caller_model, _MODELS[caller_model] + + env_model = os.environ.get("META_IMAGE_MODEL") + if env_model and env_model in _MODELS: + return env_model, _MODELS[env_model] + + try: + from hermes_cli.config import load_config + + cfg = load_config() or {} + ig = cfg.get("image_gen") or {} + scoped = (ig.get("meta-ai") or {}).get("model") + if scoped and scoped in _MODELS: + return scoped, _MODELS[scoped] + top = ig.get("model") + if top and top in _MODELS: + return top, _MODELS[top] + except Exception: + logger.debug("Could not read image_gen model from config", exc_info=True) + + return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL] + + +class MetaImageGenProvider(ImageGenProvider): + """Meta Model API ``images.generate`` backend (muse-image).""" + + @property + def name(self) -> str: + return "meta-ai" + + @property + def display_name(self) -> str: + return "Meta Model API" + + def is_available(self) -> bool: + if not _resolve_api_key(): + return False + try: + import openai # noqa: F401 + except ImportError: + return False + return True + + def list_models(self) -> List[Dict[str, Any]]: + return [ + { + "id": mid, + "display": m["display"], + "speed": m["speed"], + "strengths": m["strengths"], + "price": m["price"], + } + for mid, m in _MODELS.items() + ] + + def default_model(self) -> Optional[str]: + return DEFAULT_MODEL + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Meta Model API", + "badge": "paid", + "tag": "Muse Image via Meta Model API (api.meta.ai)", + "env_vars": [ + { + "key": API_KEY_ENV, + "prompt": "Meta Model API key (LLM|... token)", + "url": "https://api.meta.ai", + }, + ], + } + + def capabilities(self) -> Dict[str, Any]: + # Text-to-image only for now. Bump this once image-to-image is verified + # against the Meta endpoint. + return {"modalities": ["text"], "max_reference_images": 0} + + def generate( + self, + prompt: str, + aspect_ratio: str = DEFAULT_ASPECT_RATIO, + *, + image_url: Optional[str] = None, + reference_image_urls: Optional[List[str]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + prompt = (prompt or "").strip() + aspect = resolve_aspect_ratio(aspect_ratio) + + if not prompt: + return error_response( + error="Prompt is required and must be a non-empty string", + error_type="invalid_argument", + provider="meta-ai", + aspect_ratio=aspect, + ) + + api_key = _resolve_api_key() + if not api_key: + return error_response( + error=( + f"{API_KEY_ENV} not set. Run `hermes tools` -> Image " + "Generation -> Meta Model API to configure." + ), + error_type="auth_required", + provider="meta-ai", + aspect_ratio=aspect, + ) + + try: + import openai + except ImportError: + return error_response( + error="openai Python package not installed (pip install openai)", + error_type="missing_dependency", + provider="meta-ai", + aspect_ratio=aspect, + ) + + model_id, _meta = _resolve_model(kwargs.get("model")) + size = _SIZES.get(aspect, _SIZES["square"]) + + client = openai.OpenAI(api_key=api_key, base_url=_resolve_base_url()) + + payload: Dict[str, Any] = { + "model": model_id, + "prompt": prompt, + "size": size, + "n": 1, + } + + try: + response = client.images.generate(**payload) + except Exception as exc: + logger.debug("Meta image generation failed", exc_info=True) + return error_response( + error=f"Meta image generation failed: {exc}", + error_type="api_error", + provider="meta-ai", + model=model_id, + prompt=prompt, + aspect_ratio=aspect, + ) + + try: + first = response.data[0] + except (AttributeError, IndexError, TypeError): + return error_response( + error="Meta response contained no image data", + error_type="empty_response", + provider="meta-ai", + model=model_id, + prompt=prompt, + aspect_ratio=aspect, + ) + + b64 = getattr(first, "b64_json", None) + url = getattr(first, "url", None) + + try: + if b64: + path = save_b64_image(b64, prefix="meta", extension="webp") + image_ref = str(path) + elif url: + path = save_url_image(url, prefix="meta") + image_ref = str(path) + else: + return error_response( + error="Meta response contained neither b64_json nor URL", + error_type="empty_response", + provider="meta-ai", + model=model_id, + prompt=prompt, + aspect_ratio=aspect, + ) + except Exception as exc: + return error_response( + error=f"Failed to save Meta image: {exc}", + error_type="io_error", + provider="meta-ai", + model=model_id, + prompt=prompt, + aspect_ratio=aspect, + ) + + revised_prompt = getattr(first, "revised_prompt", None) + extra: Dict[str, Any] = {"size": size} + if revised_prompt: + extra["revised_prompt"] = revised_prompt + + return success_response( + image=image_ref, + model=model_id, + prompt=prompt, + aspect_ratio=aspect, + provider="meta-ai", + modality="text", + extra=extra, + ) + + +def register(ctx) -> None: + """Plugin entry point -- wire ``MetaImageGenProvider`` into the registry.""" + ctx.register_image_gen_provider(MetaImageGenProvider()) diff --git a/plugins/image_gen/meta-ai/plugin.yaml b/plugins/image_gen/meta-ai/plugin.yaml new file mode 100644 index 0000000000..2d5bb36649 --- /dev/null +++ b/plugins/image_gen/meta-ai/plugin.yaml @@ -0,0 +1,7 @@ +name: meta-ai-image-gen +version: 1.0.0 +description: "Meta Model API image generation backend (muse-image). OpenAI-compatible /v1/images/generations. Saves images to $HERMES_HOME/cache/images/." +author: Meta Platforms, Inc. +kind: backend +requires_env: + - META_MODEL_API_KEY diff --git a/plugins/model-providers/alibaba-coding-plan/__init__.py b/plugins/model-providers/alibaba-coding-plan/__init__.py index b420fbbbd9..4723606d0e 100644 --- a/plugins/model-providers/alibaba-coding-plan/__init__.py +++ b/plugins/model-providers/alibaba-coding-plan/__init__.py @@ -9,6 +9,10 @@ Region split, mirroring the base DashScope pair (#73265): Profile names match the models.dev catalog keys exactly so model metadata lines up and ``model.provider: alibaba-coding-plan-cn`` resolves at runtime. + +The CN profile checks its own ``ALIBABA_CODING_PLAN_CN_API_KEY`` first (#101122, +mirroring kimi-coding-cn) and keeps the shared vars as ordered fallbacks so +existing CN users configured with the shared key keep working. """ from providers import register_provider @@ -31,7 +35,7 @@ alibaba_coding_plan_cn = ProviderProfile( display_name="Alibaba Cloud (Coding Plan, China)", description="Alibaba Cloud Coding Plan, mainland-China endpoint", signup_url="https://help.aliyun.com/zh/model-studio/", - env_vars=("ALIBABA_CODING_PLAN_API_KEY", "DASHSCOPE_API_KEY", "ALIBABA_CODING_PLAN_CN_BASE_URL"), + env_vars=("ALIBABA_CODING_PLAN_CN_API_KEY", "ALIBABA_CODING_PLAN_API_KEY", "DASHSCOPE_API_KEY", "ALIBABA_CODING_PLAN_CN_BASE_URL"), base_url="https://coding.dashscope.aliyuncs.com/v1", auth_type="api_key", ) diff --git a/plugins/model-providers/alibaba/__init__.py b/plugins/model-providers/alibaba/__init__.py index 6135945d50..a9198d3931 100644 --- a/plugins/model-providers/alibaba/__init__.py +++ b/plugins/model-providers/alibaba/__init__.py @@ -55,7 +55,7 @@ alibaba_token_plan_cn = ProviderProfile( display_name="Alibaba Cloud (Token Plan, China)", description="Alibaba Cloud Model Studio Token Plan, mainland-China endpoint", signup_url="https://help.aliyun.com/zh/model-studio/", - env_vars=("ALIBABA_TOKEN_PLAN_API_KEY", "ALIBABA_TOKEN_PLAN_CN_BASE_URL"), + env_vars=("ALIBABA_TOKEN_PLAN_CN_API_KEY", "ALIBABA_TOKEN_PLAN_API_KEY", "ALIBABA_TOKEN_PLAN_CN_BASE_URL"), base_url="https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", auth_type="api_key", ) diff --git a/plugins/model-providers/copilot-acp/__init__.py b/plugins/model-providers/copilot-acp/__init__.py index 6e452706c3..89f51b850f 100644 --- a/plugins/model-providers/copilot-acp/__init__.py +++ b/plugins/model-providers/copilot-acp/__init__.py @@ -1,10 +1,16 @@ """GitHub Copilot ACP provider profile. -copilot-acp uses an external ACP subprocess — NOT the standard -transport. api_mode="copilot_acp" is handled separately in run_agent.py. -The profile captures auth + endpoint metadata for registry migration. +copilot-acp does not speak OpenAI-over-HTTP: it drives an external ACP +subprocess over stdio. The profile therefore supplies its own client through +:meth:`ProviderProfile.create_client` instead of letting the core build an +``openai.OpenAI``. That hook is the registration seam — this profile is its +in-tree consumer, and an out-of-tree ACP provider registered from +``~/.hermes/plugins/model-providers/`` or a pip entry point uses the exact same +three lines without touching core. """ +from typing import Any + from providers import register_provider from providers.base import ProviderProfile @@ -12,6 +18,12 @@ from providers.base import ProviderProfile class CopilotACPProfile(ProviderProfile): """GitHub Copilot ACP — external process, no REST models endpoint.""" + def create_client(self, **client_kwargs: Any) -> Any: + """Build the ACP stdio shim rather than an HTTP client.""" + from agent.copilot_acp_client import CopilotACPClient + + return CopilotACPClient(**client_kwargs) + def fetch_models( self, *, @@ -30,6 +42,13 @@ copilot_acp = CopilotACPProfile( env_vars=(), # Managed by ACP subprocess base_url="acp://copilot", # ACP internal scheme auth_type="external_process", + # How to launch the CLI. Previously hardcoded in + # hermes_cli/auth.py::resolve_external_process_provider_credentials; the env + # var names are unchanged, so existing setups keep working. + process_command="copilot", + process_args=("--acp", "--stdio"), + process_command_env_vars=("HERMES_COPILOT_ACP_COMMAND", "COPILOT_CLI_PATH"), + process_args_env_var="HERMES_COPILOT_ACP_ARGS", ) register_provider(copilot_acp) diff --git a/plugins/model-providers/meta-ai/__init__.py b/plugins/model-providers/meta-ai/__init__.py index de2a3847f5..e54b1f4f54 100644 --- a/plugins/model-providers/meta-ai/__init__.py +++ b/plugins/model-providers/meta-ai/__init__.py @@ -58,6 +58,27 @@ def _resolve_effort(reasoning_config: dict | None) -> str: class MetaAIProfile(ProviderProfile): """Meta Model API — top-level reasoning_effort, self-contained.""" + # Non-chat model prefixes excluded from the agent picker. The live + # /v1/models catalog includes image-generation and transcription models + # that are not suitable for agentic chat. + _NON_CHAT_PREFIXES = ("muse-image-", "muse-voice-") + + def fetch_models( + self, + *, + api_key: str | None = None, + base_url: str | None = None, + timeout: float = 8.0, + ) -> list[str] | None: + """Fetch and filter the live catalog, excluding non-chat models.""" + live = super().fetch_models(api_key=api_key, base_url=base_url, timeout=timeout) + if live is None: + return None + return [ + m for m in live + if not any(m.startswith(p) for p in self._NON_CHAT_PREFIXES) + ] + def build_api_kwargs_extras( self, *, @@ -100,17 +121,20 @@ meta_ai = MetaAIProfile( api_mode="codex_responses", # Muse Spark is natively multimodal (image/video/pdf/audio in, text out). supports_vision=True, + # ...but only on user turns: an image envelope inside a role:tool message + # 400s "messages[N].content did not match any supported type" (#101668). + supports_vision_tool_messages=False, # Cheap contributor tier is a good default for auxiliary tasks # (compaction, title generation, vision) when this is the main provider. default_aux_model="muse-spark-1.2-contributor", # Muse spends completion budget on hidden reasoning tokens first; a low cap # can finish with empty content. 16k is a safe floor. default_max_tokens=16384, - # Curated safety net shown in the picker when the live /v1/models fetch - # fails or no credentials are configured yet. + # Minimal fallback shown when the live /v1/models fetch fails or no + # credentials are configured yet. Keep this list small — just enough so + # the picker isn't empty when the API is unreachable. fallback_models=( "muse-spark-1.2", - "muse-spark-1.2-contributor", ), ) diff --git a/plugins/model-providers/openrouter/__init__.py b/plugins/model-providers/openrouter/__init__.py index 6a91d5258c..3ae39250ff 100644 --- a/plugins/model-providers/openrouter/__init__.py +++ b/plugins/model-providers/openrouter/__init__.py @@ -50,9 +50,12 @@ class OpenRouterProfile(ProviderProfile): """OpenRouter aggregator — provider preferences, reasoning config passthrough.""" @staticmethod - def _clamp_reasoning_to_catalog(cfg: dict[str, Any], model: str | None) -> dict[str, Any]: + def _clamp_reasoning_to_catalog(cfg: dict[str, Any], model: str | None) -> dict[str, Any] | None: """Clamp ``cfg["effort"]`` to the model's catalog-advertised levels. + Returns None when the config is a disable and the catalog marks the + route reasoning-mandatory (the caller omits the field). + OpenRouter's /v1/models entries publish ``reasoning.supported_efforts`` per model (ported from PrimeIntellect-ai/prime-agent#1258). Sending an unsupported effort (e.g. ``ultra`` to a route that stops at ``high``) @@ -61,7 +64,8 @@ class OpenRouterProfile(ProviderProfile): or no supported_efforts list is published (None = all levels accepted). """ effort = cfg.get("effort") - if not effort or cfg.get("enabled") is False: + disabled = cfg.get("enabled") is False or effort == "none" + if not effort and not disabled: return cfg try: from hermes_cli.models import ( @@ -71,6 +75,11 @@ class OpenRouterProfile(ProviderProfile): caps = openrouter_model_reasoning_capabilities(model) if not caps or not caps.get("supports_reasoning"): return cfg + # A reasoning-mandatory route 400s on a disable ("Reasoning is + # mandatory for this endpoint and cannot be disabled") — omit + # the field and let the model think, same as the Nous profile. + if disabled: + return None if caps.get("mandatory") else cfg clamped = clamp_reasoning_effort_to_supported( effort, caps.get("supported_efforts") ) @@ -215,9 +224,11 @@ class OpenRouterProfile(ProviderProfile): if cfg.get("enabled", True) is not False and effort and effort != "none": top_level["verbosity"] = effort elif reasoning_config is not None: - extra_body["reasoning"] = self._clamp_reasoning_to_catalog( + clamped = self._clamp_reasoning_to_catalog( dict(reasoning_config), model ) + if clamped is not None: + extra_body["reasoning"] = clamped else: extra_body["reasoning"] = {"enabled": True, "effort": "medium"} @@ -249,7 +260,7 @@ openrouter = OpenRouterProfile( "anthropic/claude-sonnet-4.6", "openai/gpt-5.4", "deepseek/deepseek-chat", - "google/gemini-3.7-flash", + "google/gemini-3.8-flash", "qwen/qwen3-plus", ), ) diff --git a/plugins/platforms/a2a/adapter.py b/plugins/platforms/a2a/adapter.py index 79842c88c6..7a9b6b1142 100644 --- a/plugins/platforms/a2a/adapter.py +++ b/plugins/platforms/a2a/adapter.py @@ -74,8 +74,30 @@ def _reply_timeout() -> float: return 300.0 +def _profile_scoped() -> bool: + """True when running inside a multiplexed secondary profile's scope. + + Secondary-profile adapters are constructed inside ``_profile_runtime_scope`` + (secret scope installed + multiplex active) — the same discriminator the + Buzz/SimpleX adapters use for this bug class (#98738). The DEFAULT profile + under multiplexing runs unscoped: ``os.environ`` holds its own bridge + output there and keeps its legacy precedence. + """ + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + return bool(is_multiplex_active() and current_secret_scope() is not None) + except Exception: + return False + + def _default_agent_name() -> str: - name = os.getenv("A2A_AGENT_NAME", "").strip() + # Scope-aware: inside a secondary multiplex profile, os.environ holds the + # DEFAULT profile's bridged A2A_AGENT_NAME — borrowing it would brand a + # secondary profile's Agent Card with another profile's identity. There + # is no per-profile config.yaml equivalent yet, so a scoped profile just + # falls through to the hostname-based default below instead. + name = "" if _profile_scoped() else os.getenv("A2A_AGENT_NAME", "").strip() if name: return name try: @@ -227,7 +249,7 @@ class A2ARequestHandler(BaseHTTPRequestHandler): } # Do not leak profile/tenant topology on remote unauthenticated GETs. # Agent Cards are intentionally public; health topology is not. - if security.localhost_only() or security.authenticate( + if self.adapter._security_context.localhost_only() or self.adapter._security_context.authenticate( self.headers.get("Authorization"), self.client_address[0] if self.client_address else "", ) is not None: @@ -246,7 +268,9 @@ class A2ARequestHandler(BaseHTTPRequestHandler): # Identity comes from the presented credential (or the socket in # localhost-only mode) — never from the request body. - identity = security.authenticate(self.headers.get("Authorization"), client_ip) + identity = adapter._security_context.authenticate( + self.headers.get("Authorization"), client_ip + ) if identity is None: self._json(401, protocol.jsonrpc_error(None, protocol.ERR_UNAUTHORIZED, "unauthorized")) return @@ -292,7 +316,7 @@ class A2ARequestHandler(BaseHTTPRequestHandler): self._json(429, protocol.jsonrpc_error(req_id, protocol.ERR_RATE_LIMITED, "rate limit exceeded")) return - if not security.is_trusted_peer(identity): + if not adapter._security_context.is_trusted_peer(identity): self._json(403, protocol.jsonrpc_error( req_id, protocol.ERR_UNTRUSTED_PEER, f"peer '{identity}' not trusted")) return @@ -343,8 +367,17 @@ class A2AAdapter(BasePlatformAdapter): super().__init__(config=config, platform=platform) extra = getattr(config, "extra", {}) or {} - self.port = int(os.getenv("A2A_PORT") or extra.get("port", _DEFAULT_PORT)) - self.host = security.resolve_bind_host() + # Scope-aware: a secondary multiplex profile must not borrow the + # default profile's bridged A2A_PORT (mirrors the Buzz/SimpleX fix + # for #98738) — an unconfigured profile falls closed to the module + # default port instead. (advertised_toolsets has the same env-leak + # shape but is left unscoped here — see the "Scope note" in this + # fix's PR description: open PR #98937 is actively rewriting this + # field's None-vs-empty-list semantics.) + self._security_context = security.A2ASecurityContext.capture() + _port_env = None if _profile_scoped() else os.getenv("A2A_PORT") + self.port = int(_port_env or extra.get("port", _DEFAULT_PORT)) + self.host = self._security_context.resolve_bind_host() self.agent_name = _default_agent_name() self._advertised_toolsets = [ t.strip() for t in ( @@ -440,7 +473,11 @@ class A2AAdapter(BasePlatformAdapter): self._mark_connected() - exposure = "localhost-only" if security.localhost_only() else "REMOTE (bearer auth)" + exposure = ( + "localhost-only" + if self._security_context.localhost_only() + else "REMOTE (bearer auth)" + ) logger.info( "A2A: serving Agent Card + JSON-RPC on http://%s:%s (%s) as %r; %d routed agent(s)", self.host, self.port, exposure, self.agent_name, len(self._agents), @@ -502,9 +539,15 @@ class A2AAdapter(BasePlatformAdapter): raw = cfg.get("a2a_served_agents") or (cfg.get("a2a") or {}).get("served_agents") agents: dict[str, dict] = {} - default_desc = os.getenv( - "A2A_AGENT_DESCRIPTION", - "Hermes Agent — a general-purpose agent reachable over A2A.", + # Scope-aware for the same reason as port/toolsets above: a secondary + # profile must not inherit the default profile's A2A_AGENT_DESCRIPTION. + default_desc = ( + "Hermes Agent — a general-purpose agent reachable over A2A." + if _profile_scoped() + else os.getenv( + "A2A_AGENT_DESCRIPTION", + "Hermes Agent — a general-purpose agent reachable over A2A.", + ) ) agents[""] = { "slug": "", @@ -613,7 +656,7 @@ class A2AAdapter(BasePlatformAdapter): skills=self._advertised_skills(agent), streaming=bool(agent.get("local", True)), push_notifications=True, - auth_required=not security.localhost_only(), + auth_required=not self._security_context.localhost_only(), tenant=str(agent.get("tenant") or ""), ) @@ -1192,7 +1235,10 @@ class A2AAdapter(BasePlatformAdapter): if not callback_url: return - if not security.is_safe_callback_url(callback_url): + if not security.is_safe_callback_url( + callback_url, + localhost_mode=self._security_context.localhost_only(), + ): logger.warning("A2A: push notification for task %s blocked — unsafe callback URL: %s", task_id, callback_url) protocol.metrics.push_failed += 1 @@ -1201,7 +1247,7 @@ class A2AAdapter(BasePlatformAdapter): # Push payload uses the StreamResponse format (same as streaming). payload = protocol.status_update(task_id, context_id, state, (reply or "")[:2000]) - signature = security.sign_push_payload(payload) + signature = self._security_context.sign_push_payload(payload) headers = {"Content-Type": "application/json"} if signature: headers["X-A2A-Signature"] = signature diff --git a/plugins/platforms/a2a/security.py b/plugins/platforms/a2a/security.py index 753c202a54..350031b9df 100644 --- a/plugins/platforms/a2a/security.py +++ b/plugins/platforms/a2a/security.py @@ -31,29 +31,44 @@ import logging import os import re import time +from dataclasses import dataclass from pathlib import Path from typing import Optional logger = logging.getLogger(__name__) -# -------------------------------------------------------------------------- -# Bearer auth + peer identity -# -------------------------------------------------------------------------- +def _profile_scoped() -> bool: + """True when running inside a multiplexed secondary profile's scope. -def get_bearer_token() -> str: - """Return the configured shared inbound bearer token (empty if none).""" - return os.getenv("A2A_BEARER_TOKEN", "").strip() - - -def get_peer_tokens() -> dict[str, str]: - """Parse A2A_PEER_TOKENS ("alice:tok1,bob:tok2") into {token: peer_name}. - - Per-peer tokens give each remote agent its own credential, so the identity - used for rate limiting, trust, and audit is authenticated — not whatever - the request body claims. + Same discriminator as the Buzz/SimpleX/Raft adapters (#98738): secret + scope installed + multiplex active. The DEFAULT profile under + multiplexing (and every single-profile process) runs unscoped and keeps + its legacy ``os.environ`` precedence. """ - raw = os.getenv("A2A_PEER_TOKENS", "").strip() + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + return bool(is_multiplex_active() and current_secret_scope() is not None) + except Exception: + return False + + +def _startup_env(name: str) -> str: + """Read one A2A setting from the active profile's scope, else the env. + + Inside a secondary profile's scope the scope is authoritative: a miss + yields "" and never falls through to ``os.environ`` (which holds the + default profile's tokens in a multiplexer). + """ + if _profile_scoped(): + from agent.secret_scope import get_secret + + return (get_secret(name) or "").strip() + return os.getenv(name, "").strip() + + +def _parse_peer_tokens(raw: str) -> dict[str, str]: out: dict[str, str] = {} for pair in raw.split(","): pair = pair.strip() @@ -66,6 +81,115 @@ def get_peer_tokens() -> dict[str, str]: return out +def _configured_trusted_peers() -> frozenset[str]: + raw = _startup_env("A2A_TRUSTED_PEERS") + if raw: + return frozenset(p.strip() for p in raw.split(",") if p.strip()) + try: + from hermes_cli.config import load_config + + cfg = load_config() or {} + peers = (cfg.get("a2a") or {}).get("trusted_peers", []) + if isinstance(peers, list): + return frozenset(str(peer).strip() for peer in peers if str(peer).strip()) + except Exception: + pass + return frozenset() + + +@dataclass(frozen=True) +class A2ASecurityContext: + """Immutable, profile-scoped security settings captured at adapter startup. + + ``ThreadingHTTPServer`` handles requests on fresh threads that do not inherit + the gateway's profile ContextVars. Keeping the resolved settings on the + adapter prevents those threads from falling back to another profile's + process-global environment. + """ + + bearer_token: str + peer_tokens: tuple[tuple[str, str], ...] + trusted_peers: frozenset[str] + allow_all_users: bool + requested_host: str + push_secret: str + + @classmethod + def capture(cls) -> "A2ASecurityContext": + bearer_token = _startup_env("A2A_BEARER_TOKEN") + return cls( + bearer_token=bearer_token, + peer_tokens=tuple(_parse_peer_tokens(_startup_env("A2A_PEER_TOKENS")).items()), + trusted_peers=_configured_trusted_peers(), + allow_all_users=_startup_env("A2A_ALLOW_ALL_USERS").lower() + in {"1", "true", "yes"}, + requested_host=_startup_env("A2A_HOST") or "127.0.0.1", + push_secret=_startup_env("A2A_PUSH_SECRET") or bearer_token, + ) + + def localhost_only(self) -> bool: + return not (self.bearer_token or self.peer_tokens) + + def resolve_bind_host(self) -> str: + loopback = {"127.0.0.1", "localhost", "::1"} + if self.requested_host in loopback: + return self.requested_host + if self.localhost_only(): + logger.warning( + "A2A: A2A_HOST=%s ignored — no A2A_BEARER_TOKEN or " + "A2A_PEER_TOKENS set; binding to 127.0.0.1. Configure a token " + "to expose A2A remotely.", + self.requested_host, + ) + return "127.0.0.1" + return self.requested_host + + def authenticate(self, auth_header: Optional[str], client_ip: str = "") -> Optional[str]: + if self.localhost_only(): + return f"ip:{client_ip or 'local'}" + presented = _parse_bearer(auth_header) + if presented is None: + return None + for token, name in self.peer_tokens: + if hmac.compare_digest(presented, token): + return name + if self.bearer_token and hmac.compare_digest(presented, self.bearer_token): + return f"ip:{client_ip or 'unknown'}" + return None + + def is_trusted_peer(self, identity: str) -> bool: + if self.allow_all_users or self.localhost_only() or not self.trusted_peers: + return True + return identity in self.trusted_peers + + def sign_push_payload(self, payload: dict) -> str: + if not self.push_secret: + return "" + body = json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hmac.new( + self.push_secret.encode("utf-8"), body, hashlib.sha256 + ).hexdigest() + + +# -------------------------------------------------------------------------- +# Bearer auth + peer identity +# -------------------------------------------------------------------------- + +def get_bearer_token() -> str: + """Return the configured shared inbound bearer token (empty if none).""" + return _startup_env("A2A_BEARER_TOKEN") + + +def get_peer_tokens() -> dict[str, str]: + """Parse A2A_PEER_TOKENS ("alice:tok1,bob:tok2") into {token: peer_name}. + + Per-peer tokens give each remote agent its own credential, so the identity + used for rate limiting, trust, and audit is authenticated — not whatever + the request body claims. + """ + return _parse_peer_tokens(_startup_env("A2A_PEER_TOKENS")) + + def _parse_bearer(auth_header: Optional[str]) -> Optional[str]: if not auth_header: return None @@ -85,24 +209,12 @@ def authenticate(auth_header: Optional[str], client_ip: str = "") -> Optional[st Comparisons are constant-time (hmac.compare_digest). """ - peer_tokens = get_peer_tokens() - shared = get_bearer_token() - if not peer_tokens and not shared: - return f"ip:{client_ip or 'local'}" - presented = _parse_bearer(auth_header) - if presented is None: - return None - for token, name in peer_tokens.items(): - if hmac.compare_digest(presented, token): - return name - if shared and hmac.compare_digest(presented, shared): - return f"ip:{client_ip or 'unknown'}" - return None + return A2ASecurityContext.capture().authenticate(auth_header, client_ip) def localhost_only() -> bool: """True when we must refuse non-loopback binds (no token of any kind set).""" - return not (get_bearer_token() or get_peer_tokens()) + return A2ASecurityContext.capture().localhost_only() def resolve_bind_host() -> str: @@ -112,18 +224,7 @@ def resolve_bind_host() -> str: per-peer) AND explicitly asked for a wider host. A token alone does not widen the bind — opting into remote exposure must be deliberate. """ - requested = os.getenv("A2A_HOST", "").strip() or "127.0.0.1" - loopback = {"127.0.0.1", "localhost", "::1"} - if requested in loopback: - return requested - if localhost_only(): - logger.warning( - "A2A: A2A_HOST=%s ignored — no A2A_BEARER_TOKEN or A2A_PEER_TOKENS " - "set; binding to 127.0.0.1. Configure a token to expose A2A remotely.", - requested, - ) - return "127.0.0.1" - return requested + return A2ASecurityContext.capture().resolve_bind_host() # -------------------------------------------------------------------------- @@ -138,18 +239,7 @@ def get_trusted_peers() -> set[str]: names from ``authenticate()`` — peer-token names, or ``ip:`` for shared-token callers. """ - env_peers = os.getenv("A2A_TRUSTED_PEERS", "").strip() - if env_peers: - return {p.strip() for p in env_peers.split(",") if p.strip()} - try: - from hermes_cli.config import load_config - cfg = load_config() or {} - peers_list = (cfg.get("a2a") or {}).get("trusted_peers", []) - if isinstance(peers_list, list): - return {str(p).strip() for p in peers_list if p} - except Exception: - pass - return set() + return set(_configured_trusted_peers()) def is_trusted_peer(identity: str) -> bool: @@ -160,14 +250,7 @@ def is_trusted_peer(identity: str) -> bool: otherwise any *authenticated* identity is allowed (authentication is the primary gate — the allow-list is an optional restriction on top). """ - if os.getenv("A2A_ALLOW_ALL_USERS", "").strip().lower() in ("1", "true", "yes"): - return True - if localhost_only(): - return True - trusted = get_trusted_peers() - if not trusted: - return True - return identity in trusted + return A2ASecurityContext.capture().is_trusted_peer(identity) # -------------------------------------------------------------------------- @@ -259,10 +342,7 @@ def get_push_secret() -> str: Falls back to the bearer token if no dedicated push secret is set. If neither is configured, push notifications are unsigned (localhost-only mode). """ - secret = os.getenv("A2A_PUSH_SECRET", "").strip() - if secret: - return secret - return get_bearer_token() + return A2ASecurityContext.capture().push_secret def sign_push_payload(payload: dict) -> str: @@ -304,12 +384,14 @@ _BLOCKED_PREFIXES = ( ) -def is_safe_callback_url(url: str) -> bool: +def is_safe_callback_url(url: str, *, localhost_mode: Optional[bool] = None) -> bool: """Check if a push notification callback URL is safe from SSRF. Blocks internal/private/loopback/metadata addresses. Only allows http:// and https:// schemes. """ + if localhost_mode is None: + localhost_mode = localhost_only() if not url or not isinstance(url, str): return False try: @@ -324,16 +406,16 @@ def is_safe_callback_url(url: str) -> bool: hostname_lower = hostname.lower() if hostname_lower == "localhost": # Loopback callbacks only make sense for local testing. - return localhost_only() + return localhost_mode for prefix in _BLOCKED_PREFIXES: if hostname_lower.startswith(prefix.lower()): - if localhost_only() and prefix in ("127.", "::1"): + if localhost_mode and prefix in ("127.", "::1"): return True return False try: ip = ipaddress.ip_address(hostname) if ip.is_loopback or ip.is_link_local or ip.is_private or ip.is_reserved: - if localhost_only() and ip.is_loopback: + if localhost_mode and ip.is_loopback: return True return False except ValueError: diff --git a/plugins/platforms/buzz/adapter.py b/plugins/platforms/buzz/adapter.py index aa74bb897d..94a5bd6c4a 100644 --- a/plugins/platforms/buzz/adapter.py +++ b/plugins/platforms/buzz/adapter.py @@ -173,7 +173,7 @@ from gateway.platforms.base import ( SendResult, MessageEvent, MessageType, - cache_media_bytes, + cache_media_bytes_async, ) from gateway.config import Platform @@ -2376,7 +2376,7 @@ class BuzzAdapter(BasePlatformAdapter): logger.warning("Buzz: attachment SHA-256 does not match imeta") return None try: - return cache_media_bytes( + return await cache_media_bytes_async( bytes(data), filename=metadata["filename"], mime_type=metadata["mime_type"], @@ -2863,7 +2863,7 @@ class BuzzAdapter(BasePlatformAdapter): media_kinds: List[str] = [] from gateway.platforms.base import ( - cache_media_bytes, + cache_media_bytes_async, validate_inbound_media_size, ) @@ -2894,8 +2894,10 @@ class BuzzAdapter(BasePlatformAdapter): mimetypes.guess_type(download_path.name)[0] or "application/octet-stream" ) - cached = cache_media_bytes( - download_path.read_bytes(), + # Up to the inbound media cap (128 MiB) — read off the loop too. + data = await asyncio.to_thread(download_path.read_bytes) + cached = await cache_media_bytes_async( + data, filename=download_path.name, mime_type=mime_type, ) diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index 3bdc58a5b3..971e0494cc 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -203,10 +203,10 @@ from gateway.platforms.base import ( ProcessingOutcome, SendResult, cache_image_from_url, - cache_image_from_bytes, + cache_image_from_bytes_async, cache_audio_from_url, - cache_audio_from_bytes, - cache_document_from_bytes, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, SUPPORTED_DOCUMENT_TYPES, _TEXT_INJECT_EXTENSIONS, _prefix_within_utf16_limit, @@ -382,6 +382,10 @@ class _DiscordNonConversationalMessageTracker: def __init__(self, max_tracked: int = _MAX_TRACKED): self._max_tracked = max_tracked self._ids: dict[str, None] = dict.fromkeys(self._load()) + # Serializes the offloaded flushes so two concurrent mark_many() calls + # cannot land their writes out of order (last-writer-wins would drop + # the newer ids from disk). + self._persist_lock = asyncio.Lock() def _state_path(self) -> _Path: from hermes_constants import get_hermes_home @@ -404,17 +408,21 @@ class _DiscordNonConversationalMessageTracker: logger.debug("[%s] Failed to load non-conversational Discord IDs", "Discord") return [] - def _save(self) -> None: + def _snapshot(self) -> list[str]: + """Trim in-memory state and return the ids to persist (loop-side).""" ids = list(self._ids) if len(ids) > self._max_tracked: ids = ids[-self._max_tracked:] self._ids = dict.fromkeys(ids) + return ids + + def _save(self, ids: list[str]) -> None: try: atomic_json_write(self._state_path(), ids, indent=None) except Exception: logger.debug("[%s] Failed to save non-conversational Discord IDs", "Discord", exc_info=True) - def mark_many(self, message_ids: List[str]) -> None: + async def mark_many(self, message_ids: List[str]) -> None: changed = False for message_id in message_ids: key = str(message_id or "").strip() @@ -422,7 +430,16 @@ class _DiscordNonConversationalMessageTracker: self._ids[key] = None changed = True if changed: - self._save() + # atomic_json_write() calls os.fsync(), which blocks until the + # write reaches stable storage. Both callers of mark_many() run + # on the event loop, so offload the flush the same way #83906 + # did for the other gateway persist paths. The snapshot (and the + # trim that reassigns ``_ids``) stays on the loop so the worker + # never touches the dict while another task mutates it; the lock + # keeps flushes in mutation order. + async with self._persist_lock: + ids = self._snapshot() + await asyncio.to_thread(self._save, ids) def __contains__(self, message_id: str) -> bool: return str(message_id or "") in self._ids @@ -3611,7 +3628,7 @@ class DiscordAdapter(BasePlatformAdapter): if message_ids: _target_id = thread_id or chat_id if nonconversational: - self._nonconversational_messages.mark_many(message_ids) + await self._nonconversational_messages.mark_many(message_ids) elif not _looks_like_nonconversational_history_message(content): self._last_self_message_id[_target_id] = message_ids[-1] @@ -6456,6 +6473,14 @@ class DiscordAdapter(BasePlatformAdapter): ) return (len(self._skill_entries), self._skill_group_hidden_count) + def _interaction_guild_id(self, interaction: discord.Interaction) -> Optional[str]: + """Resolve the guild id of a slash interaction (mirrors the message path).""" + guild_id = getattr(interaction, "guild_id", None) + if guild_id is None: + guild = getattr(getattr(interaction, "channel", None), "guild", None) + guild_id = getattr(guild, "id", None) + return str(guild_id) if guild_id else None + def _build_slash_event(self, interaction: discord.Interaction, text: str) -> MessageEvent: """Build a MessageEvent from a Discord slash command interaction.""" is_dm = isinstance(interaction.channel, discord.DMChannel) @@ -6480,6 +6505,12 @@ class DiscordAdapter(BasePlatformAdapter): # For forum threads, inherit the parent forum's topic. chat_topic = self._get_effective_topic(interaction.channel, is_thread=is_thread) + # guild_id/parent_chat_id feed profile_routes matching in build_source, + # exactly as on_message passes them — without them a guild- or + # channel-routed profile never matches a native slash command (#69178). + parent_id = ( + self._get_parent_channel_id(interaction.channel) if is_thread else None + ) or "" source = self.build_source( chat_id=str(interaction.channel_id), chat_name=chat_name, @@ -6488,11 +6519,12 @@ class DiscordAdapter(BasePlatformAdapter): user_name=interaction.user.display_name, thread_id=thread_id, chat_topic=chat_topic, + guild_id=self._interaction_guild_id(interaction), + parent_chat_id=parent_id or None, ) msg_type = MessageType.COMMAND if text.startswith("/") else MessageType.TEXT channel_id = str(interaction.channel_id) - parent_id = str(getattr(getattr(interaction, "channel", None), "parent_id", "") or "") return MessageEvent( text=text, message_type=msg_type, @@ -6574,6 +6606,8 @@ class DiscordAdapter(BasePlatformAdapter): _chan = getattr(interaction, "channel", None) chat_topic = self._get_effective_topic(_chan, is_thread=True) if _chan else None + _parent_channel = self._thread_parent_channel(getattr(interaction, "channel", None)) + _parent_id = str(getattr(_parent_channel, "id", "") or "") source = self.build_source( chat_id=thread_id, chat_name=chat_name, @@ -6582,10 +6616,10 @@ class DiscordAdapter(BasePlatformAdapter): user_name=interaction.user.display_name, thread_id=thread_id, chat_topic=chat_topic, + guild_id=self._interaction_guild_id(interaction), + parent_chat_id=_parent_id or None, ) - _parent_channel = self._thread_parent_channel(getattr(interaction, "channel", None)) - _parent_id = str(getattr(_parent_channel, "id", "") or "") _skills = self._resolve_channel_skills(thread_id, _parent_id or None) _channel_prompt = self._resolve_channel_prompt(thread_id, _parent_id or None) event = MessageEvent( @@ -7855,7 +7889,7 @@ class DiscordAdapter(BasePlatformAdapter): msg = await channel.send(content=content, embed=embed, view=view) view._message = msg # store for on_timeout expiration editing if _metadata_marks_nonconversational(metadata): - self._nonconversational_messages.mark_many([str(msg.id)]) + await self._nonconversational_messages.mark_many([str(msg.id)]) return SendResult(success=True, message_id=str(msg.id)) except Exception as e: return SendResult(success=False, error=str(e)) @@ -8091,7 +8125,7 @@ class DiscordAdapter(BasePlatformAdapter): raw_bytes = await self._read_attachment_bytes(att, media_type="image") if raw_bytes is not None: try: - return cache_image_from_bytes(raw_bytes, ext=ext) + return await cache_image_from_bytes_async(raw_bytes, ext=ext) except Exception as e: logger.debug( "[Discord] cache_image_from_bytes rejected att.read() data; falling back to URL: %s", @@ -8110,7 +8144,7 @@ class DiscordAdapter(BasePlatformAdapter): raw_bytes = await self._read_attachment_bytes(att, media_type="audio") if raw_bytes is not None: try: - return cache_audio_from_bytes(raw_bytes, ext=ext) + return await cache_audio_from_bytes_async(raw_bytes, ext=ext) except Exception as e: logger.debug( "[Discord] cache_audio_from_bytes failed; falling back to URL: %s", @@ -8437,7 +8471,7 @@ class DiscordAdapter(BasePlatformAdapter): else: try: raw_bytes = await self._cache_discord_document(att, ext) - cached_path = cache_document_from_bytes( + cached_path = await cache_document_from_bytes_async( raw_bytes, att.filename or f"document{ext or '.bin'}" ) if in_allowlist: diff --git a/plugins/platforms/email/adapter.py b/plugins/platforms/email/adapter.py index 89ead8a82a..228cad281f 100644 --- a/plugins/platforms/email/adapter.py +++ b/plugins/platforms/email/adapter.py @@ -7,8 +7,12 @@ Uses IMAP to receive and SMTP to send messages. Environment variables: EMAIL_IMAP_HOST — IMAP server host (e.g., imap.gmail.com) EMAIL_IMAP_PORT — IMAP server port (default: 993) + EMAIL_IMAP_SECURITY — IMAP transport: tls, starttls, or plain (default: tls) + EMAIL_IMAP_TLS_VERIFY — Verify the IMAP TLS certificate (default: true) EMAIL_SMTP_HOST — SMTP server host (e.g., smtp.gmail.com) EMAIL_SMTP_PORT — SMTP server port (default: 587) + EMAIL_SMTP_SECURITY — SMTP transport: tls, starttls, or plain (port-based default) + EMAIL_SMTP_TLS_VERIFY — Verify the SMTP TLS certificate (default: true) EMAIL_ADDRESS — Email address for the agent EMAIL_PASSWORD — Email password or app-specific password EMAIL_POLL_INTERVAL — Seconds between mailbox checks (default: 15) @@ -88,7 +92,40 @@ def _esecret_int(name: str, default: int) -> int: def _esecret_bool(name: str, default: bool = False) -> bool: """Scope-aware boolean read (``env_bool`` variant of ``_get_esecret``).""" - return is_truthy_value(_get_esecret(name, ""), default=default) + raw = str(_get_esecret(name, "")).strip() + return is_truthy_value(raw, default=default) if raw else default + + +_SECURITY_ALIASES = { + "tls": "tls", "ssl": "tls", "implicit": "tls", + "starttls": "starttls", + "plain": "plain", "none": "plain", +} + + +def _normalize_security(value: Any, default: str = "tls") -> str: + """Map an IMAP/SMTP security setting to ``tls`` | ``starttls`` | ``plain``. + + Unknown values log a warning and fall back to *default* rather than + failing the connection, so a typo never silently downgrades to plaintext. + """ + raw = str(value or "").strip().lower().replace("-", "").replace("_", "") + if not raw: + return default + mode = _SECURITY_ALIASES.get(raw) + if mode is None: + logger.warning("Unknown email security mode %r; using %r", value, default) + return default + return mode + + +def _tls_context(verify: bool, host: str) -> ssl.SSLContext: + """Verified context by default; unverified only when explicitly opted out.""" + if verify: + return ssl.create_default_context() + if host not in ("127.0.0.1", "::1", "localhost"): + logger.warning("TLS verification disabled for non-loopback host %s", host) + return ssl._create_unverified_context() # Automated sender patterns — emails from these are silently ignored @@ -554,8 +591,23 @@ class EmailAdapter(BasePlatformAdapter): self._password = _get_secret("EMAIL_PASSWORD", "") self._imap_host = (_get_secret("EMAIL_IMAP_HOST", "") or extra.get("imap_host", "")).strip() self._imap_port = _esecret_int("EMAIL_IMAP_PORT", 993) + self._imap_security = _normalize_security( + _get_secret("EMAIL_IMAP_SECURITY", "") or extra.get("imap_security", "") + ) + self._imap_tls_verify = _esecret_bool( + "EMAIL_IMAP_TLS_VERIFY", + is_truthy_value(extra.get("imap_tls_verify"), default=True), + ) self._smtp_host = (_get_secret("EMAIL_SMTP_HOST", "") or extra.get("smtp_host", "")).strip() self._smtp_port = _esecret_int("EMAIL_SMTP_PORT", 587) + self._smtp_security = _normalize_security( + _get_secret("EMAIL_SMTP_SECURITY", "") or extra.get("smtp_security", ""), + default="tls" if self._smtp_port == 465 else "starttls", + ) + self._smtp_tls_verify = _esecret_bool( + "EMAIL_SMTP_TLS_VERIFY", + is_truthy_value(extra.get("smtp_tls_verify"), default=True), + ) self._poll_interval = _esecret_int("EMAIL_POLL_INTERVAL", 15) # Skip attachments — configured via config.yaml: @@ -627,6 +679,25 @@ class EmailAdapter(BasePlatformAdapter): # Fallback: just clear old entries if sort fails self._seen_uids = set(list(self._seen_uids)[-self._seen_uids_max // 2:]) + def _connect_imap(self) -> imaplib.IMAP4: + """Create an IMAP connection using implicit TLS, STARTTLS, or plaintext.""" + if self._imap_security == "tls": + return imaplib.IMAP4_SSL( + self._imap_host, + self._imap_port, + timeout=30, + ssl_context=_tls_context(self._imap_tls_verify, self._imap_host), + ) + + imap = imaplib.IMAP4(self._imap_host, self._imap_port, timeout=30) + if self._imap_security == "starttls": + try: + imap.starttls(ssl_context=_tls_context(self._imap_tls_verify, self._imap_host)) + except Exception: + _close_imap(imap) + raise + return imap + def _connect_smtp(self) -> smtplib.SMTP: """Create an SMTP connection, selecting the correct protocol for the port. @@ -642,22 +713,24 @@ class EmailAdapter(BasePlatformAdapter): Returns a connected SMTP object with TLS established — callers can proceed directly to ``login()``. """ - ctx = ssl.create_default_context() host = self._smtp_host port = self._smtp_port + security = self._smtp_security + ctx = _tls_context(self._smtp_tls_verify, host) def _connect(*, ipv4_only: bool = False) -> smtplib.SMTP: """Attempt one SMTP connection.""" smtp_cls = _IPv4SMTP if ipv4_only else smtplib.SMTP smtp_ssl_cls = _IPv4SMTP_SSL if ipv4_only else smtplib.SMTP_SSL - if port == 465: + if security == "tls": return smtp_ssl_cls(host, port, timeout=SMTP_CONNECT_TIMEOUT, context=ctx) smtp = smtp_cls(host, port, timeout=SMTP_CONNECT_TIMEOUT) - try: - smtp.starttls(context=ctx) - except Exception: - smtp.close() - raise + if security == "starttls": + try: + smtp.starttls(context=ctx) + except Exception: + smtp.close() + raise return smtp try: @@ -711,7 +784,7 @@ class EmailAdapter(BasePlatformAdapter): # (#79889). imap = None try: - imap = imaplib.IMAP4_SSL(self._imap_host, self._imap_port, timeout=30) + imap = self._connect_imap() imap.login(self._address, self._password) _send_imap_id(imap) imap.select("INBOX") @@ -855,7 +928,7 @@ class EmailAdapter(BasePlatformAdapter): results = [] imap: Optional[imaplib.IMAP4] = None try: - imap = imaplib.IMAP4_SSL(self._imap_host, self._imap_port, timeout=30) + imap = self._connect_imap() try: imap.login(self._address, self._password) _send_imap_id(imap) @@ -1438,7 +1511,6 @@ async def _standalone_send( """Out-of-process Email delivery via SMTP (one-shot). Implements the standalone_sender_fn contract; replaces the legacy _send_email helper.""" import smtplib - import ssl as _ssl from email.mime.text import MIMEText from email.utils import formatdate @@ -1450,6 +1522,14 @@ async def _standalone_send( smtp_port = int(_get_secret("EMAIL_SMTP_PORT", "587") or "587") except (ValueError, TypeError): smtp_port = 587 + smtp_security = _normalize_security( + _get_secret("EMAIL_SMTP_SECURITY", "") or extra.get("smtp_security"), + default="tls" if smtp_port == 465 else "starttls", + ) + smtp_tls_verify = _esecret_bool( + "EMAIL_SMTP_TLS_VERIFY", + is_truthy_value(extra.get("smtp_tls_verify"), default=True), + ) if not all([address, password, smtp_host]): return {"error": "Email not configured (EMAIL_ADDRESS, EMAIL_PASSWORD, EMAIL_SMTP_HOST required)"} @@ -1461,8 +1541,17 @@ async def _standalone_send( msg["Subject"] = "Hermes Agent" msg["Date"] = formatdate(localtime=True) - server = smtplib.SMTP(smtp_host, smtp_port) - server.starttls(context=_ssl.create_default_context()) + ctx = _tls_context(smtp_tls_verify, smtp_host) + if smtp_security == "tls": + server = smtplib.SMTP_SSL(smtp_host, smtp_port, context=ctx) + else: + server = smtplib.SMTP(smtp_host, smtp_port) + if smtp_security == "starttls": + try: + server.starttls(context=ctx) + except Exception: + server.close() + raise server.login(address, password) server.send_message(msg) server.quit() diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py index 044f12d573..d8df5b6445 100644 --- a/plugins/platforms/feishu/adapter.py +++ b/plugins/platforms/feishu/adapter.py @@ -126,10 +126,10 @@ from gateway.platforms.base import ( ProcessingOutcome, SendResult, SUPPORTED_DOCUMENT_TYPES, - cache_document_from_bytes, + cache_document_from_bytes_async, cache_image_from_url, - cache_audio_from_bytes, - cache_image_from_bytes, + cache_audio_from_bytes_async, + cache_image_from_bytes_async, ) from gateway.status import acquire_scoped_lock, release_scoped_lock from hermes_constants import get_hermes_home @@ -436,6 +436,9 @@ class FeishuAdapterSettings: group_rules: Dict[str, FeishuGroupRule] = field(default_factory=dict) allow_bots: str = "none" # "none" | "mentions" | "all" require_mention: bool = True + # DM allow-all (FEISHU_ALLOW_ALL_USERS / GATEWAY_ALLOW_ALL_USERS), resolved + # per-profile so multiplexed secondary adapters honor their own .env. + allow_all_dm: bool = False @dataclass @@ -1322,16 +1325,96 @@ def _strip_edge_self_mentions( return remaining +# --------------------------------------------------------------------------- +# Multiplex isolation for the lark_oapi WebSocket client (#73779) +# --------------------------------------------------------------------------- +# +# ``lark_oapi.ws.client`` keeps the asyncio loop used by ``Client.start()`` +# and every coroutine it spawns in a *module-level global* (``loop``), and +# Hermes also monkey-patches ``websockets.connect`` on the shared +# ``websockets`` module to inject per-adapter ping settings. In multiplex +# mode every profile runs its own WS client on a dedicated thread, so the N +# threads overwrite each other's module globals (last-write-wins): a client +# ends up scheduling tasks on a sibling profile's loop ("Future attached to +# a different loop" crashes) or binds to the wrong loop at construction time +# and goes deaf from the start. +# +# The fix installs process-wide, thread-dispatching shims exactly once: +# +# * ``ws_client_module.loop`` becomes a proxy that forwards every attribute +# access to the loop registered by the *current thread*. All SDK reads of +# the global happen on the thread that owns the loop (``start()`` blocks +# in ``run_until_complete`` and every ``create_task`` callback runs on +# the loop's own thread), so each profile transparently sees its own +# loop. Threads that never registered one (single-profile installs, CLI) +# fall back to the SDK's original module loop. +# * ``websockets.connect`` becomes a single dispatcher that merges the +# per-thread ping overrides registered by the calling profile, so +# profiles no longer race over the global patch or restore each other's +# hooks while a sibling is still connected. + +_WS_ISOLATION_LOCK = threading.Lock() +_WS_ISOLATION_INSTALLED = False +# Per-WS-thread registration: ``.loop`` (the thread's asyncio loop) and +# ``.connect_kwargs`` (websockets.connect overrides, e.g. ping settings). +_ws_isolation_state = threading.local() + + +class _ThreadLocalLoopProxy: + """Forwards attribute access to the current thread's registered loop.""" + + def __init__(self, fallback: Any) -> None: + self._fallback = fallback + + def _target(self) -> Any: + return getattr(_ws_isolation_state, "loop", None) or self._fallback + + def __getattr__(self, name: str) -> Any: + return getattr(self._target(), name) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"" + + +def _install_lark_ws_isolation(ws_client_module: Any) -> None: + """Install the thread-dispatching shims once per process (idempotent).""" + global _WS_ISOLATION_INSTALLED + with _WS_ISOLATION_LOCK: + if _WS_ISOLATION_INSTALLED: + return + + ws_client_module.loop = _ThreadLocalLoopProxy(ws_client_module.loop) + + real_connect = ws_client_module.websockets.connect + + def _dispatch_connect(*args: Any, **kwargs: Any) -> Any: + overrides = getattr(_ws_isolation_state, "connect_kwargs", None) or {} + for key, value in overrides.items(): + kwargs.setdefault(key, value) + return real_connect(*args, **kwargs) + + # Keep ``inspect.signature(websockets.connect)`` honest: the SDK's + # ``_ws_connect_kwargs()`` probes the real signature to decide whether + # the installed websockets generation supports the ``proxy`` kwarg. + _dispatch_connect.__wrapped__ = real_connect + _dispatch_connect.__name__ = getattr(real_connect, "__name__", "connect") + ws_client_module.websockets.connect = _dispatch_connect + _WS_ISOLATION_INSTALLED = True + + def _run_official_feishu_ws_client(ws_client: Any, adapter: Any) -> None: - """Run the official Lark WS client in its own thread-local event loop.""" + """Run the official Lark WS client in its own thread-local event loop. + + In multiplex mode several profiles run this concurrently; the shims + installed by ``_install_lark_ws_isolation`` make each thread see its own + loop and connect overrides (see the isolation comment block above). + """ import lark_oapi.ws.client as ws_client_module loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - ws_client_module.loop = loop adapter._ws_thread_loop = loop - original_connect = ws_client_module.websockets.connect original_configure = getattr(ws_client, "_configure", None) def _apply_runtime_ws_overrides() -> None: @@ -1343,12 +1426,15 @@ def _run_official_feishu_ws_client(ws_client: Any, adapter: Any) -> None: except Exception: logger.debug("[Feishu] Failed to apply websocket runtime overrides", exc_info=True) - def _connect_with_overrides(*args: Any, **kwargs: Any) -> Any: - if adapter._ws_ping_interval is not None and "ping_interval" not in kwargs: - kwargs["ping_interval"] = adapter._ws_ping_interval - if adapter._ws_ping_timeout is not None and "ping_timeout" not in kwargs: - kwargs["ping_timeout"] = adapter._ws_ping_timeout - return original_connect(*args, **kwargs) + connect_overrides: Dict[str, Any] = {} + if adapter._ws_ping_interval is not None: + connect_overrides["ping_interval"] = adapter._ws_ping_interval + if adapter._ws_ping_timeout is not None: + connect_overrides["ping_timeout"] = adapter._ws_ping_timeout + + _install_lark_ws_isolation(ws_client_module) + _ws_isolation_state.loop = loop + _ws_isolation_state.connect_kwargs = connect_overrides def _configure_with_overrides(conf: Any) -> Any: if original_configure is None: @@ -1357,7 +1443,6 @@ def _run_official_feishu_ws_client(ws_client: Any, adapter: Any) -> None: _apply_runtime_ws_overrides() return result - ws_client_module.websockets.connect = _connect_with_overrides if original_configure is not None: setattr(ws_client, "_configure", _configure_with_overrides) _apply_runtime_ws_overrides() @@ -1366,7 +1451,8 @@ def _run_official_feishu_ws_client(ws_client: Any, adapter: Any) -> None: except Exception: pass finally: - ws_client_module.websockets.connect = original_connect + _ws_isolation_state.loop = None + _ws_isolation_state.connect_kwargs = None if original_configure is not None: setattr(ws_client, "_configure", original_configure) pending = [t for t in asyncio.all_tasks(loop) if not t.done()] @@ -1517,6 +1603,8 @@ class FeishuAdapter(BasePlatformAdapter): self._sdk_executor_closing = False self._ws_client: Optional[Any] = None self._ws_future: Optional[asyncio.Future] = None + self._ws_supervisor: Optional[asyncio.Task] = None + self._ws_restart_backoff = 5.0 self._ws_thread_loop: Optional[asyncio.AbstractEventLoop] = None self._loop: Optional[asyncio.AbstractEventLoop] = None self._webhook_runner: Optional[Any] = None @@ -1526,6 +1614,9 @@ class FeishuAdapter(BasePlatformAdapter): self._seen_message_order: List[str] = [] self._dedup_state_path = get_hermes_home() / "feishu_seen_message_ids.json" self._dedup_lock = threading.Lock() + # Serializes the offloaded dedup-state flushes so two concurrent + # inbound messages cannot land their writes out of order. + self._dedup_persist_lock = asyncio.Lock() self._sender_name_cache: Dict[str, tuple[str, float]] = {} # sender_id → (name, expire_at) self._webhook_rate_counts: Dict[str, tuple[int, float]] = {} # rate_key → (count, window_start) self._webhook_anomaly_counts: Dict[str, tuple[int, str, float]] = {} # ip → (count, last_status, first_seen) @@ -1591,7 +1682,9 @@ class FeishuAdapter(BasePlatformAdapter): # Env-only so adapter and gateway auth bypass share one source; yaml # feishu.allow_bots is bridged to this env var at config load. - allow_bots = os.getenv("FEISHU_ALLOW_BOTS", "none").strip().lower() + # Scope-aware read: under multiplex a secondary profile's .env must + # govern its own adapter (same pattern as app_secret below) — #86905. + allow_bots = _get_scoped_secret("FEISHU_ALLOW_BOTS", "none").strip().lower() if allow_bots not in {"none", "mentions", "all"}: logger.warning( "[Feishu] Unknown allow_bots=%r, falling back to 'none'. Valid: none, mentions, all.", @@ -1599,8 +1692,13 @@ class FeishuAdapter(BasePlatformAdapter): ) allow_bots = "none" + allow_all_dm = any( + _get_scoped_secret(var, "").strip().lower() in {"true", "1", "yes"} + for var in ("FEISHU_ALLOW_ALL_USERS", "GATEWAY_ALLOW_ALL_USERS") + ) + return FeishuAdapterSettings( - app_id=str(extra.get("app_id") or os.getenv("FEISHU_APP_ID", "")).strip(), + app_id=str(extra.get("app_id") or _get_scoped_secret("FEISHU_APP_ID", "")).strip(), app_secret=str(extra.get("app_secret") or _get_scoped_secret("FEISHU_APP_SECRET", "")).strip(), domain_name=str(extra.get("domain") or os.getenv("FEISHU_DOMAIN", "feishu")).strip().lower(), connection_mode=str( @@ -1610,15 +1708,15 @@ class FeishuAdapter(BasePlatformAdapter): verification_token=str( extra.get("verification_token") or _get_scoped_secret("FEISHU_VERIFICATION_TOKEN", "") ).strip(), - group_policy=os.getenv("FEISHU_GROUP_POLICY", "allowlist").strip().lower(), + group_policy=_get_scoped_secret("FEISHU_GROUP_POLICY", "allowlist").strip().lower(), allowed_group_users=frozenset( item.strip() - for item in os.getenv("FEISHU_ALLOWED_USERS", "").split(",") + for item in _get_scoped_secret("FEISHU_ALLOWED_USERS", "").split(",") if item.strip() ), - bot_open_id=os.getenv("FEISHU_BOT_OPEN_ID", "").strip(), - bot_user_id=os.getenv("FEISHU_BOT_USER_ID", "").strip(), - bot_name=os.getenv("FEISHU_BOT_NAME", "").strip(), + bot_open_id=_get_scoped_secret("FEISHU_BOT_OPEN_ID", "").strip(), + bot_user_id=_get_scoped_secret("FEISHU_BOT_USER_ID", "").strip(), + bot_name=_get_scoped_secret("FEISHU_BOT_NAME", "").strip(), dedup_cache_size=max( 32, env_int("HERMES_FEISHU_DEDUP_CACHE_SIZE", _DEFAULT_DEDUP_CACHE_SIZE), @@ -1658,8 +1756,9 @@ class FeishuAdapter(BasePlatformAdapter): default_group_policy=default_group_policy, group_rules=group_rules, allow_bots=allow_bots, + allow_all_dm=allow_all_dm, require_mention=_to_boolean( - extra.get("require_mention", os.getenv("FEISHU_REQUIRE_MENTION", "true")) + extra.get("require_mention", _get_scoped_secret("FEISHU_REQUIRE_MENTION", "true")) ), ) @@ -1692,6 +1791,7 @@ class FeishuAdapter(BasePlatformAdapter): self._ws_ping_interval = settings.ws_ping_interval self._ws_ping_timeout = settings.ws_ping_timeout self._allow_bots = settings.allow_bots + self._allow_all_dm = settings.allow_all_dm self._require_mention = settings.require_mention def _build_event_handler(self) -> Any: @@ -1814,6 +1914,13 @@ class FeishuAdapter(BasePlatformAdapter): self._loop = asyncio.get_running_loop() await self._connect_with_retry() + if self._connection_mode == "websocket": + # Supervised reconnect (#73779): the WS thread can die without + # any external signal; keep a watcher alive for as long as this + # adapter is supposed to be connected. + self._ws_supervisor = asyncio.ensure_future( + self._supervise_websocket_thread() + ) self._mark_connected() logger.info("[Feishu] Connected in %s mode (%s)", self._connection_mode, self._domain_name) # Plugin-registered native handlers (lark_oapi client). @@ -1829,6 +1936,9 @@ class FeishuAdapter(BasePlatformAdapter): async def disconnect(self) -> None: """Disconnect from Feishu/Lark.""" self._running = False + if self._ws_supervisor is not None: + self._ws_supervisor.cancel() + self._ws_supervisor = None await self._cancel_pending_tasks(self._pending_text_batch_tasks) await self._cancel_pending_tasks(self._pending_media_batch_tasks) self._reset_batch_buffers() @@ -2633,7 +2743,7 @@ class FeishuAdapter(BasePlatformAdapter): return message_id = getattr(message, "message_id", None) - if not message_id or self._is_duplicate(message_id): + if not message_id or await self._is_duplicate(message_id): logger.debug("[Feishu] Dropping duplicate/missing message_id: %s", message_id) return @@ -3549,7 +3659,7 @@ class FeishuAdapter(BasePlatformAdapter): default_name=preferred_name, default_ext=default_ext, ) - cached_path = cache_document_from_bytes(body, filename) + cached_path = await cache_document_from_bytes_async(body, filename) return cached_path, filename @staticmethod @@ -4000,7 +4110,7 @@ class FeishuAdapter(BasePlatformAdapter): content_type = self._get_response_header(response, "Content-Type") filename = getattr(response, "file_name", None) or f"{image_key}.jpg" ext = self._guess_extension(filename, content_type, ".jpg", allowed=_IMAGE_EXTENSIONS) - cached_path = cache_image_from_bytes(raw_bytes, ext=ext) + cached_path = await cache_image_from_bytes_async(raw_bytes, ext=ext) media_type = self._normalize_media_type(content_type, default=self._default_image_media_type(ext)) return cached_path, media_type except Exception: @@ -4054,26 +4164,26 @@ class FeishuAdapter(BasePlatformAdapter): if media_type.startswith("image/"): ext = self._guess_extension(filename, content_type, ".jpg", allowed=_IMAGE_EXTENSIONS) - cached_path = cache_image_from_bytes(raw_bytes, ext=ext) + cached_path = await cache_image_from_bytes_async(raw_bytes, ext=ext) logger.info("[Feishu] Cached message image resource at %s", cached_path) return cached_path, media_type or self._default_image_media_type(ext) if request_type == "audio" or media_type.startswith("audio/"): ext = self._guess_extension(filename, content_type, ".ogg", allowed=_AUDIO_EXTENSIONS) - cached_path = cache_audio_from_bytes(raw_bytes, ext=ext) + cached_path = await cache_audio_from_bytes_async(raw_bytes, ext=ext) logger.info("[Feishu] Cached message audio resource at %s", cached_path) return cached_path, (media_type or f"audio/{ext.lstrip('.') or 'ogg'}") if media_type.startswith("video/"): if not Path(filename).suffix: filename = f"{filename}.mp4" - cached_path = cache_document_from_bytes(raw_bytes, filename) + cached_path = await cache_document_from_bytes_async(raw_bytes, filename) logger.info("[Feishu] Cached message video resource at %s", cached_path) return cached_path, media_type if not Path(filename).suffix and media_type in _DOCUMENT_MIME_TO_EXT: filename = f"{filename}{_DOCUMENT_MIME_TO_EXT[media_type]}" - cached_path = cache_document_from_bytes(raw_bytes, filename) + cached_path = await cache_document_from_bytes_async(raw_bytes, filename) logger.info("[Feishu] Cached message document resource at %s", cached_path) return cached_path, (media_type or self._guess_document_media_type(filename)) except Exception: @@ -4390,9 +4500,10 @@ class FeishuAdapter(BasePlatformAdapter): return "bot_not_mentioned" if not is_group: - if os.getenv("FEISHU_ALLOW_ALL_USERS", "").strip().lower() in {"true", "1", "yes"}: - return None - if os.getenv("GATEWAY_ALLOW_ALL_USERS", "").strip().lower() in {"true", "1", "yes"}: + # Snapshotted per-profile in _load_settings: _admit runs on the + # lark_oapi WS thread with no secret scope, and a bare os.getenv + # here would read the default profile's value (#86905). + if self._allow_all_dm: return None # Empty FEISHU_ALLOWED_USERS is the pairing-mode default from setup: # forward DMs to gateway intake so the pairing handshake can run. @@ -4630,14 +4741,15 @@ class FeishuAdapter(BasePlatformAdapter): def _persist_seen_message_ids(self) -> None: try: self._dedup_state_path.parent.mkdir(parents=True, exist_ok=True) - recent = self._seen_message_order[-self._dedup_cache_size:] - # Save as {msg_id: timestamp} so TTL filtering works across restarts. - payload = {"message_ids": {k: self._seen_message_ids[k] for k in recent if k in self._seen_message_ids}} + with self._dedup_lock: + recent = self._seen_message_order[-self._dedup_cache_size:] + # Save as {msg_id: timestamp} so TTL filtering works across restarts. + payload = {"message_ids": {k: self._seen_message_ids[k] for k in recent if k in self._seen_message_ids}} atomic_json_write(self._dedup_state_path, payload, indent=None) except OSError: logger.warning("[Feishu] Failed to persist dedup state to %s", self._dedup_state_path, exc_info=True) - def _is_duplicate(self, message_id: str) -> bool: + async def _is_duplicate(self, message_id: str) -> bool: now = time.time() ttl = _FEISHU_DEDUP_TTL_SECONDS with self._dedup_lock: @@ -4650,8 +4762,23 @@ class FeishuAdapter(BasePlatformAdapter): while len(self._seen_message_order) > self._dedup_cache_size: stale = self._seen_message_order.pop(0) self._seen_message_ids.pop(stale, None) - self._persist_seen_message_ids() - return False + # atomic_json_write() calls os.fsync(), which blocks until the write + # reaches stable storage. _handle_message_event_data runs on the + # event loop for every inbound message, so offload the flush the + # same way #83906 did for the other gateway persist paths. The lock + # keeps flushes in mutation order (the snapshot inside the worker is + # taken under _dedup_lock, but the write itself is not). + async with self._dedup_persist_lock_or_create(): + await asyncio.to_thread(self._persist_seen_message_ids) + return False + + def _dedup_persist_lock_or_create(self) -> asyncio.Lock: + # Tests build bare adapters via object.__new__ and install dedup state + # by hand; create the lock lazily so those fixtures keep working. + lock = getattr(self, "_dedup_persist_lock", None) + if lock is None: + lock = self._dedup_persist_lock = asyncio.Lock() + return lock # ========================================================================= # Outbound payload construction and send pipeline @@ -4946,6 +5073,52 @@ class FeishuAdapter(BasePlatformAdapter): ) await asyncio.sleep(wait_seconds) + async def _supervise_websocket_thread(self) -> None: + """Restart the WS client thread if it dies while the adapter is up. + + ``lark_oapi``'s ``start()`` blocks forever on a healthy connection + and only returns on fatal errors. Before this watcher existed the + executor future was awaited solely by ``disconnect()``, so a dead + thread left the profile silently deaf until a gateway restart + (#73779). Watch the future and, on unexpected exit, rebuild the + client with capped exponential backoff. + """ + backoff = initial_backoff = float(self._ws_restart_backoff) + last_dead: Optional[asyncio.Future] = None + while self._running: + ws_future = self._ws_future + if ws_future is None: + return + try: + await asyncio.shield(ws_future) + except asyncio.CancelledError: + raise + except Exception: + pass + # Deliberate disconnect paths nil ``_ws_client`` / ``_running`` + # before the thread exits; only restart when the link is still + # expected to be up. + if not self._running or self._ws_client is None: + return + if ws_future is not last_dead: + logger.error( + "[Feishu] WebSocket client thread exited unexpectedly; " + "restarting in %.0fs", + backoff, + ) + last_dead = ws_future + await asyncio.sleep(backoff) + if not self._running: + return + try: + await self._connect_websocket() + backoff = initial_backoff + except Exception as exc: + logger.warning( + "[Feishu] WebSocket restart failed (retrying): %s", exc + ) + backoff = min(backoff * 2, 60.0) + async def _connect_websocket(self) -> None: if not FEISHU_WEBSOCKET_AVAILABLE: raise RuntimeError("websockets not installed; websocket mode unavailable") diff --git a/plugins/platforms/feishu/feishu_meeting_invite.py b/plugins/platforms/feishu/feishu_meeting_invite.py index 69a487c029..0648d6fca9 100644 --- a/plugins/platforms/feishu/feishu_meeting_invite.py +++ b/plugins/platforms/feishu/feishu_meeting_invite.py @@ -173,7 +173,7 @@ async def handle_meeting_invited_event(adapter: Any, data: Any) -> None: dedup_key = _dedup_key(payload) is_duplicate = getattr(adapter, "_is_duplicate", None) - if callable(is_duplicate) and is_duplicate(dedup_key): + if callable(is_duplicate) and await is_duplicate(dedup_key): logger.debug("[Feishu-MeetingInvite] Dropping duplicate event: %s", dedup_key) return diff --git a/plugins/platforms/google_chat/adapter.py b/plugins/platforms/google_chat/adapter.py index 36643553f9..627c442267 100644 --- a/plugins/platforms/google_chat/adapter.py +++ b/plugins/platforms/google_chat/adapter.py @@ -48,6 +48,45 @@ import time from pathlib import Path as _Path from typing import Any, Callable, Dict, List, Optional, Tuple +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret +from agent.secret_scope import is_multiplex_active + + +def _get_scoped_secret(name: str, default: Optional[str] = None) -> Optional[str]: + """Scope-aware config/credential read with the default-profile fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and connects + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash startup/reconnect (#70652 class); there + ``os.environ`` is that profile's own value, so fall back to it. Same + pattern as ``whatsapp_common._get_wsecret`` and the WeCom/IRC/ntfy + plugin adapters. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + +def _adc_would_borrow_foreign_credentials() -> bool: + """True when ADC would silently read another profile's SA from process env. + + ``google.auth.default()`` consults ``os.environ`` directly. Under + multiplexing a scoped profile only reaches the ADC branch after its own + scope had no service-account setting -- if the process env still carries + one (the default profile's), ADC would authenticate this profile as that + other identity. Fail closed instead. + """ + return is_multiplex_active() and bool( + os.environ.get("GOOGLE_CHAT_SERVICE_ACCOUNT_JSON") + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + ) + # Heavy google-cloud + googleapiclient imports are deferred to first # adapter use. Importing them eagerly here added ~110ms wall and ~33MB # RSS to *every* CLI invocation (the plugin loader imports this module at @@ -184,15 +223,16 @@ from gateway.config import Platform, PlatformConfig Platform("google_chat") from gateway.platforms.helpers import MessageDeduplicator from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, ProcessingOutcome, SendResult, - cache_audio_from_bytes, - cache_document_from_bytes, - cache_image_from_bytes, - cache_video_from_bytes, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, + cache_image_from_bytes_async, + cache_video_from_bytes_async, ) @@ -736,28 +776,48 @@ class GoogleChatAdapter(BasePlatformAdapter): # end-of-turn by on_processing_complete via patch-to-empty so # they don't sit in the chat forever as "Hermes is thinking…". self._orphan_typing_messages: Dict[str, List[str]] = {} - # FlowControl knobs (env-configurable). + # Snapshot profile-scoped settings while adapter construction still + # runs inside _profile_runtime_scope. Pub/Sub invokes callbacks from + # its own threads, where the ContextVar secret scope is intentionally + # unavailable; callbacks must use these instance values rather than + # consulting process-global environment state. + extra = self.config.extra try: - self._max_messages = int(os.getenv("GOOGLE_CHAT_MAX_MESSAGES", "1")) + self._max_messages = int( + extra.get("max_messages") + or _get_scoped_secret("GOOGLE_CHAT_MAX_MESSAGES", "1") + ) except (ValueError, TypeError): self._max_messages = 1 try: - self._max_bytes = int(os.getenv("GOOGLE_CHAT_MAX_BYTES", str(16 * 1024 * 1024))) + self._max_bytes = int( + extra.get("max_bytes") + or _get_scoped_secret("GOOGLE_CHAT_MAX_BYTES", str(16 * 1024 * 1024)) + ) except (ValueError, TypeError): self._max_bytes = 16 * 1024 * 1024 + self._bootstrap_spaces = str( + extra.get("bootstrap_spaces") + or _get_scoped_secret("GOOGLE_CHAT_BOOTSTRAP_SPACES", "") + or "" + ).strip() + self._debug_raw = bool( + extra.get("debug_raw") + or _get_scoped_secret("GOOGLE_CHAT_DEBUG_RAW") + ) self._http_events_url = ( - self.config.extra.get("http_events_url") - or os.getenv("GOOGLE_CHAT_HTTP_EVENTS_URL", "") + extra.get("http_events_url") + or _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_URL", "") or "" ).strip() self._http_events_audience = ( - self.config.extra.get("http_events_audience") - or os.getenv("GOOGLE_CHAT_HTTP_EVENTS_AUDIENCE", "") + extra.get("http_events_audience") + or _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_AUDIENCE", "") or self._http_events_url ).strip() self._http_events_service_account_email = ( - self.config.extra.get("http_events_service_account_email") - or os.getenv("GOOGLE_CHAT_HTTP_EVENTS_SERVICE_ACCOUNT_EMAIL", "") + extra.get("http_events_service_account_email") + or _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_SERVICE_ACCOUNT_EMAIL", "") or "" ).strip().lower() @@ -779,7 +839,7 @@ class GoogleChatAdapter(BasePlatformAdapter): """ sa_path = ( self.config.extra.get("service_account_json") - or os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + or _get_scoped_secret("GOOGLE_APPLICATION_CREDENTIALS") ) if sa_path: # Inline JSON (rare, but supported). @@ -811,6 +871,13 @@ class GoogleChatAdapter(BasePlatformAdapter): # No explicit SA configured — try ADC. This is the Cloud Run / GCE # path; google-auth picks up the workload identity automatically. + if _adc_would_borrow_foreign_credentials(): + raise ValueError( + "Google Chat ADC skipped for this profile: service-account " + "credentials are set in the process environment but not in " + "this profile's secret scope. Set " + "GOOGLE_CHAT_SERVICE_ACCOUNT_JSON in this profile's .env." + ) try: import google.auth as google_auth except ImportError: @@ -916,8 +983,12 @@ class GoogleChatAdapter(BasePlatformAdapter): # ------------------------------------------------------------------ def _bot_id_cache_path(self) -> _Path: """Location where the resolved bot user_id is cached across restarts.""" - base = os.getenv("HERMES_HOME", str(_Path.home() / ".hermes")) - return _Path(base) / "google_chat_bot_id.json" + # Resolve at call time (connect() runs inside the profile scope) so + # multiplexed profiles do not share one bot-identity cache file; the + # thread-count store above already resolves the same way. + from hermes_constants import get_hermes_home as _get_hermes_home + + return _get_hermes_home() / "google_chat_bot_id.json" def _load_cached_bot_id(self) -> Optional[str]: path = self._bot_id_cache_path() @@ -952,7 +1023,7 @@ class GoogleChatAdapter(BasePlatformAdapter): if self.config.home_channel and self.config.home_channel.chat_id: candidate_spaces.append(self.config.home_channel.chat_id) # Env-configured allowed spaces (comma-separated). Optional. - extra_spaces = os.getenv("GOOGLE_CHAT_BOOTSTRAP_SPACES", "").strip() + extra_spaces = self._bootstrap_spaces if extra_spaces: candidate_spaces.extend( s.strip() for s in extra_spaces.split(",") if s.strip() @@ -1401,7 +1472,7 @@ class GoogleChatAdapter(BasePlatformAdapter): list(envelope.keys()), ce_type, ) - if os.getenv("GOOGLE_CHAT_DEBUG_RAW"): + if self._debug_raw: # Dangerous flag: contains message text and sender email. Route # through the global redaction filter and gate at DEBUG level so # default log configurations never surface it. Operators must @@ -2044,13 +2115,13 @@ class GoogleChatAdapter(BasePlatformAdapter): else: ext = "" if mime.startswith("image/"): - local = cache_image_from_bytes(data, ext=ext or ".jpg") + local = await cache_image_from_bytes_async(data, ext=ext or ".jpg") elif mime.startswith("audio/"): - local = cache_audio_from_bytes(data, ext=ext or ".ogg") + local = await cache_audio_from_bytes_async(data, ext=ext or ".ogg") elif mime.startswith("video/"): - local = cache_video_from_bytes(data, ext=ext or ".mp4") + local = await cache_video_from_bytes_async(data, ext=ext or ".mp4") else: - local = cache_document_from_bytes(data, filename) + local = await cache_document_from_bytes_async(data, filename) return local, mime # ------------------------------------------------------------------ @@ -3362,14 +3433,14 @@ def _check_for_registry() -> bool: if not check_google_chat_requirements(): return False project = ( - os.getenv("GOOGLE_CHAT_PROJECT_ID") - or os.getenv("GOOGLE_CLOUD_PROJECT") + _get_scoped_secret("GOOGLE_CHAT_PROJECT_ID") + or _get_scoped_secret("GOOGLE_CLOUD_PROJECT") ) subscription = ( - os.getenv("GOOGLE_CHAT_SUBSCRIPTION_NAME") - or os.getenv("GOOGLE_CHAT_SUBSCRIPTION") + _get_scoped_secret("GOOGLE_CHAT_SUBSCRIPTION_NAME") + or _get_scoped_secret("GOOGLE_CHAT_SUBSCRIPTION") ) - http_events_url = os.getenv("GOOGLE_CHAT_HTTP_EVENTS_URL") + http_events_url = _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_URL") return bool(http_events_url or (project and subscription)) @@ -3393,14 +3464,14 @@ def _env_enablement() -> Optional[Dict[str, Any]]: ``PlatformConfig`` rather than being merged into ``extra``. """ project = ( - os.getenv("GOOGLE_CHAT_PROJECT_ID") - or os.getenv("GOOGLE_CLOUD_PROJECT") + _get_scoped_secret("GOOGLE_CHAT_PROJECT_ID") + or _get_scoped_secret("GOOGLE_CLOUD_PROJECT") ) subscription = ( - os.getenv("GOOGLE_CHAT_SUBSCRIPTION_NAME") - or os.getenv("GOOGLE_CHAT_SUBSCRIPTION") + _get_scoped_secret("GOOGLE_CHAT_SUBSCRIPTION_NAME") + or _get_scoped_secret("GOOGLE_CHAT_SUBSCRIPTION") ) - http_events_url = os.getenv("GOOGLE_CHAT_HTTP_EVENTS_URL") + http_events_url = _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_URL") if not (http_events_url or (project and subscription)): return None seed: Dict[str, Any] = {} @@ -3410,23 +3481,32 @@ def _env_enablement() -> Optional[Dict[str, Any]]: seed["subscription_name"] = subscription if http_events_url: seed["http_events_url"] = http_events_url - http_events_audience = os.getenv("GOOGLE_CHAT_HTTP_EVENTS_AUDIENCE") + http_events_audience = _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_AUDIENCE") if http_events_audience: seed["http_events_audience"] = http_events_audience - http_events_sa_email = os.getenv("GOOGLE_CHAT_HTTP_EVENTS_SERVICE_ACCOUNT_EMAIL") + http_events_sa_email = _get_scoped_secret("GOOGLE_CHAT_HTTP_EVENTS_SERVICE_ACCOUNT_EMAIL") if http_events_sa_email: seed["http_events_service_account_email"] = http_events_sa_email + for env_name, extra_name in ( + ("GOOGLE_CHAT_MAX_MESSAGES", "max_messages"), + ("GOOGLE_CHAT_MAX_BYTES", "max_bytes"), + ("GOOGLE_CHAT_BOOTSTRAP_SPACES", "bootstrap_spaces"), + ("GOOGLE_CHAT_DEBUG_RAW", "debug_raw"), + ): + value = _get_scoped_secret(env_name) + if value: + seed[extra_name] = value sa_json = ( - os.getenv("GOOGLE_CHAT_SERVICE_ACCOUNT_JSON") - or os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + _get_scoped_secret("GOOGLE_CHAT_SERVICE_ACCOUNT_JSON") + or _get_scoped_secret("GOOGLE_APPLICATION_CREDENTIALS") ) if sa_json: seed["service_account_json"] = sa_json - home = os.getenv("GOOGLE_CHAT_HOME_CHANNEL") + home = _get_scoped_secret("GOOGLE_CHAT_HOME_CHANNEL") if home: seed["home_channel"] = { "chat_id": home, - "name": os.getenv("GOOGLE_CHAT_HOME_CHANNEL_NAME", "Home"), + "name": _get_scoped_secret("GOOGLE_CHAT_HOME_CHANNEL_NAME", "Home"), } return seed @@ -3576,8 +3656,8 @@ async def _standalone_send( extra = getattr(pconfig, "extra", {}) or {} sa_value = ( extra.get("service_account_json") - or os.getenv("GOOGLE_CHAT_SERVICE_ACCOUNT_JSON") - or os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + or _get_scoped_secret("GOOGLE_CHAT_SERVICE_ACCOUNT_JSON") + or _get_scoped_secret("GOOGLE_APPLICATION_CREDENTIALS") ) if service_account is None: @@ -3607,6 +3687,12 @@ async def _standalone_send( return {"error": f"Google Chat standalone send: SA JSON file is invalid: {exc}"} creds = service_account.Credentials.from_service_account_info(info, scopes=_CHAT_SCOPES) else: + if _adc_would_borrow_foreign_credentials(): + return {"error": ( + "Google Chat standalone send: ADC skipped for this profile: " + "service-account credentials are set in the process environment " + "but not in this profile's secret scope" + )} try: import google.auth as _google_auth except ImportError: @@ -3655,7 +3741,7 @@ async def _standalone_send( return {"error": "Google Chat standalone send: aiohttp not installed"} try: - async with _aiohttp.ClientSession(timeout=_aiohttp.ClientTimeout(total=30.0), trust_env=True) as session: + async with _aiohttp.ClientSession(timeout=_aiohttp.ClientTimeout(total=30.0), trust_env=gateway_trust_env()) as session: async with session.post( url, json=body, diff --git a/plugins/platforms/homeassistant/adapter.py b/plugins/platforms/homeassistant/adapter.py index 37a7397d4b..bfdd136cdf 100644 --- a/plugins/platforms/homeassistant/adapter.py +++ b/plugins/platforms/homeassistant/adapter.py @@ -30,6 +30,7 @@ except ImportError: from gateway.config import Platform, PlatformConfig from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, @@ -141,7 +142,8 @@ class HomeAssistantAdapter(BasePlatformAdapter): # Dedicated REST session for send() calls self._rest_session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=30) + timeout=aiohttp.ClientTimeout(total=30), + trust_env=gateway_trust_env(), ) # Warn if no event filters are configured @@ -171,7 +173,8 @@ class HomeAssistantAdapter(BasePlatformAdapter): ws_url = f"{ws_url}/api/websocket" self._session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=30) + timeout=aiohttp.ClientTimeout(total=30), + trust_env=gateway_trust_env(), ) self._ws = await self._session.ws_connect(ws_url, heartbeat=30, timeout=30) @@ -447,7 +450,7 @@ class HomeAssistantAdapter(BasePlatformAdapter): body = await resp.text() return SendResult(success=False, error=f"HTTP {resp.status}: {body}") else: - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(trust_env=gateway_trust_env()) as session: async with session.post( url, headers=headers, @@ -532,7 +535,8 @@ async def _standalone_send( try: async with aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=30) + timeout=aiohttp.ClientTimeout(total=30), + trust_env=gateway_trust_env(), ) as session: async with session.post(url, headers=headers, json=payload) as resp: if resp.status not in {200, 201}: diff --git a/plugins/platforms/irc/adapter.py b/plugins/platforms/irc/adapter.py index ce3ec4ed59..8afce0ff19 100644 --- a/plugins/platforms/irc/adapter.py +++ b/plugins/platforms/irc/adapter.py @@ -130,16 +130,17 @@ class IRCAdapter(BasePlatformAdapter): extra = getattr(config, "extra", {}) or {} # Connection settings (env vars override config.yaml) - self.server = os.getenv("IRC_SERVER") or extra.get("server", "") + self.server = _get_scoped_secret("IRC_SERVER") or extra.get("server", "") try: - self.port = int(os.getenv("IRC_PORT") or extra.get("port", 6697)) + self.port = int(_get_scoped_secret("IRC_PORT") or extra.get("port", 6697)) except (ValueError, TypeError): self.port = 6697 - self.nickname = os.getenv("IRC_NICKNAME") or extra.get("nickname", "hermes-bot") - self.channel = os.getenv("IRC_CHANNEL") or extra.get("channel", "") + self.nickname = _get_scoped_secret("IRC_NICKNAME") or extra.get("nickname", "hermes-bot") + self.channel = _get_scoped_secret("IRC_CHANNEL") or extra.get("channel", "") + _use_tls_raw = _get_scoped_secret("IRC_USE_TLS") self.use_tls = ( - os.getenv("IRC_USE_TLS", "").lower() in {"1", "true", "yes"} - if os.getenv("IRC_USE_TLS") + _use_tls_raw.lower() in {"1", "true", "yes"} + if _use_tls_raw else extra.get("use_tls", True) ) self.server_password = _get_scoped_secret("IRC_SERVER_PASSWORD") or extra.get("server_password", "") @@ -545,8 +546,8 @@ def check_requirements() -> bool: Only requires the server and channel — no external pip packages needed. """ - server = os.getenv("IRC_SERVER", "") - channel = os.getenv("IRC_CHANNEL", "") + server = _get_scoped_secret("IRC_SERVER", "") + channel = _get_scoped_secret("IRC_CHANNEL", "") # Also accept config.yaml-only configuration (no env vars). # The gateway passes PlatformConfig; we just check env for the # hermes setup / requirements check path. @@ -556,8 +557,8 @@ def check_requirements() -> bool: def validate_config(config) -> bool: """Validate that the platform config has enough info to connect.""" extra = getattr(config, "extra", {}) or {} - server = os.getenv("IRC_SERVER") or extra.get("server", "") - channel = os.getenv("IRC_CHANNEL") or extra.get("channel", "") + server = _get_scoped_secret("IRC_SERVER") or extra.get("server", "") + channel = _get_scoped_secret("IRC_CHANNEL") or extra.get("channel", "") return bool(server and channel) @@ -671,8 +672,8 @@ def interactive_setup() -> None: def is_connected(config) -> bool: """Check whether IRC is configured (env or config.yaml).""" extra = getattr(config, "extra", {}) or {} - server = os.getenv("IRC_SERVER") or extra.get("server", "") - channel = os.getenv("IRC_CHANNEL") or extra.get("channel", "") + server = _get_scoped_secret("IRC_SERVER") or extra.get("server", "") + channel = _get_scoped_secret("IRC_CHANNEL") or extra.get("channel", "") return bool(server and channel) @@ -689,24 +690,24 @@ def _env_enablement() -> dict | None: the core hook — it becomes a proper ``HomeChannel`` dataclass on the ``PlatformConfig`` rather than being merged into ``extra``. """ - server = os.getenv("IRC_SERVER", "").strip() - channel = os.getenv("IRC_CHANNEL", "").strip() + server = _get_scoped_secret("IRC_SERVER", "").strip() + channel = _get_scoped_secret("IRC_CHANNEL", "").strip() if not (server and channel): return None seed: dict = { "server": server, "channel": channel, } - port = os.getenv("IRC_PORT", "").strip() + port = _get_scoped_secret("IRC_PORT", "").strip() if port: try: seed["port"] = int(port) except ValueError: pass - nickname = os.getenv("IRC_NICKNAME", "").strip() + nickname = _get_scoped_secret("IRC_NICKNAME", "").strip() if nickname: seed["nickname"] = nickname - use_tls = os.getenv("IRC_USE_TLS", "").strip().lower() + use_tls = _get_scoped_secret("IRC_USE_TLS", "").strip().lower() if use_tls: seed["use_tls"] = use_tls in {"1", "true", "yes"} # Passwords live in PlatformConfig.extra as well for back-compat with @@ -718,11 +719,11 @@ def _env_enablement() -> dict | None: # Optional home-channel (usually the same as IRC_CHANNEL, but can be a # dedicated reports channel). Defaults to IRC_CHANNEL so cron jobs # with ``deliver=irc`` have a sensible target without extra config. - home = os.getenv("IRC_HOME_CHANNEL") or channel + home = _get_scoped_secret("IRC_HOME_CHANNEL") or channel if home: seed["home_channel"] = { "chat_id": home, - "name": os.getenv("IRC_HOME_CHANNEL_NAME", home), + "name": _get_scoped_secret("IRC_HOME_CHANNEL_NAME", home), } return seed @@ -770,19 +771,19 @@ async def _standalone_send( primitive. """ extra = getattr(pconfig, "extra", {}) or {} - server = os.getenv("IRC_SERVER") or extra.get("server", "") - channel = os.getenv("IRC_CHANNEL") or extra.get("channel", "") + server = _get_scoped_secret("IRC_SERVER") or extra.get("server", "") + channel = _get_scoped_secret("IRC_CHANNEL") or extra.get("channel", "") if not server or not channel: return {"error": "IRC standalone send: IRC_SERVER and IRC_CHANNEL must be configured"} - port_value = os.getenv("IRC_PORT") or extra.get("port", 6697) + port_value = _get_scoped_secret("IRC_PORT") or extra.get("port", 6697) try: port = int(port_value) except (TypeError, ValueError): return {"error": f"IRC standalone send: invalid port {port_value!r}"} - nickname = os.getenv("IRC_NICKNAME") or extra.get("nickname", "hermes-bot") - use_tls_env = os.getenv("IRC_USE_TLS") + nickname = _get_scoped_secret("IRC_NICKNAME") or extra.get("nickname", "hermes-bot") + use_tls_env = _get_scoped_secret("IRC_USE_TLS") if use_tls_env is not None: use_tls = use_tls_env.lower() in {"1", "true", "yes"} else: diff --git a/plugins/platforms/line/adapter.py b/plugins/platforms/line/adapter.py index b8d3ae10cd..e253e87840 100644 --- a/plugins/platforms/line/adapter.py +++ b/plugins/platforms/line/adapter.py @@ -113,14 +113,15 @@ logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult, - cache_audio_from_bytes, - cache_document_from_bytes, - cache_image_from_bytes, - cache_video_from_bytes, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, + cache_image_from_bytes_async, + cache_video_from_bytes_async, ) from gateway.config import Platform @@ -514,7 +515,7 @@ class _LineClient: async def reply(self, reply_token: str, messages: List[Dict[str, Any]]) -> None: import aiohttp timeout = aiohttp.ClientTimeout(total=self._timeout) - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=gateway_trust_env()) as session: async with session.post( LINE_REPLY_URL, headers=self._headers, @@ -527,7 +528,7 @@ class _LineClient: async def push(self, chat_id: str, messages: List[Dict[str, Any]]) -> None: import aiohttp timeout = aiohttp.ClientTimeout(total=self._timeout) - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=gateway_trust_env()) as session: async with session.post( LINE_PUSH_URL, headers=self._headers, @@ -546,7 +547,7 @@ class _LineClient: clamped = max(5, min(60, (seconds // 5) * 5 or 5)) try: timeout = aiohttp.ClientTimeout(total=5.0) - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=gateway_trust_env()) as session: await session.post( LINE_LOADING_URL, headers=self._headers, @@ -560,7 +561,7 @@ class _LineClient: import aiohttp url = LINE_CONTENT_URL_FMT.format(message_id=message_id) timeout = aiohttp.ClientTimeout(total=30.0) - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=gateway_trust_env()) as session: async with session.get(url, headers={"Authorization": f"Bearer {self._token}"}) as resp: if resp.status >= 400: raise RuntimeError(f"LINE content {resp.status}") @@ -571,7 +572,7 @@ class _LineClient: import aiohttp timeout = aiohttp.ClientTimeout(total=10.0) try: - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=gateway_trust_env()) as session: async with session.get(LINE_BOT_INFO_URL, headers=self._headers) as resp: if resp.status >= 400: return None @@ -1153,16 +1154,16 @@ class LineAdapter(BasePlatformAdapter): }.get(msg_type, ".bin") try: if msg_type == "image": - return cache_image_from_bytes(data, ext=ext), "image/jpeg" + return await cache_image_from_bytes_async(data, ext=ext), "image/jpeg" if msg_type == "audio": media_type = mimetypes.guess_type(f"audio{ext}")[0] or "audio/mp4" - return cache_audio_from_bytes(data, ext=ext), media_type + return await cache_audio_from_bytes_async(data, ext=ext), media_type if msg_type == "video": media_type = mimetypes.guess_type(f"video{ext}")[0] or "video/mp4" - return cache_video_from_bytes(data, ext=ext), media_type + return await cache_video_from_bytes_async(data, ext=ext), media_type document_name = filename or f"line_file{ext}" return ( - cache_document_from_bytes(data, document_name), + await cache_document_from_bytes_async(data, document_name), mimetypes.guess_type(document_name)[0] or "application/octet-stream", ) except Exception as exc: diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py index 92076b8223..a0faa97dbf 100644 --- a/plugins/platforms/matrix/adapter.py +++ b/plugins/platforms/matrix/adapter.py @@ -128,6 +128,7 @@ except ImportError: from gateway.config import Platform, PlatformConfig from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, @@ -593,13 +594,14 @@ def _resolve_max_message_length(config) -> int: # Back-compat alias for callers/tests that import the module constant. MAX_MESSAGE_LENGTH = DEFAULT_MAX_MESSAGE_LENGTH -# Store directory for E2EE keys and sync state. -# Uses get_hermes_home() so each profile gets its own Matrix store. +# Store directory for E2EE keys and sync state. Resolved per adapter in +# ``connect()`` (see ``_resolve_store_dir``), NOT at module scope: the +# multiplex gateway imports this module once, so a module-level constant +# would pin the root HERMES_HOME for every profile and all bots' Olm +# identities would collide in one crypto.db (#89168). Mirrors the +# pairing-store fix (a6397c379). from hermes_constants import get_hermes_dir as _get_hermes_dir -_STORE_DIR = _get_hermes_dir("platforms/matrix/store", "matrix/store") -_CRYPTO_DB_PATH = _STORE_DIR / "crypto.db" - # Grace period: ignore messages older than this many seconds before startup. _STARTUP_GRACE_SECONDS = 5 @@ -763,7 +765,7 @@ def _create_matrix_session(proxy_url: str | None): import aiohttp if not proxy_url: - return aiohttp.ClientSession(trust_env=True) + return aiohttp.ClientSession(trust_env=gateway_trust_env()) if proxy_url.split("://")[0].lower().startswith("socks"): try: @@ -778,7 +780,7 @@ def _create_matrix_session(proxy_url: str | None): "Run: pip install aiohttp-socks", proxy_url, ) - return aiohttp.ClientSession(trust_env=True) + return aiohttp.ClientSession(trust_env=gateway_trust_env()) return aiohttp.ClientSession(proxy=proxy_url) @@ -1018,7 +1020,7 @@ def check_matrix_requirements() -> bool: """ token = _startup_env_secret("MATRIX_ACCESS_TOKEN") password = _startup_env_secret("MATRIX_PASSWORD") - homeserver = os.getenv("MATRIX_HOMESERVER", "") + homeserver = _startup_env_secret("MATRIX_HOMESERVER") if not token and not password: logger.debug("Matrix: neither MATRIX_ACCESS_TOKEN nor MATRIX_PASSWORD set") @@ -1185,6 +1187,23 @@ class MatrixAdapter(BasePlatformAdapter): max_message_length = DEFAULT_MAX_MESSAGE_LENGTH _split_threshold = DEFAULT_MAX_MESSAGE_LENGTH - 100 + def _resolve_store_dir(self) -> Path: + """Pin this adapter's crypto-store directory to the active profile. + + Called from ``connect()``, which the multiplex gateway runs inside + ``_profile_runtime_scope`` -- ``get_hermes_dir`` honors that + context-local HERMES_HOME, so each profile's adapter gets its own + store. Cached on the instance so later reads (diagnostics, error + logs) outside the scope still report the store actually in use. + """ + self._store_dir = _get_hermes_dir("platforms/matrix/store", "matrix/store") + return self._store_dir + + @property + def _crypto_db_path(self) -> Path: + store_dir = self._store_dir or _get_hermes_dir("platforms/matrix/store", "matrix/store") + return store_dir / "crypto.db" + def __init__(self, config: PlatformConfig): super().__init__(config, Platform.MATRIX) @@ -1215,6 +1234,7 @@ class MatrixAdapter(BasePlatformAdapter): self._client: Any = None # mautrix.client.Client self._crypto_db: Any = None # mautrix.util.async_db.Database + self._store_dir: Optional[Path] = None # pinned per profile in connect() self._sync_task: Optional[asyncio.Task] = None self._invite_join_tasks: Dict[str, asyncio.Task] = {} self._closing = False @@ -1670,7 +1690,7 @@ class MatrixAdapter(BasePlatformAdapter): "Matrix: server has different identity keys for device %s — " "local crypto state is stale. Delete %s and restart.", client.device_id, - _CRYPTO_DB_PATH, + str(self._crypto_db_path), ) return False @@ -1726,8 +1746,9 @@ class MatrixAdapter(BasePlatformAdapter): logger.error("Matrix: homeserver URL not configured") return False - # Ensure store dir exists for E2EE key persistence. - _STORE_DIR.mkdir(parents=True, exist_ok=True) + # Ensure store dir exists for E2EE key persistence (resolved here, + # inside the profile scope, so multiplexed profiles never share it). + self._resolve_store_dir().mkdir(parents=True, exist_ok=True) # Create the HTTP API layer. client_session = _create_matrix_session(self._proxy_url) @@ -1884,7 +1905,7 @@ class MatrixAdapter(BasePlatformAdapter): from mautrix.crypto.store.asyncpg import PgCryptoStore from mautrix.util.async_db import Database - _STORE_DIR.mkdir(parents=True, exist_ok=True) + self._store_dir.mkdir(parents=True, exist_ok=True) except Exception as exc: if self._e2ee_mode == "optional": logger.warning( @@ -1905,7 +1926,7 @@ class MatrixAdapter(BasePlatformAdapter): if self._encryption: try: # Remove legacy pickle file from pre-SQLite era. - legacy_pickle = _STORE_DIR / "crypto_store.pickle" + legacy_pickle = self._store_dir / "crypto_store.pickle" if legacy_pickle.exists(): logger.info( "Matrix: removing legacy crypto_store.pickle (migrated to SQLite)" @@ -1913,7 +1934,7 @@ class MatrixAdapter(BasePlatformAdapter): legacy_pickle.unlink() crypto_db = Database.create( - f"sqlite:///{_CRYPTO_DB_PATH}", + f"sqlite:///{self._crypto_db_path}", upgrade_table=PgCryptoStore.upgrade_table, ) await crypto_db.start() @@ -2041,7 +2062,7 @@ class MatrixAdapter(BasePlatformAdapter): client.crypto = olm logger.info( "Matrix: E2EE enabled (store: %s%s)", - str(_CRYPTO_DB_PATH), + str(self._crypto_db_path), f", device_id={client.device_id}" if client.device_id else "", ) except Exception as exc: @@ -2285,7 +2306,7 @@ class MatrixAdapter(BasePlatformAdapter): "mode": self._e2ee_mode, "enabled": bool(self._encryption), "deps_available": _check_e2ee_deps(), - "crypto_store_path": str(_CRYPTO_DB_PATH), + "crypto_store_path": str(self._crypto_db_path), "recovery_key_configured": bool( _scoped_recovery_key().strip() ), @@ -3679,9 +3700,9 @@ class MatrixAdapter(BasePlatformAdapter): if file_bytes is not None: from gateway.platforms.base import ( - cache_audio_from_bytes, - cache_document_from_bytes, - cache_image_from_bytes, + cache_audio_from_bytes_async, + cache_document_from_bytes_async, + cache_image_from_bytes_async, ) if msg_type == MessageType.PHOTO: @@ -3692,7 +3713,7 @@ class MatrixAdapter(BasePlatformAdapter): "image/webp": ".webp", } ext = ext_map.get(media_type, ".jpg") - cached_path = cache_image_from_bytes(file_bytes, ext=ext) + cached_path = await cache_image_from_bytes_async(file_bytes, ext=ext) logger.info("[Matrix] Cached user image at %s", cached_path) elif msg_type in {MessageType.AUDIO, MessageType.VOICE}: ext = ( @@ -3704,14 +3725,14 @@ class MatrixAdapter(BasePlatformAdapter): ).suffix or ".ogg" ) - cached_path = cache_audio_from_bytes(file_bytes, ext=ext) + cached_path = await cache_audio_from_bytes_async(file_bytes, ext=ext) else: filename = body or ( "video.mp4" if msg_type == MessageType.VIDEO else "document" ) - cached_path = cache_document_from_bytes( + cached_path = await cache_document_from_bytes_async( file_bytes, filename ) except Exception as e: diff --git a/plugins/platforms/mattermost/adapter.py b/plugins/platforms/mattermost/adapter.py index 6962fbf615..45797f0268 100644 --- a/plugins/platforms/mattermost/adapter.py +++ b/plugins/platforms/mattermost/adapter.py @@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional, Tuple from gateway.config import Platform, PlatformConfig from gateway.platforms.helpers import MessageDeduplicator from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, @@ -100,7 +101,7 @@ def validate_mattermost_config(config: PlatformConfig) -> bool: """Return True when Mattermost has enough config to connect.""" extra = getattr(config, "extra", {}) or {} token = (getattr(config, "token", None) or _get_scoped_secret("MATTERMOST_TOKEN", "")).strip() - url = (extra.get("url", "") or os.getenv("MATTERMOST_URL", "")).strip() + url = (extra.get("url", "") or _get_scoped_secret("MATTERMOST_URL", "")).strip() if not token: logger.debug("Mattermost: MATTERMOST_TOKEN not set") return False @@ -120,7 +121,7 @@ class MattermostAdapter(BasePlatformAdapter): self._base_url: str = ( config.extra.get("url", "") - or os.getenv("MATTERMOST_URL", "") + or _get_scoped_secret("MATTERMOST_URL", "") ).rstrip("/") self._token: str = config.token or _get_scoped_secret("MATTERMOST_TOKEN", "") @@ -137,7 +138,7 @@ class MattermostAdapter(BasePlatformAdapter): # Reply mode: "thread" to nest replies, "off" for flat messages. self._reply_mode: str = ( config.extra.get("reply_mode", "") - or os.getenv("MATTERMOST_REPLY_MODE", "off") + or _get_scoped_secret("MATTERMOST_REPLY_MODE", "off") ).lower() self._last_post_status: Optional[int] = None @@ -316,7 +317,8 @@ class MattermostAdapter(BasePlatformAdapter): return False self._session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=30) + timeout=aiohttp.ClientTimeout(total=30), + trust_env=gateway_trust_env(), ) self._closing = False @@ -870,7 +872,7 @@ class MattermostAdapter(BasePlatformAdapter): # ignored, even if @mentioned. DMs are already excluded above. allowed_raw = self.config.extra.get("allowed_channels") if self.config.extra else None if allowed_raw is None: - allowed_raw = os.getenv("MATTERMOST_ALLOWED_CHANNELS", "") + allowed_raw = _get_scoped_secret("MATTERMOST_ALLOWED_CHANNELS", "") if isinstance(allowed_raw, list): allowed_channels = {str(c).strip() for c in allowed_raw if str(c).strip()} else: @@ -884,12 +886,18 @@ class MattermostAdapter(BasePlatformAdapter): ) return - require_mention = os.getenv( - "MATTERMOST_REQUIRE_MENTION", "true" - ).lower() not in {"false", "0", "no"} + require_mention_raw = self.config.extra.get("require_mention") if self.config.extra else None + if require_mention_raw is None: + require_mention_raw = _get_scoped_secret("MATTERMOST_REQUIRE_MENTION", "true") + require_mention = str(require_mention_raw).lower() not in {"false", "0", "no"} - free_channels_raw = os.getenv("MATTERMOST_FREE_RESPONSE_CHANNELS", "") - free_channels = {ch.strip() for ch in free_channels_raw.split(",") if ch.strip()} + free_channels_raw = self.config.extra.get("free_response_channels") if self.config.extra else None + if free_channels_raw is None: + free_channels_raw = _get_scoped_secret("MATTERMOST_FREE_RESPONSE_CHANNELS", "") + if isinstance(free_channels_raw, list): + free_channels = {str(ch).strip() for ch in free_channels_raw if str(ch).strip()} + else: + free_channels = {ch.strip() for ch in str(free_channels_raw).split(",") if ch.strip()} is_free_channel = channel_id in free_channels mention_patterns = [ @@ -958,18 +966,18 @@ class MattermostAdapter(BasePlatformAdapter): ) as resp: if resp.status < 400: file_data = await resp.read() - from gateway.platforms.base import cache_image_from_bytes, cache_document_from_bytes + from gateway.platforms.base import cache_image_from_bytes_async, cache_document_from_bytes_async if mime.startswith("image/"): - local_path = cache_image_from_bytes(file_data, ext or ".png") + local_path = await cache_image_from_bytes_async(file_data, ext or ".png") media_urls.append(local_path) media_types.append(mime) elif mime.startswith("audio/"): - from gateway.platforms.base import cache_audio_from_bytes - local_path = cache_audio_from_bytes(file_data, ext or ".ogg") + from gateway.platforms.base import cache_audio_from_bytes_async + local_path = await cache_audio_from_bytes_async(file_data, ext or ".ogg") media_urls.append(local_path) media_types.append(mime) else: - local_path = cache_document_from_bytes(file_data, fname) + local_path = await cache_document_from_bytes_async(file_data, fname) media_urls.append(local_path) media_types.append(mime) else: @@ -1057,7 +1065,7 @@ async def _standalone_send( base_url = ( (getattr(pconfig, "extra", {}) or {}).get("url") - or os.getenv("MATTERMOST_URL", "") + or _get_scoped_secret("MATTERMOST_URL", "") ).rstrip("/") token = (getattr(pconfig, "token", None) or _get_scoped_secret("MATTERMOST_TOKEN", "")).strip() if not base_url or not token: @@ -1232,40 +1240,62 @@ def interactive_setup() -> None: # --------------------------------------------------------------------------- +def _profile_scoped_config_load() -> bool: + """True when running inside a multiplexed secondary profile's scope. + + Secondary-profile adapters are constructed and connected inside + ``_profile_runtime_scope`` (secret scope installed + multiplex active) -- + the same discriminator the Buzz/Discord/Telegram/WhatsApp/LINE/DingTalk + adapters use for this bug class (#98738 / #72348 / #80099). The DEFAULT + profile under multiplexing runs unscoped: ``os.environ`` holds its own + bridge output there and keeps its legacy precedence. + """ + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + return bool(is_multiplex_active() and current_secret_scope() is not None) + except Exception: + return False + + def _apply_yaml_config(yaml_cfg: dict, mattermost_cfg: dict) -> dict | None: - """Translate ``config.yaml`` ``mattermost:`` keys into env vars. + """Translate ``config.yaml`` ``mattermost:`` keys into env vars and + ``PlatformConfig.extra`` entries. Implements the ``apply_yaml_config_fn`` contract (#24836 / #25443). Mirrors the legacy ``mattermost_cfg`` block that used to live in ``gateway/config.py::load_gateway_config()`` before this migration. - The MattermostAdapter reads its runtime configuration via - ``os.getenv()`` for ``MATTERMOST_REQUIRE_MENTION``, - ``MATTERMOST_FREE_RESPONSE_CHANNELS``, and - ``MATTERMOST_ALLOWED_CHANNELS``. Rather than rewrite those call sites - to read from ``PlatformConfig.extra``, this hook keeps the env-driven - model and merely owns the YAML→env translation here, next to the - adapter that consumes it. - - Env vars take precedence over YAML — every assignment is guarded - by ``not os.getenv(...)`` so an explicit env var survives a config.yaml - update. Returns ``None`` because no extras are seeded into - ``PlatformConfig.extra`` directly (everything flows through env). + Env vars take precedence over YAML for single-profile deployments -- + each env write is guarded by ``not os.getenv(...)`` so an explicit env + var survives a config.yaml update. Under a multiplexed secondary + profile's scope, the env write is skipped entirely (it would otherwise + leak into the process-global ``os.environ`` and be inherited by every + other profile); instead the values are returned so the caller merges + them into this profile's own ``PlatformConfig.extra``, which the + require_mention/free_response_channels/allowed_channels read sites now + check first. """ - if "require_mention" in mattermost_cfg and not os.getenv("MATTERMOST_REQUIRE_MENTION"): - os.environ["MATTERMOST_REQUIRE_MENTION"] = str(mattermost_cfg["require_mention"]).lower() + _skip_env_bridge = _profile_scoped_config_load() + seeded: dict = {} + if "require_mention" in mattermost_cfg: + seeded["require_mention"] = mattermost_cfg["require_mention"] + if not _skip_env_bridge and not os.getenv("MATTERMOST_REQUIRE_MENTION"): + os.environ["MATTERMOST_REQUIRE_MENTION"] = str(mattermost_cfg["require_mention"]).lower() frc = mattermost_cfg.get("free_response_channels") - if frc is not None and not os.getenv("MATTERMOST_FREE_RESPONSE_CHANNELS"): - if isinstance(frc, list): - frc = ",".join(str(v) for v in frc) - os.environ["MATTERMOST_FREE_RESPONSE_CHANNELS"] = str(frc) + if frc is not None: + seeded["free_response_channels"] = frc + if not _skip_env_bridge and not os.getenv("MATTERMOST_FREE_RESPONSE_CHANNELS"): + _frc = ",".join(str(v) for v in frc) if isinstance(frc, list) else str(frc) + os.environ["MATTERMOST_FREE_RESPONSE_CHANNELS"] = _frc # allowed_channels: if set, bot ONLY responds in these channels (whitelist) ac = mattermost_cfg.get("allowed_channels") - if ac is not None and not os.getenv("MATTERMOST_ALLOWED_CHANNELS"): - if isinstance(ac, list): - ac = ",".join(str(v) for v in ac) - os.environ["MATTERMOST_ALLOWED_CHANNELS"] = str(ac) - return None # all settings flow through env; nothing to merge into extras + if ac is not None: + seeded["allowed_channels"] = ac + if not _skip_env_bridge and not os.getenv("MATTERMOST_ALLOWED_CHANNELS"): + _ac = ",".join(str(v) for v in ac) if isinstance(ac, list) else str(ac) + os.environ["MATTERMOST_ALLOWED_CHANNELS"] = _ac + return seeded or None # --------------------------------------------------------------------------- diff --git a/plugins/platforms/ntfy/adapter.py b/plugins/platforms/ntfy/adapter.py index b9fb08c7ef..87986416ef 100644 --- a/plugins/platforms/ntfy/adapter.py +++ b/plugins/platforms/ntfy/adapter.py @@ -155,21 +155,21 @@ def check_requirements() -> bool: """ if not HTTPX_AVAILABLE: return False - topic = os.getenv("NTFY_TOPIC", "").strip() + topic = _get_scoped_secret("NTFY_TOPIC", "").strip() return bool(topic) def validate_config(config) -> bool: """Validate that the configured ntfy platform has a topic set.""" extra = getattr(config, "extra", {}) or {} - topic = extra.get("topic") or os.getenv("NTFY_TOPIC", "") + topic = extra.get("topic") or _get_scoped_secret("NTFY_TOPIC", "") return bool(topic) def is_connected(config) -> bool: """Check whether ntfy is configured (env or config.yaml).""" extra = getattr(config, "extra", {}) or {} - topic = os.getenv("NTFY_TOPIC") or extra.get("topic", "") + topic = _get_scoped_secret("NTFY_TOPIC") or extra.get("topic", "") return bool(topic) @@ -189,12 +189,12 @@ class NtfyAdapter(BasePlatformAdapter): extra = config.extra or {} self._server: str = ( extra.get("server") - or os.getenv("NTFY_SERVER_URL", DEFAULT_SERVER) + or _get_scoped_secret("NTFY_SERVER_URL", DEFAULT_SERVER) ).rstrip("/") - self._topic: str = extra.get("topic") or os.getenv("NTFY_TOPIC", "") + self._topic: str = extra.get("topic") or _get_scoped_secret("NTFY_TOPIC", "") self._publish_topic: str = ( extra.get("publish_topic") - or os.getenv("NTFY_PUBLISH_TOPIC", "") + or _get_scoped_secret("NTFY_PUBLISH_TOPIC", "") or self._topic ) self._token: str = extra.get("token") or _get_scoped_secret("NTFY_TOKEN", "") @@ -488,27 +488,27 @@ def _env_enablement() -> dict | None: core hook — it becomes a proper ``HomeChannel`` dataclass on the ``PlatformConfig`` rather than being merged into ``extra``. """ - topic = os.getenv("NTFY_TOPIC", "").strip() + topic = _get_scoped_secret("NTFY_TOPIC", "").strip() if not topic: return None seed: dict = { "topic": topic, - "server": os.getenv("NTFY_SERVER_URL", DEFAULT_SERVER).rstrip("/"), + "server": _get_scoped_secret("NTFY_SERVER_URL", DEFAULT_SERVER).rstrip("/"), } - publish_topic = os.getenv("NTFY_PUBLISH_TOPIC", "").strip() + publish_topic = _get_scoped_secret("NTFY_PUBLISH_TOPIC", "").strip() if publish_topic: seed["publish_topic"] = publish_topic token = _get_scoped_secret("NTFY_TOKEN", "").strip() if token: seed["token"] = token - markdown = os.getenv("NTFY_MARKDOWN", "").strip().lower() + markdown = _get_scoped_secret("NTFY_MARKDOWN", "").strip().lower() if markdown: seed["markdown"] = markdown in ("1", "true", "yes") - home = os.getenv("NTFY_HOME_CHANNEL", "").strip() or topic + home = _get_scoped_secret("NTFY_HOME_CHANNEL", "").strip() or topic if home: seed["home_channel"] = { "chat_id": home, - "name": os.getenv("NTFY_HOME_CHANNEL_NAME", home), + "name": _get_scoped_secret("NTFY_HOME_CHANNEL_NAME", home), } return seed @@ -540,20 +540,20 @@ async def _standalone_send( extra = getattr(pconfig, "extra", {}) or {} server = ( extra.get("server") - or os.getenv("NTFY_SERVER_URL", DEFAULT_SERVER) + or _get_scoped_secret("NTFY_SERVER_URL", DEFAULT_SERVER) ).rstrip("/") publish_topic = ( chat_id or extra.get("publish_topic") - or os.getenv("NTFY_PUBLISH_TOPIC", "").strip() + or _get_scoped_secret("NTFY_PUBLISH_TOPIC", "").strip() or extra.get("topic") - or os.getenv("NTFY_TOPIC", "").strip() + or _get_scoped_secret("NTFY_TOPIC", "").strip() ) if not publish_topic: return {"error": "ntfy standalone send: NTFY_TOPIC not configured"} token = extra.get("token") or _get_scoped_secret("NTFY_TOKEN", "") - markdown_env = os.getenv("NTFY_MARKDOWN", "").strip().lower() + markdown_env = _get_scoped_secret("NTFY_MARKDOWN", "").strip().lower() markdown_enabled = bool(extra.get("markdown")) or markdown_env in ("1", "true", "yes") headers = {"Content-Type": "text/plain; charset=utf-8", "X-Tags": _ECHO_TAG, **_build_auth_header(token)} diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index 341071fb70..c35ebf1401 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -460,8 +460,8 @@ def check_requirements() -> bool: if not HTTPX_AVAILABLE: logger.warning("photon: httpx not installed — pip install httpx") return False - if not _node_command("node"): - logger.warning("photon: node binary not found on PATH or in the pm store") + if not (_get_scoped_secret("PHOTON_NODE_BIN") or _node_command("node")): + logger.warning("photon: node binary not found on PATH, in the pm store, or via PHOTON_NODE_BIN") return False if not sidecar_deps_installed(): # spectrum-ts not installed yet, or node_modules/ was partially created @@ -585,7 +585,7 @@ def _reinstall_sidecar_deps() -> None: def validate_config(cfg: PlatformConfig) -> bool: extra = cfg.extra or {} - project_id = extra.get("project_id") or os.getenv("PHOTON_PROJECT_ID") + project_id = extra.get("project_id") or _get_scoped_secret("PHOTON_PROJECT_ID") project_secret = extra.get("project_secret") or _get_scoped_secret("PHOTON_PROJECT_SECRET") if not project_id or not project_secret: # Fall back to auth.json @@ -608,11 +608,11 @@ def _env_enablement() -> Optional[dict]: if not (project_id and project_secret): return None seed: dict = {"project_id": project_id, "project_secret": project_secret} - home = os.getenv("PHOTON_HOME_CHANNEL", "").strip() + home = _get_scoped_secret("PHOTON_HOME_CHANNEL", "").strip() if home: seed["home_channel"] = { "chat_id": home, - "name": os.getenv("PHOTON_HOME_CHANNEL_NAME", "Home"), + "name": _get_scoped_secret("PHOTON_HOME_CHANNEL_NAME", "Home"), } return seed @@ -625,7 +625,7 @@ def _markdown_enabled() -> bool: ``PHOTON_MARKDOWN=false`` is the kill-switch back to stripped plain text without a release. """ - return os.getenv("PHOTON_MARKDOWN", "true").strip().lower() not in { + return _get_scoped_secret("PHOTON_MARKDOWN", "true").strip().lower() not in { "false", "0", "no", } @@ -763,7 +763,7 @@ class PhotonAdapter(BasePlatformAdapter): # the spectrum-ts SDK authenticates with. stored_id, stored_sec = load_project_credentials() self._project_id: str = ( - os.getenv("PHOTON_PROJECT_ID") + _get_scoped_secret("PHOTON_PROJECT_ID") or extra.get("project_id") or stored_id or "" @@ -777,7 +777,7 @@ class PhotonAdapter(BasePlatformAdapter): # Sidecar self._sidecar_port = _coerce_port( - extra.get("sidecar_port") or os.getenv("PHOTON_SIDECAR_PORT"), + extra.get("sidecar_port") or _get_scoped_secret("PHOTON_SIDECAR_PORT"), _DEFAULT_SIDECAR_PORT, ) self._sidecar_bind = _DEFAULT_SIDECAR_BIND @@ -785,9 +785,9 @@ class PhotonAdapter(BasePlatformAdapter): _get_scoped_secret("PHOTON_SIDECAR_TOKEN") or secrets.token_hex(16) ) self._autostart_sidecar = str( - os.getenv("PHOTON_SIDECAR_AUTOSTART", "true") + _get_scoped_secret("PHOTON_SIDECAR_AUTOSTART", "true") ).lower() not in ("0", "false", "no") - self._node_bin = _node_command("node") or "node" + self._node_bin = _get_scoped_secret("PHOTON_NODE_BIN") or _node_command("node") or "node" # Presence watchdog. spectrum-ts only reconnects when its inbound # iterator throws or ends; a half-open ("zombie") gRPC socket makes the @@ -810,21 +810,21 @@ class PhotonAdapter(BasePlatformAdapter): self._probe_interval = _coerce_float( _first_set( extra.get("probe_interval_seconds"), - os.getenv("PHOTON_PROBE_INTERVAL_SECONDS"), + _get_scoped_secret("PHOTON_PROBE_INTERVAL_SECONDS"), ), 600.0, ) self._probe_timeout = _coerce_float( _first_set( extra.get("probe_timeout_seconds"), - os.getenv("PHOTON_PROBE_TIMEOUT_SECONDS"), + _get_scoped_secret("PHOTON_PROBE_TIMEOUT_SECONDS"), ), 10.0, ) self._probe_max_failures = _coerce_int( _first_set( extra.get("probe_max_failures"), - os.getenv("PHOTON_PROBE_MAX_FAILURES"), + _get_scoped_secret("PHOTON_PROBE_MAX_FAILURES"), ), 3, ) @@ -877,14 +877,14 @@ class PhotonAdapter(BasePlatformAdapter): # always processed. Config key wins, then env var. _require_mention = extra.get("require_mention") if _require_mention is None: - _require_mention = os.getenv("PHOTON_REQUIRE_MENTION") + _require_mention = _get_scoped_secret("PHOTON_REQUIRE_MENTION") self.require_mention = str(_require_mention).strip().lower() in { "true", "1", "yes", "on", } self._mention_patterns = self._compile_mention_patterns( extra["mention_patterns"] if "mention_patterns" in extra - else os.getenv("PHOTON_MENTION_PATTERNS") + else _get_scoped_secret("PHOTON_MENTION_PATTERNS") ) # -- Group-mention gating (parity with BlueBubbles) ------------------- @@ -1288,7 +1288,7 @@ class PhotonAdapter(BasePlatformAdapter): media_urls: List[str] = [] media_types: List[str] = [] - def _normalize_binary_payload( + async def _normalize_binary_payload( payload: Dict[str, Any] ) -> tuple[str, MessageType, List[str], List[str]]: is_voice = payload.get("type") == "voice" @@ -1300,8 +1300,10 @@ class PhotonAdapter(BasePlatformAdapter): if not is_voice and (name.lower().endswith(".caf") or mime == "audio/x-caf"): is_voice = True mtype = MessageType.VOICE if is_voice else _attachment_message_type(mime) - cached = _cache_inbound_attachment( - payload, name, mime, force_audio=is_voice + # Base64 decode + media-cache write (fsync-free but still disk + # I/O on possibly multi-MB payloads) — keep it off the event loop. + cached = await asyncio.to_thread( + _cache_inbound_attachment, payload, name, mime, force_audio=is_voice ) if cached: return ( @@ -1449,7 +1451,7 @@ class PhotonAdapter(BasePlatformAdapter): text = content.get("text") or "" mtype = MessageType.TEXT elif ctype in {"attachment", "voice"}: - text, mtype, media_urls, media_types = _normalize_binary_payload(content) + text, mtype, media_urls, media_types = await _normalize_binary_payload(content) elif ctype == "richlink": text = _format_richlink_content(content) mtype = MessageType.TEXT @@ -1472,7 +1474,7 @@ class PhotonAdapter(BasePlatformAdapter): text_parts.append(_format_richlink_content(item_content)) continue if item_type in {"attachment", "voice"}: - marker, item_mtype, item_urls, item_types = _normalize_binary_payload( + marker, item_mtype, item_urls, item_types = await _normalize_binary_payload( item_content ) if mtype == MessageType.TEXT: @@ -2308,7 +2310,7 @@ class PhotonAdapter(BasePlatformAdapter): return True def _reactions_enabled(self) -> bool: - return os.getenv("PHOTON_REACTIONS", "false").strip().lower() in { + return _get_scoped_secret("PHOTON_REACTIONS", "false").strip().lower() in { "true", "1", "yes", "on", } @@ -2839,7 +2841,7 @@ async def _standalone_send( if not HTTPX_AVAILABLE: return {"error": "httpx not installed"} port = _coerce_port( - (pconfig.extra or {}).get("sidecar_port") or os.getenv("PHOTON_SIDECAR_PORT"), + (pconfig.extra or {}).get("sidecar_port") or _get_scoped_secret("PHOTON_SIDECAR_PORT"), _DEFAULT_SIDECAR_PORT, ) token = _get_scoped_secret("PHOTON_SIDECAR_TOKEN") diff --git a/plugins/platforms/photon/auth.py b/plugins/platforms/photon/auth.py index 3b18cbd7d8..b9f389f59b 100644 --- a/plugins/platforms/photon/auth.py +++ b/plugins/platforms/photon/auth.py @@ -254,7 +254,7 @@ def load_project_credentials() -> Tuple[Optional[str], Optional[str]]: use. This is the pair the Node sidecar feeds to ``spectrum-ts``; the id is the unified project id (dashboard id == spectrumProjectId). """ - env_id = os.getenv("PHOTON_PROJECT_ID") + env_id = _get_scoped_secret("PHOTON_PROJECT_ID") env_sec = _get_scoped_secret("PHOTON_PROJECT_SECRET") if env_id and env_sec: return env_id, env_sec @@ -277,7 +277,7 @@ def load_dashboard_project_id() -> Optional[str]: rewrote (it now 404s), while the Spectrum id always matches the live row. Falls back to the legacy keys for older records. """ - env_id = os.getenv("PHOTON_DASHBOARD_PROJECT_ID") + env_id = _get_scoped_secret("PHOTON_DASHBOARD_PROJECT_ID") if env_id: return env_id auth = _load_auth() diff --git a/plugins/platforms/raft/adapter.py b/plugins/platforms/raft/adapter.py index d31ee4601a..49f9224675 100644 --- a/plugins/platforms/raft/adapter.py +++ b/plugins/platforms/raft/adapter.py @@ -97,6 +97,52 @@ _RAFT_TURN_IDS: set[str] = set() _RAFT_PROMPT_TURN_IDS: set[str] = set() +def _profile_scoped() -> bool: + """True when running inside a multiplexed secondary profile's scope. + + Secondary-profile adapters are constructed, connected, and reloaded + inside ``_profile_runtime_scope`` (secret scope installed + multiplex + active) — the same discriminator the Buzz/SimpleX adapters use for this + bug class (#98738). The DEFAULT profile under multiplexing runs + unscoped: ``os.environ`` holds its own bridge output there and keeps its + legacy precedence. + """ + try: + from agent.secret_scope import current_secret_scope, is_multiplex_active + + return bool(is_multiplex_active() and current_secret_scope() is not None) + except Exception: + return False + + +def _resolve_raft_profile() -> str: + """Scope-aware resolution of the ``RAFT_PROFILE`` slug. + + Raft has no ``config.yaml`` equivalent for this value (env-only), so a + secondary multiplex profile's only way to configure Raft is via its own + ``.env`` file — which the installed secret scope (built from that + profile's ``.env`` by ``_profile_runtime_scope``) already carries. + Reading raw ``os.environ.get("RAFT_PROFILE")`` here would instead return + the DEFAULT profile's bridged value, misdirecting the bridge subprocess + or CLI hint at another profile's external Raft workspace/agent identity. + + ``get_secret()`` is only called when ``_profile_scoped()`` is True — the + callers of this helper (``connect()``/``register()``) run inside + ``_profile_runtime_scope`` for secondary profiles, but the DEFAULT + profile's own startup path never installs a scope, where ``get_secret()`` + would raise ``UnscopedSecretError``; the guard keeps that path on the + unchanged ``os.environ`` read. + """ + if _profile_scoped(): + try: + from agent.secret_scope import get_secret + + return (get_secret("RAFT_PROFILE") or "").strip() + except Exception: + return "" + return os.environ.get("RAFT_PROFILE", "").strip() + + def check_raft_requirements() -> bool: """Check if Raft channel dependencies are available. @@ -533,7 +579,7 @@ class RaftAdapter(BasePlatformAdapter): logger.warning("[raft] raft CLI not found in PATH; bridge not spawned — wake-only polling mode") return - profile = os.environ.get("RAFT_PROFILE", "") + profile = _resolve_raft_profile() if not profile: logger.warning("[raft] RAFT_PROFILE not set; bridge not spawned") return @@ -777,8 +823,12 @@ def _env_enablement() -> Optional[dict]: """Seed PlatformConfig.extra from env vars during gateway config load. Auto-enables when RAFT_PROFILE is set (the adapter needs it anyway). + Scope-aware: consults the active profile's own RAFT_PROFILE (env, or a + secondary profile's own .env via the secret scope) instead of the + default profile's bridged env value (mirrors the Buzz/SimpleX fix for + #98738) — see ``_resolve_raft_profile``. """ - if not os.getenv("RAFT_PROFILE"): + if not _resolve_raft_profile(): return None return {"enabled": True} @@ -839,12 +889,18 @@ def register(ctx) -> None: setup_fn=interactive_setup, env_enablement_fn=_env_enablement, emoji="🔔", + # Scope-aware (mirrors _resolve_raft_profile's docstring): register() + # runs inside _profile_runtime_scope for a secondary multiplex + # profile (via discover_plugins() in + # gateway/run.py::_start_one_profile_adapters), so this resolves + # that profile's own RAFT_PROFILE instead of the default profile's + # bridged env value baked into a shared registry entry. platform_hint=( "You are connected to Raft via an external-agent channel. " "Run `raft --profile {profile} profile show` to confirm which agent profile is active. " "Run `raft --profile {profile} manual get raft-cli-overview` to learn available Raft commands. " "Always pass `--profile {profile}` to every raft CLI call." - ).format(profile=os.environ.get("RAFT_PROFILE", "your-agent-profile")), + ).format(profile=_resolve_raft_profile() or "your-agent-profile"), ) ctx.register_hook("on_session_start", _on_session_start) ctx.register_hook("pre_llm_call", _on_pre_llm_call) diff --git a/plugins/platforms/simplex/adapter.py b/plugins/platforms/simplex/adapter.py index b4f493e456..979c1e6ea3 100644 --- a/plugins/platforms/simplex/adapter.py +++ b/plugins/platforms/simplex/adapter.py @@ -56,6 +56,28 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware env read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which holds the DEFAULT + profile's YAML-to-env bridge output under multiplexing). The default + profile's adapter constructs *unscoped*, where a bare ``get_secret`` + would raise ``UnscopedSecretError``; there ``os.environ`` is that + profile's own value, so fall back to it. Same helper as the IRC/ntfy/ + Mattermost plugins. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + # Lazy import: BasePlatformAdapter and friends live in the main repo. # Imported at module top because they're stdlib-only inside Hermes — no # external dependency that would block the plugin from loading. @@ -153,7 +175,7 @@ class SimplexAdapter(BasePlatformAdapter): # Contact-request auto-accept (on by default — matches the way most # bot deployments expect to behave). Read from env first, then fall # back to the value seeded by ``_env_enablement``. - env_auto = os.getenv("SIMPLEX_AUTO_ACCEPT") + env_auto = _get_scoped_secret("SIMPLEX_AUTO_ACCEPT") if env_auto is not None: self.auto_accept = env_auto.strip().lower() not in {"0", "false", "no", ""} else: @@ -162,7 +184,7 @@ class SimplexAdapter(BasePlatformAdapter): # Group allowlist. Without ``SIMPLEX_GROUP_ALLOWED``, group messages # are ignored entirely (safer default — a bot in a group otherwise # processes every member's traffic). Use ``*`` to accept any group. - group_allowed_str = os.getenv("SIMPLEX_GROUP_ALLOWED", "") or extra.get( + group_allowed_str = _get_scoped_secret("SIMPLEX_GROUP_ALLOWED", "") or extra.get( "group_allowed", "" ) self.group_allow_from = set(_parse_comma_list(group_allowed_str)) @@ -1172,7 +1194,7 @@ def check_requirements() -> bool: so the gateway never instantiates the adapter when the dependency is missing or no daemon URL is configured. """ - if not os.getenv("SIMPLEX_WS_URL"): + if not _get_scoped_secret("SIMPLEX_WS_URL"): return False try: import websockets # noqa: F401 @@ -1184,14 +1206,14 @@ def check_requirements() -> bool: def validate_config(config) -> bool: """Validate that the platform config has enough info to connect.""" extra = getattr(config, "extra", {}) or {} - ws_url = os.getenv("SIMPLEX_WS_URL") or extra.get("ws_url", "") + ws_url = _get_scoped_secret("SIMPLEX_WS_URL") or extra.get("ws_url", "") return bool(ws_url) def is_connected(config) -> bool: """Check whether SimpleX is configured (env or config.yaml).""" extra = getattr(config, "extra", {}) or {} - ws_url = os.getenv("SIMPLEX_WS_URL") or extra.get("ws_url", "") + ws_url = _get_scoped_secret("SIMPLEX_WS_URL") or extra.get("ws_url", "") return bool(ws_url) @@ -1207,24 +1229,24 @@ def _env_enablement() -> Optional[dict]: becomes a proper ``HomeChannel`` dataclass on the ``PlatformConfig`` rather than being merged into ``extra``. """ - ws_url = os.getenv("SIMPLEX_WS_URL", "").strip() + ws_url = _get_scoped_secret("SIMPLEX_WS_URL", "").strip() if not ws_url: return None seed: dict = {"ws_url": ws_url} - auto_accept = os.getenv("SIMPLEX_AUTO_ACCEPT", "").strip().lower() + auto_accept = _get_scoped_secret("SIMPLEX_AUTO_ACCEPT", "").strip().lower() if auto_accept: seed["auto_accept"] = auto_accept not in {"0", "false", "no"} - group_allowed = os.getenv("SIMPLEX_GROUP_ALLOWED", "").strip() + group_allowed = _get_scoped_secret("SIMPLEX_GROUP_ALLOWED", "").strip() if group_allowed: seed["group_allowed"] = group_allowed - home = os.getenv("SIMPLEX_HOME_CHANNEL", "").strip() + home = _get_scoped_secret("SIMPLEX_HOME_CHANNEL", "").strip() if home: seed["home_channel"] = { "chat_id": home, - "name": os.getenv("SIMPLEX_HOME_CHANNEL_NAME", "").strip() or home, + "name": _get_scoped_secret("SIMPLEX_HOME_CHANNEL_NAME", "").strip() or home, } return seed @@ -1257,7 +1279,7 @@ async def _standalone_send( return {"error": "websockets not installed. Run: pip install websockets"} extra = getattr(pconfig, "extra", {}) or {} - ws_url = os.getenv("SIMPLEX_WS_URL") or extra.get( + ws_url = _get_scoped_secret("SIMPLEX_WS_URL") or extra.get( "ws_url", "ws://127.0.0.1:5225" ) if not ws_url: diff --git a/plugins/platforms/slack/adapter.py b/plugins/platforms/slack/adapter.py index e672affb04..e1d9ac7380 100644 --- a/plugins/platforms/slack/adapter.py +++ b/plugins/platforms/slack/adapter.py @@ -43,6 +43,7 @@ from agent.secret_scope import UnscopedSecretError, get_secret from gateway.config import Platform, PlatformConfig from gateway.platforms.helpers import MessageDeduplicator from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, @@ -55,8 +56,8 @@ from gateway.platforms.base import ( resolve_proxy_url, safe_url_for_log, _ssrf_redirect_guard, - cache_document_from_bytes, - cache_video_from_bytes, + cache_document_from_bytes_async, + cache_video_from_bytes_async, ) try: # sibling module; support both package and flat plugin-dir import @@ -1825,7 +1826,7 @@ class SlackAdapter(BasePlatformAdapter): "Slack's ephemeral reply limit.]_" ) try: - async with aiohttp.ClientSession(trust_env=True) as session: + async with aiohttp.ClientSession(trust_env=gateway_trust_env()) as session: for idx, chunk in enumerate(chunks): payload = { "response_type": "ephemeral", @@ -3838,6 +3839,30 @@ class SlackAdapter(BasePlatformAdapter): return "none" return value + def _slack_api_human_users(self) -> frozenset: + """Slack user IDs whose Web-API posts count as human-authored. + + A message posted with a *user* token (``xoxp-``) is authored by a real + person, but Slack still stamps it with the posting ``app_id`` and it + carries no ``client_msg_id`` — exactly the #35777 app/bot signature in + ``_event_declares_bot_sender``. Operators running their own front-end + (dashboard, mobile shell) allowlist those *users* via + ``platforms.slack.extra.api_human_users`` (``SLACK_API_HUMAN_USERS`` + fallback) instead of ``allow_bots: all``. Users only — an app-id + allowlist would also admit the app's own ``xoxb`` bot posts, which + carry the same user+app_id shape. + """ + cached = getattr(self, "_api_human_users_cache", None) + if cached is None: + raw = self.config.extra.get("api_human_users") + if raw is None: + raw = os.getenv("SLACK_API_HUMAN_USERS", "") + parts = raw if isinstance(raw, (list, tuple, set)) else str(raw).split(",") + cached = self._api_human_users_cache = frozenset( + str(p).strip() for p in parts if str(p).strip() + ) + return cached + def _event_declares_bot_sender(self, event: dict) -> bool: """Return True when the Slack event itself identifies a bot sender.""" if event.get("bot_id") or event.get("bot_profile"): @@ -3852,7 +3877,11 @@ class SlackAdapter(BasePlatformAdapter): # human-authored messages normally carry client_msg_id, so treat the # combination as app/bot-authored (#35777). if event.get("app_id") and not event.get("client_msg_id"): - return True + # ...unless the operator allowlisted this user's API posts + # (_slack_api_human_users). ``user`` is required so classic bot + # posts (no ``user``) never match; bot_message/bot_id already + # returned True above. + return event.get("user") not in self._slack_api_human_users() return False def _resolve_thread_ts( @@ -6324,9 +6353,19 @@ class SlackAdapter(BasePlatformAdapter): # or file downloads. The final gateway runner auth check happens # after MessageEvent construction, so adapter-side media fetches need # the same auth chain up front. + # Prefer the injected profile-bound check (survives the multiplex + # closure handler, which has no ``__self__``); fall back to runner + # introspection for adapters wired without one. + _early_decision = ( + self._is_sender_authorized( + user_id, "dm" if is_dm else "group", channel_id + ) + if user_id and getattr(self, "_authorization_check", None) is not None + else None + ) _runner = getattr(getattr(self, "_message_handler", None), "__self__", None) _auth_fn = getattr(_runner, "_is_user_authorized", None) - if user_id and callable(_auth_fn): + if _early_decision is None and user_id and callable(_auth_fn): _source = self.build_source( chat_id=channel_id, chat_name="", @@ -6334,13 +6373,14 @@ class SlackAdapter(BasePlatformAdapter): user_id=user_id, user_name="", ) - if not _auth_fn(_source): - logger.warning( - "[Slack] Early reject of unauthorized user %s in channel %s", - user_id, - channel_id, - ) - return + _early_decision = bool(_auth_fn(_source)) + if _early_decision is False: + logger.warning( + "[Slack] Early reject of unauthorized user %s in channel %s", + user_id, + channel_id, + ) + return # Build thread_ts for session keying. # In channels: fall back to ts so each top-level @mention starts a @@ -6891,7 +6931,7 @@ class SlackAdapter(BasePlatformAdapter): raw_bytes = await self._download_slack_file_bytes( url, team_id=team_id ) - cached_path = cache_video_from_bytes(raw_bytes, ext=ext) + cached_path = await cache_video_from_bytes_async(raw_bytes, ext=ext) media_urls.append(cached_path) media_types.append( SUPPORTED_VIDEO_TYPES.get(ext, mimetype or "video/mp4") @@ -6945,7 +6985,7 @@ class SlackAdapter(BasePlatformAdapter): raw_bytes = await self._download_slack_file_bytes( url, team_id=team_id ) - cached_path = cache_document_from_bytes( + cached_path = await cache_document_from_bytes_async( raw_bytes, original_filename or f"document{ext or '.bin'}" ) if in_allowlist: @@ -7049,7 +7089,9 @@ class SlackAdapter(BasePlatformAdapter): # subtype=bot_message with user=None; flag them so the # gateway SLACK_ALLOW_BOTS bypass can authorize them # (they carry no user_id to match against the allowlist). - is_bot=bool(event.get("bot_id")) or event.get("subtype") == "bot_message", + # Same predicate as the drop gate above, so an api_human_users + # post is a plain human here too. + is_bot=self._event_declares_bot_sender(event), ) # Per-channel ephemeral prompt @@ -7459,6 +7501,23 @@ class SlackAdapter(BasePlatformAdapter): if not normalized_user_id: return False + chat_type = "dm" if str(channel_id or "").startswith("D") else "group" + + # Preferred path: the auth callback GatewayRunner injects at connect + # time (``set_authorization_check``) runs the full, profile-bound + # ``_is_user_authorized`` chain. Unlike the ``__self__`` introspection + # below it also resolves on a multiplexed adapter, whose message + # handler is a profile closure with no ``__self__`` (#72657, same + # class as Telegram's #86296). + # ``getattr``: adapters built via ``object.__new__`` never ran + # ``BasePlatformAdapter.__init__``. + if getattr(self, "_authorization_check", None) is not None: + injected = self._is_sender_authorized( + normalized_user_id, chat_type, str(channel_id or "") + ) + if injected is not None: + return injected + runner = getattr(getattr(self, "_message_handler", None), "__self__", None) auth_fn = getattr(runner, "_is_user_authorized", None) if callable(auth_fn): @@ -7468,7 +7527,7 @@ class SlackAdapter(BasePlatformAdapter): source = SessionSource( platform=Platform.SLACK, chat_id=str(channel_id or normalized_user_id), - chat_type="dm" if str(channel_id or "").startswith("D") else "group", + chat_type=chat_type, user_id=normalized_user_id, user_name=str(user_name).strip() if user_name else None, scope_id=str(team_id) if team_id else None, @@ -7481,21 +7540,15 @@ class SlackAdapter(BasePlatformAdapter): exc_info=True, ) - if os.getenv("SLACK_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"}: + # Env-only fallback (no injected check, no bound runner). Gate reads go + # through the shared per-profile accessor: under multiplex a scoped + # miss returns "" instead of falling through to ``os.environ``, which + # holds the DEFAULT profile's allow-all flag / allowlist. + from gateway.authz_mixin import _platform_gate_env as _env + + if _env("SLACK_ALLOW_ALL_USERS").lower() in {"true", "1", "yes"}: return True - def _env(name: str) -> str: - # Multiplex: profile .env is in secret_scope, not process environ. - try: - from agent.secret_scope import get_secret - - val = get_secret(name) - if val is not None and str(val).strip(): - return str(val).strip() - except Exception: - pass - return (os.getenv(name) or "").strip() - allowed_ids = set() platform_allowlist = _env("SLACK_ALLOWED_USERS") if platform_allowlist: @@ -7507,8 +7560,6 @@ class SlackAdapter(BasePlatformAdapter): if allowed_ids: return "*" in allowed_ids or normalized_user_id in allowed_ids - if _env("SLACK_ALLOW_ALL_USERS").lower() in {"true", "1", "yes"}: - return True return _env("GATEWAY_ALLOW_ALL_USERS").lower() in {"true", "1", "yes"} async def _handle_slash_confirm_action(self, ack, body, action) -> None: @@ -8184,7 +8235,7 @@ class SlackAdapter(BasePlatformAdapter): skip_for_delta = bool(after_ts and msg_ts and msg_ts <= after_ts) if skip_for_delta and not is_parent: continue - is_bot = bool(msg.get("bot_id")) or msg.get("subtype") == "bot_message" + is_bot = self._event_declares_bot_sender(msg) msg_user = msg.get("user", "") # Identify "our own" bot for this workspace (multi-workspace safe). @@ -8953,13 +9004,13 @@ class SlackAdapter(BasePlatformAdapter): ) if audio: - from gateway.platforms.base import cache_audio_from_bytes + from gateway.platforms.base import cache_audio_from_bytes_async - return cache_audio_from_bytes(response.content, ext) + return await cache_audio_from_bytes_async(response.content, ext) else: - from gateway.platforms.base import cache_image_from_bytes + from gateway.platforms.base import cache_image_from_bytes_async - return cache_image_from_bytes(response.content, ext) + return await cache_image_from_bytes_async(response.content, ext) except (httpx.TimeoutException, httpx.HTTPStatusError) as exc: if ( isinstance(exc, httpx.HTTPStatusError) diff --git a/plugins/platforms/sms/adapter.py b/plugins/platforms/sms/adapter.py index 37db336e7a..8d2592bc7b 100644 --- a/plugins/platforms/sms/adapter.py +++ b/plugins/platforms/sms/adapter.py @@ -29,6 +29,7 @@ from typing import Any, Dict, Optional from gateway.config import Platform, PlatformConfig from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, @@ -156,7 +157,7 @@ class SmsAdapter(BasePlatformAdapter): await site.start() self._http_session = aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=30), - trust_env=True, + trust_env=gateway_trust_env(), ) self._running = True @@ -200,7 +201,7 @@ class SmsAdapter(BasePlatformAdapter): session = self._http_session or aiohttp.ClientSession( timeout=aiohttp.ClientTimeout(total=30), - trust_env=True, + trust_env=gateway_trust_env(), ) try: for chunk in chunks: diff --git a/plugins/platforms/teams/adapter.py b/plugins/platforms/teams/adapter.py index 42f424a90a..583623ae33 100644 --- a/plugins/platforms/teams/adapter.py +++ b/plugins/platforms/teams/adapter.py @@ -103,12 +103,13 @@ TextBlock = None # type: ignore[assignment,misc] from gateway.config import Platform, PlatformConfig from gateway.platforms.helpers import MessageDeduplicator from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult, cache_image_from_url, - cache_media_bytes, + cache_media_bytes_async, ) from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError @@ -641,7 +642,7 @@ async def _standalone_send( # Per-request timeouts so a slow STS endpoint cannot starve the # subsequent activity POST of its budget. per_request_timeout = _aiohttp.ClientTimeout(total=15.0) - async with _aiohttp.ClientSession(trust_env=True) as session: + async with _aiohttp.ClientSession(trust_env=gateway_trust_env()) as session: async with session.post( token_url, data={ @@ -1095,7 +1096,7 @@ class TeamsAdapter(BasePlatformAdapter): filename = att_name or (f"document.{file_type}" if file_type else "document") try: data = await self._fetch_attachment_bytes(download_url) - cached = cache_media_bytes(data, filename=filename, mime_type="") + cached = await cache_media_bytes_async(data, filename=filename, mime_type="") if cached: media_urls.append(cached.path) media_types.append(cached.media_type) @@ -1116,7 +1117,7 @@ class TeamsAdapter(BasePlatformAdapter): # bearer token; the generic cache helper sends none. data = await self._fetch_attachment_bytes(content_url) ext = content_type.split("/")[-1].split(";")[0] or "png" - cached_m = cache_media_bytes( + cached_m = await cache_media_bytes_async( data, filename=att_name or f"image.{ext}", mime_type=content_type, @@ -1144,7 +1145,7 @@ class TeamsAdapter(BasePlatformAdapter): # Direct-URL non-image attachment (video/audio/document). try: data = await self._fetch_attachment_bytes(content_url) - cached = cache_media_bytes( + cached = await cache_media_bytes_async( data, filename=att_name, mime_type=content_type ) if cached: diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index 7f122a065e..62ca27eea0 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -223,9 +223,9 @@ from gateway.platforms.base import ( ProcessingOutcome, SendResult, classify_send_error, - cache_image_from_bytes, - cache_audio_from_bytes, - cache_video_from_bytes, + cache_image_from_bytes_async, + cache_audio_from_bytes_async, + cache_video_from_bytes_async, cache_document_from_bytes, resolve_proxy_url, SUPPORTED_VIDEO_TYPES, @@ -900,6 +900,13 @@ class TelegramAdapter(BasePlatformAdapter): # blow the gateway's connect timeout (#46298). self._post_connect_task: Optional[asyncio.Task] = None + @property + def send_path_degraded(self) -> bool: + # True from polling-generation start until the first getUpdates + # round-trip is proven (_record_polling_progress), and again at every + # polling-death site. getattr: tests build adapters via object.__new__(). + return bool(getattr(self, "_send_path_degraded", False)) + def _mark_connected(self) -> None: self._drop_delayed_deliveries = False super()._mark_connected() @@ -1213,18 +1220,40 @@ class TelegramAdapter(BasePlatformAdapter): if not normalized_user_id: return False + normalized_chat_type = str(chat_type or "dm").strip().lower() or "dm" + if normalized_chat_type == "private": + normalized_chat_type = "dm" + elif normalized_chat_type == "supergroup": + normalized_chat_type = "forum" if thread_id is not None else "group" + + # Preferred path: the auth callback GatewayRunner injects at + # connection time (set_authorization_check), which delegates to the + # full _is_user_authorized chain -- env allowlists, group allowlists, + # pairing store, allow-all flags. Unlike the __self__ introspection + # below, this also works for a secondary multiplexed adapter, whose + # _message_handler is a profile closure with no __self__ (the same + # gap the admin-tier check had -- resolved the same way). The getattr + # tolerates partially-constructed adapters (object.__new__ in tests) + # that never ran BasePlatformAdapter.__init__. + if getattr(self, "_authorization_check", None) is not None: + injected = self._is_sender_authorized( + normalized_user_id, + chat_type=normalized_chat_type, + chat_id=str(chat_id or normalized_user_id), + thread_id=str(thread_id) if thread_id is not None else None, + ) + if injected is not None: + return injected + + # Legacy path: resolve the runner off the bound message handler. + # Still reachable for adapters wired without set_authorization_check + # (bare-adapter tests, direct embedding). runner = getattr(getattr(self, "_message_handler", None), "__self__", None) auth_fn = getattr(runner, "_is_user_authorized", None) if callable(auth_fn): try: from gateway.session import SessionSource - normalized_chat_type = str(chat_type or "dm").strip().lower() or "dm" - if normalized_chat_type == "private": - normalized_chat_type = "dm" - elif normalized_chat_type == "supergroup": - normalized_chat_type = "forum" if thread_id is not None else "group" - source = SessionSource( platform=Platform.TELEGRAM, chat_id=str(chat_id or normalized_user_id), @@ -1264,6 +1293,9 @@ class TelegramAdapter(BasePlatformAdapter): user = getattr(message, "from_user", None) chat = getattr(message, "chat", None) user_id = str(getattr(user, "id", "")).strip() or None + # Carry the bot flag so the runner's ``*_ALLOW_BOTS`` policy branch is + # reachable from this prefilter, exactly as it is for ``build_source``. + is_bot = bool(getattr(user, "is_bot", False)) if user is not None else False user_name = ( str(getattr(user, "username", "") or getattr(user, "full_name", "") or "").strip() or None @@ -1309,6 +1341,7 @@ class TelegramAdapter(BasePlatformAdapter): user_id=user_id, user_name=user_name, thread_id=thread_id, + is_bot=is_bot, ) def _source_from_reaction_for_auth(self, update): @@ -1390,14 +1423,20 @@ class TelegramAdapter(BasePlatformAdapter): if source.chat_type != "dm": return False - runner = getattr(getattr(self, "_message_handler", None), "__self__", None) + # The bound-handler ``__self__`` is None under multiplex (the handler is + # a profile closure); ``gateway_runner`` is injected on every adapter + # by ``GatewayRunner._create_adapter`` and survives that wrapping. + runner = getattr( + getattr(self, "_message_handler", None), "__self__", None + ) or getattr(self, "gateway_runner", None) behavior_fn = getattr(runner, "_get_unauthorized_dm_behavior", None) if callable(behavior_fn): try: return ( behavior_fn( Platform.TELEGRAM, - profile=getattr(source, "profile", None), + profile=getattr(source, "profile", None) + or getattr(self, "_owner_profile", None), ) == "pair" ) @@ -1490,6 +1529,8 @@ class TelegramAdapter(BasePlatformAdapter): user_id, chat_type=source.chat_type, chat_id=source.chat_id, + is_bot=source.is_bot, + thread_id=source.thread_id, ) if has_callback else None @@ -1686,7 +1727,11 @@ class TelegramAdapter(BasePlatformAdapter): return "thread not found" in str(error).lower() def _prune_stale_dm_topic_binding( - self, chat_id: Any, thread_id: Any, + self, + chat_id: Any, + thread_id: Any, + *, + metadata: Optional[Dict[str, Any]] = None, ) -> None: """Drop the stale ``telegram_dm_topic_bindings`` row for a topic Telegram has confirmed deleted. @@ -1699,6 +1744,12 @@ class TelegramAdapter(BasePlatformAdapter): on to a fresh topic). Best-effort: we never raise from a send-fallback path — a failed cleanup must not turn into a failed user-facing send. + + Rows are namespaced by profile (#76423). Under + ``gateway.profile_routes`` the transport adapter may not be the + profile that wrote the binding, so the send's ``hermes_profile`` + metadata wins over the adapter's own profile stamp; single-profile + bots fall back to ``"default"``. """ if chat_id is None or thread_id is None: return @@ -1709,8 +1760,15 @@ class TelegramAdapter(BasePlatformAdapter): if db is None or not hasattr(db, "delete_telegram_topic_binding"): return try: + profile_name = ( + (metadata or {}).get("hermes_profile") + or getattr(self, "_hermes_profile_name", None) + or "default" + ) removed = db.delete_telegram_topic_binding( - chat_id=str(chat_id), thread_id=str(thread_id), + chat_id=str(chat_id), + thread_id=str(thread_id), + profile_name=profile_name, ) except Exception: logger.debug( @@ -2643,6 +2701,13 @@ class TelegramAdapter(BasePlatformAdapter): self._polling_conflict_recovery_generation = None else: self._polling_conflict_count = 0 + # First proof getUpdates is flowing for this generation: flip a + # published "retrying" (degraded connect, reconnect stamp, or the + # mid-session recovery below) back to "connected" (#101391). + if self._send_path_degraded and getattr(self, "_running", False) and not self.has_fatal_error: + self._write_runtime_status_safe( + "connected", platform_state="connected", error_code=None, error_message=None, + ) self._send_path_degraded = False def _observe_polling_request_result(self, request, generation, result): @@ -2861,6 +2926,11 @@ class TelegramAdapter(BasePlatformAdapter): ) return self._send_path_degraded = True + # Polling died mid-session on an adapter that published "connected" + # at connect time. Without this, gateway_state.json keeps saying + # connected for as long as the recovery ladder runs (#101391: 11 h). + if getattr(self, "_running", False): + self._mark_degraded() logger.warning( "[%s] Telegram polling degraded (%s); gateway stays alive and will retry. Error: %s", self.name, reason, _redact_telegram_error_text(error), @@ -5590,7 +5660,9 @@ class TelegramAdapter(BasePlatformAdapter): self.name, effective_thread_id, ) self._prune_stale_dm_topic_binding( - chat_id, effective_thread_id, + chat_id, + effective_thread_id, + metadata=metadata, ) used_thread_fallback = True effective_thread_id = None @@ -6380,7 +6452,8 @@ class TelegramAdapter(BasePlatformAdapter): # Same prune as the streaming send path — the # control-message retry tells us the topic is gone, # so the binding row in state.db must go too - # (#31501). + # (#31501). Control sends carry no gateway metadata, so + # the prune namespaces by this adapter's profile stamp. self._prune_stale_dm_topic_binding( kwargs.get("chat_id"), message_thread_id, ) @@ -9569,7 +9642,7 @@ class TelegramAdapter(BasePlatformAdapter): ``_max_doc_bytes`` limit as the addressed document path. Oversized or unsupported attachments are noted in the transcript without downloading. """ - from gateway.platforms.base import cache_media_bytes + from gateway.platforms.base import cache_media_bytes_async source, filename, mime, kind = self._observed_media_source(msg) if source is None: @@ -9595,7 +9668,7 @@ class TelegramAdapter(BasePlatformAdapter): data = bytes(await file_obj.download_as_bytearray()) if not filename: filename = os.path.basename(getattr(file_obj, "file_path", "") or "") - cached = cache_media_bytes(data, filename=filename, mime_type=mime, default_kind=kind) + cached = await cache_media_bytes_async(data, filename=filename, mime_type=mime, default_kind=kind) except Exception as exc: logger.warning("[Telegram] Failed to cache observed group media: %s", _redact_telegram_error_text(exc), exc_info=True) return @@ -9622,7 +9695,7 @@ class TelegramAdapter(BasePlatformAdapter): async def _cache_replied_media(self, msg: Any, event: MessageEvent) -> None: """Cache media from the message this turn replies to, if any.""" - from gateway.platforms.base import cache_media_bytes + from gateway.platforms.base import cache_media_bytes_async reply_msg = getattr(msg, "reply_to_message", None) if reply_msg is None: @@ -9645,7 +9718,7 @@ class TelegramAdapter(BasePlatformAdapter): data = bytes(await file_obj.download_as_bytearray()) if not filename: filename = os.path.basename(getattr(file_obj, "file_path", "") or "") - cached = cache_media_bytes(data, filename=filename, mime_type=mime, default_kind=kind) + cached = await cache_media_bytes_async(data, filename=filename, mime_type=mime, default_kind=kind) except Exception as exc: logger.warning("[Telegram] Failed to cache replied-to media: %s", _redact_telegram_error_text(exc), exc_info=True) return @@ -10263,7 +10336,7 @@ class TelegramAdapter(BasePlatformAdapter): ext = candidate break # Save to local cache (for vision tool access) - cached_path = cache_image_from_bytes(bytes(image_bytes), ext=ext) + cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=ext) event.media_urls = [cached_path] event.media_types = [f"image/{ext.lstrip('.')}" ] logger.info("[Telegram] Cached user photo at %s", cached_path) @@ -10290,7 +10363,7 @@ class TelegramAdapter(BasePlatformAdapter): return file_obj = await msg.voice.get_file() audio_bytes = await file_obj.download_as_bytearray() - cached_path = cache_audio_from_bytes(bytes(audio_bytes), ext=".ogg") + cached_path = await cache_audio_from_bytes_async(bytes(audio_bytes), ext=".ogg") event.media_urls = [cached_path] event.media_types = ["audio/ogg"] logger.info("[Telegram] Cached user voice at %s", cached_path) @@ -10307,7 +10380,7 @@ class TelegramAdapter(BasePlatformAdapter): return file_obj = await msg.audio.get_file() audio_bytes = await file_obj.download_as_bytearray() - cached_path = cache_audio_from_bytes(bytes(audio_bytes), ext=".mp3") + cached_path = await cache_audio_from_bytes_async(bytes(audio_bytes), ext=".mp3") event.media_urls = [cached_path] event.media_types = ["audio/mp3"] logger.info("[Telegram] Cached user audio at %s", cached_path) @@ -10331,7 +10404,7 @@ class TelegramAdapter(BasePlatformAdapter): if file_obj.file_path.lower().endswith(candidate): ext = candidate break - cached_path = cache_video_from_bytes(bytes(video_bytes), ext=ext) + cached_path = await cache_video_from_bytes_async(bytes(video_bytes), ext=ext) event.media_urls = [cached_path] event.media_types = [SUPPORTED_VIDEO_TYPES.get(ext, "video/mp4")] logger.info("[Telegram] Cached user video at %s", cached_path) @@ -10381,7 +10454,7 @@ class TelegramAdapter(BasePlatformAdapter): image_bytes = await file_obj.download_as_bytearray() image_ext = ext if ext in _TELEGRAM_IMAGE_EXTENSIONS else _TELEGRAM_IMAGE_MIME_TO_EXT.get(doc_mime, ".jpg") try: - cached_path = cache_image_from_bytes(bytes(image_bytes), ext=image_ext) + cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=image_ext) except ValueError as e: logger.warning("[Telegram] Failed to cache image document: %s", _redact_telegram_error_text(e), exc_info=True) event.text = ( @@ -10419,7 +10492,7 @@ class TelegramAdapter(BasePlatformAdapter): if ext in SUPPORTED_VIDEO_TYPES: file_obj = await doc.get_file() video_bytes = await file_obj.download_as_bytearray() - cached_path = cache_video_from_bytes(bytes(video_bytes), ext=ext) + cached_path = await cache_video_from_bytes_async(bytes(video_bytes), ext=ext) event.media_urls = [cached_path] event.media_types = [SUPPORTED_VIDEO_TYPES[ext]] event.message_type = MessageType.VIDEO @@ -10440,9 +10513,9 @@ class TelegramAdapter(BasePlatformAdapter): file_obj = await doc.get_file() doc_bytes = await file_obj.download_as_bytearray() raw_bytes = bytes(doc_bytes) - from gateway.platforms.base import cache_media_bytes + from gateway.platforms.base import cache_media_bytes_async - cached = cache_media_bytes( + cached = await cache_media_bytes_async( raw_bytes, filename=original_filename or f"document{ext or '.bin'}", mime_type=doc_mime, @@ -10591,7 +10664,7 @@ class TelegramAdapter(BasePlatformAdapter): try: file_obj = await sticker.get_file() image_bytes = await file_obj.download_as_bytearray() - cached_path = cache_image_from_bytes(bytes(image_bytes), ext=".webp") + cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=".webp") logger.info("[Telegram] Analyzing sticker at %s", cached_path) from tools.vision_tools import vision_analyze_tool diff --git a/plugins/platforms/wecom/adapter.py b/plugins/platforms/wecom/adapter.py index c26d4a8350..71a6b843d3 100644 --- a/plugins/platforms/wecom/adapter.py +++ b/plugins/platforms/wecom/adapter.py @@ -63,12 +63,13 @@ except ImportError: from gateway.config import Platform, PlatformConfig from gateway.platforms.helpers import MessageDeduplicator from gateway.platforms.base import ( + gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult, - cache_document_from_bytes, - cache_image_from_bytes, + cache_document_from_bytes_async, + cache_image_from_bytes_async, ) from utils import env_float @@ -317,12 +318,12 @@ class WeComAdapter(BasePlatformAdapter): super().__init__(config, Platform.WECOM) extra = config.extra or {} - self._bot_id = str(extra.get("bot_id") or os.getenv("WECOM_BOT_ID", "")).strip() + self._bot_id = str(extra.get("bot_id") or _get_scoped_secret("WECOM_BOT_ID", "")).strip() self._secret = str(extra.get("secret") or _get_scoped_secret("WECOM_SECRET", "")).strip() self._ws_url = str( extra.get("websocket_url") or extra.get("websocketUrl") - or os.getenv("WECOM_WEBSOCKET_URL", DEFAULT_WS_URL) + or _get_scoped_secret("WECOM_WEBSOCKET_URL", DEFAULT_WS_URL) ).strip() or DEFAULT_WS_URL self._dm_policy = str(extra.get("dm_policy") or _get_scoped_secret("WECOM_DM_POLICY", "pairing")).strip().lower() @@ -723,7 +724,7 @@ class WeComAdapter(BasePlatformAdapter): except ImportError: _ssl_ctx = _ssl.create_default_context() _connector = aiohttp.TCPConnector(ssl=_ssl_ctx) - self._session = aiohttp.ClientSession(trust_env=True, connector=_connector) + self._session = aiohttp.ClientSession(trust_env=gateway_trust_env(), connector=_connector) self._ws = await self._session.ws_connect( self._ws_url, heartbeat=HEARTBEAT_INTERVAL_SECONDS * 2, @@ -1611,13 +1612,13 @@ class WeComAdapter(BasePlatformAdapter): if kind == "image": ext = self._detect_image_ext(raw) try: - return cache_image_from_bytes(raw, ext), self._mime_for_ext(ext, fallback="image/jpeg") + return await cache_image_from_bytes_async(raw, ext), self._mime_for_ext(ext, fallback="image/jpeg") except ValueError as exc: logger.warning("[%s] Rejected non-image bytes: %s", self.name, exc) return None filename = str(media.get("filename") or media.get("name") or "wecom_file") - return cache_document_from_bytes(raw, filename), mimetypes.guess_type(filename)[0] or "application/octet-stream" + return await cache_document_from_bytes_async(raw, filename), mimetypes.guess_type(filename)[0] or "application/octet-stream" url = str(media.get("url") or "").strip() if not url: @@ -1641,13 +1642,13 @@ class WeComAdapter(BasePlatformAdapter): if kind == "image": ext = self._guess_extension(url, content_type, fallback=self._detect_image_ext(raw)) try: - return cache_image_from_bytes(raw, ext), content_type or self._mime_for_ext(ext, fallback="image/jpeg") + return await cache_image_from_bytes_async(raw, ext), content_type or self._mime_for_ext(ext, fallback="image/jpeg") except ValueError as exc: logger.warning("[%s] Rejected non-image bytes from %s: %s", self.name, url, exc) return None filename = self._guess_filename(url, headers.get("content-disposition"), content_type) - return cache_document_from_bytes(raw, filename), content_type + return await cache_document_from_bytes_async(raw, filename), content_type @staticmethod def _decode_base64(data: str) -> bytes: diff --git a/providers/__init__.py b/providers/__init__.py index 011e84afa7..939d05ebac 100644 --- a/providers/__init__.py +++ b/providers/__init__.py @@ -108,6 +108,57 @@ def _user_plugins_dir() -> Path | None: return None +def _installed_plugins_dir() -> Path | None: + """Return ``$HERMES_HOME/plugins/`` if it exists. + + This is where ``hermes plugins install`` clones a plugin — flat, one + directory per plugin, NOT under ``model-providers/``. See + :func:`_discover_installed_provider_plugins`. + """ + try: + from hermes_constants import get_hermes_home + + d = get_hermes_home() / "plugins" + return d if d.is_dir() else None + except Exception: + return None + + +def _declares_model_provider_kind(plugin_dir: Path) -> bool: + """Whether ``plugin_dir``'s manifest declares ``kind: model-provider``. + + Only that kind is imported from the flat install directory — every other + plugin there belongs to ``PluginManager``, which owns its lifecycle and + consent flow. Parsed with PyYAML when available, falling back to a line + scan so provider discovery never hard-depends on it. + """ + for filename in ("plugin.yaml", "plugin.yml"): + manifest = plugin_dir / filename + if not manifest.is_file(): + continue + try: + text = manifest.read_text(encoding="utf-8", errors="replace") + except Exception: + return False + try: + import yaml + + data = yaml.safe_load(text) + if isinstance(data, dict): + return str(data.get("kind", "")).strip() == "model-provider" + except Exception: + pass + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("#") or ":" not in stripped: + continue + key, _, value = stripped.partition(":") + if key.strip() == "kind": + return value.strip().strip("\"'") == "model-provider" + return False + return False + + def _import_plugin_dir(plugin_dir: Path, source: str) -> None: """Import a single plugin directory so it self-registers. @@ -274,6 +325,8 @@ def _discover_providers() -> None: Order: 1. Bundled plugins at ``/plugins/model-providers//`` 2. User plugins at ``$HERMES_HOME/plugins/model-providers//`` + 2b. Plugins installed by ``hermes plugins install`` at + ``$HERMES_HOME/plugins//`` that declare ``kind: model-provider`` 3. Legacy per-file modules at ``providers/.py`` (back-compat) Each step imports its plugins, which call ``register_provider()`` at @@ -317,6 +370,25 @@ def _discover_providers() -> None: continue _import_plugin_dir(child, "user") + # 2b. Plugins installed by ``hermes plugins install`` / the plugin index. + # Those clone into $HERMES_HOME/plugins// — flat, NOT under + # model-providers/ — so step 2 never sees them. PluginManager does not + # import them either: it classifies ``kind: model-provider`` and routes + # it here on purpose. Without this step the documented install path + # silently half-works — the CLI reports success and the provider does + # not exist. Only manifests declaring that kind are imported; every + # other plugin in this directory belongs to PluginManager. + installed_dir = _installed_plugins_dir() + if installed_dir is not None: + for child in sorted(installed_dir.iterdir()): + if not child.is_dir() or child.name.startswith(("_", ".")): + continue + if child.name == "model-providers": + continue # handled by step 2 + if not _declares_model_provider_kind(child): + continue + _import_plugin_dir(child, "user") + # 3. Legacy single-file profiles at providers/.py. Kept for # back-compat — if someone drops a ``providers/foo.py`` into an # editable install, it still works without the plugin layout. diff --git a/providers/base.py b/providers/base.py index 5d14868eb1..b76748f0dc 100644 --- a/providers/base.py +++ b/providers/base.py @@ -78,6 +78,17 @@ class ProviderProfile: # top-level fields rather than ignoring them. supports_prompt_cache_key: bool = False + # ── External-process providers (auth_type="external_process") ── + # An agent CLI driven over stdio (ACP) rather than an HTTP endpoint. These + # describe how to launch it; hermes_cli/auth.py's + # resolve_external_process_provider_credentials() reads them instead of + # hardcoding one vendor's binary. Env vars are checked in order and win + # over the static defaults, so an operator can point at a custom build. + process_command: str = "" # default binary, e.g. "copilot" + process_args: tuple = () # default argv tail, e.g. ("--acp", "--stdio") + process_command_env_vars: tuple = () # env overrides for the binary, in priority order + process_args_env_var: str = "" # env override for argv (shlex-split) + # ── Model catalog ───────────────────────────────────────── # fallback_models: curated list shown in /model picker when live fetch fails. # Only agentic models that support tool calling should appear here. @@ -223,6 +234,31 @@ class ProviderProfile: """ return None + def create_client(self, **client_kwargs: Any) -> Any | None: + """Return a provider-specific client, or ``None`` for the standard one. + + Most providers speak OpenAI-compatible HTTP and want the shared + ``openai.OpenAI`` client the core builds — they inherit this and return + ``None``. A provider whose wire protocol is not HTTP at all (the ACP + subprocess shims) or which needs a native SDK overrides this and + returns its own client object. + + ``client_kwargs`` is the same mapping the core would have passed to + ``openai.OpenAI`` (``api_key``, ``base_url``, ``command``, ``args``, + timeouts, headers…). Unknown keys must be tolerated: the core adds to + this mapping over time, so an override should accept ``**kwargs`` and + pick what it needs rather than enumerate. + + Returning ``None`` (the default) is always safe — the caller falls + through to its existing construction path. + + This is the hook that lets a provider ship *outside* this tree: with it, + a profile registered from ``~/.hermes/plugins/model-providers/`` or a + pip entry point can supply its own transport without any core edit. See + ``plugins/model-providers/copilot-acp/`` for the in-tree example. + """ + return None + def fetch_models( self, *, diff --git a/pyproject.toml b/pyproject.toml index 076f34c50a..2e7f40d0ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,13 +175,13 @@ dependencies = [ # Hence the ``sys_platform == 'win32'`` marker: the dep (and its portalocker # / pywin32 tree) ships only where it's actually used. "concurrent-log-handler==0.9.29; sys_platform == 'win32'", - # First-party lifecycle and shared-metrics runtime. Relay 0.6 is the minimum - # lossless provider-codec contract. Managed calls pass request/response data - # through this native module in-process; shared metrics installs no network - # exporter and consumes only its bounded projection. Relay publishes wheels - # only (no sdist), so this marker must stay false anywhere no wheel tag can - # match — otherwise installing Python dependencies fails resolution outright - # instead of falling back to the no-op Relay host (#76469, Termux). + # First-party lifecycle and shared-metrics runtime. Relay 0.8 is the supported + # native runtime and provider-codec baseline. Managed calls pass request and + # response data through this native module in-process; shared metrics installs + # no network exporter and consumes only its bounded projection. This marker + # must stay false anywhere no compatible native wheel tag can match; otherwise + # installing Python dependencies fails instead of falling back to the no-op + # Relay host (#76469, Termux). # Termux Python reports plain linux/aarch64 but runs on Bionic # libc, which satisfies neither manylinux nor musllinux, hence the # `'android' not in platform_release` guard on the linux arms: Android GKI @@ -189,7 +189,7 @@ dependencies = [ # 738 CPython reports sys_platform == 'android' and never matched.) Pre-GKI # devices can still slip through; they get the same resolution failure as # before, worked around by installing with `--no-deps` or an older release. - "nemo-relay>=0.7.1,<0.8; (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'linux' and platform_machine == 'x86_64' and 'android' not in platform_release) or (sys_platform == 'linux' and platform_machine == 'aarch64' and 'android' not in platform_release) or (sys_platform == 'win32' and platform_machine == 'AMD64') or (sys_platform == 'win32' and platform_machine == 'ARM64')", + "nemo-relay>=0.8.3,<0.9; (sys_platform == 'darwin' and platform_machine == 'arm64') or (sys_platform == 'linux' and platform_machine == 'x86_64' and 'android' not in platform_release) or (sys_platform == 'linux' and platform_machine == 'aarch64' and 'android' not in platform_release) or (sys_platform == 'win32' and platform_machine == 'AMD64') or (sys_platform == 'win32' and platform_machine == 'ARM64')", ] [project.optional-dependencies] diff --git a/run_agent.py b/run_agent.py index eb904ccc71..2b44775af6 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1885,12 +1885,26 @@ class AIAgent: (LiteLLM/sglang/vLLM/LM Studio proxies, Tailscale boxes), which report finish_reason correctly and were the source of #13971's false-positive truncation continuations. + + Also excludes Ollama Cloud — the hosted service correctly reports + finish_reason and is not affected by the local Ollama stop-reason + bug (GH-72316). Two signatures identify it: the ``ollama.com`` host + (provider ``ollama-cloud``) and the ``:cloud`` model suffix (cloud + generation proxied through a local 11434 endpoint, #98406). Applying + the stop→length rewrite to them manufactures false truncations and + causes the continuation nudge to consume the model's output budget + on the next retry, making further false-positives more likely. """ model_lower = (self.model or "").lower() provider_lower = (self.provider or "").lower() if "glm" not in model_lower and provider_lower != "zai": return False - if "ollama" in self._base_url_lower or ":11434" in self._base_url_lower: + base = self._base_url_lower + # Ollama Cloud (hosted service or :cloud proxy) forwards finish_reason + # faithfully — do not rewrite. + if "ollama.com" in base or ":cloud" in model_lower: + return False + if "ollama" in base or ":11434" in base: return True return provider_lower == "ollama" @@ -2007,6 +2021,13 @@ class AIAgent: if not enabled: return + # Structural clone at the single chokepoint every review path + # (automatic, /refine, idle-queue deferral) goes through. The fork + # sanitizes its transcript in place; a shallow copy would alias the + # nested tool_calls/content containers of the live history (#100795). + from agent.turn_finalizer import _clone_background_review_messages + messages_snapshot = _clone_background_review_messages(messages_snapshot) + kwargs = dict( messages_snapshot=messages_snapshot, review_memory=review_memory, @@ -2649,13 +2670,17 @@ class AIAgent: # ("storage was busy, send it again") from disk-full/read-only. from hermes_state import ( CompressionSessionClosedError, + StateDbCorruptError, StateDbReplacedError, classify_persistence_error, divert_session_transcript_jsonl, ) self._last_persistence_error_cause = classify_persistence_error(e) - if isinstance(e, StateDbReplacedError): + if isinstance(e, (StateDbReplacedError, StateDbCorruptError)): + # Replaced generation or quarantined (structurally corrupt) + # handle: SQLite will not take this batch again, so keep it + # on disk instead of only in RAM. try: divert_session_transcript_jsonl( getattr(self, "session_id", "") or "", @@ -2663,7 +2688,8 @@ class AIAgent: ) except Exception: logger.warning( - "JSONL divert failed after state.db replace for %s", + "JSONL divert failed after state.db %s for %s", + self._last_persistence_error_cause, getattr(self, "session_id", None), exc_info=True, ) @@ -4397,6 +4423,11 @@ class AIAgent: "pending_messages/pending-*.json." ) if cause == "corrupt": + from hermes_state import _default_db_path + + # Copy-pasteable, so name the real store (profiles / + # HERMES_HOME do not live under ~/.hermes). + db_path = _default_db_path() return ( prefix + "the turn was stopped because the state database " @@ -4404,8 +4435,15 @@ class AIAgent: "have been lost on restart). Freeing disk space will " "not help. Recovery options:\n" "1. Run `hermes doctor --fix`\n" - "2. Salvage with: sqlite3 ~/.hermes/state.db \".recover\" " - "(then replace state.db)\n" + "2. Stop the gateway, then recover with:\n" + f" hermes sessions recover --source {db_path} " + "--inspect-only\n" + " (if it reports recoverable) hermes sessions recover " + f"--source {db_path} --output recovered-state.db\n" + " — recovery snapshots the damaged file first; do NOT " + "run `sqlite3 ... \".recover\"` against the live " + "state.db, a vulnerable sqlite3 CLI can corrupt it " + "further\n" "3. Restore from a backup in ~/.hermes/backups/\n" "Then send your message again." ) @@ -5138,6 +5176,14 @@ class AIAgent: # still holds the closed agent (e.g. a draining background task). try: self._session_messages = [] + # Shadow copies of the same transcript: the DB-flush settled-prefix + # snapshot (a shallow copy of the whole list, see + # _flush_session_to_db) and the streamed-text accumulator. On a + # closed delegate child these were the only remaining owners of + # every message dict, so a retained child kept its full history + # alive in the parent's heap. + self._db_flush_scan_prefix = None + self._streamed_assistant_text_parts = [] except Exception: pass @@ -6397,9 +6443,12 @@ class AIAgent: try: from hermes_cli.auth import resolve_nous_runtime_credentials + # Pass the bearer that just 401'd so a refresh already done by a + # sibling process is adopted instead of rotating the grant again. creds = resolve_nous_runtime_credentials( timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=force, + stale_access_token=self.api_key or None, ) except Exception as exc: logger.debug("Nous credential refresh failed: %s", exc) @@ -7123,6 +7172,26 @@ class AIAgent: self._record_streamed_assistant_text(tail) self._current_streamed_assistant_text = "" + @property + def _current_streamed_assistant_text(self) -> str: + """Visible assistant text streamed so far this turn. + + Backed by a list of pieces rather than one growing string. Adding to + a string with ``+=`` on an attribute copies the whole thing every + time, so a long reply costs the square of its length in copying. The + pieces are joined here when a caller needs the full text. Emptiness + checks on the hot path should look at ``_streamed_assistant_text_parts`` + instead, so they do not join on every delta. + """ + parts = getattr(self, "_streamed_assistant_text_parts", None) + if not parts: + return "" + return "".join(parts) + + @_current_streamed_assistant_text.setter + def _current_streamed_assistant_text(self, value: str) -> None: + self._streamed_assistant_text_parts = [value] if value else [] + def _record_streamed_assistant_text(self, text: str) -> None: """Accumulate visible assistant text emitted through stream callbacks.""" # Single-writer guard (#65991): a superseded stream must not pollute the @@ -7132,9 +7201,11 @@ class AIAgent: if self._stream_writer_superseded(): return if isinstance(text, str) and text: - self._current_streamed_assistant_text = ( - getattr(self, "_current_streamed_assistant_text", "") + text - ) + parts = getattr(self, "_streamed_assistant_text_parts", None) + if parts is None: + parts = [] + self._streamed_assistant_text_parts = parts + parts.append(text) @staticmethod def _normalize_interim_visible_text(text: str) -> str: @@ -7481,9 +7552,12 @@ class AIAgent: else: # Defensive: legacy callers without the scrubber attribute. text = sanitize_context(text) - # Only strip leading newlines on the first delta — mid-stream "\n" is legitimate markdown. + # Only strip leading newlines on the first delta. Mid-stream + # newlines are legitimate markdown. Look at the parts list, not + # the joined property: joining on every token would copy the + # whole reply again. if not prepended_break and not getattr( - self, "_current_streamed_assistant_text", "" + self, "_streamed_assistant_text_parts", None ): text = text.lstrip("\n") if not text: @@ -8520,6 +8594,7 @@ class AIAgent: task_id: str = "default", focus_topic: str = None, force: bool = False, + bypass_cooldown: bool = False, defer_context_engine_notification: bool = False, commit_fence=None, ) -> tuple: @@ -8528,7 +8603,9 @@ class AIAgent: ``force=True`` is passed by the manual ``/compress`` slash command so users can bypass the summary-failure cooldown after an auto-compress abort. Auto-compress callers use the default - ``force=False``. + ``force=False``. ``bypass_cooldown=True`` is passed by the + provider-proven overflow recovery path so one real attempt runs while + the cooldown is armed (#100661) — without clearing it. """ # Per-attempt signal consumed by turn-start preflight (#98424) and the # in-loop pre-API/overflow consumers. A stalled compression must not @@ -8609,6 +8686,7 @@ class AIAgent: approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, force=force, + bypass_cooldown=bypass_cooldown, defer_context_engine_notification=( defer_context_engine_notification ), @@ -8989,6 +9067,13 @@ class AIAgent: function_result = append_toolguard_guidance(function_result, decision) if decision.should_halt: self._set_tool_guardrail_halt(decision) + else: + # observe_call may have raised the identical-call streak halt + # (hard_stop_enabled, tool-agnostic) — surface it the same way. + streak_halt = self._tool_guardrails.halt_decision + if streak_halt is not None and streak_halt.code == "identical_call_streak_halt": + function_result = append_toolguard_guidance(function_result, streak_halt) + self._set_tool_guardrail_halt(streak_halt) if stall_notice: function_result = (function_result or "") + "\n\n" + stall_notice return function_result @@ -9239,8 +9324,11 @@ class AIAgent: relay_turn = None durable_turn_lease = None durable_turn_lease_stop = None - durable_turn_lease_thread = None - durable_turn_liveness_thread = None + durable_turn_lease_refresh = None + durable_turn_liveness_watchdog = None + # Handles on the shared periodic scheduler thread (one per process, + # agent/periodic_scheduler.py) instead of 1-2 daemon threads per turn. + durable_turn_timer_handles = [] durable_turn_lease_activity_lock = threading.Lock() durable_turn_lease_turn_active = False durable_turn_lease_interrupt_message = None @@ -9451,7 +9539,7 @@ class AIAgent: ) # Long model/tool/compression turns outlive a fixed TTL. Refresh - # in a daemon thread; holder-qualified UPDATE and DELETE fence a + # on the shared periodic scheduler; holder-qualified UPDATE and DELETE fence a # late refresher/release from a successor lease. durable_turn_lease_stop = threading.Event() _lease_refresh_interval = float( @@ -9468,10 +9556,10 @@ class AIAgent: # "active", and never be force-aborted. # # The watchdog policy (config resolution, sampling state - # machine, thread mechanics) lives in agent/turn_liveness.py; + # machine, polling mechanics) lives in agent/turn_liveness.py; # this block is only the integration seam: resolve the # config.yaml settings, wire the commit/deactivate callbacks - # that own turn-lease state, and start the thread. + # that own turn-lease state, and schedule the poll. try: from hermes_cli.config import ( load_config_readonly as _liveness_load_config, @@ -9599,60 +9687,57 @@ class AIAgent: with durable_turn_lease_activity_lock: return durable_turn_lease_turn_active - def _refresh_durable_turn_lease() -> None: - while not durable_turn_lease_stop.wait(_lease_refresh_interval): - try: - if not _turn_db.refresh_session_turn_lease( - getattr(self, "session_id", None) or session_id, - durable_turn_lease, - ttl_seconds=_lease_ttl, - ): - # finally sets the stop event then releases. - # A late holder-fenced miss after that join - # timeout must not hard-interrupt the next turn. - if durable_turn_lease_stop.is_set(): - return - logger.error( - "Lost session turn lease while turn is active: %s", - getattr(self, "session_id", None) or session_id, - ) - _interrupt_turn( - "Session turn lease lost; stopping to protect " - "the transcript." - ) - return - except Exception: + def _refresh_durable_turn_lease(): + # One periodic tick on the shared scheduler thread every + # _lease_refresh_interval; returning False stops it. + if durable_turn_lease_stop.is_set(): + return False + try: + if not _turn_db.refresh_session_turn_lease( + getattr(self, "session_id", None) or session_id, + durable_turn_lease, + ttl_seconds=_lease_ttl, + ): + # finally sets the stop event then releases. + # A late holder-fenced miss after that cancel + # wait must not hard-interrupt the next turn. if durable_turn_lease_stop.is_set(): - return - logger.warning( - "Failed to refresh session turn lease: %s", + return False + logger.error( + "Lost session turn lease while turn is active: %s", getattr(self, "session_id", None) or session_id, - exc_info=True, ) _interrupt_turn( - "Session turn lease could not be refreshed; " - "stopping to protect the transcript." + "Session turn lease lost; stopping to protect " + "the transcript." ) - return + return False + except Exception: + if durable_turn_lease_stop.is_set(): + return False + logger.warning( + "Failed to refresh session turn lease: %s", + getattr(self, "session_id", None) or session_id, + exc_info=True, + ) + _interrupt_turn( + "Session turn lease could not be refreshed; " + "stopping to protect the transcript." + ) + return False - durable_turn_lease_thread = threading.Thread( - target=_refresh_durable_turn_lease, - name="session-turn-lease-refresh", - daemon=True, - ) + durable_turn_lease_refresh = _refresh_durable_turn_lease if _liveness_timeout is not None: - durable_turn_liveness_thread = ( - turn_liveness.TurnLivenessWatchdog( - self, - session_id=getattr(self, "session_id", None) or session_id, - timeout_s=_liveness_timeout, - poll_s=_liveness_poll, - stop_event=durable_turn_lease_stop, - activity_lock=self._liveness_activity_lock(), - is_turn_active=_turn_is_active, - commit_abort=_commit_turn_liveness_abort, - deactivate_turn=_deactivate_turn_after_liveness_abort, - ).make_thread() + durable_turn_liveness_watchdog = turn_liveness.TurnLivenessWatchdog( + self, + session_id=getattr(self, "session_id", None) or session_id, + timeout_s=_liveness_timeout, + poll_s=_liveness_poll, + stop_event=durable_turn_lease_stop, + activity_lock=self._liveness_activity_lock(), + is_turn_active=_turn_is_active, + commit_abort=_commit_turn_liveness_abort, + deactivate_turn=_deactivate_turn_after_liveness_abort, ) @@ -9705,7 +9790,7 @@ class AIAgent: # which may be observed from another thread. with bind_subagent_parent(self), scoped_runtime_main({}): try: - if durable_turn_lease_thread is not None: + if durable_turn_lease_refresh is not None: with durable_turn_lease_activity_lock: durable_turn_lease_turn_active = True # Stamp the activity clock at turn entry (#95663 @@ -9718,9 +9803,17 @@ class AIAgent: # first poll whenever the agent had been idle longer # than the watchdog bound. self._touch_activity("starting new turn") - durable_turn_lease_thread.start() - if durable_turn_liveness_thread is not None: - durable_turn_liveness_thread.start() + from agent.periodic_scheduler import schedule as _schedule_periodic + + durable_turn_timer_handles.append( + _schedule_periodic( + durable_turn_lease_refresh, _lease_refresh_interval + ) + ) + if durable_turn_liveness_watchdog is not None: + durable_turn_timer_handles.append( + durable_turn_liveness_watchdog.schedule() + ) result = run_conversation( self, user_message, @@ -9789,17 +9882,12 @@ class AIAgent: ) finally: _stop_durable_turn_lease_refresher() - for _durable_thread in ( - durable_turn_lease_thread, - durable_turn_liveness_thread, - ): - if ( - _durable_thread is not None - and _durable_thread.is_alive() - ): - _durable_thread.join(timeout=1.0) + # wait=1.0 mirrors the old thread join(timeout=1.0): an + # in-flight tick on the scheduler thread finishes first. + for _durable_handle in durable_turn_timer_handles: + _durable_handle.cancel(wait=1.0) # Clear any interrupt the refresher may have fired between - # the inner stop and this join. Must run AFTER join so a + # the inner stop and this cancel. Must run AFTER it so a # late interrupt does not survive into the next turn. _clear_durable_turn_lease_interrupt() if durable_turn_lease is not None: diff --git a/scripts/add_contributor.py b/scripts/add_contributor.py index f8ef9ac611..28f0a2772f 100644 --- a/scripts/add_contributor.py +++ b/scripts/add_contributor.py @@ -55,6 +55,24 @@ def _legacy_login(email: str) -> str | None: return None +def _case_collision(email: str) -> str | None: + """An existing mapping whose filename differs from `email` only in case. + + Returns the colliding filename, or None. Exact matches are not collisions -- + that is the ordinary "already mapped" path handled by the caller. + """ + if not EMAILS_DIR.is_dir(): + return None + + # casefold (not lower) matches how macOS/Windows fold non-ASCII text — + # same key scripts/check-case-collisions.py uses repo-wide. + folded = email.casefold() + for entry in EMAILS_DIR.iterdir(): + if entry.name != email and entry.name.casefold() == folded: + return entry.name + return None + + def add_contributor(email: str, login: str, comment: str = "") -> int: email = email.strip() login = login.strip().lstrip("@") @@ -67,6 +85,23 @@ def add_contributor(email: str, login: str, comment: str = "") -> int: return 2 path = EMAILS_DIR / email + + # One file per email means the FILENAME is the key, and on a + # case-insensitive filesystem (Windows, default macOS) two emails differing + # only in case are the same file. Creating both makes the repo impossible to + # check out cleanly there -- `git status` reports a phantom modification + # forever, because whichever file git wrote second wins on disk. Refuse for + # the same reason a conflicting login is refused: resolve it deliberately. + collision = _case_collision(email) + if collision is not None: + print( + f"error: {email} collides with existing mapping {collision} on " + "case-insensitive filesystems (Windows/macOS) — the two are the same " + "file there. Reuse that mapping, or resolve manually.", + file=sys.stderr, + ) + return 1 + existing = read_mapping_file(path) if path.is_file() else None if existing is None: existing = _legacy_login(email) diff --git a/scripts/ci/classify_changes.py b/scripts/ci/classify_changes.py index 415a3307ba..4aeb35d4c9 100644 --- a/scripts/ci/classify_changes.py +++ b/scripts/ci/classify_changes.py @@ -27,6 +27,12 @@ Lanes: * ``installer`` — PowerShell installer tests (Windows runner). * ``bootstrap`` — the bootstrap installer lane: install.sh sandbox install, pin-fragment drift check, and shipped version-stamp verification. +* ``desktop_updater`` — the Windows desktop-update hand-off script and the + tests that drive the REAL ``windows.ps1`` (``-SelfTestUi`` / pipe drain / + retry policy). These are integration tests of a PowerShell process on a + shared runner; running them on every Python PR made their timing noise + everyone's problem. They still run on push (fail-open) and whenever the + script, its siblings, or their tests change. * ``rust`` — ``cargo test`` for the Tauri bootstrap installer. ``.rs`` lives under ``apps/``, so without this lane a Rust change matched ``frontend`` and only the TypeScript matrix ran. @@ -118,6 +124,16 @@ _INSTALLER_FILES = {"scripts/install.ps1", "scripts/install.cmd"} # bootstrap-installer.yml lane — a real sandboxed install + stamp check. _BOOTSTRAP_PATHS = ("apps/bootstrap-installer/",) _BOOTSTRAP_FILES = {"scripts/install.sh", "setup-hermes.sh"} +# Windows desktop-update hand-off (scripts/desktop-update/windows.ps1 + the +# Electron side that launches it) and the pytest files that spawn it. +_DESKTOP_UPDATER_PATHS = ("scripts/desktop-update/",) +_DESKTOP_UPDATER_TEST_PREFIX = "tests/test_desktop_update_" +_DESKTOP_UPDATER_FILES = { + "apps/desktop/electron/updater-process.ts", + "apps/desktop/electron/managed-ssh-update.ts", + "tests/conftest.py", + "pyproject.toml", +} # Rust crates — currently just the Tauri bootstrap installer (Hermes-Setup). # These live under ``apps/``, so before this lane existed a ``.rs`` edit matched @@ -172,6 +188,14 @@ def _is_installer(p: str) -> bool: return p.startswith(_INSTALLER_PATHS) or p in _INSTALLER_FILES +def _is_desktop_updater(p: str) -> bool: + return ( + p.startswith(_DESKTOP_UPDATER_PATHS) + or p.startswith(_DESKTOP_UPDATER_TEST_PREFIX) + or p in _DESKTOP_UPDATER_FILES + ) + + def _is_rust(p: str) -> bool: return ( p.endswith(".rs") @@ -218,6 +242,7 @@ def classify(files: list[str]) -> dict[str, bool]: "bootstrap": any( f.startswith(_BOOTSTRAP_PATHS) or f in _BOOTSTRAP_FILES for f in files ), + "desktop_updater": any(_is_desktop_updater(f) for f in files), "rust": any(_is_rust(f) for f in files), "mcp_catalog": any(_is_mcp_catalog(f) for f in files), "ci_review": any(_is_ci_review(f) for f in files), @@ -236,6 +261,7 @@ def classify(files: list[str]) -> dict[str, bool]: ret["npm_lock"] = True ret["installer"] = True ret["bootstrap"] = True + ret["desktop_updater"] = True ret["rust"] = True ret["nix"] = True ret["ci_review"] = True diff --git a/scripts/desktop-update/windows.ps1 b/scripts/desktop-update/windows.ps1 index 7067cff335..19c9d8615a 100644 --- a/scripts/desktop-update/windows.ps1 +++ b/scripts/desktop-update/windows.ps1 @@ -212,6 +212,36 @@ function Start-UiServer([string]$HtmlPath) { }) [void]$ps.BeginInvoke() + # Readiness handshake. BeginInvoke returns before the runspace has + # opened its pipeline and JIT'd the script block — on a loaded machine + # that is seconds, during which the kernel ACCEPTS connections into + # the listener's backlog and nobody answers them. Anything that + # trusted "listener bound" as "server serving" (the browser window + # opening to a page that never loads; the -SelfTestUi URL that CI + # polls) raced that gap. Prove one /progress round-trip before + # handing the port out, so the URL means "serving", not "bound". + $ready = $false + $readyDeadline = [DateTime]::UtcNow.AddSeconds(15) + while (-not $ready -and [DateTime]::UtcNow -lt $readyDeadline) { + try { + $probe = [System.Net.HttpWebRequest]::Create("http://127.0.0.1:$port/progress") + $probe.Timeout = 1000 + $probe.ReadWriteTimeout = 1000 + $probe.KeepAlive = $false + $resp = $probe.GetResponse() + try { $ready = ([int]$resp.StatusCode -eq 200) } finally { $resp.Close() } + } catch { + Start-Sleep -Milliseconds 100 + } + } + if (-not $ready) { + Write-HandoffLog "progress server did not answer /progress within 15s; continuing without UI" + try { $listener.Stop() } catch {} + try { $ps.Stop() } catch {} + try { $rs.Close() } catch {} + return $null + } + return @{ Listener = $listener; Runspace = $rs; PowerShell = $ps; Port = $port; BrowserProc = $null; Profile = $null } } catch { try { if ($listener) { $listener.Stop() } } catch {} diff --git a/scripts/install.ps1 b/scripts/install.ps1 index f3c684b716..9ba37d11f7 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -318,6 +318,899 @@ function Stage-Complete { pinnedCommit = "$commit" pinnedBranch = $Branch completedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ") + +function Install-DesktopVoiceDeps { + # Desktop ships with working voice out of the box: eagerly install the + # wake-word + local-STT stacks ([wake] + [voice] extras) instead of + # leaving them to lazy first-use install. Policy change (Teknium, July + # 2026, #70509 testing): the first ear-click used to trigger a + # multi-minute onnxruntime pip install that froze the UI and blew RPC + # timeouts. Best-effort -- lazy install remains the fallback for anything + # this step fails to fetch. + if (-not $script:UvCmd) { Resolve-UvCmd } + if (-not $script:UvCmd) { + Write-Warn "uv unavailable -- voice/wake deps will lazy-install at first use instead" + return + } + $env:VIRTUAL_ENV = "$InstallDir\venv" + Write-Info "Installing voice + wake-word dependencies (onnxruntime, faster-whisper -- 1-3min)..." + Push-Location $InstallDir + try { + Invoke-NativeWithRelaxedErrorAction { & $UvCmd pip install -e ".[wake,voice]" } + if ($LASTEXITCODE -eq 0) { + Write-Success "Voice + wake-word dependencies installed" + } else { + Write-Warn "Voice/wake dependency install failed (exit $LASTEXITCODE) -- they will lazy-install at first use" + } + } finally { + Pop-Location + } +} + +function Install-Desktop { + # Build apps/desktop into a launchable Hermes.exe. Only called from + # Stage-Desktop, which is itself only included in the manifest when + # -IncludeDesktop was passed to install.ps1. + # + # The workspace npm install at repo root (done by Install-NodeDeps for + # browser tools) does NOT pull apps/desktop's dependencies, because the + # browser-tools workspace at $InstallDir\package.json is a separate + # workspace from apps/*. We do a full root-level `npm install` here + # so the workspace resolves apps/desktop's deps (including Electron + # itself, ~150MB), then run `npm run pack` in apps/desktop which + # produces the unpacked binary at apps/desktop/release/-unpacked/. + # + # The Tauri bootstrap installer's launch_hermes_desktop command + # resolves apps/desktop/release/win-unpacked/Hermes.exe directly, + # so an "unpacked" build (electron-builder --dir) is enough -- we + # don't need to produce an NSIS/MSI artifact here. + + # Always re-resolve Node here. Stages run in separate PowerShell processes, + # so $script:HasNode from Stage-Node isn't visible; more importantly Test-Node + # enforces the supported Node lines and prepends the Hermes-managed Node to + # PATH, so the build never runs on an unsupported system Node -- the cause + # of the opaque "Build desktop app ... exit code 1" failure (Vite crashes on + # old Node). + Test-Node | Out-Null + if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { + Write-Warn "Skipping desktop build (Node.js / npm not on PATH)" + $script:_StageSkippedReason = "Node.js not available" + return + } + + $desktopDir = "$InstallDir\apps\desktop" + if (-not (Test-Path "$desktopDir\package.json")) { + Write-Warn "Skipping desktop build (apps/desktop not present in checkout)" + $script:_StageSkippedReason = "apps/desktop not present" + return + } + + $npmCmd = Get-Command npm -ErrorAction SilentlyContinue + if (-not $npmCmd) { + Write-Warn "Skipping desktop build (npm not on PATH)" + $script:_StageSkippedReason = "npm not found" + return + } + $npmExe = $npmCmd.Source + if ($npmExe -like "*.ps1") { + $sibling = Join-Path (Split-Path $npmExe -Parent) "npm.cmd" + if (Test-Path $sibling) { $npmExe = $sibling } + } + + # 1. Workspace-level install so apps/desktop's deps (Electron, Vite, + # node-pty prebuilds, etc.) actually land in node_modules. This is + # the SAME `npm install` Install-NodeDeps does for browser tools, + # but at the root rather than the browser-tools workspace, so all + # apps/* workspaces resolve. + Write-Info "Installing desktop workspace dependencies (this includes Electron ~150MB, takes 1-3min)..." + Push-Location $InstallDir + $prevEAP = $ErrorActionPreference + try { + $ErrorActionPreference = "Continue" + # Drop --silent so npm emits its full progress + error trail. + # When this fails on a non-dev box (e.g. native-module build + # without VS Build Tools, ETARGET on a transitive, etc.), the + # actual reason needs to reach the Tauri installer's log; with + # --silent it was completely suppressed and the user just saw + # "exit 1" with no actionable detail. + # + # The streaming sink in bootstrap.rs's run_install_script + # captures every stdout/stderr line as it's emitted, so we don't + # need a side TEMP log file -- the installer's bootstrap log + # IS the artifact a support engineer reads. + # + # Prefer `npm ci`: it wipes node_modules and reinstalls from the + # lockfile, always producing a complete tree. Bare `npm install` + # can report "up to date" against a stale + # node_modules\.package-lock.json marker while node_modules is + # actually empty (Windows workspace-hoisting flake), leaving + # tsc/typescript unresolved so `npm run pack`'s `tsc -b` dies with + # no obvious cause. Fall back to `npm install` only if `npm ci` + # fails (lockfile out of sync / very old npm without ci). + # + # Tee the merged output into $npmOut while still emitting every line + # live. We don't need a side log file (the bootstrap streaming sink + # is the artifact), but on failure we scan $npmOut for the TLS-trust + # signature so corporate-proxy users get the NODE_EXTRA_CA_CERTS hint + # instead of an opaque "exit 1" (issue #38016). + & $npmExe ci 2>&1 | ForEach-Object { "$_" } | Tee-Object -Variable npmOut + $code = $LASTEXITCODE + if ($code -ne 0) { + Write-Info " npm ci failed (exit $code) -- retrying with npm install..." + & $npmExe install 2>&1 | ForEach-Object { "$_" } | Tee-Object -Variable npmOut + $code = $LASTEXITCODE + } + $ErrorActionPreference = $prevEAP + if ($code -ne 0) { + if (Test-ElectronPkgStagedMissingDist -InstallDir $InstallDir) { + Write-Warn "Desktop dependency install failed with a missing Electron dist; attempting self-heal..." + Try-RestoreElectronDist -InstallDir $InstallDir | Out-Null + } else { + Show-NpmCertHint ($npmOut -join "`n") | Out-Null + # Replay npm's own debug log into our stream: the terse + # summary above rarely contains the postinstall stderr + # (e.g. Electron's install.js) that explains the failure. + Write-NpmDebugLogTail -NpmOutput ($npmOut -join "`n") + throw "desktop workspace npm install failed (exit $code) -- see lines above for cause" + } + } else { + Write-Success "Desktop workspace dependencies installed" + } + } catch { + if ($prevEAP) { $ErrorActionPreference = $prevEAP } + Pop-Location + throw + } + Pop-Location + + # 2. Build apps/desktop. `npm run pack` runs: + # assert-root-install + write-build-stamp + stage-native-deps + + # tsc -b + vite build + electron-builder --dir + # The --dir mode produces an unpacked Hermes.exe in + # apps/desktop/release/win-unpacked/ without bundling NSIS/MSI; + # we don't need a distributable installer artifact, just a + # launchable binary the Tauri installer can spawn. + # + # CSC_IDENTITY_AUTO_DISCOVERY=false tells electron-builder we are + # NOT signing the output. Combined with signAndEditExecutable=false in + # apps/desktop/package.json's build.win block, electron-builder never + # invokes signtool and therefore never fetches/extracts winCodeSign + # (whose macOS symlinks crash 7-Zip on non-admin Windows -- a dead end we + # are NOT trying to work around). The Hermes icon + product name are + # stamped onto Hermes.exe by our own rcedit step (Set-DesktopExeIdentity) + # AFTER this build, completely decoupled from electron-builder signing. + # + # WIN_CSC_LINK and WIN_CSC_KEY_PASSWORD explicitly cleared as + # belt-and-suspenders: if the user's environment has them set + # for some other tool, electron-builder would still try to sign. + Write-Info "Building desktop app (this takes 1-3 minutes)..." + $buildLog = "$env:TEMP\hermes-desktop-build-$(Get-Random).log" + # Seed GITHUB_SHA for write-build-stamp.mjs. The stamp prefers CI env vars + # over `git rev-parse`, so this covers: (1) node can't find git.exe on PATH + # even though this PowerShell session can, (2) ZIP/init trees that still + # lack a HEAD after a failed post-extract fetch. Without it the desktop + # pack dies with "could not determine git commit" (#50823). + if (-not $env:GITHUB_SHA) { + if ($Commit) { + $env:GITHUB_SHA = $Commit + } else { + Push-Location $InstallDir + try { + $global:LASTEXITCODE = 0 + $resolvedSha = & git -c windows.appendAtomically=false rev-parse HEAD 2>$null + if ($LASTEXITCODE -ne 0 -or -not $resolvedSha) { + # ZIP path may have FETCH_HEAD after a fetch even when HEAD is unset. + $global:LASTEXITCODE = 0 + $resolvedSha = & git -c windows.appendAtomically=false rev-parse FETCH_HEAD 2>$null + } + if ($LASTEXITCODE -eq 0 -and $resolvedSha) { + $env:GITHUB_SHA = ("$resolvedSha").Trim() + } + } catch { } finally { + Pop-Location + } + } + } + if (-not $env:GITHUB_REF_NAME) { + $env:GITHUB_REF_NAME = if ($Branch) { $Branch } else { "main" } + } + if ($env:GITHUB_SHA) { + $shaPreview = if ($env:GITHUB_SHA.Length -ge 12) { $env:GITHUB_SHA.Substring(0, 12) } else { $env:GITHUB_SHA } + Write-Info "Desktop build stamp: $shaPreview ($($env:GITHUB_REF_NAME))" + } else { + Write-Warn "Could not resolve a git commit for the desktop stamp -- write-build-stamp will use its non-git fallback" + } + Push-Location $desktopDir + $prevEAP = $ErrorActionPreference + $prevCSCAuto = $env:CSC_IDENTITY_AUTO_DISCOVERY + $prevWinCscLink = $env:WIN_CSC_LINK + $prevWinCscKeyPassword = $env:WIN_CSC_KEY_PASSWORD + try { + $ErrorActionPreference = "Continue" + $env:CSC_IDENTITY_AUTO_DISCOVERY = "false" + $env:WIN_CSC_LINK = "" + $env:WIN_CSC_KEY_PASSWORD = "" + & $npmExe run pack 2>&1 | ForEach-Object { "$_" } | Tee-Object -FilePath $buildLog + $code = $LASTEXITCODE + if ($code -ne 0) { + $purged = @() + $restored = $false + if (-not (Test-ElectronDist -InstallDir $InstallDir)) { + $purged = @(Clear-ElectronBuildCache -DesktopDir $desktopDir) + $restored = Restore-ElectronDist -InstallDir $InstallDir + } + if ($restored) { + Write-Warn "Desktop build failed - refreshed the Electron download, retrying once:" + foreach ($p in $purged) { Write-Info " - $p" } + & $npmExe run pack 2>&1 | ForEach-Object { "$_" } | Tee-Object -FilePath $buildLog + $code = $LASTEXITCODE + } + } + if ($code -ne 0 -and -not $env:ELECTRON_MIRROR) { + $mirror = $script:DesktopElectronFallbackMirror + Write-Warn "Desktop build still failing - the Electron download from GitHub looks blocked." + Write-Warn "Re-downloading Electron via a public mirror ($mirror), then rebuilding:" + Write-Info " (set ELECTRON_MIRROR yourself to use a different/trusted mirror)" + if (-not (Test-ElectronDist -InstallDir $InstallDir)) { + Restore-ElectronDist -InstallDir $InstallDir -Mirror $mirror | Out-Null + } + $prevMirror = $env:ELECTRON_MIRROR + $env:ELECTRON_MIRROR = $mirror + try { + & $npmExe run pack 2>&1 | ForEach-Object { "$_" } | Tee-Object -FilePath $buildLog + $code = $LASTEXITCODE + } finally { + $env:ELECTRON_MIRROR = $prevMirror + } + } + $ErrorActionPreference = $prevEAP + if ($code -ne 0) { + $errText = Get-Content $buildLog -Raw -ErrorAction SilentlyContinue + if ($errText) { + $snippet = if ($errText.Length -gt 1800) { $errText.Substring(0, 1800) + "..." } else { $errText } + Write-Info " desktop build output:" + foreach ($line in $snippet -split "`n") { Write-Host " $line" -ForegroundColor DarkGray } + Write-Info " Full log: $buildLog" + } + # `npm run pack` failures (lifecycle script exits) also land in + # npm's debug log; replay it so the bootstrap log carries the + # full evidence even when $buildLog's tail cuts off the cause. + Write-NpmDebugLogTail -NpmOutput $errText + throw "apps/desktop build failed (exit $code)" + } + Write-Success "Desktop app built" + Remove-Item -LiteralPath $buildLog -Force -ErrorAction SilentlyContinue + } catch { + if ($prevEAP) { $ErrorActionPreference = $prevEAP } + Pop-Location + throw + } finally { + # Restore env to whatever the caller had -- don't leak our + # signing-off override into anything install.ps1 invokes later + # (Stage-PlatformSdks, etc.). + $env:CSC_IDENTITY_AUTO_DISCOVERY = $prevCSCAuto + $env:WIN_CSC_LINK = $prevWinCscLink + $env:WIN_CSC_KEY_PASSWORD = $prevWinCscKeyPassword + } + Pop-Location + + # 3. Sanity-check the produced binary. Probe both arches so this works + # on x64 and arm64 build machines. + $exeCandidates = @( + "$desktopDir\release\win-unpacked\Hermes.exe", + "$desktopDir\release\win-arm64-unpacked\Hermes.exe" + ) + $found = $false + $desktopExe = $null + foreach ($cand in $exeCandidates) { + if (Test-Path $cand) { + Write-Success "Desktop ready: $cand" + $desktopExe = $cand + $found = $true + break + } + } + if (-not $found) { + throw "Desktop build completed but no Hermes.exe was found under $desktopDir\release\*-unpacked\" + } + + # 3b. The Hermes icon + identity are stamped onto Hermes.exe by the + # electron-builder `afterPack` hook (apps/desktop/scripts/after-pack.mjs) + # during `npm run pack` above -- for every build, so the installer's + # --update rebuild stays branded too. No separate stamp step needed here. + # electron-builder's own rcedit step stays disabled (signAndEditExecutable + # =false) because enabling it drags in signtool -> winCodeSign -> the + # unfixable symlink crash; the afterPack hook runs rcedit directly. + + # 3c. Grant ALL APPLICATION PACKAGES (S-1-15-2-2) RX on the unpacked app + # directory. Chromium's GPU/renderer sandboxes CHECK-fail with + # 0x80000003 when this ACE is missing alongside orphan AppContainer + # SIDs under %LOCALAPPDATA% (electron/electron#51761, hermes-agent#38216). + # Best-effort -- never fail an otherwise-good install over ACL repair. + try { + $appDir = Split-Path -Parent $desktopExe + & icacls $appDir /grant "*S-1-15-2-2:(OI)(CI)(RX)" /T /C /Q | Out-Null + if ($LASTEXITCODE -eq 0) { + Write-Success "Granted AppContainer read access on $appDir" + } else { + Write-Warn "icacls AppContainer grant returned exit $LASTEXITCODE for $appDir" + } + } catch { + Write-Warn "Could not grant AppContainer ACL: $($_.Exception.Message)" + } + + # 4. Create Start Menu + Desktop shortcuts pointing DIRECTLY at the packed + # Hermes.exe. We deliberately do NOT point them at `hermes desktop`: that + # command rebuilds (npm install + electron-builder) on every launch, + # which would cost minutes each time. The packed exe is the consumer -- + # launching it directly is instant, and updates flow through the + # installer's --update path (which rebuilds once, then relaunches). + New-DesktopShortcuts -TargetExe $desktopExe +} + +function New-DesktopShortcuts { + param([Parameter(Mandatory = $true)][string]$TargetExe) + + # Best-effort: a shortcut failure must never fail an otherwise-good install. + try { + $shell = New-Object -ComObject WScript.Shell + $workDir = Split-Path -Parent $TargetExe + + # Prefer the standalone icon.ico (shipped beside the exe via + # electron-builder extraResources -> resources/icon.ico) over the exe's + # embedded resource. An explicit .ico path is more stable across update + # cycles: pointing at "$TargetExe,0" makes Windows cache the icon it + # extracted from the exe at shortcut-creation time, and that cached + # bitmap can persist (showing the OLD/Electron icon) even after the exe + # is re-stamped on update. A dedicated .ico sidesteps that extraction. + $iconIco = Join-Path $workDir 'resources\icon.ico' + if (Test-Path $iconIco) { + $iconLocation = "$iconIco,0" + } else { + $iconLocation = "$TargetExe,0" + } + + $targets = @( + (Join-Path ([Environment]::GetFolderPath('Programs')) 'Hermes.lnk'), + (Join-Path ([Environment]::GetFolderPath('Desktop')) 'Hermes.lnk') + ) + + foreach ($lnkPath in $targets) { + try { + $parent = Split-Path -Parent $lnkPath + if (-not (Test-Path $parent)) { + New-Item -ItemType Directory -Force -Path $parent | Out-Null + } + $sc = $shell.CreateShortcut($lnkPath) + $sc.TargetPath = $TargetExe + $sc.WorkingDirectory = $workDir + $sc.IconLocation = $iconLocation + $sc.Description = 'Hermes Agent' + $sc.Save() + Write-Success "Shortcut created: $lnkPath" + } catch { + Write-Warn "Could not create shortcut $lnkPath : $($_.Exception.Message)" + } + } + + # Bust the Windows shell icon cache so the desktop/Start-Menu shortcut + # repaints with the (possibly newly-stamped) icon instead of a stale + # cached bitmap. Critical on the --update path: the exe was re-stamped + # with the Hermes icon, but without this the shortcut can keep drawing + # the old Electron icon until the user manually refreshes / reboots. + # Best-effort and silent -- never fail the install over a cosmetic cache. + try { + & ie4uinit.exe -show 2>$null + } catch { + # ie4uinit may be absent/renamed on some SKUs -- ignore. + } + } catch { + Write-Warn "Skipping shortcut creation: $($_.Exception.Message)" + } +} + +function Install-PlatformSdks { + # Ensure messaging-platform SDKs matching tokens the user added to + # ~/.hermes/.env are importable. Two problems this solves: + # + # 1. The tiered `uv pip install` cascade above can fall through to a + # lower tier when the first fails (common when RL git deps choke), + # which silently skips some messaging SDKs from [messaging]. + # 2. `uv` creates the venv without pip. If a messaging SDK ends up + # missing, the user can't `pip install python-telegram-bot` to + # recover -- pip simply isn't in their venv. + # + # Strategy: bootstrap pip via `python -m ensurepip` (idempotent), then + # for each token set in .env, verify the matching SDK imports. If not, + # run one targeted `pip install` as last-chance recovery. Keeps fresh + # Windows installs from hitting silent "python-telegram-bot not installed" + # at runtime. + if ($NoVenv) { + Write-Info "Skipping platform-SDK verification (-NoVenv: no venv to bootstrap)" + return + } + + $pythonExe = "$InstallDir\venv\Scripts\python.exe" + if (-not (Test-Path $pythonExe)) { + Write-Warn "Skipping platform-SDK verification: $pythonExe not found" + return + } + + $envPath = "$HermesHome\.env" + if (-not (Test-Path $envPath)) { return } + $envLines = Get-Content $envPath -ErrorAction SilentlyContinue + + # Map: env var set in .env -> (import name, pip spec matching [messaging] extra). + # Specs mirror pyproject.toml to avoid version drift. + $sdkMap = @( + @{ Var = "TELEGRAM_BOT_TOKEN"; Import = "telegram"; Spec = "python-telegram-bot[webhooks]>=22.6,<23" }, + @{ Var = "DISCORD_BOT_TOKEN"; Import = "discord"; Spec = "discord.py[voice]>=2.7.1,<3" }, + @{ Var = "SLACK_BOT_TOKEN"; Import = "slack_sdk"; Spec = "slack-sdk>=3.27.0,<4" }, + @{ Var = "SLACK_APP_TOKEN"; Import = "slack_bolt";Spec = "slack-bolt>=1.18.0,<2" }, + @{ Var = "WHATSAPP_ENABLED"; Import = "qrcode"; Spec = "qrcode>=7.0,<8" } + ) + + # Which tokens are actually set (not placeholder)? + $needed = @() + foreach ($sdk in $sdkMap) { + $match = $envLines | Where-Object { + $_ -match ("^" + [regex]::Escape($sdk.Var) + "=.+") ` + -and $_ -notmatch "your-token-here" ` + -and $_ -notmatch "^\s*#" + } + if ($match) { $needed += $sdk } + } + if ($needed.Count -eq 0) { return } + + Write-Host "" + Write-Info "Verifying platform SDKs for tokens found in $envPath ..." + + # Verify each SDK's import without triggering side-effect imports. + # Quirk: PowerShell wraps non-zero-exit native stderr as a + # NativeCommandError that prints even with `2>$null` / `*> $null` + # unless we set $ErrorActionPreference to SilentlyContinue for the + # span. Save + restore rather than nuking globally. + $prevEAP = $ErrorActionPreference + $ErrorActionPreference = "SilentlyContinue" + try { + $missing = @() + foreach ($sdk in $needed) { + & $pythonExe -c "import $($sdk.Import)" 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + $missing += $sdk + Write-Warn " $($sdk.Import) NOT importable (needed for $($sdk.Var))" + } else { + Write-Success " $($sdk.Import) OK" + } + } + } finally { + $ErrorActionPreference = $prevEAP + } + if ($missing.Count -eq 0) { return } + + # Bootstrap pip into the venv if it isn't there. `uv` creates venvs + # without pip; ensurepip is the stdlib-blessed way to add it. + $prevEAP = $ErrorActionPreference + $ErrorActionPreference = "SilentlyContinue" + try { + & $pythonExe -m pip --version 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Info "Bootstrapping pip into venv (uv doesn't ship pip)..." + & $pythonExe -m ensurepip --upgrade 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Warn "ensurepip failed -- can't auto-install missing SDKs." + Write-Info "Manual recovery: $UvCmd pip install `"$($missing[0].Spec)`"" + return + } + } + + foreach ($sdk in $missing) { + Write-Info " Installing $($sdk.Spec) ..." + & $pythonExe -m pip install $sdk.Spec 2>&1 | ForEach-Object { Write-Host " $_" } + if ($LASTEXITCODE -eq 0) { + Write-Success " Installed $($sdk.Import)" + } else { + Write-Warn " Failed to install $($sdk.Spec). Recover manually: $pythonExe -m pip install `"$($sdk.Spec)`"" + } + } + } finally { + $ErrorActionPreference = $prevEAP + } +} + +function Invoke-SetupWizard { + if ($SkipSetup) { + Write-Info "Skipping setup wizard (-SkipSetup)" + return + } + + if ($NonInteractive) { + # The setup wizard prompts for API keys, model choice, persona, etc. + # Non-interactive callers (GUI installer) own that UX themselves; let + # them drive it after install.ps1 returns. + Write-Info "Skipping setup wizard (non-interactive). Configure via the GUI or 'hermes setup'." + return + } + + Write-Host "" + Write-Info "Starting setup wizard..." + Write-Host "" + + Push-Location $InstallDir + + # Run hermes setup using the venv Python directly (no activation needed) + if (-not $NoVenv) { + & ".\venv\Scripts\python.exe" -m hermes_cli.main setup + } else { + python -m hermes_cli.main setup + } + + Pop-Location +} + +function Start-GatewayIfConfigured { + $envPath = "$HermesHome\.env" + if (-not (Test-Path $envPath)) { return } + + $hasMessaging = $false + $content = Get-Content $envPath -ErrorAction SilentlyContinue + foreach ($var in @("TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN", "WHATSAPP_ENABLED")) { + $match = $content | Where-Object { $_ -match "^${var}=.+" -and $_ -notmatch "your-token-here" } + if ($match) { $hasMessaging = $true; break } + } + + if (-not $hasMessaging) { return } + + $hermesCmd = "$InstallDir\venv\Scripts\hermes.exe" + if (-not (Test-Path $hermesCmd)) { + $hermesCmd = "hermes" + } + + # If WhatsApp is enabled but not yet paired, run foreground for QR scan + $whatsappEnabled = $content | Where-Object { $_ -match "^WHATSAPP_ENABLED=true" } + $whatsappSession = "$HermesHome\whatsapp\session\creds.json" + if ($whatsappEnabled -and -not (Test-Path $whatsappSession)) { + Write-Host "" + Write-Info "WhatsApp is enabled but not yet paired." + Write-Info "Running 'hermes whatsapp' to pair via QR code..." + Write-Host "" + # Non-interactive callers (GUI installer, CI) skip the QR-pair prompt; + # WhatsApp pairing requires a human looking at a phone camera, so the + # downstream UI is responsible for surfacing this when it makes sense. + if (-not $NonInteractive) { + $response = Read-Host "Pair WhatsApp now? [Y/n]" + if ($response -eq "" -or $response -match "^[Yy]") { + try { + & $hermesCmd whatsapp + } catch { + # Expected after pairing completes + } + } + } else { + Write-Info "Skipping WhatsApp pairing prompt (non-interactive)." + } + } + + Write-Host "" + Write-Info "Messaging platform token detected!" + Write-Info "The gateway handles messaging platforms and cron job execution." + Write-Host "" + + # In non-interactive mode the gateway lifecycle is the caller's problem + # (the GUI manages its own gateway process, CI doesn't want background + # services on the build agent, etc.). Treat it like the user declined. + if ($NonInteractive) { + Write-Info "Skipping gateway autostart prompt (non-interactive)." + Write-Info "Start the gateway later with: hermes gateway" + return + } + + $response = Read-Host "Would you like to start the gateway now? [Y/n]" + + if ($response -eq "" -or $response -match "^[Yy]") { + Write-Info "Starting gateway in background..." + try { + $logFile = "$HermesHome\logs\gateway.log" + Start-Process -FilePath $hermesCmd -ArgumentList "gateway" ` + -RedirectStandardOutput $logFile ` + -RedirectStandardError "$HermesHome\logs\gateway-error.log" ` + -WindowStyle Hidden + Write-Success "Gateway started! Your bot is now online." + Write-Info "Logs: $logFile" + Write-Info "To stop: close the gateway process from Task Manager" + } catch { + Write-Warn "Failed to start gateway. Run manually: hermes gateway" + } + } else { + Write-Info "Skipped. Start the gateway later with: hermes gateway" + } +} + +function Write-Completion { + Write-Host "" + Write-Host "+---------------------------------------------------------+" -ForegroundColor Green + Write-Host "| [OK] Installation Complete! |" -ForegroundColor Green + Write-Host "+---------------------------------------------------------+" -ForegroundColor Green + Write-Host "" + + # Show file locations + Write-Host "* Your files:" -ForegroundColor Cyan + Write-Host "" + Write-Host " Config: " -NoNewline -ForegroundColor Yellow + Write-Host "$HermesHome\config.yaml" + Write-Host " API Keys: " -NoNewline -ForegroundColor Yellow + Write-Host "$HermesHome\.env" + Write-Host " Data: " -NoNewline -ForegroundColor Yellow + Write-Host "$HermesHome\cron\, sessions\, logs\" + Write-Host " Code: " -NoNewline -ForegroundColor Yellow + Write-Host "$HermesHome\hermes-agent\" + Write-Host "" + + Write-Host "---------------------------------------------------------" -ForegroundColor Cyan + Write-Host "" + Write-Host "* Commands:" -ForegroundColor Cyan + Write-Host "" + Write-Host " hermes " -NoNewline -ForegroundColor Green + Write-Host "Start chatting" + Write-Host " hermes setup " -NoNewline -ForegroundColor Green + Write-Host "Configure API keys & settings" + Write-Host " hermes config " -NoNewline -ForegroundColor Green + Write-Host "View/edit configuration" + Write-Host " hermes config edit " -NoNewline -ForegroundColor Green + Write-Host "Open config in editor" + Write-Host " hermes gateway " -NoNewline -ForegroundColor Green + Write-Host "Start messaging gateway (Telegram, Discord, etc.)" + Write-Host " hermes update " -NoNewline -ForegroundColor Green + Write-Host "Update to latest version" + Write-Host "" + + Write-Host "---------------------------------------------------------" -ForegroundColor Cyan + Write-Host "" + Write-Host "[*] Restart your terminal for PATH changes to take effect" -ForegroundColor Yellow + Write-Host "" + + if (-not $HasNode) { + Write-Host "Note: Node.js could not be installed automatically." -ForegroundColor Yellow + Write-Host "Browser tools need Node.js. Install manually:" -ForegroundColor Yellow + Write-Host " https://nodejs.org/en/download/" -ForegroundColor Yellow + Write-Host "" + } + + if (-not $HasRipgrep) { + Write-Host "Note: ripgrep (rg) was not installed. For faster file search:" -ForegroundColor Yellow + Write-Host " winget install BurntSushi.ripgrep.MSVC" -ForegroundColor Yellow + Write-Host "" + } +} + +# ============================================================================ +# Stage protocol +# ============================================================================ +# +# install.ps1 supports a small, stable "stage protocol" that lets programmatic +# callers (the desktop GUI's onboarding wizard, CI, future install.sh, etc.) +# drive the install one step at a time and surface progress/errors with their +# own UI. CLI users running the canonical `irm | iex` one-liner never +# encounter this -- default invocation behaves exactly as before. +# +# Entry points: +# +# install.ps1 Interactive install (today's behavior). +# install.ps1 -ProtocolVersion Emit the protocol version integer. +# install.ps1 -Manifest Emit the stage manifest as JSON. +# install.ps1 -Stage Run one stage and emit its result. +# install.ps1 -NonInteractive Disable all Read-Host prompts (also +# skips the setup wizard and the gateway +# autostart prompt). Can be combined +# with default invocation to do a full +# non-interactive install. +# install.ps1 -Json Emit machine-readable JSON instead of +# the human-readable success banner at +# the end of a full install. +# +# Manifest schema (the JSON returned by -Manifest): +# +# { +# "protocol_version": 1, +# "stages": [ +# { +# "name": "uv", +# "title": "Installing uv package manager", +# "category": "prereqs", +# "needs_user_input": false +# }, +# ... +# ] +# } +# +# Stage result (the JSON written by -Stage ): +# +# { +# "stage": "uv", +# "ok": true, +# "skipped": false, +# "reason": null, +# "duration_ms": 1234 +# } +# +# Exit codes: +# +# 0 -- success (stage ran, or stage was deliberately skipped). +# 1 -- generic failure; the stage threw. +# 2 -- unknown stage name passed to -Stage. +# +# Adding a stage: +# +# 1. Append an entry to $InstallStages below. +# 2. Make sure the worker function it points at is idempotent and respects +# $NonInteractive when it has prompts. Add it before "configure" +# (the wizard) or "gateway" (autostart) if it should run unconditionally; +# after those if it's optional post-install glue. +# 3. Do NOT bump $InstallStageProtocolVersion -- adding stages is additive. +# Drivers iterate the manifest dynamically. +# +# ============================================================================ + +# Stage definitions -- the single source of truth. Each entry maps a stable +# stage name (the API contract drivers depend on) to the worker function that +# implements it. ``Title`` is what UIs show; ``Category`` lets UIs group +# stages; ``NeedsUserInput`` tells UIs "this stage prompts -- either skip it +# or arrange to provide answers another way." +$InstallStages = @( + @{ Name = "uv"; Title = "Installing uv package manager"; Category = "prereqs"; NeedsUserInput = $false; Worker = "Stage-Uv" } + @{ Name = "git"; Title = "Installing Git"; Category = "prereqs"; NeedsUserInput = $false; Worker = "Stage-Git" } + @{ Name = "node"; Title = "Detecting Node.js"; Category = "prereqs"; NeedsUserInput = $false; Worker = "Stage-Node" } + @{ Name = "system-packages"; Title = "Installing ripgrep and ffmpeg"; Category = "prereqs"; NeedsUserInput = $false; Worker = "Stage-SystemPackages" } + @{ Name = "repository"; Title = "Cloning Hermes repository"; Category = "install"; NeedsUserInput = $false; Worker = "Stage-Repository" } + # Managed Python lives under $InstallDir\.hermes-runtime, so the checkout + # must exist before this stage creates that directory. Otherwise the later + # repository stage treats the runtime-only directory as a broken checkout, + # parks it, and leaves Stage-Venv with no managed interpreter. + @{ Name = "python"; Title = "Verifying Python $PythonVersion"; Category = "prereqs"; NeedsUserInput = $false; Worker = "Stage-Python" } + @{ Name = "venv"; Title = "Creating Python virtual environment"; Category = "install"; NeedsUserInput = $false; Worker = "Stage-Venv" } + @{ Name = "dependencies"; Title = "Installing Python dependencies"; Category = "install"; NeedsUserInput = $false; Worker = "Stage-Dependencies" } + @{ Name = "node-deps"; Title = "Installing Node.js dependencies"; Category = "install"; NeedsUserInput = $false; Worker = "Stage-NodeDeps" } +) +if ($IncludeDesktop) { + # Insert AFTER node-deps so workspace npm is already installed when + # the desktop build runs. Inserted only when explicitly requested + # (Hermes-Setup.exe), never via the irm|iex CLI one-liner. + $InstallStages += @{ Name = "desktop"; Title = "Building desktop app"; Category = "install"; NeedsUserInput = $false; Worker = "Stage-Desktop" } +} +$InstallStages += @( + @{ Name = "path"; Title = "Adding Hermes to PATH"; Category = "finalize"; NeedsUserInput = $false; Worker = "Stage-Path" } + @{ Name = "config-templates"; Title = "Writing configuration templates"; Category = "finalize"; NeedsUserInput = $false; Worker = "Stage-ConfigTemplates" } + @{ Name = "platform-sdks"; Title = "Installing messaging platform SDKs"; Category = "finalize"; NeedsUserInput = $false; Worker = "Stage-PlatformSdks" } + @{ Name = "bootstrap-marker"; Title = "Marking install complete"; Category = "finalize"; NeedsUserInput = $false; Worker = "Stage-BootstrapMarker" } + # Interactive stages. In non-interactive mode these become no-ops; the + # caller (GUI / CI) handles the equivalent UX themselves. + @{ Name = "configure"; Title = "Configuring API keys and models"; Category = "post-install"; NeedsUserInput = $true; Worker = "Stage-Configure" } + @{ Name = "gateway"; Title = "Starting messaging gateway"; Category = "post-install"; NeedsUserInput = $true; Worker = "Stage-Gateway" } +) + +# Stage workers -- thin wrappers that delegate to the existing Install-* / +# Test-* / Invoke-* functions while preserving their error semantics. Kept +# as a separate layer so the existing functions remain callable directly +# (helpful for one-off recovery: ``. install.ps1; Install-Venv``). +# +# Stages that depend on uv (anything after Stage-Uv) call Resolve-UvCmd +# first so they work in cross-process driver mode where $script:UvCmd +# set by Stage-Uv in a sibling powershell process is not visible here. +# Resolve-UvCmd is a fast no-op when $script:UvCmd is already populated +# (the default-invocation case where Main runs everything in one +# process), and throws cleanly if uv truly isn't installed yet. +function Stage-Uv { if (-not (Install-Uv)) { throw "uv installation failed" } } +function Stage-Python { Resolve-UvCmd; if (-not (Test-Python)) { throw "Python $PythonVersion not available" } } +function Stage-Git { + if (-not (Install-Git)) { + if ($script:GitInstallFailureReason) { throw $script:GitInstallFailureReason } + throw "Git not available and auto-install failed -- install from https://git-scm.com/download/win then re-run" + } +} +# Node is optional (browser tools degrade gracefully without it). Surface +# failure to the JSON contract as skipped=true / reason rather than ok=true, +# so a GUI driver consuming the manifest can distinguish "node ready" from +# "node missing". Install flow continues either way -- matches the +# existing Write-Completion behavior that prints a "Note: Node.js could +# not be installed" hint instead of aborting. +function Stage-Node { + if (-not (Test-Node)) { + $script:_StageSkippedReason = "Node.js not available; browser tools will be unavailable until node is installed manually from https://nodejs.org/en/download/" + } +} +function Stage-SystemPackages { Install-SystemPackages } +function Stage-Repository { Install-Repository } +function Stage-Venv { Resolve-UvCmd; Install-Venv } +function Stage-Dependencies { Resolve-UvCmd; Install-Dependencies } +function Stage-NodeDeps { Install-NodeDeps } +function Stage-Desktop { Install-DesktopVoiceDeps; Install-Desktop } +function Stage-Path { Set-PathVariable } +function Stage-ConfigTemplates { Copy-ConfigTemplates } +function Stage-PlatformSdks { Resolve-UvCmd; Install-PlatformSdks } +function Stage-BootstrapMarker { Write-BootstrapMarker } +function Stage-Configure { Invoke-SetupWizard } +function Stage-Gateway { Start-GatewayIfConfigured } + +function Get-InstallStage { + param([string]$Name) + foreach ($s in $InstallStages) { + if ($s.Name -eq $Name) { return $s } + } + return $null +} + +function Step-OutOfInstallDir { + # Windows refuses to delete a directory any shell is currently cd'd + # inside -- and silently leaves orphan files behind, which then wedge + # "is this a valid git repo" probes on re-install. Harmless when the + # caller ran the installer from somewhere else. + try { + $currentResolved = (Get-Location).ProviderPath + $installResolved = $null + if (Test-Path $InstallDir) { + $installResolved = (Resolve-Path $InstallDir -ErrorAction SilentlyContinue).ProviderPath + } + if ($installResolved -and $currentResolved.ToLower().StartsWith($installResolved.ToLower())) { + Write-Info "Stepping out of $InstallDir so Windows can replace files there if needed..." + Set-Location $env:USERPROFILE + } + } catch {} +} + +function Invoke-Stage { + param( + [Parameter(Mandatory=$true)] [hashtable]$StageDef + ) + + # Refresh PATH from registry so this stage sees binaries installed by + # prior stages, even when each stage runs in its own powershell process. + # No-op in cost-relevant cases (default invocation path syncs once per + # foreach pass; cross-process drivers get the necessary freshening). + Sync-EnvPath + + # Per-stage soft-skip channel. A worker can populate + # $script:_StageSkippedReason to surface "ran, but the thing it was + # supposed to set up is not available" as skipped=true in the JSON + # frame, without throwing. Used by Stage-Node so the install flow + # doesn't abort when an optional capability is missing while still + # being honest in the protocol contract. Reset before each stage so + # a prior stage's reason can never leak into a later stage's frame. + $script:_StageSkippedReason = $null + + $start = [DateTime]::UtcNow + $result = @{ + stage = $StageDef.Name + ok = $false + skipped = $false + reason = $null + duration_ms = 0 + } + + try { + & $StageDef.Worker + $result.ok = $true + if ($script:_StageSkippedReason) { + $result.skipped = $true + $result.reason = $script:_StageSkippedReason + } + } catch { + $result.ok = $false + $result.reason = "$_" + throw + } finally { + $result.duration_ms = [int]([DateTime]::UtcNow - $start).TotalMilliseconds + if ($Json -or $Stage) { + # In stage-driver mode every stage emits a JSON line so the + # caller can stream progress. In default interactive mode we + # stay silent here (the worker already wrote human output). + $result | ConvertTo-Json -Compress | Write-Output + # Tell the entry-point catch that we've already emitted a + # frame for this failure (when $result.ok = $false), so it + # doesn't double-emit a second JSON object and break the + # one-line-per-stage contract the driver protocol promises. + if (-not $result.ok) { + $script:_StageEmittedErrorFrame = $true + } } $markerPath = Join-Path $InstallDir ".hermes-bootstrap-complete" $tmp = "$markerPath.tmp" diff --git a/tests/agent/lsp/_mock_lsp_server.py b/tests/agent/lsp/_mock_lsp_server.py index d7ce410151..57bfedbe19 100644 --- a/tests/agent/lsp/_mock_lsp_server.py +++ b/tests/agent/lsp/_mock_lsp_server.py @@ -103,6 +103,14 @@ def main(): if msg.get("method") == "workspace/didChangeWatchedFiles": continue + if msg.get("method") == "workspace/didChangeWorkspaceFolders": + # Multi-root tests observe attached folders through this log. + log_path = os.environ.get("MOCK_LSP_FOLDERS_LOG") + if log_path: + with open(log_path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(msg.get("params")) + "\n") + continue + if msg.get("method") in {"textDocument/didOpen", "textDocument/didChange"}: params = msg.get("params") or {} td = params.get("textDocument") or {} diff --git a/tests/agent/lsp/test_multi_root.py b/tests/agent/lsp/test_multi_root.py new file mode 100644 index 0000000000..4f05aaac80 --- /dev/null +++ b/tests/agent/lsp/test_multi_root.py @@ -0,0 +1,125 @@ +"""Multi-root servers share ONE process across project roots. + +A profiled session with subagents editing across ~30 git worktrees ran +30-60 pyright processes. Pyright supports multi-root workspaces, so +the service keys such clients by ``server_id`` alone and attaches each +new root via ``workspace/didChangeWorkspaceFolders``. Single-root +servers keep the one-client-per-root behaviour. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +from agent.lsp.manager import LSPService +from agent.lsp.servers import SERVERS, ServerContext, ServerDef, SpawnSpec +from agent.lsp.workspace import clear_cache + +MOCK_SERVER = str(Path(__file__).parent / "_mock_lsp_server.py") + + +@pytest.fixture(autouse=True) +def _clear_workspace_cache(): + clear_cache() + yield + clear_cache() + + +def _make_repo(tmp_path: Path, name: str) -> Path: + repo = tmp_path / name + repo.mkdir() + (repo / ".git").mkdir() + (repo / "pyproject.toml").write_text("", encoding="utf-8") + (repo / "x.py").write_text("print('hi')\n", encoding="utf-8") + return repo + + +@pytest.fixture +def two_repos(tmp_path): + return _make_repo(tmp_path, "repo-a"), _make_repo(tmp_path, "repo-b") + + +@pytest.fixture +def mock_pyright(monkeypatch, tmp_path): + """Install the mock as ``pyright``; yield (spawn_count, folders_log, set_multi_root).""" + idx = next(i for i, s in enumerate(SERVERS) if s.server_id == "pyright") + original = SERVERS[idx] + spawns = {"value": 0} + folders_log = tmp_path / "folders.jsonl" + + def _spawn(root: str, ctx: ServerContext) -> SpawnSpec: + spawns["value"] += 1 + return SpawnSpec( + command=[sys.executable, MOCK_SERVER], + workspace_root=root, + cwd=root, + env={"MOCK_LSP_SCRIPT": "errors", "MOCK_LSP_FOLDERS_LOG": str(folders_log)}, + ) + + def _install(multi_root: bool) -> None: + SERVERS[idx] = ServerDef( + server_id="pyright", + extensions=original.extensions, + resolve_root=lambda fp, ws: ws, + build_spawn=_spawn, + multi_root=multi_root, + description="mock pyright", + ) + + yield spawns, folders_log, _install + SERVERS[idx] = original + + +def _service() -> LSPService: + return LSPService( + enabled=True, wait_mode="document", wait_timeout=3.0, install_strategy="manual" + ) + + +def test_multi_root_server_shares_one_client_across_roots(two_repos, mock_pyright, monkeypatch): + repo_a, repo_b = two_repos + spawns, folders_log, install = mock_pyright + install(multi_root=True) + svc = _service() + try: + monkeypatch.chdir(str(repo_a)) + diags_a = svc.get_diagnostics_sync(str(repo_a / "x.py")) + monkeypatch.chdir(str(repo_b)) + diags_b = svc.get_diagnostics_sync(str(repo_b / "x.py")) + + # Exactly one process; the second root arrived as a folder change. + assert spawns["value"] == 1 + assert len(svc._clients) == 1 + client = next(iter(svc._clients.values())) + assert client.workspace_folders == [str(repo_a), str(repo_b)] + events = [json.loads(line) for line in folders_log.read_text(encoding="utf-8").splitlines()] + assert [f["uri"] for e in events for f in e["event"]["added"]] == [ + Path(repo_b).as_uri() + ] + # Diagnostics still resolve per file in both folders. + assert len(diags_a) == 1 and len(diags_b) == 1 + status = svc.get_status()["clients"][0] + assert status["workspace_root"] == str(repo_a) + assert status["workspace_folders"] == [str(repo_a), str(repo_b)] + finally: + svc.shutdown() + + +def test_single_root_server_still_spawns_per_root(two_repos, mock_pyright, monkeypatch): + repo_a, repo_b = two_repos + spawns, folders_log, install = mock_pyright + install(multi_root=False) + svc = _service() + try: + monkeypatch.chdir(str(repo_a)) + svc.get_diagnostics_sync(str(repo_a / "x.py")) + monkeypatch.chdir(str(repo_b)) + svc.get_diagnostics_sync(str(repo_b / "x.py")) + assert spawns["value"] == 2 + assert set(svc._clients) == {("pyright", str(repo_a)), ("pyright", str(repo_b))} + assert not folders_log.exists() + finally: + svc.shutdown() diff --git a/tests/agent/lsp/test_workspace.py b/tests/agent/lsp/test_workspace.py index a59d9976ca..801148cc35 100644 --- a/tests/agent/lsp/test_workspace.py +++ b/tests/agent/lsp/test_workspace.py @@ -50,6 +50,19 @@ def test_nearest_root_finds_first_marker(tmp_path: Path): assert found == str(root) +def test_nearest_root_skips_package_dirs(tmp_path: Path): + # hermes_cli/setup.py is a module inside a package, not a project + # marker; treating it as one spawned a second pyright per worktree. + root = tmp_path / "p" + pkg = root / "hermes_cli" + pkg.mkdir(parents=True) + (root / "pyproject.toml").write_text("") + (pkg / "__init__.py").write_text("") + (pkg / "setup.py").write_text("") + found = nearest_root(str(pkg / "main.py"), ["pyproject.toml", "setup.py"]) + assert found == str(root) + + diff --git a/tests/agent/test_acp_openai_bridge.py b/tests/agent/test_acp_openai_bridge.py index d1c0402604..f687461818 100644 --- a/tests/agent/test_acp_openai_bridge.py +++ b/tests/agent/test_acp_openai_bridge.py @@ -200,7 +200,10 @@ def test_copilot_prompt_still_carries_the_contract_and_the_tools(): assert "{...}" in prompt assert '"name": "memory"' in prompt assert '"name": "read_file"' in prompt # copilot forwards everything - assert "Hermes requested model hint: gpt-5" in prompt + # No prompt-text model mention: the model is applied via ACP + # session/set_model, and a prompt hint makes a substituted backend + # falsely self-identify as the requested model. + assert "model hint" not in prompt assert "hi" in prompt diff --git a/tests/agent/test_anthropic_adapter.py b/tests/agent/test_anthropic_adapter.py index 9619925194..61754a4307 100644 --- a/tests/agent/test_anthropic_adapter.py +++ b/tests/agent/test_anthropic_adapter.py @@ -983,19 +983,23 @@ class TestBuildAnthropicKwargs: def test_supports_fast_mode_predicate(self): - """Fast mode is Opus 4.6 only — Opus 4.7 and others must be excluded. + """The speed-param allowlist tracks the live fast-mode docs. - For Opus 4.8 the fast variant is a separate model ID - (anthropic/claude-opus-4.8-fast) routed through the normal model - field, NOT via the ``speed: "fast"`` request parameter. So - ``_supports_fast_mode`` (which gates the parameter) must stay - False for both opus-4-8 and opus-4-8-fast. + Per https://platform.claude.com/docs/en/build-with-claude/fast-mode: + Opus 4.8 and Opus 5 support ``speed: "fast"``. Opus 4.6 LOST fast + mode (param silently ignored → standard speed at standard billing); + Opus 4.7 hard-400s. Dedicated ``…-fast`` model ids select fast + inference via the model field and must not also get the param. """ from agent.anthropic_adapter import _supports_fast_mode - assert _supports_fast_mode("claude-opus-4-6") is True - assert _supports_fast_mode("anthropic/claude-opus-4-6") is True + assert _supports_fast_mode("claude-opus-4-8") is True + assert _supports_fast_mode("claude-opus-4.8") is True + assert _supports_fast_mode("anthropic/claude-opus-4-8") is True + assert _supports_fast_mode("claude-opus-5") is True + assert _supports_fast_mode("anthropic/claude-opus-5") is True + assert _supports_fast_mode("claude-opus-4-6") is False + assert _supports_fast_mode("anthropic/claude-opus-4-6") is False assert _supports_fast_mode("claude-opus-4-7") is False - assert _supports_fast_mode("claude-opus-4-8") is False assert _supports_fast_mode("claude-opus-4-8-fast") is False assert _supports_fast_mode("claude-sonnet-4-6") is False assert _supports_fast_mode("claude-haiku-4-5") is False diff --git a/tests/agent/test_aux_stream_host_deadline.py b/tests/agent/test_aux_stream_host_deadline.py new file mode 100644 index 0000000000..924b0766a8 --- /dev/null +++ b/tests/agent/test_aux_stream_host_deadline.py @@ -0,0 +1,295 @@ +"""#99692 — the streamed auxiliary summary must not outlive its compression host. + +Background +---------- +``run_compress_context_with_progress_timeout`` arms a wall-clock deadline on the +``CompressionCommitFence`` (``set_total_ceiling_seconds``), whose docstring calls +it "the wall-clock deadline **shared by the host and worker**". Only the host +ever read it. + +``8207862212`` (fix(compression): stop timeout paths from blocking retries) +closed the first half: a cancelled fence now releases the compression OWNER, +which frees the pool slot and the session lease. It left the second half open +by design — its own comment says the isolated provider daemon runs on "until +the auxiliary stream's longer absolute ceiling expires". + +That ceiling is ``_aux_stream_total_ceiling`` = ``max(600, 4 * aux_timeout)``: +>= the default host ceiling (600s) for every configured timeout, and it starts +counting later (after pool admission, serialization, prompt build and TTFT). +So the daemon holding the socket is *always* still streaming when its host gives +up — 2400s with the reporter's ``auxiliary.compression.timeout: 600`` — billing +every token of a summary the fence is already guaranteed to refuse, and stacking +one fresh orphan per turn because the session never shrank. + +These tests pin the missing half of that shared deadline: the stream consumer +must stop at the host's deadline, including on the isolated provider daemon +that ``_run_protected_sync_provider_call`` spawns. +""" + +from __future__ import annotations + +import ast +import asyncio +import inspect +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from agent import auxiliary_client as aux +from agent.conversation_compression import ( + DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS, + CompressionCommitFence, +) + + +def _chunk(text: str) -> SimpleNamespace: + return SimpleNamespace( + id="resp-1", + model="test-model", + usage=None, + choices=[ + SimpleNamespace( + index=0, + finish_reason=None, + delta=SimpleNamespace(content=text, tool_calls=None), + ) + ], + ) + + +class _Stream: + """Chunk iterator that records how far the consumer drained it.""" + + def __init__(self, count: int = 50) -> None: + self._count = count + self.yielded = 0 + self.closed = False + + def __iter__(self): + for _ in range(self._count): + self.yielded += 1 + yield _chunk("x") + + def close(self) -> None: + self.closed = True + + +class _AsyncStream(_Stream): + async def __aiter__(self): # pragma: no cover - exercised via asyncio.run + for _ in range(self._count): + self.yielded += 1 + yield _chunk("x") + + +# ── The structural gap the bug lives in ────────────────────────────────── + + +def test_stream_ceiling_structurally_outlives_the_default_host_ceiling(): + """The worker's own budget is >= the host's for every configured timeout. + + This is the arithmetic that guarantees the orphan: there is no aux timeout + for which ``_aux_stream_total_ceiling`` lands below the 600s default host + ceiling, and the reporter's ``auxiliary.compression.timeout: 600`` puts it + at 2400s — a 30-minute window in which an abandoned provider daemon keeps + streaming a summary nobody can commit. + """ + for aux_timeout in (None, 0, 30.0, 120.0, 300.0): + assert ( + aux._aux_stream_total_ceiling(aux_timeout) + >= DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS + ) + assert aux._aux_stream_total_ceiling(600.0) == 2400.0 + assert ( + aux._aux_stream_total_ceiling(600.0) + - DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS + == 1800.0 + ) + + +# ── The fence must publish the deadline it already owns ────────────────── + + +def test_commit_fence_publishes_its_shared_deadline(): + fence = CompressionCommitFence() + assert fence.deadline_monotonic is None + + fence.set_total_ceiling_seconds(600.0) + published = fence.deadline_monotonic + assert published is not None + assert 590.0 < published - time.monotonic() <= 600.0 + assert not fence.deadline_exceeded + + fence.set_total_ceiling_seconds(0.001) + time.sleep(0.01) + assert fence.deadline_exceeded + assert fence.deadline_monotonic <= time.monotonic() + + +# ── The stream consumer must honour it ─────────────────────────────────── + + +def test_streamed_summary_stops_at_an_elapsed_host_deadline(): + """A host that already gave up must not leave the worker streaming on.""" + stream = _Stream(count=50) + with aux.aux_stream_deadline(time.monotonic() - 1.0): + with pytest.raises(TimeoutError) as excinfo: + aux._aggregate_chat_stream(stream, model="m", total_ceiling=2400.0) + + # "timed out" keeps _is_timeout_error classification identical to a + # request timeout, so the existing recovery chains are unchanged. + assert "timed out" in str(excinfo.value) + assert "host compression deadline" in str(excinfo.value) + # Stopped on the first frame instead of draining the whole stream, and the + # HTTP response was closed rather than left dangling. + assert stream.yielded == 1 + assert stream.closed is True + + +def test_streamed_summary_runs_to_completion_under_a_live_host_deadline(): + stream = _Stream(count=5) + with aux.aux_stream_deadline(time.monotonic() + 600.0): + response = aux._aggregate_chat_stream( + stream, model="m", total_ceiling=2400.0 + ) + assert response.choices[0].message.content == "xxxxx" + assert stream.yielded == 5 + + +def test_no_host_deadline_keeps_the_historical_ceiling_behaviour(): + """Every non-compression aux caller must be byte-for-byte unchanged.""" + stream = _Stream(count=5) + response = aux._aggregate_chat_stream(stream, model="m", total_ceiling=2400.0) + assert response.choices[0].message.content == "xxxxx" + assert stream.yielded == 5 + + # An installed-then-exited scope must not leak into the next call. + with aux.aux_stream_deadline(time.monotonic() - 1.0): + pass + stream2 = _Stream(count=3) + assert ( + aux._aggregate_chat_stream( + stream2, model="m", total_ceiling=2400.0 + ).choices[0].message.content + == "xxx" + ) + + +def test_none_deadline_is_a_no_op_passthrough(): + """Callers wire the scope unconditionally; a fenceless call must not break.""" + stream = _Stream(count=3) + with aux.aux_stream_deadline(None): + response = aux._aggregate_chat_stream( + stream, model="m", total_ceiling=2400.0 + ) + assert response.choices[0].message.content == "xxx" + + +def test_nested_none_inherits_rather_than_escaping_the_host_deadline(): + """A fenceless aux call nested inside a fenced one stays bounded. + + ``None`` means "I have no deadline of my own", not "clear the one in + force" — mirroring ``_aux_thread_local_hook``'s passthrough contract. If it + cleared, any nested auxiliary call made during compression would escape the + host ceiling that the whole attempt is supposed to live inside. + """ + outer = time.monotonic() - 1.0 + stream = _Stream(count=50) + with aux.aux_stream_deadline(outer): + with aux.aux_stream_deadline(None): + assert aux._current_aux_stream_deadline() == outer + with pytest.raises(TimeoutError): + aux._aggregate_chat_stream(stream, model="m", total_ceiling=2400.0) + assert stream.yielded == 1 + + +def test_deadline_scope_restores_the_previous_value(): + outer = time.monotonic() + 900.0 + with aux.aux_stream_deadline(outer): + assert aux._current_aux_stream_deadline() == outer + with aux.aux_stream_deadline(time.monotonic() + 10.0): + assert aux._current_aux_stream_deadline() != outer + assert aux._current_aux_stream_deadline() == outer + assert aux._current_aux_stream_deadline() is None + + +def test_async_stream_mirror_honours_the_host_deadline(): + """The async consumer must not drift from the sync one.""" + stream = _AsyncStream(count=50) + + async def _run(): + with aux.aux_stream_deadline(time.monotonic() - 1.0): + return await aux._aggregate_chat_stream_async( + stream, model="m", total_ceiling=2400.0 + ) + + with pytest.raises(TimeoutError): + asyncio.run(_run()) + assert stream.yielded == 1 + + +# ── The isolated provider daemon must inherit it ───────────────────────── + + +def test_protected_provider_daemon_inherits_the_host_deadline(): + """``_run_protected_sync_provider_call`` runs the stream on ANOTHER thread. + + Thread-locals do not cross that boundary, so without explicit propagation + the fix would be inert on exactly the path large-session compression takes + (protected + hard-cancel source installed). + """ + seen: dict[str, object] = {} + + def _callback(_kwargs): + seen["deadline"] = aux._current_aux_stream_deadline() + seen["thread"] = threading.current_thread().name + return "ok" + + deadline = time.monotonic() + 42.0 + cancel_event = threading.Event() + with aux.aux_progress_hook(lambda: None), aux.aux_interrupt_protection( + cancel_event=cancel_event + ), aux.aux_stream_deadline(deadline): + assert aux._run_protected_sync_provider_call(_callback, {}) == "ok" + + assert seen["thread"] == "hermes-protected-aux-provider" + assert seen["deadline"] == deadline + + +# ── The compression worker must actually install it ────────────────────── + + +def _summary_dispatch_source() -> str: + from agent import conversation_compression + + path = Path(inspect.getsourcefile(conversation_compression)) + return path.read_text(encoding="utf-8") + + +def test_compression_summary_dispatch_installs_the_fence_deadline(): + """Source guard: the wiring is one line and trivially droppable. + + A behavioural test would have to drive the whole ``compress_context`` body + (durable lock, watermark, telemetry, commit). This asserts the seam itself: + the same ``with`` statement that installs the progress hook must also + install the stream deadline. + """ + tree = ast.parse(_summary_dispatch_source()) + wired = False + for node in ast.walk(tree): + if not isinstance(node, ast.With): + continue + names = set() + for item in node.items: + call = item.context_expr + if isinstance(call, ast.Call) and isinstance(call.func, ast.Name): + names.add(call.func.id) + if "aux_progress_hook" in names: + assert "aux_stream_deadline" in names, ( + "the summary dispatch scope installs the progress hook but not " + "the host stream deadline — #99692 would regress" + ) + wired = True + assert wired, "summary dispatch scope not found" diff --git a/tests/agent/test_aux_stream_host_deadline_sibling_wires.py b/tests/agent/test_aux_stream_host_deadline_sibling_wires.py new file mode 100644 index 0000000000..250b5ac132 --- /dev/null +++ b/tests/agent/test_aux_stream_host_deadline_sibling_wires.py @@ -0,0 +1,189 @@ +"""#99692 sibling wires — the host compression deadline must stop EVERY aux +stream consumer, not only the chat.completions accumulator. + +``aux_stream_deadline`` (salvaged from PR #99779 by @JoaoMarcos44) publishes +the ``CompressionCommitFence`` ceiling to the streamed chat.completions path. +Two other auxiliary wires consume their streams internally and were left with +their own, always-larger budgets: + +* the Codex Responses adapter (``_CodexCompletionsAdapter.create``) — its + re-armable watchdog only knew ``_aux_stream_total_ceiling`` (>= 600s); +* the Anthropic Messages adapter — its ``on_stream_event`` hook only ticked + progress and never stopped the stream at all (nor honoured a hard cancel). + +Both now stop at the host's absolute deadline, so an abandoned summary is not +billed to completion on a socket nobody is waiting for. +""" + +from __future__ import annotations + +import time +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from agent import auxiliary_client as aux +from agent.anthropic_adapter import create_anthropic_message + + +# ── Codex Responses wire ───────────────────────────────────────────────── + + +def _codex_content_event(text="tok"): + return SimpleNamespace(type="response.output_text.delta", delta=text) + + +def _consume_codex(stream, *, model, on_event): + del model + for event in stream: + on_event(event) + return SimpleNamespace( + output=[SimpleNamespace( + type="message", + content=[SimpleNamespace(type="output_text", text="summary")], + )], + usage=None, + ) + + +def _make_codex_adapter(event_iter): + real_client = SimpleNamespace( + base_url="https://chatgpt.com/backend-api/codex", + responses=SimpleNamespace(create=lambda **_kwargs: event_iter), + close=lambda: None, + ) + return aux._CodexCompletionsAdapter(real_client, "gpt-5.6-sol") + + +def test_codex_stream_stops_at_the_host_deadline_not_its_own_ceiling(): + """A live (re-arming) Codex stream must die at the host's deadline even + though its own hard ceiling is >= 600s and every token re-arms the + no-progress window.""" + yielded = [0] + + def _live_forever(): + while True: + time.sleep(0.02) + yielded[0] += 1 + yield _codex_content_event() + + adapter = _make_codex_adapter(_live_forever()) + start = time.monotonic() + with ( + patch("agent.codex_runtime._consume_codex_event_stream", _consume_codex), + aux.aux_stream_deadline(time.monotonic() + 0.4), + pytest.raises(TimeoutError, match="hard ceiling"), + ): + adapter.create( + messages=[{"role": "user", "content": "summarize"}], + timeout=300, + ) + elapsed = time.monotonic() - start + assert elapsed < 5.0, f"stream outlived the host deadline by {elapsed:.1f}s" + assert yielded[0] < 100 + + +def test_codex_stream_without_host_deadline_keeps_its_ceiling(): + def _short(): + for _ in range(3): + yield _codex_content_event() + + adapter = _make_codex_adapter(_short()) + with patch("agent.codex_runtime._consume_codex_event_stream", _consume_codex): + response = adapter.create( + messages=[{"role": "user", "content": "summarize"}], timeout=300, + ) + assert response.choices[0].message.content == "summary" + + +# ── Anthropic Messages wire ────────────────────────────────────────────── + + +class _AnthropicStream: + def __init__(self, count=10_000, delay=0.01): + self._count, self._delay = count, delay + self.yielded = 0 + self.exited = False + self.response = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.exited = True + return False + + def __iter__(self): + for _ in range(self._count): + time.sleep(self._delay) + self.yielded += 1 + yield SimpleNamespace( + type="content_block_delta", delta=SimpleNamespace(text="tok"), + ) + + def get_final_message(self): + return SimpleNamespace(content=[SimpleNamespace(type="text", text="summary")]) + + +def _anthropic_client(stream): + return SimpleNamespace( + messages=SimpleNamespace( + stream=lambda **_kw: stream, + create=lambda **_kw: pytest.fail("must not fall back to create()"), + ) + ) + + +def test_anthropic_stream_stops_at_the_host_deadline(): + stream = _AnthropicStream() + ticks = [] + with ( + aux.aux_progress_hook(lambda: ticks.append(1)), + aux.aux_stream_deadline(time.monotonic() + 0.3), + ): + hook = aux._anthropic_aux_stream_event_hook() + start = time.monotonic() + with pytest.raises(TimeoutError, match="timed out at the host compression deadline"): + create_anthropic_message( + _anthropic_client(stream), {"model": "m", "messages": []}, + on_stream_event=hook, + ) + assert time.monotonic() - start < 5.0 + assert stream.exited, "stream context must be closed on the deadline" + assert ticks, "substantive deltas must still tick the progress hook" + assert stream.yielded < 1000 + + +def test_anthropic_stream_honours_an_explicit_hard_cancel(): + stream = _AnthropicStream() + cancelled = {"v": False} + with ( + aux.aux_progress_hook(lambda: None), + aux.aux_interrupt_protection(cancel_check=lambda: cancelled["v"]), + ): + hook = aux._anthropic_aux_stream_event_hook() + + def _flip_after_first(event, _inner=hook): + cancelled["v"] = True + _inner(event) + + with pytest.raises(aux.AuxiliaryExplicitCancellation): + create_anthropic_message( + _anthropic_client(stream), {"model": "m", "messages": []}, + on_stream_event=_flip_after_first, + ) + assert stream.yielded == 1 + assert stream.exited + + +def test_anthropic_stream_without_host_deadline_runs_to_completion(): + stream = _AnthropicStream(count=5, delay=0) + with aux.aux_progress_hook(lambda: None): + hook = aux._anthropic_aux_stream_event_hook() + message = create_anthropic_message( + _anthropic_client(stream), {"model": "m", "messages": []}, + on_stream_event=hook, + ) + assert message.content[0].text == "summary" + assert stream.yielded == 5 diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index bd970dd972..c0c485ea72 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -2118,6 +2118,139 @@ class TestTransientTransportRetry: assert primary.chat.completions.create.call_count == 1 assert fb_client.chat.completions.create.call_count == 1 + def test_vision_skips_same_provider_retry_on_timeout(self): + """Vision is on the interactive critical path: the turn holding the + image cannot answer, and because turns are serialised the following + user messages stall behind it. A full-budget timeout must therefore + fall straight through to fallback rather than spending a second + ``timeout`` window on the same provider (same reasoning as #54465). + """ + class _Timeout(Exception): + pass + _Timeout.__name__ = "APITimeoutError" + + primary = MagicMock() + primary.base_url = "https://openrouter.ai/api/v1" + primary.chat.completions.create.side_effect = _Timeout("Request timed out.") + + fb_client = MagicMock() + fb_client.base_url = "https://api.openai.com/v1" + fb_client.chat.completions.create.return_value = {"fallback": True} + + p1, p2, p3 = self._patches(primary) + with ( + p1, p2, p3, + # Vision resolves its client through resolve_vision_provider_client(), + # not _get_cached_client(); the retry block under test is shared. + patch( + "agent.auxiliary_client.resolve_vision_provider_client", + return_value=("openrouter", primary, "some-model"), + ), + patch( + "agent.auxiliary_client._try_configured_fallback_chain", + return_value=(None, None, ""), + ), + patch( + "agent.auxiliary_client._try_main_agent_model_fallback", + return_value=(fb_client, "fb-model", "openai"), + ), + ): + result = call_llm(task="vision", messages=[{"role": "user", "content": "hi"}]) + assert result == {"fallback": True} + assert primary.chat.completions.create.call_count == 1 + assert fb_client.chat.completions.create.call_count == 1 + + @pytest.mark.asyncio + async def test_vision_skips_same_provider_retry_on_timeout_async(self): + """Async twin of the sync guard: tools/vision_tools.py drives + ``async_call_llm``, so the skip must hold on the async site too.""" + class _Timeout(Exception): + pass + _Timeout.__name__ = "APITimeoutError" + + primary = MagicMock() + primary.base_url = "https://openrouter.ai/api/v1" + primary.chat.completions.create = AsyncMock( + side_effect=_Timeout("Request timed out.") + ) + expected = {"fallback": True} + + p1, p2, p3 = self._patches(primary) + with ( + p1, p2, p3, + patch( + "agent.auxiliary_client.resolve_vision_provider_client", + return_value=("openrouter", primary, "some-model"), + ), + patch( + "agent.auxiliary_client._try_configured_fallback_chain", + return_value=(MagicMock(), "fb-model", "configured-fallback"), + ), + patch( + "agent.auxiliary_client._to_async_client", + return_value=(MagicMock(), "fb-model"), + ), + patch( + "agent.auxiliary_client._call_fallback_candidate_async", + new=AsyncMock(return_value=expected), + ), + ): + result = await async_call_llm( + task="vision", messages=[{"role": "user", "content": "hi"}] + ) + assert result == expected + assert primary.chat.completions.create.call_count == 1 + + @pytest.mark.asyncio + async def test_no_progress_timeout_still_retries_same_provider_async(self): + """A stillborn stream (no-progress window, zero output) is cheap: it + keeps the same-provider retry on the async site, mirroring sync.""" + primary = MagicMock() + primary.base_url = "https://chatgpt.com/backend-api/codex" + primary.chat.completions.create = AsyncMock(side_effect=[ + TimeoutError( + "Codex auxiliary Responses stream produced no output within " + "60.0s (no-progress timeout, 60.2s elapsed)" + ), + {"retried": True}, + ]) + + p1, p2, p3 = self._patches(primary) + with ( + p1, p2, p3, + patch( + "agent.auxiliary_client.resolve_vision_provider_client", + return_value=("openrouter", primary, "some-model"), + ), + ): + result = await async_call_llm( + task="vision", messages=[{"role": "user", "content": "hi"}] + ) + assert result == {"retried": True} + assert primary.chat.completions.create.call_count == 2 + + def test_non_critical_task_still_retries_same_provider_on_timeout(self): + """The skip is scoped to critical-path tasks. Everything else keeps the + existing one-shot same-provider retry, so this is not a blanket change. + """ + class _Timeout(Exception): + pass + _Timeout.__name__ = "APITimeoutError" + + primary = MagicMock() + primary.base_url = "https://openrouter.ai/api/v1" + primary.chat.completions.create.side_effect = [ + _Timeout("Request timed out."), + {"retried": True}, + ] + + p1, p2, p3 = self._patches(primary) + with p1, p2, p3: + result = call_llm(task="title", messages=[{"role": "user", "content": "hi"}]) + assert result == {"retried": True} + # Same provider was retried once — unchanged behaviour off the critical path. + assert primary.chat.completions.create.call_count == 2 + def test_timeout_forwards_failed_model_to_configured_chain(self): """A timeout is model-specific, so call_llm must forward the failed model to the configured chain (failed_model=, not None). This diff --git a/tests/agent/test_auxiliary_client_nous_401_cache_key.py b/tests/agent/test_auxiliary_client_nous_401_cache_key.py new file mode 100644 index 0000000000..dcfde071b5 --- /dev/null +++ b/tests/agent/test_auxiliary_client_nous_401_cache_key.py @@ -0,0 +1,150 @@ +"""A Nous 401 refresh must replace the auxiliary client under the SAME cache key +``call_llm`` acquired it with (model dimension #56889, task dimension #58894). +Otherwise the expired client is never evicted and every auxiliary call 401s, +refreshes, and retries forever (#91023). + +End-to-end through the REAL call_llm / async_call_llm and the REAL module +cache; only the client factories are patched. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import agent.auxiliary_client as ac + + +NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1" + + +@pytest.fixture(autouse=True) +def _clean_client_cache(): + ac._client_cache.clear() + yield + ac._client_cache.clear() + + +class _Auth401(Exception): + """A 401 the auth-error classifier recognizes (``status_code`` attribute).""" + + status_code = 401 + + +def _nous_mock_client(*, async_mode, raises=None, returns=None): + """A stand-in OpenAI client whose ``chat.completions.create`` 401s or returns.""" + client = MagicMock() + client.base_url = NOUS_BASE_URL + create = AsyncMock() if async_mode else MagicMock() + if raises is not None: + create.side_effect = raises + else: + create.return_value = returns + client.chat.completions.create = create + return client + + +def test_call_llm_auto_provider_evicts_stale_client_end_to_end(monkeypatch): + """End-to-end: a default auto-provider 401 must evict the stale client. + + The integration guard the unit refresh tests structurally cannot give: it + runs the REAL primary acquisition (``_get_cached_client`` at call_llm's + acquisition site) and the REAL 401 refresh against the REAL module cache, + patching only the client *factories* -- never ``_get_cached_client``, whose + wholesale patching in the pre-existing call_llm 401 tests is exactly why the + acquisition-vs-refresh key divergence went unseen. The stale client is + acquired under the auto+task cache key; the 401 refresh must land the fresh + client under that SAME key and evict the stale one. If the acquisition site + stops threading ``task`` (the #58894 regression) the refresh rebuilds a + divergent key, the stale expired-credential client survives, and the + stale-absence assertion fails. + """ + task = "compression" + stale = _nous_mock_client(async_mode=False, raises=_Auth401("stale creds")) + fresh = _nous_mock_client(async_mode=False, returns={"ok": True}) + + # Force the default auto path and make the primary acquisition build `stale`. + monkeypatch.setattr( + ac, "_resolve_task_provider_model", + lambda *a, **k: ("auto", None, None, None, None), + ) + monkeypatch.setattr( + ac, "resolve_provider_client", + lambda *a, **k: (stale, "nous-model"), + ) + # The 401 refresh rebuilds a fresh client from refreshed runtime creds. + monkeypatch.setattr( + ac, "_resolve_nous_runtime_api", + lambda *, force_refresh=False, stale_access_token=None: ("fresh-key", NOUS_BASE_URL), + ) + monkeypatch.setattr( + ac, "_create_openai_client", + lambda *, api_key, base_url, **kwargs: fresh, + ) + monkeypatch.setattr(ac, "_validate_llm_response", lambda resp, _task: resp) + + result = ac.call_llm(task=task, messages=[{"role": "user", "content": "hi"}]) + + assert result == {"ok": True} + assert stale.chat.completions.create.call_count == 1 + assert fresh.chat.completions.create.call_count == 1 + # The stale expired-credential client must be gone from the cache, not merely + # shadowed by the fresh client under a divergent (task-dropped) key. + assert not any(entry[0] is stale for entry in ac._client_cache.values()), ( + "stale auto-provider client survived the 401 refresh: the acquisition " + "site dropped the task dimension so the refresh keyed the fresh client " + "under a different cache entry (#58894)" + ) + assert any(entry[0] is fresh for entry in ac._client_cache.values()) + + +@pytest.mark.asyncio +async def test_async_call_llm_auto_provider_evicts_stale_client_end_to_end(monkeypatch): + """Async twin of the end-to-end auto-provider eviction guard. + + Passing a non-None ``main_runtime`` also pins the async acquisition site's + ``main_runtime`` threading: for ``provider == "auto"`` the runtime is part of + the key, so if the async acquisition rebuilds without it (while the refresh + passes it) the fresh client again lands under a divergent key -- the same bug + class one element over. Reverting either the ``task`` or the ``main_runtime`` + kwarg at the async acquisition site fails this test. + """ + task = "session_search" + main_runtime = {"provider": "nous", "model": "Hermes-4-405B"} + stale = _nous_mock_client(async_mode=True, raises=_Auth401("stale creds")) + fresh = _nous_mock_client(async_mode=True, returns={"ok": True}) + + monkeypatch.setattr( + ac, "_resolve_task_provider_model", + lambda *a, **k: ("auto", None, None, None, None), + ) + monkeypatch.setattr( + ac, "resolve_provider_client", + lambda *a, **k: (stale, "nous-model"), + ) + monkeypatch.setattr( + ac, "_resolve_nous_runtime_api", + lambda *, force_refresh=False, stale_access_token=None: ("fresh-key", NOUS_BASE_URL), + ) + # Async refresh builds a sync client then wraps it; patch the wrap to `fresh`. + monkeypatch.setattr( + ac, "_create_openai_client", + lambda *, api_key, base_url, **kwargs: MagicMock(), + ) + monkeypatch.setattr(ac, "_to_async_client", lambda *a, **k: (fresh, "nous-model")) + monkeypatch.setattr(ac, "_validate_llm_response", lambda resp, _task: resp) + + result = await ac.async_call_llm( + task=task, + messages=[{"role": "user", "content": "hi"}], + main_runtime=main_runtime, + ) + + assert result == {"ok": True} + assert stale.chat.completions.create.await_count == 1 + assert fresh.chat.completions.create.await_count == 1 + assert not any(entry[0] is stale for entry in ac._client_cache.values()), ( + "stale auto-provider async client survived the 401 refresh: the async " + "acquisition site dropped the task/main_runtime dimension so the refresh " + "keyed the fresh client under a different cache entry (#58894)" + ) + assert any(entry[0] is fresh for entry in ac._client_cache.values()) diff --git a/tests/agent/test_cjk_token_estimation.py b/tests/agent/test_cjk_token_estimation.py index 003d39d1eb..3c39c54cd0 100644 --- a/tests/agent/test_cjk_token_estimation.py +++ b/tests/agent/test_cjk_token_estimation.py @@ -49,15 +49,16 @@ def test_cjk_tail_does_not_expand_to_english_char_budget(): def _reference_per_char_estimate(text: str) -> int: - """The pre-perf-gate per-character reference implementation.""" + """Per-character reference: CJK ~1 token/char, everything else UTF-8 + bytes/4 (the byte width corrects Cyrillic/Greek/Arabic under-counting).""" dense = 0 - sparse = 0 + sparse_bytes = 0 for ch in text: if _is_cjk_token_dense_char(ch): dense += 1 else: - sparse += 1 - return dense + ((sparse + 3) // 4) + sparse_bytes += len(ch.encode("utf-8")) + return dense + ((sparse_bytes + 3) // 4) def test_perf_gated_estimator_matches_per_char_reference(): @@ -79,3 +80,40 @@ def test_perf_gated_estimator_matches_per_char_reference(): +def test_cyrillic_counts_by_utf8_bytes(): + # «русский текст» = 12 Cyrillic chars (2 bytes each) + 1 ASCII space: + # 25 bytes -> ceil(25/4) = 7 tokens; the old chars/4 rule said 4 — + # the ~2x under-count that let real prompts ride the context ceiling. + from agent.model_metadata import estimate_tokens_rough + assert estimate_tokens_rough("русский текст") == 7 + # Pure ASCII unchanged. + assert estimate_tokens_rough("a" * 400) == 100 + + +def test_accented_latin_is_not_inflated_by_byte_counting(): + # Byte-counting must not punish Western-European text: only the accented + # chars are 2 bytes, so the estimate moves by a few percent, not 2x. + from agent.model_metadata import estimate_tokens_rough + fr = "La compression du contexte permet aux longues sessions de rester dans la fenêtre du fournisseur sans perdre le fil de la tâche." + ascii_rule = (len(fr) + 3) // 4 + est = estimate_tokens_rough(fr) + assert ascii_rule <= est <= int(ascii_rule * 1.10), (ascii_rule, est) + + +def test_mixed_cyrillic_and_ascii_code_counts_ascii_at_one_byte(): + from agent.model_metadata import estimate_tokens_rough + code = "def compress(ctx):\n # Сжимаем контекст\n return summarize(ctx)\n" + ascii_part = "def compress(ctx):\n # \n return summarize(ctx)\n" + cyr = "Сжимаем контекст" + expected = (len(ascii_part.encode()) + len(cyr.encode()) + 3) // 4 + assert estimate_tokens_rough(code) == expected + # and strictly more than the old chars/4 rule for the same text + assert estimate_tokens_rough(code) > (len(code) + 3) // 4 + + +def test_lone_surrogates_do_not_raise(): + # main's estimator was total (len/regex never raise); byte-counting must + # stay total too — tool output routinely carries unpaired surrogates. + from agent.model_metadata import estimate_tokens_rough + assert estimate_tokens_rough("abc\ud800def") >= 2 + assert estimate_tokens_rough("漢字\udfff") >= 2 diff --git a/tests/agent/test_codex_ttfb_watchdog.py b/tests/agent/test_codex_ttfb_watchdog.py index 66208a8e1a..bdf53061f3 100644 --- a/tests/agent/test_codex_ttfb_watchdog.py +++ b/tests/agent/test_codex_ttfb_watchdog.py @@ -108,6 +108,89 @@ def test_ttfb_includes_silent_hang_hint_for_gpt_5_5(tmp_path, monkeypatch): stop["flag"] = True +def test_ttfb_installs_and_retires_the_codex_request_token(tmp_path, monkeypatch): + """The watchdog must publish a per-request token and clear it on the kill. + + ``run_codex_stream`` reads ``agent._active_codex_stream_request_token`` to + tell whether it is still the owning attempt. Without an install here the + whole retirement guard would be inert, and without the clear on kill a + retired worker would keep normalizing partial deltas into a "completed" + response. + + The worker also unwinds with its own local error after the force-close; + that error must not replace the watchdog's retryable ``TimeoutError``. + """ + from agent import chat_completion_helpers as h + + agent = _make_codex_agent(tmp_path, monkeypatch) + monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1") + + closes: list = [] + seen = {"token_while_running": None} + dummy_client = SimpleNamespace() + monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client) + monkeypatch.setattr( + agent, + "_abort_request_openai_client", + lambda c, reason=None: closes.append(reason), + ) + monkeypatch.setattr( + agent, + "_close_request_openai_client", + lambda c, reason=None: closes.append(reason), + ) + + def fake_stream(api_kwargs, client=None, on_first_delta=None): + seen["token_while_running"] = getattr( + agent, "_active_codex_stream_request_token", None + ) + deadline = time.time() + 30 + while time.time() < deadline: + if getattr(agent, "_active_codex_stream_request_token", None) is None: + # Retired by the watchdog — mimic the transport unwinding. + raise RuntimeError("retired worker stream ended without terminal") + time.sleep(0.02) + raise RuntimeError("test timed out waiting for retirement") + + monkeypatch.setattr(agent, "_run_codex_stream", fake_stream) + + with pytest.raises(TimeoutError) as excinfo: + h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"}) + + assert seen["token_while_running"] is not None, ( + "interruptible_api_call must install a request token before the worker runs" + ) + assert "TTFB" in str(excinfo.value) + assert "retired worker" not in str(excinfo.value) + assert "codex_ttfb_kill" in closes + assert getattr(agent, "_active_codex_stream_request_token", None) is None + + +def test_non_codex_api_mode_installs_no_request_token(tmp_path, monkeypatch): + """The token is codex_responses-only — other api_modes stay untouched.""" + from agent import chat_completion_helpers as h + + agent = _make_codex_agent(tmp_path, monkeypatch) + agent.api_mode = "chat_completions" + + seen = {"token": "unset"} + dummy_client = SimpleNamespace() + monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client) + + def fake_dispatch(_agent, _api_kwargs, *, make_client): + make_client("test") + seen["token"] = getattr( + _agent, "_active_codex_stream_request_token", "absent" + ) + return SimpleNamespace(choices=[]) + + monkeypatch.setattr(h, "_dispatch_nonstreaming_api_request", fake_dispatch) + + h.interruptible_api_call(agent, {"model": "gpt-5.5", "messages": []}) + + assert seen["token"] in (None, "absent") + + def test_ttfb_does_not_kill_when_events_flow(tmp_path, monkeypatch): diff --git a/tests/agent/test_compression_anti_thrash_recovery.py b/tests/agent/test_compression_anti_thrash_recovery.py index 109f23c18e..cf245ac9a5 100644 --- a/tests/agent/test_compression_anti_thrash_recovery.py +++ b/tests/agent/test_compression_anti_thrash_recovery.py @@ -17,10 +17,13 @@ The recovery contract pinned here: next recovery waits a FULL fresh window (no immediate re-probe loop). * An effective probe (or any fitting real-usage reading) fully clears the counters through the existing ``update_from_response`` path. -* The recovery clock is armed lazily on the first blocked evaluation and is - NOT durable: a process restart that loads a durable tripped counter - (#69872) starts a full fresh window blocked — a restart must never disarm - or shorten the guard (#54923). +* The recovery clock is armed lazily on the first blocked evaluation and + persisted on the session row as a wall-clock deadline (#100185): a fresh + compressor that loads a durable tripped counter (#69872) with NO stored + deadline starts a full window blocked — a restart must never disarm or + shorten the guard (#54923) — while one that loads an armed deadline + resumes that window instead of restarting it, so gateway agent rebuilds + cannot block a session forever. * The protection itself is preserved: inside the window the gate stays blocked exactly as before. """ @@ -57,10 +60,10 @@ class TestRecoveryWindow: cc = _compressor() _trip(cc) base = 1000.0 - with patch("agent.context_compressor.time.monotonic", return_value=base): + with patch("agent.context_compressor.time.time", return_value=base): assert cc.should_compress(cc.threshold_tokens + 1) is False with patch( - "agent.context_compressor.time.monotonic", + "agent.context_compressor.time.time", return_value=base + cc._ANTI_THRASH_RECOVERY_SECONDS + 1, ): assert cc.should_compress(cc.threshold_tokens + 1) is True @@ -73,10 +76,10 @@ class TestRecoveryWindow: cc = _compressor() cc._fallback_compression_streak = 2 base = 1000.0 - with patch("agent.context_compressor.time.monotonic", return_value=base): + with patch("agent.context_compressor.time.time", return_value=base): assert cc.should_compress(cc.threshold_tokens + 1) is False with patch( - "agent.context_compressor.time.monotonic", + "agent.context_compressor.time.time", return_value=base + cc._ANTI_THRASH_RECOVERY_SECONDS + 1, ): assert cc.should_compress(cc.threshold_tokens + 1) is True @@ -95,13 +98,13 @@ class TestRestartSemantics: cc = _compressor() cc.bind_session_state(session_db=db, session_id="sess-1") assert cc._ineffective_compression_count == 2 - # The recovery clock is process-local and must come up disarmed. + # No stored deadline yet -> the clock comes up disarmed. assert cc._anti_thrash_recovery_deadline == 0.0 base = 5000.0 - with patch("agent.context_compressor.time.monotonic", return_value=base): + with patch("agent.context_compressor.time.time", return_value=base): assert cc.should_compress(cc.threshold_tokens + 1) is False with patch( - "agent.context_compressor.time.monotonic", + "agent.context_compressor.time.time", return_value=base + cc._ANTI_THRASH_RECOVERY_SECONDS + 1, ): assert cc.should_compress(cc.threshold_tokens + 1) is True @@ -113,9 +116,92 @@ class TestRestartSemantics: cc = _compressor() _trip(cc) base = 1000.0 - with patch("agent.context_compressor.time.monotonic", return_value=base): + with patch("agent.context_compressor.time.time", return_value=base): assert cc.should_compress(cc.threshold_tokens + 1) is False assert cc._anti_thrash_recovery_deadline > 0.0 cc.on_session_reset() assert cc._anti_thrash_recovery_deadline == 0.0 assert cc._ineffective_compression_count == 0 + + +class TestDurableDeadline: + """#100185: the gateway rebuilds the compressor on every cache eviction.""" + + def _bound(self, db, session_id="sess-1"): + cc = _compressor() + cc.bind_session_state(session_db=db, session_id=session_id) + return cc + + def test_fresh_compressors_resume_the_same_window(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-1", source="telegram") + db.set_compression_ineffective_count("sess-1", 2) + base = 5000.0 + first = self._bound(db) + with patch("agent.context_compressor.time.time", return_value=base): + assert first.should_compress(first.threshold_tokens + 1) is False + # Deadline is durable, as a wall-clock epoch. + assert db.get_compression_recovery_deadline("sess-1") == ( + base + first._ANTI_THRASH_RECOVERY_SECONDS + ) + # Fresh compressor (gateway rebuilt the agent) well past the window: + # before the fix it re-armed a new window and stayed blocked forever. + second = self._bound(db) + assert second._anti_thrash_recovery_deadline == ( + base + first._ANTI_THRASH_RECOVERY_SECONDS + ) + with patch( + "agent.context_compressor.time.time", + return_value=base + first._ANTI_THRASH_RECOVERY_SECONDS + 1, + ): + assert second.should_compress(second.threshold_tokens + 1) is True + assert db.get_compression_ineffective_count("sess-1") == 1 + assert db.get_compression_recovery_deadline("sess-1") == 0.0 + + def test_fresh_compressor_inside_window_stays_blocked(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-1", source="telegram") + db.set_compression_ineffective_count("sess-1", 2) + base = 5000.0 + first = self._bound(db) + with patch("agent.context_compressor.time.time", return_value=base): + assert first.should_compress(first.threshold_tokens + 1) is False + second = self._bound(db) + with patch("agent.context_compressor.time.time", return_value=base + 10): + assert second.should_compress(second.threshold_tokens + 1) is False + assert db.get_compression_ineffective_count("sess-1") == 2 + + def test_backward_clock_jump_is_bounded_to_one_window(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-1", source="telegram") + db.set_compression_ineffective_count("sess-1", 2) + window = ContextCompressor._ANTI_THRASH_RECOVERY_SECONDS + db.set_compression_recovery_deadline("sess-1", 1_000_000.0) + cc = self._bound(db) + # Wall clock now far BEFORE the stored deadline (clock stepped back). + with patch("agent.context_compressor.time.time", return_value=100.0): + assert cc.should_compress(cc.threshold_tokens + 1) is False + assert db.get_compression_recovery_deadline("sess-1") == 100.0 + window + + def test_clearing_the_guard_disarms_the_durable_deadline(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-1", source="telegram") + db.set_compression_ineffective_count("sess-1", 2) + cc = self._bound(db) + with patch("agent.context_compressor.time.time", return_value=5000.0): + assert cc.should_compress(cc.threshold_tokens + 1) is False + assert db.get_compression_recovery_deadline("sess-1") > 0.0 + cc._record_ineffective_compression_verdict(0) + with patch("agent.context_compressor.time.time", return_value=5001.0): + assert cc.should_compress(cc.threshold_tokens + 1) is True + assert db.get_compression_recovery_deadline("sess-1") == 0.0 + + def test_session_db_round_trip(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-1", source="cli") + assert db.get_compression_recovery_deadline("sess-1") == 0.0 + db.set_compression_recovery_deadline("sess-1", 1234.5) + assert db.get_compression_recovery_deadline("sess-1") == 1234.5 + db.set_compression_recovery_deadline("sess-1", 0.0) + assert db.get_compression_recovery_deadline("sess-1") == 0.0 + assert db.get_compression_recovery_deadline("missing") == 0.0 diff --git a/tests/agent/test_compression_attempt_lifecycle.py b/tests/agent/test_compression_attempt_lifecycle.py index d7cf68be4f..84879151fb 100644 --- a/tests/agent/test_compression_attempt_lifecycle.py +++ b/tests/agent/test_compression_attempt_lifecycle.py @@ -287,3 +287,73 @@ class TestTransientBlockIsNotExhaustion: mock_agent = MagicMock() # MagicMock auto-attributes are truthy but not str. assert compression_blocked_transiently(mock_agent) is False + + +def _summary_response(content: str): + from unittest.mock import MagicMock + + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message.content = content + return response + + +class TestProviderOverflowBypassesCooldown: + """#100661: a provider-proven overflow must get one REAL summary attempt + while the summary-failure cooldown is armed. Before the fix every turn of + a wedged session hit the cooldown gate, returned the soft "temporarily + paused" deferral, and the next failure extended the ladder — 4 long + sessions were lost this way. Ordinary (non-overflow) automatic passes + must still defer.""" + + def _armed_agent(self, tmp_path: Path, session_id: str): + db, agent = _build_agent(tmp_path, session_id) + # Realistic arming: a failed/stalled attempt recorded the ladder. + agent.context_compressor.record_timeout_failure( + "stall", failure_kind="stalled" + ) + assert agent.context_compressor.should_compress_info(500_000)[0] is False + return db, agent + + def test_overflow_attempt_invokes_summarizer_while_cooldown_armed( + self, tmp_path: Path + ): + db, agent = self._armed_agent(tmp_path, "OVERFLOW_BYPASS") + calls = [] + + def fake_call_llm(**kwargs): + calls.append(kwargs) + return _summary_response("## Goal\nRecovered after overflow.") + + # Bulky turns so the compacted transcript is genuinely smaller. + live = [ + {"role": "user" if i % 2 == 0 else "assistant", "content": f"m{i} " * 400} + for i in range(20) + ] + with patch("agent.context_compressor.call_llm", fake_call_llm): + out, _ = compress_context( + agent, live, "sys", approx_tokens=500_000, bypass_cooldown=True + ) + assert len(calls) == 1, ( + "provider-proven overflow must reach the summary LLM even while " + "the failure cooldown is armed (#100661)" + ) + assert compression_blocked_transiently(agent) is False + assert len(out) < len(live), "the attempt must actually compact" + + def test_non_overflow_pass_still_deferred_by_cooldown(self, tmp_path: Path): + db, agent = self._armed_agent(tmp_path, "OVERFLOW_ORDINARY") + calls = [] + + def fake_call_llm(**kwargs): # pragma: no cover - must not run + calls.append(kwargs) + return _summary_response("unexpected") + + live = _messages() + before = copy.deepcopy(live) + with patch("agent.context_compressor.call_llm", fake_call_llm): + out, _ = compress_context(agent, live, "sys", approx_tokens=500_000) + assert calls == [] and out == before + assert compression_blocked_transiently(agent) is True, ( + "ordinary threshold pressure keeps honoring the cooldown (#11529)" + ) diff --git a/tests/agent/test_compression_busy_steer_anchor.py b/tests/agent/test_compression_busy_steer_anchor.py new file mode 100644 index 0000000000..3aed06bece --- /dev/null +++ b/tests/agent/test_compression_busy_steer_anchor.py @@ -0,0 +1,146 @@ +"""Regression coverage for busy-steer preservation across compaction (#100053). + +With ``display.busy_input_mode: steer`` the follow-up rides inside the latest +``role=tool`` result (``apply_pending_steer_to_tool_results``), never as a +``role=user`` row. ``_ensure_compressed_has_user_turn`` must treat that marker +as live user intent — and must pick whichever intent-bearing row is LAST in +the original transcript, so an older steer never outranks a newer real user +request. +""" + +import pytest + +from agent.context_compressor import ( + COMPRESSION_CONTINUATION_USER_CONTENT, + SUMMARY_PREFIX, +) +from agent.conversation_compression import ( + _compressed_has_busy_steer, + _ensure_compressed_has_user_turn, +) +from agent.prompt_builder import STEER_MARKER_OPEN, format_steer_marker + +REQUEST_A = "Historical request A: audit the auth module." +STEER_B = "Steer B: stop, switch to fixing the login bug instead." +REQUEST_C = "Newer real user request C: now write the release notes." + + +def _tool_turns(start: int, count: int, *, steer_at: int | None = None) -> list[dict]: + turns: list[dict] = [] + for idx in range(start, start + count): + turns.append( + { + "role": "assistant", + "content": "Working.", + "tool_calls": [ + { + "id": f"call-{idx}", + "function": {"name": "terminal", "arguments": "{}"}, + } + ], + } + ) + content = f"tool output {idx}" + if steer_at == idx: + content += format_steer_marker(STEER_B) + turns.append({"role": "tool", "tool_call_id": f"call-{idx}", "content": content}) + return turns + + +def _summary_row() -> dict: + return {"role": "user", "content": f"{SUMMARY_PREFIX}\n\nEarlier work summarized."} + + +def _assert_alternation(messages: list[dict]) -> None: + roles = [m.get("role") for m in messages] + for left, right in zip(roles, roles[1:]): + assert not (left == right == "user"), f"user/user adjacency in {roles}" + assert not (left == right == "assistant"), f"assistant/assistant adjacency in {roles}" + + +def _user_rows(messages: list[dict]) -> list[str]: + return [str(m.get("content")) for m in messages if m.get("role") == "user"] + + +def test_s1_steer_summarized_away_becomes_anchor_not_historical_request(): + """S1: the steer lived in a tool row that compaction dropped; the only + ``role=user`` row in history is the already-consumed request A. The steer + must be restored as the anchor, and A must not be replayed.""" + original = [{"role": "user", "content": REQUEST_A}] + _tool_turns(0, 6, steer_at=2) + compressed = [_summary_row(), *_tool_turns(5, 1)] + + outcome = _ensure_compressed_has_user_turn(original, compressed) + + assert outcome == "inserted" + _assert_alternation(compressed) + users = _user_rows(compressed) + assert STEER_B in users, users + assert REQUEST_A not in users, "historical request replayed as new input" + assert COMPRESSION_CONTINUATION_USER_CONTENT not in users + # Steer text is used exactly once across the whole compressed transcript. + assert sum(str(m.get("content")).count(STEER_B) for m in compressed) == 1 + + +def test_s2_steer_surviving_in_tail_tool_row_counts_as_present(): + """S2: the steer-bearing tool row survived into the tail. No anchor may be + inserted (the intent is already there) and A must not be cloned.""" + original = [{"role": "user", "content": REQUEST_A}] + _tool_turns(0, 6, steer_at=5) + compressed = [_summary_row(), *_tool_turns(5, 1, steer_at=5)] + before = [dict(m) for m in compressed] + + outcome = _ensure_compressed_has_user_turn(original, compressed) + + assert outcome == "already_present" + assert compressed == before, "transcript mutated despite live steer present" + assert REQUEST_A not in _user_rows(compressed) + assert sum(str(m.get("content")).count(STEER_B) for m in compressed) == 1 + + +def test_s3_newer_real_user_turn_outranks_older_steer(): + """S3: ``[user A, tool(steer B), ..., user C]`` — C is the newest intent. + A steer-first scan would anchor the consumed steer B and replay it.""" + original = ( + [{"role": "user", "content": REQUEST_A}] + + _tool_turns(0, 3, steer_at=1) + + [{"role": "user", "content": REQUEST_C}] + + _tool_turns(3, 4) + ) + compressed = [_summary_row(), *_tool_turns(6, 1)] + + outcome = _ensure_compressed_has_user_turn(original, compressed) + + assert outcome == "inserted" + _assert_alternation(compressed) + users = _user_rows(compressed) + assert REQUEST_C in users, users + assert STEER_B not in users, "older consumed steer replayed over newer user turn" + assert REQUEST_A not in users + assert not any(STEER_B in u for u in users) + + +def test_newer_steer_outranks_older_real_user_turn(): + """Mirror of S3: ``[user A, ..., tool(steer B)]`` — the steer is newest.""" + original = [{"role": "user", "content": REQUEST_A}] + _tool_turns(0, 4, steer_at=3) + compressed = [_summary_row(), *_tool_turns(4, 1)] + + outcome = _ensure_compressed_has_user_turn(original, compressed) + + assert outcome == "inserted" + _assert_alternation(compressed) + users = _user_rows(compressed) + assert STEER_B in users + assert REQUEST_A not in users + + +@pytest.mark.parametrize( + "role", + ["user", "assistant"], +) +def test_compressed_steer_presence_only_counts_tool_rows(role): + """A summary or assistant row that merely quotes the marker text is not a + live steer delivery — only ``role=tool`` rows carry real steers.""" + quoted = {"role": role, "content": f"{SUMMARY_PREFIX}\n{format_steer_marker(STEER_B)}"} + assert _compressed_has_busy_steer([quoted]) is False + assert STEER_MARKER_OPEN in quoted["content"] + live = {"role": "tool", "tool_call_id": "c", "content": f"ok{format_steer_marker(STEER_B)}"} + assert _compressed_has_busy_steer([live]) is True diff --git a/tests/agent/test_compression_concurrent_fork.py b/tests/agent/test_compression_concurrent_fork.py index a155b5ab65..831f1f9584 100644 --- a/tests/agent/test_compression_concurrent_fork.py +++ b/tests/agent/test_compression_concurrent_fork.py @@ -167,6 +167,105 @@ def test_compression_activity_heartbeat_touches_agent_during_long_compress(tmp_p assert db.get_compression_lock_holder(session_id) is None +def test_compression_activity_heartbeat_emits_client_status_events(tmp_path: Path) -> None: + """The heartbeat must re-emit the compacting status, not just DB touches. + + Remote transports (e.g. the Android relay app) run idle-progress turn + watchdogs that ``session.interrupt`` a turn after ~180s with no gateway + events. Compression is silent on the event stream, so without periodic + status heartbeats a long compression is killed mid-flight and retriggers + forever on sessions near the context ceiling. + """ + from agent.conversation_compression import ( + COMPACTION_HEARTBEAT_STATUS, + is_compaction_progress_status, + ) + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HEARTBEAT_STATUS_TEST" + db.create_session(session_id, source="test") + + agent = _build_agent_with_db(db, session_id) + agent._compression_activity_heartbeat_interval = 0.1 + touch_calls: list[str] = [] + agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc) + status_events: list[tuple[str, str]] = [] + setattr( + agent, + "status_callback", + lambda event, message: status_events.append((event, message)), + ) + + def _slow_compress(*_a, **_kw): + _wait_for_touch(touch_calls, "context compression in progress") + return [ + {"role": "user", "content": "[CONTEXT COMPACTION] summary"}, + {"role": "user", "content": "tail"}, + ] + + agent.context_compressor.compress.side_effect = _slow_compress + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + agent._compress_context(messages, "sys", approx_tokens=120_000) + + heartbeats = [e for e in status_events if e[1] == COMPACTION_HEARTBEAT_STATUS] + assert heartbeats, "no heartbeat status reached the client" + # Same "lifecycle" key as the other compaction statuses so the TUI gateway + # re-tags it to kind="compacting" and Telegram edits one bubble in place. + assert {event for event, _ in heartbeats} == {"lifecycle"} + assert is_compaction_progress_status(COMPACTION_HEARTBEAT_STATUS) + # Exactly one routine start line precedes the first heartbeat; the + # heartbeat no longer re-emits a start of its own (adapters without + # send_or_update_status would otherwise post two messages). + assert status_events[0][1] != COMPACTION_HEARTBEAT_STATUS + # Every heartbeat is a periodic tick: none may precede the first + # "in progress" DB touch, which is what start() would have produced. + first_tick_touch = touch_calls.index("context compression in progress") + assert first_tick_touch >= 1 # "started" touch came first + assert len(heartbeats) <= touch_calls.count("context compression in progress") + + +def test_compression_heartbeat_is_silent_for_quiet_context_engines(tmp_path: Path) -> None: + """A context engine that suppresses the routine start status opens no + visible compaction phase; the heartbeat must not open one either (there + would be no terminal edge to close it).""" + from agent.conversation_compression import COMPACTION_HEARTBEAT_STATUS + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HEARTBEAT_QUIET_TEST" + db.create_session(session_id, source="test") + + agent = _build_agent_with_db(db, session_id) + agent._compression_activity_heartbeat_interval = 0.1 + touch_calls: list[str] = [] + agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc) + status_events: list[tuple[str, str]] = [] + setattr( + agent, + "status_callback", + lambda event, message: status_events.append((event, message)), + ) + + def _slow_compress(*_a, **_kw): + _wait_for_touch(touch_calls, "context compression in progress") + return [ + {"role": "user", "content": "[CONTEXT COMPACTION] summary"}, + {"role": "user", "content": "tail"}, + ] + + agent.context_compressor.compress.side_effect = _slow_compress + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + with patch( + "agent.conversation_compression.automatic_compaction_status_message", + return_value="", + ): + agent._compress_context(messages, "sys", approx_tokens=120_000) + + assert all(m != COMPACTION_HEARTBEAT_STATUS for _, m in status_events) + assert "context compression in progress" in touch_calls # DB touches still ran + + def test_lock_contender_preserves_terminal_compaction_lifecycle(tmp_path: Path) -> None: """A lock loser still closes the structured compaction lifecycle. diff --git a/tests/agent/test_context_compressor.py b/tests/agent/test_context_compressor.py index 7e373f209e..10997fe94a 100644 --- a/tests/agent/test_context_compressor.py +++ b/tests/agent/test_context_compressor.py @@ -855,6 +855,19 @@ class TestAuthFailureAborts: ) assert _is_summary_access_or_quota_error(err) is True + def test_unscoped_secret_read_is_terminal_access_failure(self): + # Multiplexed gateway: a credential read reached get_secret() from a + # worker thread without the profile scope. The summary model is + # unreachable until the spawn site is fixed — abort and preserve the + # session rather than truncating the middle window (#100849 bundle). + from agent.secret_scope import UnscopedSecretError + + err = UnscopedSecretError( + "get_secret('SURPLUS_API_KEY') called with no profile secret scope " + "active while multiplexing is on." + ) + assert _is_summary_access_or_quota_error(err) is True + diff --git a/tests/agent/test_copilot_acp_client.py b/tests/agent/test_copilot_acp_client.py index 20d086d451..ec4ae1a60d 100644 --- a/tests/agent/test_copilot_acp_client.py +++ b/tests/agent/test_copilot_acp_client.py @@ -319,3 +319,107 @@ def test_probe_skipped_for_custom_args_without_acp(): with _patch("agent.copilot_acp_client.subprocess.run") as run_mock: assert _acp_supported("mycli", ["--custom-transport"]) is True run_mock.assert_not_called() + + +# --- session/set_model: honor the picker-selected model ---------------------- +# +# `copilot --acp` validates but IGNORES the `--model` spawn flag; the ACP +# session runs the CLI's own default unless the client issues the ACP-native +# `session/set_model` call. Without it, picking gpt-5.6-terra in Hermes +# visibly answers as the CLI's default model. + + +# --- session model selection ------------------------------------------------- + + +def _session_with_config_options(): + return { + "sessionId": "s1", + "configOptions": [ + { + "id": "model", + "category": "model", + "type": "select", + "currentValue": "auto", + "options": [ + {"value": "auto", "name": "Auto"}, + {"value": "gpt-5.6-terra", "name": "GPT-5.6 Terra"}, + { + "value": "claude-fable-5", + "name": "Claude Fable 5", + "_meta": {"copilotEnablement": "disabled"}, + }, + ], + } + ], + } + + +def test_model_selection_prefers_stable_config_option(): + from agent.copilot_acp_client import _model_selection_request + + assert _model_selection_request( + _session_with_config_options(), "gpt-5.6-terra" + ) == ( + "session/set_config_option", + {"sessionId": "s1", "configId": "model", "value": "gpt-5.6-terra"}, + ) + + +def test_model_selection_rejects_disabled_config_option(): + from agent.copilot_acp_client import _model_selection_request + + assert _model_selection_request( + _session_with_config_options(), "claude-fable-5" + ) is None + + +def test_model_selection_rejects_unknown_config_option(): + from agent.copilot_acp_client import _model_selection_request + + assert _model_selection_request( + _session_with_config_options(), "not-served-here" + ) is None + + +def test_model_selection_falls_back_to_legacy_extension(): + from agent.copilot_acp_client import _model_selection_request + + legacy_session = { + "sessionId": "s1", + "models": { + "availableModels": [ + {"modelId": "auto"}, + {"modelId": "gpt-5.6-terra"}, + ] + }, + } + assert _model_selection_request(legacy_session, "gpt-5.6-terra") == ( + "session/set_model", + {"sessionId": "s1", "modelId": "gpt-5.6-terra"}, + ) + + +def test_model_selection_skips_provider_virtual_slug(): + from agent.copilot_acp_client import _model_selection_request + + assert _model_selection_request( + _session_with_config_options(), "copilot-acp" + ) is None + + +def test_run_prompt_receives_picker_model(): + # _create_chat_completion must forward `model` into _run_prompt — the + # original wiring dropped it, reducing the selection to prompt text. + client = CopilotACPClient(acp_cwd="/tmp") + seen = {} + + def fake_run_prompt(prompt_text, *, timeout_seconds, model=None): + seen["model"] = model + return "ok", "" + + with patch.object(CopilotACPClient, "_run_prompt", side_effect=fake_run_prompt): + client._create_chat_completion( + model="gpt-5.6-terra", messages=[{"role": "user", "content": "hi"}] + ) + assert seen["model"] == "gpt-5.6-terra" diff --git a/tests/agent/test_credential_pool_nous_refresh_stampede.py b/tests/agent/test_credential_pool_nous_refresh_stampede.py new file mode 100644 index 0000000000..34a3128e69 --- /dev/null +++ b/tests/agent/test_credential_pool_nous_refresh_stampede.py @@ -0,0 +1,130 @@ +"""Concurrent Nous 401 recovery must not stampede the shared OAuth grant. + +Sep 2 2026 incident: ~120 subagent processes shared one Nous OAuth pool entry +whose access token hit its hourly expiry. Every process got a 401, every +process force-refreshed, and each rotation invalidated the token a sibling had +just adopted — 81 refreshes and ~540 401s in eight minutes. Processes that +lost the auth-store flock race had their only entry benched ("matched no nous +entry ... pool size 0") and surfaced the 401 to the user as "out of funds". + +Two invariants pinned here: + +1. ``resolve_nous_runtime_credentials(force_refresh=True, stale_access_token=X)`` + does NOT POST a refresh when the store already holds a usable token that + is not X — a peer already rotated; adopt it. +2. A lock-timeout during a pool-level Nous refresh leaves the entry + untouched instead of marking it exhausted. +""" + +import json +import logging + +import hermes_cli.auth as auth_mod +from agent.credential_pool import CredentialPool, PooledCredential + +from tests.hermes_cli.test_auth_nous_provider import _invoke_jwt, _setup_nous_auth + + +def test_forced_refresh_adopts_peer_rotation_instead_of_reposting(tmp_path, monkeypatch): + hermes_home = tmp_path / "hermes" + peer_token = _invoke_jwt(seconds=3600) + failed_token = _invoke_jwt(seconds=3000) # what THIS process still holds + _setup_nous_auth( + hermes_home, + access_token=peer_token, + refresh_token="rt-after-peer-rotation", + scope=auth_mod.DEFAULT_NOUS_SCOPE, + expires_at=auth_mod.datetime.fromtimestamp( + auth_mod.time.time() + 3600, tz=auth_mod.timezone.utc + ).isoformat(), + expires_in=3600, + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + posts = [] + + def _fake_refresh_access_token(*, client, portal_base_url, client_id, refresh_token): + posts.append(refresh_token) + return { + "access_token": _invoke_jwt(seconds=7200), + "refresh_token": "rt-should-not-happen", + "expires_in": 7200, + "token_type": "Bearer", + "scope": auth_mod.DEFAULT_NOUS_SCOPE, + } + + monkeypatch.setattr(auth_mod, "_refresh_access_token", _fake_refresh_access_token) + + creds = auth_mod.resolve_nous_runtime_credentials( + force_refresh=True, stale_access_token=failed_token + ) + + assert posts == [], "peer already rotated — must not consume the refresh token again" + assert creds["api_key"] == peer_token + + # Same call WITHOUT the hint keeps the pre-existing force semantics. + auth_mod.resolve_nous_runtime_credentials(force_refresh=True) + assert posts == ["rt-after-peer-rotation"] + + +def test_lock_timeout_during_nous_refresh_does_not_bench_entry(monkeypatch, caplog): + entry = PooledCredential( + id="267aed", + provider="nous", + auth_type="oauth", + access_token=_invoke_jwt(seconds=3600), + refresh_token="rt", + label="test@nous", + source="device_code", + priority=0, + ) + pool = CredentialPool.__new__(CredentialPool) + pool._lock = __import__("threading").RLock() + pool._entries = [entry] + pool._active_leases = {} + pool._current_id = None + pool._max_concurrent = 2 + pool._unmatched_rotation_streak = 0 + pool.provider = "nous" + + monkeypatch.setattr(pool, "_sync_nous_entry_from_auth_store", lambda e: e) + monkeypatch.setattr(pool, "_persist", lambda *a, **k: None) + + def _busy(*a, **k): + raise TimeoutError("Timed out waiting for auth store lock") + + monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _busy) + + result = pool._refresh_entry_impl(entry, force=True) + + assert result is entry + assert pool._entries[0].last_status is None, "lock contention is not a credential failure" + + +def test_agent_401_refresh_passes_failed_bearer_as_stale_hint(monkeypatch): + """Every 401-recovery caller must hand the auth store the bearer that + failed — without it ``_already_rotated_by_peer`` can never fire and each + subagent rotates the shared grant again (the "no crash, N refreshes" + variant of the Sep 2 stampede). + """ + from run_agent import AIAgent + + agent = AIAgent.__new__(AIAgent) + agent.provider = "nous" + agent.api_mode = "chat_completions" + agent.api_key = "jwt-that-just-401d" + agent.base_url = "https://inference-api.nousresearch.com/v1" + agent._client_kwargs = {} + monkeypatch.setattr(agent, "_replace_primary_openai_client", lambda **k: True) + + seen = {} + + def _fake_resolve(**kwargs): + seen.update(kwargs) + return {"api_key": "fresh", "base_url": agent.base_url} + + monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _fake_resolve) + + assert agent._try_refresh_nous_client_credentials(force=True) is True + assert seen["force_refresh"] is True + assert seen["stale_access_token"] == "jwt-that-just-401d" diff --git a/tests/agent/test_credential_pool_profile_oauth_fork.py b/tests/agent/test_credential_pool_profile_oauth_fork.py new file mode 100644 index 0000000000..eec0c6c52e --- /dev/null +++ b/tests/agent/test_credential_pool_profile_oauth_fork.py @@ -0,0 +1,561 @@ +"""Regression tests for #100339: cloned / borrowed single-use Anthropic OAuth +grants must never fork across profiles. + +Real imports, real temp HERMES_HOME root + named profile, real auth.json I/O. +The Anthropic token endpoint is replaced at the ``urllib.request.urlopen`` +boundary with genuine single-use semantics (a refresh token redeems once; +a second POST returns ``invalid_grant``). +""" +from __future__ import annotations + +import io +import json +import os +import time +import urllib.error +import urllib.request + +import pytest + + +@pytest.fixture +def fleet(tmp_path, monkeypatch): + """Root HERMES_HOME with an expired-but-refreshable Anthropic pool row.""" + root = tmp_path / "hermes-root" + root.mkdir() + (tmp_path / "fakehome").mkdir() + # Keep host ~/.claude and host auth.json out of the picture. + monkeypatch.setenv("HOME", str(tmp_path / "fakehome")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "fakehome")) + for var in ("ANTHROPIC_TOKEN", "ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("HERMES_HOME", str(root)) + # The pytest seat-belt in the root write-through compares the global path + # against $HOME/.hermes/auth.json; our root is elsewhere, so writes go. + import hermes_constants + hermes_constants._default_hermes_root_memo = None # type: ignore[attr-defined] + + expired = int((time.time() - 3600) * 1000) + store = { + "version": 1, + "providers": {}, + "credential_pool": { + "anthropic": [{ + "id": "abc123", "label": "team-grant", "auth_type": "oauth", + "priority": 0, "source": "manual:hermes_pkce", + "access_token": "sk-ant-oat01-AT0", "refresh_token": "sk-ant-ort-RT0", + "expires_at_ms": expired, "base_url": "https://api.anthropic.com", + }], + "openai": [{ + "id": "key001", "label": "static", "auth_type": "api_key", + "priority": 0, "source": "manual", "access_token": "sk-static-key", + }], + }, + } + (root / "auth.json").write_text(json.dumps(store)) + + server = {"valid": {"sk-ant-ort-RT0"}, "spent": set(), "n": 0, "log": []} + + class _Resp(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def fake_urlopen(req, timeout=None): + assert "oauth/token" in req.full_url + body = req.data.decode() + if req.get_header("Content-type", "").startswith("application/json"): + rt = json.loads(body)["refresh_token"] + else: + from urllib.parse import parse_qsl + rt = dict(parse_qsl(body))["refresh_token"] + if rt in server["spent"] or rt not in server["valid"]: + server["log"].append(("REUSE", rt)) + raise urllib.error.HTTPError( + req.full_url, 400, "Bad Request", {}, + io.BytesIO(b'{"error":"invalid_grant","error_description":"refresh_token_reused"}'), + ) + server["n"] += 1 + server["spent"].add(rt) + server["valid"].discard(rt) + new_rt = f"sk-ant-ort-RT{server['n']}" + server["valid"].add(new_rt) + server["log"].append(("ROTATE", rt, new_rt)) + return _Resp(json.dumps({ + "access_token": f"sk-ant-oat01-AT{server['n']}", + "refresh_token": new_rt, "expires_in": 28800, "token_type": "Bearer", + }).encode()) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + def use(home): + """Switch the process to *home* (root or a profile dir).""" + monkeypatch.setenv("HERMES_HOME", str(home)) + hermes_constants._default_hermes_root_memo = None # type: ignore[attr-defined] + import hermes_cli.auth as auth_mod + auth_mod._global_auth_store_cache = None + auth_mod._oauth_heal_clean_marks.clear() + + # Process-wide notice buffer: start each test clean. + import hermes_cli.auth as _auth_mod + _auth_mod._oauth_heal_notices.clear() + _auth_mod._oauth_heal_clean_marks.clear() + + def pool_rows(home): + p = home / "auth.json" + if not p.exists(): + return None + return (json.loads(p.read_text()).get("credential_pool") or {}).get("anthropic") + + return {"root": root, "server": server, "use": use, "rows": pool_rows} + + +def _profile(fleet, name, **kw): + from hermes_cli.profiles import create_profile + fleet["use"](fleet["root"]) + return create_profile(name, **kw) + + +# ── A. cloning never copies single-use OAuth grants ────────────────────── + +def test_clone_all_strips_oauth_grant_but_keeps_api_keys(fleet): + (fleet["root"] / ".anthropic_oauth.json").write_text( + json.dumps({"accessToken": "sk-ant-oat01-AT0", "refreshToken": "sk-ant-ort-RT0", "expiresAt": 1}) + ) + pdir = _profile(fleet, "forge", clone_all=True) + store = json.loads((pdir / "auth.json").read_text()) + assert "anthropic" not in store["credential_pool"], "OAuth grant was forked into the clone" + assert store["credential_pool"]["openai"][0]["access_token"] == "sk-static-key" + assert not (pdir / ".anthropic_oauth.json").exists() + + +def test_strip_helper_drops_device_code_blocks_and_reports(tmp_path): + from hermes_cli.auth import strip_cloned_single_use_oauth_grants + pdir = tmp_path / "p" + pdir.mkdir() + (pdir / "auth.json").write_text(json.dumps({ + "version": 1, + "providers": {"openai-codex": {"access_token": "a", "refresh_token": "r"}, "nous": {"agent_key": "k"}}, + "credential_pool": { + "xai-oauth": [{"id": "x", "auth_type": "oauth", "access_token": "t", "refresh_token": "r"}], + "anthropic": [ + {"id": "legacy", "access_token": "sk-ant-oat01-legacy"}, # no auth_type field + {"id": "key", "auth_type": "api_key", "access_token": "sk-ant-api03-x"}, + ], + }, + })) + summary = strip_cloned_single_use_oauth_grants(pdir) + store = json.loads((pdir / "auth.json").read_text()) + assert sorted(summary["pool"]) == ["anthropic", "xai-oauth"] + assert summary["providers"] == ["openai-codex"] + assert "xai-oauth" not in store["credential_pool"] + assert [e["id"] for e in store["credential_pool"]["anthropic"]] == ["key"] + assert "openai-codex" not in store["providers"] and "nous" in store["providers"] + + +def test_strip_helper_is_a_noop_without_credentials(tmp_path): + from hermes_cli.auth import strip_cloned_single_use_oauth_grants + assert strip_cloned_single_use_oauth_grants(tmp_path) == {"pool": [], "providers": [], "files": []} + + +# ── B. borrowed rotation commits to root, never a profile copy ─────────── + +def test_first_profile_rotation_does_not_strand_root_or_siblings(fleet): + from agent.credential_pool import load_pool + + forge = _profile(fleet, "forge") + atlas = _profile(fleet, "atlas") + + fleet["use"](forge) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT1" + # The rotated pair landed in ROOT; forge did not grow a local copy. + assert fleet["rows"](forge) is None + assert fleet["rows"](fleet["root"])[0]["refresh_token"] == "sk-ant-ort-RT1" + + for home in (atlas, fleet["root"], forge): + fleet["use"](home) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT1", home + assert [e[0] for e in fleet["server"]["log"]] == ["ROTATE"], fleet["server"]["log"] + assert fleet["rows"](atlas) is None and fleet["rows"](forge) is None + + +def test_agent_init_resolver_sees_sibling_rotation(fleet): + from agent.anthropic_credentials import resolve_anthropic_token + from agent.credential_pool import load_pool + + forge = _profile(fleet, "forge") + atlas = _profile(fleet, "atlas") + fleet["use"](forge) + load_pool("anthropic").select() + fleet["use"](atlas) + assert resolve_anthropic_token() == "sk-ant-oat01-AT1" + + +def test_borrowing_profile_load_pool_does_not_materialize_local_copy(fleet): + from agent.credential_pool import load_pool + + fresh = _profile(fleet, "fresh") + fleet["use"](fresh) + pool = load_pool("anthropic") + assert [e.id for e in pool.entries()] == ["abc123"] + assert pool._borrowed_root_ids == {"abc123"} + assert fleet["rows"](fresh) is None + + +def test_borrower_prune_never_deletes_root_singleton_grant(fleet, tmp_path): + """Root's hermes_pkce row is seeded from ROOT's .anthropic_oauth.json; a + profile without that file must not prune (and write-through-delete) it.""" + from agent.credential_pool import load_pool + + root = fleet["root"] + (root / ".anthropic_oauth.json").write_text(json.dumps({ + "accessToken": "sk-ant-oat01-AT0", "refreshToken": "sk-ant-ort-RT0", + "expiresAt": int((time.time() - 3600) * 1000), + })) + store = json.loads((root / "auth.json").read_text()) + store["active_provider"] = "anthropic" + del store["credential_pool"]["anthropic"] + (root / "auth.json").write_text(json.dumps(store)) + fleet["use"](root) + root_rows = [e for e in load_pool("anthropic").entries()] + assert [e.source for e in root_rows] == ["hermes_pkce"] + + kid = _profile(fleet, "kid") + fleet["use"](kid) + pool = load_pool("anthropic") + assert [e.source for e in pool.entries()] == ["hermes_pkce"], "borrowed root grant was pruned" + assert fleet["rows"](root) and fleet["rows"](root)[0]["source"] == "hermes_pkce" + assert fleet["rows"](kid) is None + + # Rotating from the profile commits BOTH the pool row and the singleton at ROOT. + sel = pool.select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT1" + assert json.loads((root / ".anthropic_oauth.json").read_text())["refreshToken"] == "sk-ant-ort-RT1" + assert not (kid / ".anthropic_oauth.json").exists() + assert fleet["rows"](root)[0]["refresh_token"] == "sk-ant-ort-RT1" + + +def test_profile_auth_add_owns_only_its_own_rows(fleet): + from agent.credential_pool import AUTH_TYPE_OAUTH, PooledCredential, load_pool + + kid = _profile(fleet, "kid") + fleet["use"](kid) + pool = load_pool("anthropic") + pool.add_entry(PooledCredential( + provider="anthropic", id="own001", label="mine", auth_type=AUTH_TYPE_OAUTH, + priority=0, source="manual:hermes_pkce", access_token="sk-ant-oat01-MINE", + refresh_token="rt-mine", + )) + assert [e["id"] for e in fleet["rows"](kid)] == ["own001"], "borrowed root row was copied into the profile" + assert [e["id"] for e in fleet["rows"](fleet["root"])] == ["abc123"] + + +def test_classic_mode_persist_is_unchanged(fleet): + from agent.credential_pool import load_pool + + fleet["use"](fleet["root"]) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT1" + assert fleet["rows"](fleet["root"])[0]["refresh_token"] == "sk-ant-ort-RT1" + + +# ── C. one-time heal for installs that ALREADY forked the grant ────────── +# +# Fleets created on pre-fix code hold profile-local copies of the root grant +# (verbatim --clone-all, or the old borrowed-persist). The heal runs inside +# the profile's load_pool(): consolidate to ROOT (freshest rotation wins), +# strip the profile copy, borrow root from then on. + +def _fork(fleet, name, *, rotated_to=None): + """Create *name* with a pre-fix style verbatim copy of root's auth.json. + + ``rotated_to=N`` makes the copy the LIVE pair (RT, spent RT0 server-side) + to emulate a profile that already refreshed on the old code. + """ + pdir = _profile(fleet, name) + pdir.mkdir(parents=True, exist_ok=True) + store = json.loads((fleet["root"] / "auth.json").read_text()) + if rotated_to is not None: + row = store["credential_pool"]["anthropic"][0] + row["access_token"] = f"sk-ant-oat01-AT{rotated_to}" + row["refresh_token"] = f"sk-ant-ort-RT{rotated_to}" + row["expires_at_ms"] = int((time.time() - 60) * 1000) # newer, still expired + srv = fleet["server"] + srv["spent"].add("sk-ant-ort-RT0") + srv["valid"].discard("sk-ant-ort-RT0") + srv["valid"].add(f"sk-ant-ort-RT{rotated_to}") + srv["n"] = rotated_to + (pdir / "auth.json").write_text(json.dumps(store)) + return pdir + + +def test_heal_consolidates_existing_forks_to_the_live_copy(fleet, caplog): + """root + atlas hold spent RT0; forge already rotated to RT1 on old code.""" + import logging + from agent.credential_pool import load_pool + + forge = _fork(fleet, "forge", rotated_to=1) + atlas = _fork(fleet, "atlas") + assert fleet["rows"](forge)[0]["refresh_token"] == "sk-ant-ort-RT1" + assert fleet["rows"](atlas)[0]["refresh_token"] == "sk-ant-ort-RT0" + + with caplog.at_level(logging.INFO, logger="hermes_cli.auth"): + fleet["use"](forge) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT2" + # forge's live pair was adopted by ROOT, then rotated there; forge holds nothing. + assert fleet["rows"](forge) is None + assert fleet["rows"](fleet["root"])[0]["refresh_token"] == "sk-ant-ort-RT2" + assert fleet["rows"](fleet["root"])[0]["id"] == "abc123" + healed = [r.message for r in caplog.records if "consolidated forked anthropic OAuth grant" in r.message] + assert len(healed) == 1 and "profile forge" in healed[0] and "root updated" in healed[0] + + for home in (atlas, fleet["root"], forge): + fleet["use"](home) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT2", home + assert fleet["rows"](atlas) is None and fleet["rows"](forge) is None + # Exactly one rotation by us (RT1 -> RT2); the spent RT0 was never replayed. + assert [e[0] for e in fleet["server"]["log"]] == ["ROTATE"], fleet["server"]["log"] + # API-key rows in the profiles were not touched. + for home in (forge, atlas): + store = json.loads((home / "auth.json").read_text()) + assert store["credential_pool"]["openai"][0]["access_token"] == "sk-static-key" + + +def test_heal_is_idempotent_and_logs_once(fleet, caplog): + import logging + from agent.credential_pool import load_pool + from hermes_cli.auth import consume_oauth_heal_notices, heal_forked_single_use_oauth_grants + + kid = _fork(fleet, "kid") + fleet["use"](kid) + with caplog.at_level(logging.INFO, logger="hermes_cli.auth"): + load_pool("anthropic") + assert fleet["rows"](kid) is None + notices = consume_oauth_heal_notices() + assert len(notices) == 1 and "profile kid" in notices[0] + root_before = (fleet["root"] / "auth.json").read_text() + # Second and third loads: nothing to do, nothing written, nothing logged. + assert heal_forked_single_use_oauth_grants("anthropic") is None + load_pool("anthropic") + assert consume_oauth_heal_notices() == [] + assert (fleet["root"] / "auth.json").read_text() == root_before + assert sum("consolidated forked" in r.message for r in caplog.records) == 1 + + +def test_heal_never_deletes_the_only_surviving_copy(fleet): + """Root lost its grant (user ran `hermes auth remove` at root); the profile's + copy is the only one left — and an independent second account stays put.""" + from agent.credential_pool import load_pool + + kid = _fork(fleet, "kid", rotated_to=1) + store = json.loads((fleet["root"] / "auth.json").read_text()) + del store["credential_pool"]["anthropic"] + (fleet["root"] / "auth.json").write_text(json.dumps(store)) + + fleet["use"](kid) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT2" + assert fleet["rows"](kid) and fleet["rows"](kid)[0]["refresh_token"] == "sk-ant-ort-RT2" + assert "anthropic" not in (json.loads((fleet["root"] / "auth.json").read_text())["credential_pool"]) + + +def test_heal_leaves_a_different_account_alone(fleet): + """A profile row whose JWT identity names ANOTHER account is not root's grant.""" + import base64 + from agent.credential_pool import load_pool + + def jwt(sub): + payload = base64.urlsafe_b64encode(json.dumps({"sub": sub, "exp": int(time.time()) + 3600}).encode()).rstrip(b"=") + return "h." + payload.decode() + ".s" + + root_store = json.loads((fleet["root"] / "auth.json").read_text()) + root_store["credential_pool"]["xai-oauth"] = [{ + "id": "rootx", "auth_type": "oauth", "priority": 0, "source": "manual:device_code", + "access_token": jwt("alice"), "refresh_token": "xr-alice", + }] + (fleet["root"] / "auth.json").write_text(json.dumps(root_store)) + kid = _profile(fleet, "kid") + kid.mkdir(parents=True, exist_ok=True) + (kid / "auth.json").write_text(json.dumps({ + "version": 1, "providers": {}, + "credential_pool": {"xai-oauth": [ + {"id": "kidx", "auth_type": "oauth", "priority": 0, "source": "manual:device_code", + "access_token": jwt("bob"), "refresh_token": "xr-bob"}, + {"id": "kidk", "auth_type": "api_key", "priority": 1, "source": "manual", + "access_token": "xai-static"}, + ]}, + })) + fleet["use"](kid) + load_pool("xai-oauth") + rows = (json.loads((kid / "auth.json").read_text())["credential_pool"])["xai-oauth"] + assert [r["id"] for r in rows] == ["kidx", "kidk"] + assert json.loads((fleet["root"] / "auth.json").read_text())["credential_pool"]["xai-oauth"][0]["refresh_token"] == "xr-alice" + + +def test_heal_pkce_singleton_shape_commits_live_pair_to_root_singleton(fleet): + """`hermes auth` PKCE shape: root + profile each have .anthropic_oauth.json + + a hermes_pkce-seeded row; the profile's copy is the rotated (live) one.""" + from agent.credential_pool import load_pool + + root = fleet["root"] + store = json.loads((root / "auth.json").read_text()) + store["active_provider"] = "anthropic" + del store["credential_pool"]["anthropic"] + (root / "auth.json").write_text(json.dumps(store)) + (root / ".anthropic_oauth.json").write_text(json.dumps({ + "accessToken": "sk-ant-oat01-AT0", "refreshToken": "sk-ant-ort-RT0", + "expiresAt": int((time.time() - 3600) * 1000), + })) + fleet["use"](root) + load_pool("anthropic") # seeds root's hermes_pkce row from the singleton + + kid = _profile(fleet, "kid") + kid.mkdir(parents=True, exist_ok=True) + import shutil + shutil.copy2(root / "auth.json", kid / "auth.json") + (kid / ".anthropic_oauth.json").write_text(json.dumps({ + "accessToken": "sk-ant-oat01-AT1", "refreshToken": "sk-ant-ort-RT1", + "expiresAt": int((time.time() - 60) * 1000), + })) + kstore = json.loads((kid / "auth.json").read_text()) + kstore["credential_pool"]["anthropic"][0].update( + access_token="sk-ant-oat01-AT1", refresh_token="sk-ant-ort-RT1", + expires_at_ms=int((time.time() - 60) * 1000), + ) + (kid / "auth.json").write_text(json.dumps(kstore)) + srv = fleet["server"] + srv["spent"].add("sk-ant-ort-RT0"); srv["valid"] = {"sk-ant-ort-RT1"}; srv["n"] = 1 + + fleet["use"](kid) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT2" + assert not (kid / ".anthropic_oauth.json").exists() + assert fleet["rows"](kid) is None + assert json.loads((root / ".anthropic_oauth.json").read_text())["refreshToken"] == "sk-ant-ort-RT2" + fleet["use"](root) + sel = load_pool("anthropic").select() + assert sel is not None and sel.access_token == "sk-ant-oat01-AT2" + assert [e[0] for e in srv["log"]] == ["ROTATE"], srv["log"] + + +def test_heal_is_a_noop_in_classic_mode(fleet): + from hermes_cli.auth import heal_forked_single_use_oauth_grants + fleet["use"](fleet["root"]) + before = (fleet["root"] / "auth.json").read_text() + assert heal_forked_single_use_oauth_grants("anthropic") is None + assert (fleet["root"] / "auth.json").read_text() == before + + +# ── C. a SHARED root store is not a fork (#101356) ─────────────────────── + +def _seed_codex_grant(root): + """Give the root store an openai-codex pool row AND a providers block.""" + fresh = int((time.time() + 3600) * 1000) + store = json.loads((root / "auth.json").read_text()) + store["credential_pool"]["openai-codex"] = [{ + "id": "cdx001", "label": "codex", "auth_type": "oauth", "priority": 0, + "source": "manual:device_code", "access_token": "cdx-AT0", + "refresh_token": "cdx-RT0", "expires_at_ms": fresh, + }] + store["providers"]["openai-codex"] = { + "tokens": {"access_token": "cdx-AT0", "refresh_token": "cdx-RT0", "expires_at_ms": fresh}, + "last_refresh": fresh / 1000.0, + } + (root / "auth.json").write_text(json.dumps(store)) + + +def _shared_profile(fleet, name, *, link): + """Profile whose auth.json IS the root store (``link`` makes the alias).""" + pdir = _profile(fleet, name) + pdir.mkdir(parents=True, exist_ok=True) + alias = pdir / "auth.json" + if alias.is_symlink() or alias.exists(): + alias.unlink() + link(fleet["root"] / "auth.json", alias) + return pdir + + +def test_heal_skips_profile_auth_json_symlinked_to_the_root_store(fleet): + """#101356: `ln -s ~/.hermes/auth.json /auth.json` shares ONE store. + Both sides of the consolidation read the same file, so every row looks like + a fork of itself — healing would strip the shared grant through the link.""" + from hermes_cli.auth import consume_oauth_heal_notices, heal_forked_single_use_oauth_grants + + root = fleet["root"] + _seed_codex_grant(root) + before = (root / "auth.json").read_text() + + shared = _shared_profile(fleet, "shared", link=lambda target, alias: alias.symlink_to(target)) + fleet["use"](shared) + + assert heal_forked_single_use_oauth_grants("openai-codex") is None + assert (root / "auth.json").read_text() == before + assert (shared / "auth.json").is_symlink() + assert consume_oauth_heal_notices() == [] + store = json.loads((root / "auth.json").read_text()) + assert [r["id"] for r in store["credential_pool"]["openai-codex"]] == ["cdx001"] + assert store["providers"]["openai-codex"]["tokens"]["refresh_token"] == "cdx-RT0" + + +def test_heal_skips_profile_auth_json_hardlinked_to_the_root_store(fleet): + """Same class as the symlink: a hardlink resolves to a different name but + is the same inode, so it is still one store, not a forked copy.""" + from hermes_cli.auth import heal_forked_single_use_oauth_grants + + root = fleet["root"] + _seed_codex_grant(root) + before = (root / "auth.json").read_text() + + shared = _shared_profile(fleet, "twin", link=lambda target, alias: os.link(target, alias)) + fleet["use"](shared) + + assert heal_forked_single_use_oauth_grants("openai-codex") is None + assert (root / "auth.json").read_text() == before + assert (shared / "auth.json").samefile(root / "auth.json") + + +def test_heal_leaves_an_aliased_anthropic_singleton_alone(fleet): + """Separate auth.jsons but a profile `.anthropic_oauth.json` symlinked to + root's: one shared grant, not a fork. The heal must not self-compare it + or unlink the alias (#101356 sibling site).""" + from hermes_cli.auth import heal_forked_single_use_oauth_grants + + root = fleet["root"] + (root / ".anthropic_oauth.json").write_text(json.dumps({ + "accessToken": "AT-shared", "refreshToken": "RT-shared", + "expiresAt": int((time.time() + 3600) * 1000), + })) + kid = _profile(fleet, "kid") + kid.mkdir(parents=True, exist_ok=True) + (kid / "auth.json").write_text(json.dumps({"providers": {}, "credential_pool": {}})) + (kid / ".anthropic_oauth.json").symlink_to(root / ".anthropic_oauth.json") + before = (root / ".anthropic_oauth.json").read_text() + + fleet["use"](kid) + assert heal_forked_single_use_oauth_grants("anthropic") is None + assert (kid / ".anthropic_oauth.json").is_symlink() + assert (root / ".anthropic_oauth.json").read_text() == before + + +def test_heal_same_store_skip_is_memoized_off_the_hot_path(fleet, monkeypatch): + """The shared-store skip must record the clean mark so load_pool()'s + per-call heal does not re-stat/resolve both paths every model call.""" + from hermes_cli import auth as auth_mod + + root = fleet["root"] + _seed_codex_grant(root) + shared = _shared_profile(fleet, "shared", link=lambda target, alias: alias.symlink_to(target)) + fleet["use"](shared) + + assert auth_mod.heal_forked_single_use_oauth_grants("openai-codex") is None + assert "openai-codex" in auth_mod._oauth_heal_clean_marks + calls = [] + monkeypatch.setattr(auth_mod, "_is_same_auth_store", lambda *a: calls.append(a) or True) + assert auth_mod.heal_forked_single_use_oauth_grants("openai-codex") is None + assert calls == [], "same-store check ran again despite the clean mark" diff --git a/tests/agent/test_cron_inflight_prompt_reappend_100818.py b/tests/agent/test_cron_inflight_prompt_reappend_100818.py new file mode 100644 index 0000000000..ef1f8ccf24 --- /dev/null +++ b/tests/agent/test_cron_inflight_prompt_reappend_100818.py @@ -0,0 +1,333 @@ +"""Regression coverage for #100818: compaction of a single-prompt session +(the cron shape) must not leave the model with nothing to obey. + +A cron run is one user message — the job prompt — followed by nothing but +assistant/tool turns. When ContextCompressor fires mid-run, that prompt is +folded into the handoff summary and no user message survives *after* it. +SUMMARY_PREFIX then reads literally: + + If no user message appears AFTER this summary, do nothing. + +so the model correctly does nothing, the scheduler sees the ``[SILENT]`` +sentinel, and records ``last_status: ok`` — a silent failure. + +The fix re-appends the in-flight user task after the handoff so the prefix's +"latest user message" pointer resolves to the job prompt again. The +#80622 contract is unchanged: an idle session with no in-flight task must +still be left with nothing to act on. +""" + +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +from agent.context_compressor import ( + _SUMMARY_END_MARKER, + SUMMARY_PREFIX, + ContextCompressor, +) + + +JOB_SENTINEL = "CRON_JOB_PROMPT_sentinel_brief_the_inbox_and_write_a_digest" + + +def _make_compressor() -> ContextCompressor: + with patch( + "agent.context_compressor.get_model_context_length", return_value=100_000 + ): + compressor = ContextCompressor( + model="test", + quiet_mode=True, + protect_first_n=2, + protect_last_n=2, + ) + compressor.tail_token_budget = 500 + return compressor + + +def _tool_pairs(count: int, start: int = 0) -> List[Dict[str, Any]]: + """``count`` assistant(tool_calls) + tool result pairs.""" + turns: List[Dict[str, Any]] = [] + for i in range(start, start + count): + turns.append( + { + "role": "assistant", + "content": f"step {i}", + "tool_calls": [ + {"id": f"c{i}", "function": {"name": "terminal", "arguments": "{}"}} + ], + } + ) + turns.append( + { + "role": "tool", + "tool_call_id": f"c{i}", + "content": ("tool output " * 200) + f" {i}", + } + ) + return turns + + +def _cron_transcript() -> List[Dict[str, Any]]: + """system + one user job prompt + many tool turns, NO trailing user.""" + return [ + { + "role": "system", + "content": "You are Hermes. Cron preamble: if nothing to report, " + "return [SILENT].", + }, + {"role": "user", "content": JOB_SENTINEL}, + *_tool_pairs(40), + ] + + +def _compress(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message.content = ( + "## Historical Task Snapshot\nUser asked: '" + JOB_SENTINEL + "'\n" + "## Summary\nRan a bunch of terminal steps." + ) + compressor = _make_compressor() + with patch("agent.context_compressor.call_llm", return_value=response): + return compressor.compress(messages, current_tokens=200_000, force=True) + + +def _handoff_idx(compressed: List[Dict[str, Any]]) -> int: + """Index of the handoff row (standalone summary or merged carrier).""" + for idx in range(len(compressed) - 1, -1, -1): + content = compressed[idx].get("content") + text = content if isinstance(content, str) else str(content) + if SUMMARY_PREFIX[:60] in text or _SUMMARY_END_MARKER in text: + return idx + return -1 + + +def _text(message: Dict[str, Any]) -> str: + content = message.get("content") + return content if isinstance(content, str) else str(content) + + +def _actionable_user_rows(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return [ + m + for m in rows + if ContextCompressor._is_actionable_user_turn(m) + and not ContextCompressor._is_synthetic_compression_user_turn(m) + ] + + +def test_cron_job_prompt_survives_after_the_handoff(): + """The in-flight job prompt must be readable AFTER the summary boundary.""" + compressed = _compress(_cron_transcript()) + + idx = _handoff_idx(compressed) + assert idx >= 0, "expected a compaction handoff in the compressed transcript" + + after = compressed[idx + 1:] + # The re-append may also land inside the handoff carrier itself, after the + # end marker (the alternation-safe merge layout) — accept either shape. + carrier_tail = _text(compressed[idx]).split(_SUMMARY_END_MARKER)[-1] + + job_after_summary = any( + JOB_SENTINEL in _text(m) for m in _actionable_user_rows(after) + ) or (JOB_SENTINEL in carrier_tail) + assert job_after_summary, ( + "the in-flight cron job prompt must appear in a user message AFTER the " + "handoff summary — SUMMARY_PREFIX orders the model to do nothing " + "otherwise (#100818)" + ) + + +def test_model_is_not_left_without_a_user_message_after_the_handoff(): + """The 'no user message after this summary → do nothing' branch of + SUMMARY_PREFIX must not be what a mid-run cron compaction produces.""" + compressed = _compress(_cron_transcript()) + + idx = _handoff_idx(compressed) + after = compressed[idx + 1:] + has_user_after = bool(_actionable_user_rows(after)) or bool( + _text(compressed[idx]).split(_SUMMARY_END_MARKER)[-1].strip() + ) + assert has_user_after, ( + "compaction left no user message after the handoff; the model is " + "instructed to do nothing and the cron run fails silently" + ) + + +def test_role_alternation_and_head_are_preserved(): + """The re-append must not create two same-role rows in a row, and must + not disturb the cached head prefix.""" + messages = _cron_transcript() + compressed = _compress([dict(m) for m in messages]) + + assert compressed[0]["role"] == "system" + visible = [ + m.get("role") + for m in compressed + if not ( + m.get("role") == "tool" + or (m.get("role") == "assistant" and m.get("tool_calls")) + ) + ] + for previous, current in zip(visible, visible[1:]): + assert not (previous == current == "user"), ( + f"consecutive user rows in compressed transcript: {visible}" + ) + + +def test_idle_session_without_inflight_task_is_not_reanimated(): + """#80622 must hold: a session whose only user-role row is an inherited + handoff has no in-flight task, so compaction must not manufacture one.""" + messages: List[Dict[str, Any]] = [ + {"role": "system", "content": "You are Hermes."}, + { + "role": "user", + "content": ( + SUMMARY_PREFIX + + "\n## Historical Task Snapshot\nUser asked: 'a finished task'\n\n" + + _SUMMARY_END_MARKER + ), + }, + *_tool_pairs(40), + ] + compressed = _compress(messages) + + assert not _actionable_user_rows(compressed), ( + "no real user turn existed before compaction — none may be invented" + ) + + +def test_completed_exchange_is_not_replayed(): + """Only an in-flight task is re-appended. A turn that already produced a + final assistant reply must not be handed back to the model as a fresh + instruction.""" + messages = [ + *_cron_transcript(), + {"role": "assistant", "content": "Digest written. Nothing else to do."}, + ] + compressed = _compress(messages) + + idx = _handoff_idx(compressed) + after = compressed[idx + 1:] + assert not any( + JOB_SENTINEL in _text(m) for m in _actionable_user_rows(after) + ), "a completed exchange must not be re-appended as a new user instruction" + + +# --------------------------------------------------------------------------- +# Follow-up coverage (salvage of #101170) +# --------------------------------------------------------------------------- + + +def _pending_tail_transcript() -> List[Dict[str, Any]]: + """Cron shape whose LAST row is an assistant tool_calls turn still awaiting + its result — compaction fired inside the tool-execution window.""" + msgs = _cron_transcript() + msgs.append( + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "pending", "function": {"name": "terminal", "arguments": "{}"}} + ], + } + ) + return msgs + + +def test_pending_trailing_tool_call_survives_the_replay(): + """The replay row must not make a genuinely in-flight tool_call look + orphaned: _sanitize_tool_pairs runs before the re-append, so the trailing + assistant(tool_calls) keeps its calls and the late tool result still pairs.""" + out = _compress(_pending_tail_transcript()) + pending = [ + m + for m in out + if m.get("role") == "assistant" + and any(tc.get("id") == "pending" for tc in (m.get("tool_calls") or [])) + ] + assert pending, "trailing in-flight tool_calls were stripped" + replay_idx = max( + i for i, m in enumerate(out) + if m.get("role") == "user" and JOB_SENTINEL in str(m.get("content")) + ) + assert replay_idx > out.index(pending[0]) + + +def _compress_with(protect_first_n: int, compression_count: int, messages): + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message.content = SUMMARY_PREFIX + "\n## Summary\nran steps." + with patch( + "agent.context_compressor.get_model_context_length", return_value=100_000 + ): + compressor = ContextCompressor( + model="test", + quiet_mode=True, + protect_first_n=protect_first_n, + protect_last_n=2, + ) + compressor.tail_token_budget = 500 + compressor.compression_count = compression_count + with patch.object( + compressor, "_generate_summary", return_value=response.choices[0].message.content + ): + return compressor.compress(messages, current_tokens=90_000) + + +def _job_copies(messages) -> int: + return sum( + str(m.get("content")).count(JOB_SENTINEL) + for m in messages + if m.get("role") == "user" + ) + + +def test_merged_restatement_is_not_anchored_twice(): + """Later cycle: protect_first_n has decayed, the prompt is summarised away + and the summary is pinned to role=user followed by an exempt tool tail, so + the restatement is MERGED onto the carrier. The later + _ensure_compressed_has_user_turn pass must see intent as present instead + of inserting a second copy of the job prompt.""" + from agent.conversation_compression import _ensure_compressed_has_user_turn + + original = [{"role": "user", "content": JOB_SENTINEL}, *_tool_pairs(40)] + out = _compress_with(2, 1, original) + assert any(m.get("_inflight_replay_merged") for m in out), "expected merge layout" + assert _job_copies(out) == 1 + assert _ensure_compressed_has_user_turn(original, out) == "already_present" + assert _job_copies(out) == 1 + + +def test_restatement_survives_repeated_compactions_without_stacking(): + """A task alive across three compactions is restated exactly once per + output — one copy, one header, always after the summary.""" + from agent.context_compressor import _INFLIGHT_TASK_REPLAY_HEADER + + out = [{"role": "user", "content": JOB_SENTINEL}, *_tool_pairs(40)] + for cycle in (1, 2, 3): + extra = _tool_pairs(40, start=100 * cycle) if cycle > 1 else [] + out = _compress_with(2, cycle, out + extra) + users = [m for m in out if m.get("role") == "user"] + assert _job_copies(out) == 1, cycle + assert max( + str(m.get("content")).count(_INFLIGHT_TASK_REPLAY_HEADER) for m in users + ) == 1, cycle + last = str(users[-1].get("content")) + assert last.rfind(JOB_SENTINEL) > last.rfind(_SUMMARY_END_MARKER), cycle + + +def test_flagged_scaffolding_row_is_never_the_inflight_task(): + """A trailing user-role scaffolding row flagged synthetic (todo snapshot) + must not be mistaken for the live request and replayed as an instruction.""" + msgs = _cron_transcript() + msgs.append( + { + "role": "user", + "content": "[Your active task list was preserved across context compression]\n- x", + "_todo_snapshot_synthetic": True, + } + ) + found = ContextCompressor._find_inflight_user_task(msgs) + assert found is not None + assert JOB_SENTINEL in str(found.get("content")) diff --git a/tests/agent/test_curator.py b/tests/agent/test_curator.py index eac61e01c5..f14ef73ea9 100644 --- a/tests/agent/test_curator.py +++ b/tests/agent/test_curator.py @@ -931,7 +931,7 @@ def test_review_fork_toolset_surface_excludes_execution_tools(): # The incident class stays out: no command execution, no background # process steering (stdin is a second unguarded write sink), and no # generic filesystem-write tool. - for tool in ("terminal", "process", "write_file", "patch", + for tool in ("terminal", "process_manage", "write_file", "patch", "execute_code", "computer_use", "browser_exec"): assert tool not in surface, ( f"execution/write tool {tool!r} leaked into the curator fork's " diff --git a/tests/agent/test_curator_backup.py b/tests/agent/test_curator_backup.py index fcd55112b8..12826e49b8 100644 --- a/tests/agent/test_curator_backup.py +++ b/tests/agent/test_curator_backup.py @@ -484,3 +484,116 @@ def test_rollback_recovers_cleanly_from_a_partial_extract(backup_env, monkeypatc assert present == ["alpha", "beta"], f"tree not restored: {present}" assert "current copy" in (skills / "alpha" / "SKILL.md").read_text(encoding="utf-8") assert "current only" in (skills / "beta" / "SKILL.md").read_text(encoding="utf-8") + + +def test_snapshot_excludes_git_and_curator_backups_and_hub(backup_env): + """Tar snapshots must exclude .git, .curator_backups, and .hub (top-level and nested).""" + cb = backup_env["cb"] + skills = backup_env["skills"] + + # Top-level excluded structures + (skills / ".git").mkdir() + (skills / ".git" / "config").write_text("[core]\nrepositoryformatversion = 0\n", encoding="utf-8") + (skills / ".hub").mkdir() + (skills / ".hub" / "lock.json").write_text("{}", encoding="utf-8") + + # Regular skill with nested .git + _write_skill(skills, "alpha", body="alpha body") + nested_git = skills / "alpha" / ".git" + nested_git.mkdir() + (nested_git / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + + snap_dir = cb.snapshot_skills(reason="test-exclude-git") + assert snap_dir is not None + + archive = snap_dir / "skills.tar.gz" + assert archive.exists() + + with tarfile.open(archive, "r:gz") as tf: + members = tf.getnames() + + # Ensure no member contains .git, .curator_backups, or .hub + for name in members: + parts = Path(name).parts + assert ".git" not in parts, f".git found in archive: {name}" + assert ".curator_backups" not in parts, f".curator_backups found in archive: {name}" + assert ".hub" not in parts, f".hub found in archive: {name}" + + assert "alpha/SKILL.md" in members + + +def test_rollback_preserves_top_level_git(backup_env): + """Rollback must preserve repository .git metadata untouched in the skills root.""" + cb = backup_env["cb"] + skills = backup_env["skills"] + + git_dir = skills / ".git" + git_dir.mkdir() + (git_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + _write_skill(skills, "alpha", body="v1") + + snap_dir = cb.snapshot_skills(reason="snap-v1") + assert snap_dir is not None + + # Mutate skills tree and git metadata + _write_skill(skills, "alpha", body="v2") + _write_skill(skills, "beta", body="new") + (git_dir / "HEAD").write_text("ref: refs/heads/feature\n", encoding="utf-8") + + ok, msg, _ = cb.rollback(backup_id=snap_dir.name) + assert ok, f"rollback failed: {msg}" + + assert (skills / "alpha" / "SKILL.md").read_text(encoding="utf-8").find("v1") != -1 + assert not (skills / "beta").exists() + assert (git_dir / "HEAD").exists() + assert (git_dir / "HEAD").read_text(encoding="utf-8") == "ref: refs/heads/feature\n" + + + +def test_rollback_preserves_nested_git_inside_skill(backup_env): + """A skill that is itself a git checkout keeps its .git across rollback: + the snapshot excludes it, so rollback must carry it over from the live tree.""" + cb = backup_env["cb"] + skills = backup_env["skills"] + + _write_skill(skills, "alpha", body="v1") + nested_git = skills / "alpha" / ".git" + nested_git.mkdir() + (nested_git / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + + snap_dir = cb.snapshot_skills(reason="snap-v1") + assert snap_dir is not None + + _write_skill(skills, "alpha", body="v2") + (nested_git / "HEAD").write_text("ref: refs/heads/feature\n", encoding="utf-8") + + ok, msg, _ = cb.rollback(backup_id=snap_dir.name) + assert ok, f"rollback failed: {msg}" + assert "v1" in (skills / "alpha" / "SKILL.md").read_text(encoding="utf-8") + # Live .git state (post-snapshot) is what survives — it was never archived. + assert (nested_git / "HEAD").read_text(encoding="utf-8") == "ref: refs/heads/feature\n" + staging = list((skills / ".curator_backups").glob(".rollback-staging-*")) + assert staging == [], f"staging dir left behind: {staging}" + + +def test_rollback_preserves_nested_git_file_pointer(backup_env): + """Submodule / worktree checkouts use a ``.git`` FILE (gitdir: pointer); + it must be carried across rollback like the dir form.""" + cb = backup_env["cb"] + skills = backup_env["skills"] + + _write_skill(skills, "alpha", body="v1") + git_ptr = skills / "alpha" / ".git" + git_ptr.write_text("gitdir: ../../.git/modules/alpha\n", encoding="utf-8") + + snap_dir = cb.snapshot_skills(reason="snap-v1") + assert snap_dir is not None + with tarfile.open(snap_dir / "skills.tar.gz", "r:gz") as tf: + assert "alpha/.git" not in tf.getnames() + + _write_skill(skills, "alpha", body="v2") + ok, msg, _ = cb.rollback(backup_id=snap_dir.name) + assert ok, f"rollback failed: {msg}" + assert "v1" in (skills / "alpha" / "SKILL.md").read_text(encoding="utf-8") + assert git_ptr.is_file() + assert git_ptr.read_text(encoding="utf-8").startswith("gitdir:") diff --git a/tests/agent/test_display_todo_progress.py b/tests/agent/test_display_todo_progress.py index d182be9269..3d6d657ca5 100644 --- a/tests/agent/test_display_todo_progress.py +++ b/tests/agent/test_display_todo_progress.py @@ -26,7 +26,7 @@ class TestTodoRead: """get_cute_tool_message(…, result=…) when todos_arg is None (read path).""" def test_read_no_result(self): - msg = get_cute_tool_message("todo", {}, 0.5) + msg = get_cute_tool_message("todo_list", {}, 0.5) assert "reading tasks" in msg assert "0.5s" in msg @@ -34,7 +34,7 @@ class TestTodoRead: def test_read_zero_total(self): """Edge case: empty todo list returns summary with total=0.""" - msg = get_cute_tool_message("todo", {}, 0.5, + msg = get_cute_tool_message("todo_list", {}, 0.5, result=_todo_result(0, 0)) assert "reading tasks" in msg @@ -46,7 +46,7 @@ class TestTodoCreate: def test_create_default(self): """Brand-new plan: all pending, no result — plain count.""" - msg = get_cute_tool_message("todo", + msg = get_cute_tool_message("todo_list", {"todos": [ {"id": "a", "content": "x", "status": "pending"}, ]}, 0.3) @@ -58,7 +58,7 @@ class TestTodoCreate: def test_create_with_result_zero_done(self): """New plan with 0 done — plain count, no progress fraction.""" - msg = get_cute_tool_message("todo", + msg = get_cute_tool_message("todo_list", {"todos": [ {"id": "a", "content": "x", "status": "pending"}, {"id": "b", "content": "y", "status": "pending"}, @@ -74,7 +74,7 @@ class TestTodoUpdate: def test_update_no_result(self): """No result available — plain update N task(s).""" - msg = get_cute_tool_message("todo", + msg = get_cute_tool_message("todo_list", {"todos": [{"id": "a", "status": "completed"}], "merge": True}, 0.5) assert "update 1 task(s)" in msg @@ -82,7 +82,7 @@ class TestTodoUpdate: def test_update_halfway(self): """2/4 — midpoint progress.""" - msg = get_cute_tool_message("todo", + msg = get_cute_tool_message("todo_list", {"todos": [{"id": "b", "status": "in_progress"}], "merge": True}, 0.7, @@ -96,7 +96,7 @@ class TestTodoUpdate: def test_update_total_not_in_summary(self): """Result summary missing total key.""" - msg = get_cute_tool_message("todo", + msg = get_cute_tool_message("todo_list", {"todos": [{"id": "a", "status": "completed"}], "merge": True}, 0.3, @@ -111,7 +111,7 @@ class TestTodoEdgeCases: def test_merge_default_value(self): """merge defaults to False in function signature, should be False when absent.""" - msg = get_cute_tool_message("todo", + msg = get_cute_tool_message("todo_list", {"todos": [{"id": "a", "content": "x", "status": "pending"}]}, 1.0) assert "1 task(s)" in msg @@ -120,7 +120,7 @@ class TestTodoEdgeCases: def test_large_task_count(self): """Many tasks should not break formatting.""" many = [{"id": str(i), "content": "x", "status": "pending"} for i in range(50)] - msg = get_cute_tool_message("todo", {"todos": many}, 0.5) + msg = get_cute_tool_message("todo_list", {"todos": many}, 0.5) assert "50 task(s)" in msg @@ -131,7 +131,7 @@ class TestTodoSkinIntegration: """ def test_default_skin_prefix(self): - msg = get_cute_tool_message("todo", {}, 0.5) + msg = get_cute_tool_message("todo_list", {}, 0.5) assert msg.startswith("┊") diff --git a/tests/agent/test_empty_response_guard.py b/tests/agent/test_empty_response_guard.py index 3ec1a0c437..c8e6f0dac4 100644 --- a/tests/agent/test_empty_response_guard.py +++ b/tests/agent/test_empty_response_guard.py @@ -5,7 +5,8 @@ deterministic empties (unsignaled provider refusals with zero output tokens) while never tightening behaviour on ambiguous evidence. Fail-open contract under test: -- Missing usage -> never deterministic, default budget. +- Missing usage + no observed generation -> deterministic after two attempts. +- Missing usage + observed reasoning -> never deterministic. - Any generated tokens (output or reasoning) -> never deterministic. - Different model/provider/finish_reason across attempts -> not deterministic. - Guard disabled via config (agent.empty_response_guard.enabled: false) -> @@ -42,11 +43,21 @@ def _response(prompt_tokens=25_900, completion_tokens=0, usage_present=True): return SimpleNamespace(usage=usage) -def _record_streak(agent, responses, finish_reasons=None): +def _record_streak( + agent, responses, finish_reasons=None, observed_generations=None +): """Record attempts the way the loop does: record, then increment.""" finish_reasons = finish_reasons or ["stop"] * len(responses) - for resp, reason in zip(responses, finish_reasons): - guard.record_empty_attempt(agent, finish_reason=reason, response=resp) + observed_generations = observed_generations or [False] * len(responses) + for resp, reason, observed_generation in zip( + responses, finish_reasons, observed_generations + ): + guard.record_empty_attempt( + agent, + finish_reason=reason, + response=resp, + observed_generation=observed_generation, + ) agent._empty_content_retries += 1 @@ -62,12 +73,21 @@ class TestDeterministicEmpty: _record_streak(agent, [_response()]) assert guard.deterministic_empty(agent) is False - def test_missing_usage_fails_open(self): + def test_missing_usage_without_observed_generation_is_deterministic(self): agent = _agent() _record_streak( agent, [_response(usage_present=False), _response(usage_present=False)], ) + assert guard.deterministic_empty(agent) is True + + def test_missing_usage_with_observed_reasoning_fails_open(self): + agent = _agent() + _record_streak( + agent, + [_response(usage_present=False), _response(usage_present=False)], + observed_generations=[True, True], + ) assert guard.deterministic_empty(agent) is False def test_mixed_usage_presence_fails_open(self): diff --git a/tests/agent/test_error_classifier.py b/tests/agent/test_error_classifier.py index 5aebaa2eb4..8b29100a48 100644 --- a/tests/agent/test_error_classifier.py +++ b/tests/agent/test_error_classifier.py @@ -65,6 +65,7 @@ class TestFailoverReason: "model_not_found", "format_error", "invalid_encrypted_content", "multimodal_tool_content_unsupported", + "reasoning_mandatory", "provider_policy_blocked", "content_policy_blocked", "thinking_signature", "long_context_tier", @@ -690,6 +691,21 @@ class TestClassifyApiError: assert result.retryable is True assert result.should_fallback is False + # ── Reasoning-mandatory route rejecting a disable ── + + def test_reasoning_mandatory_400_is_retryable_not_format_error(self): + e = MockAPIError( + "Error code: 400 - This request is not valid. Check the model name " + "and other parameters. Additional info: Reasoning is mandatory for " + "this endpoint and cannot be disabled.", + status_code=400, + ) + result = classify_api_error(e, provider="nous", model="z-ai/glm-5.3-flash") + assert result.reason == FailoverReason.reasoning_mandatory + assert result.retryable is True + assert result.should_fallback is False + assert result.should_compress is False + # ── Provider-specific: llama.cpp grammar-parse ── def test_llama_cpp_unable_to_generate_parser_template(self): diff --git a/tests/agent/test_fast_mode_auto.py b/tests/agent/test_fast_mode_auto.py new file mode 100644 index 0000000000..a9af81a8df --- /dev/null +++ b/tests/agent/test_fast_mode_auto.py @@ -0,0 +1,143 @@ +"""Bounded /fast auto|cold windows and the shared route-aware gate.""" + +from types import SimpleNamespace + +from agent import fast_mode + + +def _agent(**kw): + base = dict( + service_tier="auto", + model="gpt-5.4", + provider="openai", + base_url="https://api.openai.com/v1", + api_mode="chat_completions", + request_overrides={"extra_body": {"keep": 1}}, + fast_auto_seconds=60, + ) + base.update(kw) + return SimpleNamespace(**base) + + +def test_bounded_fast_window_policy(monkeypatch): + clock = [1000.0] + monkeypatch.setattr(fast_mode.time, "monotonic", lambda: clock[0]) + + # auto: window open -> fast override layered over existing overrides + agent = _agent() + fast_mode.begin_turn(agent, conversation_history=[]) + assert fast_mode.effective_request_overrides(agent) == { + "extra_body": {"keep": 1}, + "service_tier": "priority", + } + assert agent.request_overrides == {"extra_body": {"keep": 1}} # never mutated + + # window expired -> override absent + clock[0] += 61 + assert fast_mode.effective_request_overrides(agent) == {"extra_body": {"keep": 1}} + + # auto re-opens on the next turn + fast_mode.begin_turn(agent, conversation_history=[{"role": "user", "content": "x"}]) + assert "service_tier" in fast_mode.effective_request_overrides(agent) + + # cold: prior history -> no window at all + cold = _agent(service_tier="cold") + fast_mode.begin_turn(cold, conversation_history=[{"role": "user", "content": "x"}]) + assert "service_tier" not in fast_mode.effective_request_overrides(cold) + fast_mode.begin_turn(cold, conversation_history=None) + assert fast_mode.effective_request_overrides(cold)["service_tier"] == "priority" + + # Anthropic route uses the speed param + anth = _agent( + service_tier="auto", + model="claude-opus-5", + provider="anthropic", + base_url="https://api.anthropic.com", + api_mode="anthropic_messages", + ) + fast_mode.begin_turn(anth, conversation_history=[]) + assert fast_mode.effective_request_overrides(anth)["speed"] == "fast" + + # unsupported routes never get fast params, in auto or static mode + from hermes_cli.models import resolve_fast_mode_overrides + + for provider, base_url in ( + ("openrouter", "https://openrouter.ai/api/v1"), + ("nous", "https://inference-api.nousresearch.com/v1"), + ("copilot", "https://api.githubcopilot.com"), + ("azure", "https://foo.openai.azure.com"), + ("custom", "http://10.0.0.1:8000/v1"), + ("openai", "https://proxy.example.com/v1"), + ): + proxied = _agent(provider=provider, base_url=base_url) + fast_mode.begin_turn(proxied, conversation_history=[]) + assert "service_tier" not in fast_mode.effective_request_overrides(proxied), provider + assert resolve_fast_mode_overrides("gpt-5.4", provider=provider, base_url=base_url) is None + assert resolve_fast_mode_overrides( + "claude-opus-5", provider="bedrock", base_url="https://bedrock-runtime.us-east-1.amazonaws.com" + ) is None + # first-party routes (and the legacy model-only call) still resolve + assert resolve_fast_mode_overrides("gpt-5.4", provider="openai-codex", base_url="https://chatgpt.com/backend-api/codex") + assert resolve_fast_mode_overrides("grok-4.6", provider="xai", base_url="https://api.x.ai/v1") + assert resolve_fast_mode_overrides("gpt-5.4") == {"service_tier": "priority"} + + # normal / static modes are untouched by the window logic + static = _agent(service_tier="priority", request_overrides={"service_tier": "priority"}) + fast_mode.begin_turn(static, conversation_history=[]) + assert fast_mode.effective_request_overrides(static) == {"service_tier": "priority"} + off = _agent(service_tier=None) + fast_mode.begin_turn(off, conversation_history=[]) + assert fast_mode.effective_request_overrides(off) == {"extra_body": {"keep": 1}} + + +def test_fast_auto_and_cold_parse_and_slash_command(monkeypatch): + import hermes_cli.config as config_mod + + if not hasattr(config_mod, "save_env_value_secure"): + config_mod.save_env_value_secure = lambda key, value: {"success": True} + import cli as cli_mod + from gateway.run import GatewayRunner + from hermes_cli.commands import COMMAND_REGISTRY + from hermes_cli.config import DEFAULT_CONFIG + + # config parsing: CLI, gateway, TUI all accept auto/cold; default stays off + for raw, expected in (("auto", "auto"), ("COLD", "cold"), ("fast", "priority"), ("", None), ("bogus", None)): + assert cli_mod._parse_service_tier_config(raw) == expected + monkeypatch.setattr( + "gateway.run._load_gateway_runtime_config", lambda: {"agent": {"service_tier": raw}} + ) + assert GatewayRunner._load_service_tier() == expected + assert DEFAULT_CONFIG["agent"]["service_tier"] == "" + assert DEFAULT_CONFIG["agent"]["fast_auto_seconds"] == 60 + + # /fast auto — session-scoped, agent rebuilt, status reports the mode + fast_cmd = next(c for c in COMMAND_REGISTRY if c.name == "fast") + assert {"auto", "cold"} <= set(fast_cmd.subcommands) + printed = [] + monkeypatch.setattr(cli_mod, "_cprint", lambda *a, **k: printed.append(" ".join(map(str, a)))) + monkeypatch.setattr(cli_mod, "save_config_value", lambda *a, **k: (_ for _ in ()).throw(AssertionError("no config write"))) + stub = SimpleNamespace( + service_tier=None, model="gpt-5.4", agent=object(), _fast_command_available=lambda: True + ) + cli_mod.HermesCLI._handle_fast_command(stub, "/fast auto") + assert stub.service_tier == "auto" + assert stub.agent is None + cli_mod.HermesCLI._handle_fast_command(stub, "/fast status") + assert any("auto" in line for line in printed) + cli_mod.HermesCLI._handle_fast_command(stub, "/fast cold") + assert stub.service_tier == "cold" + + # auto/cold do NOT pin a static override into the turn route + route_stub = SimpleNamespace( + model="gpt-5.4", api_key="k", base_url="https://api.openai.com/v1", provider="openai", + api_mode="chat_completions", acp_command=None, acp_args=[], _credential_pool=None, + service_tier="auto", + ) + assert cli_mod.HermesCLI._resolve_turn_agent_config(route_stub, "hi")["request_overrides"] is None + route_stub.service_tier = "priority" + assert cli_mod.HermesCLI._resolve_turn_agent_config(route_stub, "hi")["request_overrides"] == { + "service_tier": "priority" + } + route_stub.base_url = "https://openrouter.ai/api/v1" + route_stub.provider = "openrouter" + assert cli_mod.HermesCLI._resolve_turn_agent_config(route_stub, "hi")["request_overrides"] is None diff --git a/tests/agent/test_idle_compaction.py b/tests/agent/test_idle_compaction.py index f4de6c819a..ba89099cfe 100644 --- a/tests/agent/test_idle_compaction.py +++ b/tests/agent/test_idle_compaction.py @@ -39,3 +39,66 @@ class TestShouldIdleCompact: def test_fires_just_above_floor(self): assert _decide(tokens=40_001, floor_tokens=40_000) is True + +class TestPostCompactionFloor: + """The floor also honours what the previous pass actually produced (#97239). + + ``floor_tokens`` is the theoretical target (threshold × target_ratio); a + real pass lands well above it because the system prompt, the tool schemas + and the protected head/tail are incompressible. Without this, an already + compacted session re-summarises itself on every idle resume forever. + """ + + def test_unrecorded_last_compaction_keeps_original_floor(self): + # 0 = nothing compacted yet (or state reset) — original semantics. + assert _decide(tokens=40_001, floor_tokens=40_000, + last_compaction_tokens=0) is True + + def test_skips_when_transcript_has_not_grown_since_last_compaction(self): + # Previous pass produced 44,000; the transcript is still ~that size. + assert _decide(tokens=44_100, floor_tokens=40_000, + last_compaction_tokens=44_000) is False + + def test_fires_once_a_full_floor_of_new_content_accumulated(self): + assert _decide(tokens=84_001, floor_tokens=40_000, + last_compaction_tokens=44_000) is True + + def test_does_not_fire_at_exactly_the_raised_floor(self): + assert _decide(tokens=84_000, floor_tokens=40_000, + last_compaction_tokens=44_000) is False + + def test_reported_session_stops_recompacting_itself(self): + """Exact numbers from issue #97239. + + The 17:01 pass reduced 64,105 -> 44,579 tokens; the 17:36 resume + re-fired on that same transcript because 44,579 > the 25,502 + theoretical floor, blocking the prompt for another 256 s. + """ + common = dict(idle_after_seconds=1, idle_gap_seconds=747.0, + floor_tokens=25_502) + # Before the fix the second resume fired: 44,579 > 25,502. + assert _decide(tokens=44_579, last_compaction_tokens=0, **common) is True + # With the previous pass's real output known, it sits the round out. + assert _decide(tokens=44_579, last_compaction_tokens=44_579, + **common) is False + + def test_an_effective_pass_still_raises_the_floor(self): + # 100K -> 10K is a good pass; another one is worth it only once about + # a floor's worth of new content has landed on top of the 10K. + assert _decide(tokens=30_000, floor_tokens=25_000, + last_compaction_tokens=10_000) is False + assert _decide(tokens=35_001, floor_tokens=25_000, + last_compaction_tokens=10_000) is True + + def test_other_gates_still_win_over_the_raised_floor(self): + # Growth alone must not defeat the cooldown / opt-out gates. + assert _decide(tokens=200_000, floor_tokens=40_000, + last_compaction_tokens=44_000, + cooldown_active=True) is False + assert _decide(tokens=200_000, floor_tokens=40_000, + last_compaction_tokens=44_000, + idle_after_seconds=0) is False + assert _decide(tokens=200_000, floor_tokens=40_000, + last_compaction_tokens=44_000, + idle_gap_seconds=0.5) is False + diff --git a/tests/agent/test_idle_compaction_lock_and_guards.py b/tests/agent/test_idle_compaction_lock_and_guards.py index 613488abe4..d1c30a024a 100644 --- a/tests/agent/test_idle_compaction_lock_and_guards.py +++ b/tests/agent/test_idle_compaction_lock_and_guards.py @@ -53,18 +53,20 @@ def _prep_idle_agent(db: SessionDB, session_id: str, *, idle_after: int = 60, return agent -def _run_prologue(agent, history, user_message="hello again"): +def _run_prologue(agent, history, user_message="hello again", + rough_tokens: int = 999_999): """Invoke ``build_turn_context`` the way ``conversation_loop`` does. The token-threshold preflight gate is pinned False so these tests exercise the IDLE trigger in isolation (the preflight path has its own - coverage in ``test_turn_context.py``). + coverage in ``test_turn_context.py``). ``rough_tokens`` pins the estimate + that the idle floor is compared against. """ with patch("agent.auxiliary_client.set_runtime_main", lambda *a, **k: None), \ patch("agent.turn_context._should_run_preflight_estimate", return_value=False), \ patch("agent.turn_context.estimate_request_tokens_rough", - return_value=999_999): + return_value=rough_tokens): return build_turn_context( agent=agent, user_message=user_message, @@ -145,6 +147,96 @@ def test_idle_compaction_defers_to_held_compression_lock(tmp_path: Path) -> None assert ctx.messages[ctx.current_turn_user_idx]["content"] == "hello again" +def _prep_recompaction_agent(db: SessionDB, sid: str): + """Idle-eligible agent with the #97239 threshold/floor numbers. + + threshold 127,510 x target_ratio 0.20 => a 25,502 theoretical floor, the + same one the reported session kept clearing while never actually + shrinking below ~44,579. + """ + agent = _prep_idle_agent(db, sid, idle_after=1, idle_gap=747.0) + agent.context_compressor.threshold_tokens = 127_510 + agent.context_compressor.summary_target_ratio = 0.20 + agent.context_compressor.emit_automatic_compaction_status = True + del agent.context_compressor.get_automatic_compaction_status_message + return agent + + +def _pin_compress_seam(agent): + """Stub the forwarder so these tests assert the idle DECISION only. + + Whether ``compress_context`` then rotates, locks or aborts is covered by + the tests above; here the question is purely whether the idle floor let + the turn through. Returning the input list is the documented "skipped" + shape, so the caller's re-baseline stays disarmed either way. + """ + seam = MagicMock(side_effect=lambda messages, *a, **k: (messages, "SYSTEM")) + agent._compress_context = seam + return seam + + +def test_idle_compaction_skips_a_transcript_that_has_not_grown(tmp_path: Path) -> None: + """The reported loop: re-compacting a session the last pass just produced. + + ``last_compression_rough_tokens`` records what the previous pass actually + emitted (44,579). The theoretical floor (25,502) is far below it, so the + old predicate re-fired a full multi-minute summary on every idle resume + even though the transcript had not grown at all (#97239). + """ + db = SessionDB(db_path=tmp_path / "state.db") + sid = "IDLE_RECOMPACT" + db.create_session(sid, source="cli") + agent = _prep_recompaction_agent(db, sid) + agent.context_compressor.last_compression_rough_tokens = 44_579 + seam = _pin_compress_seam(agent) + + ctx = _run_prologue(agent, _history(), rough_tokens=44_579) + + seam.assert_not_called() + agent.context_compressor.compress.assert_not_called() + assert agent.session_id == sid + assert len(ctx.messages) == len(_history()) + 1 + assert ctx.current_turn_user_idx == len(ctx.messages) - 1 + + +def test_idle_compaction_fires_again_once_the_transcript_grows(tmp_path: Path) -> None: + """The raised floor is a deferral, not an off switch.""" + db = SessionDB(db_path=tmp_path / "state.db") + sid = "IDLE_REGROWN" + db.create_session(sid, source="cli") + agent = _prep_recompaction_agent(db, sid) + agent.context_compressor.last_compression_rough_tokens = 44_579 + seam = _pin_compress_seam(agent) + + # 44,579 + 25,502 = 70,081 — one floor's worth of new content on top. + _run_prologue(agent, _history(), rough_tokens=70_082) + + seam.assert_called_once() + + +def test_idle_compaction_ignores_a_non_int_last_compaction_reading( + tmp_path: Path, +) -> None: + """Compressor doubles expose a Mock here — it must not raise the floor. + + An unset/derived attribute falls back to 0, which restores the original + ``tokens > floor_tokens`` semantics exactly. + """ + db = SessionDB(db_path=tmp_path / "state.db") + sid = "IDLE_MOCKREAD" + db.create_session(sid, source="cli") + agent = _prep_recompaction_agent(db, sid) + # Left as the MagicMock auto-attribute (a truthy non-int). + assert not isinstance( + agent.context_compressor.last_compression_rough_tokens, int + ) + seam = _pin_compress_seam(agent) + + _run_prologue(agent, _history(), rough_tokens=44_579) + + seam.assert_called_once() + + def test_idle_compaction_respects_anti_thrash_breaker(tmp_path: Path) -> None: """A tripped ineffective-compression breaker must block the idle trigger. diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index a6334eb71c..f2f954675b 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -303,6 +303,21 @@ class TestDefaultContextLengths: model, provider="kimi-coding", base_url=base_url ) == 1_048_576 + @pytest.mark.parametrize("model, provider, base_url", [ + ("muse-spark-1.3-contributor-free", "opencode-free", "https://opencode.ai/zen/v1"), + ("muse-spark-1.3-contributor", "opencode-go", "https://opencode.ai/zen/go/v1"), + ("muse-spark-1.3", "meta-ai", "https://api.meta.ai/v1"), + ("meta/muse-spark-1.3", "commandcode", "https://api.commandcode.ai/provider/v1"), + ]) + def test_muse_spark_resolves_1m_without_network(self, model, provider, base_url): + """Muse Spark is 1,048,576 on every host even when models.dev and the + live /models probe are unavailable (fresh HERMES_HOME, offline).""" + with patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata._query_ollama_api_show", return_value=None), \ + patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \ + patch("agent.models_dev.fetch_models_dev", return_value={}): + assert get_model_context_length(model, provider=provider, base_url=base_url) == 1_048_576 + def test_empty_model_uses_fallback_context(self): assert get_model_context_length("") == DEFAULT_FALLBACK_CONTEXT assert get_model_context_length(None) == DEFAULT_FALLBACK_CONTEXT # type: ignore[arg-type] @@ -799,6 +814,36 @@ class TestFetchEndpointModelMetadata: not_found.close.assert_called_once() success.close.assert_called_once() + def test_remote_probe_is_memoized_on_disk_across_processes(self, tmp_path, monkeypatch): + """A fresh process (cleared in-memory cache) must answer from the disk + memo within the TTL instead of re-probing the endpoint — the cost every + one-shot Bot Mode DM hop paid on startup. Expired memos re-probe.""" + import agent.model_metadata as mm + + monkeypatch.setattr( + mm, "_get_endpoint_metadata_cache_path", lambda: tmp_path / "endpoint_model_metadata.json" + ) + success = MagicMock() + success.status_code = 200 + success.json.return_value = {"data": [{"id": "test/model", "context_length": 32768}]} + + with patch("agent.model_metadata.requests.get", return_value=success) as mock_get: + assert mm.fetch_endpoint_model_metadata("https://custom.example/v1")["test/model"]["context_length"] == 32768 + # "New process": drop the in-memory cache only. + mm._endpoint_model_metadata_cache.clear() + mm._endpoint_model_metadata_cache_time.clear() + assert mm.fetch_endpoint_model_metadata("https://custom.example/v1")["test/model"]["context_length"] == 32768 + mock_get.assert_called_once() + + # Past the TTL the memo is stale and the endpoint is probed again. + mm._endpoint_model_metadata_cache.clear() + mm._endpoint_model_metadata_cache_time.clear() + with patch("agent.model_metadata.time.time", return_value=time.time() + mm._ENDPOINT_MODEL_CACHE_TTL + 1), patch( + "agent.model_metadata.requests.get", return_value=success + ) as mock_get: + mm.fetch_endpoint_model_metadata("https://custom.example/v1") + mock_get.assert_called_once() + # ========================================================================= # Nous Portal context-window resolution (provider="nous") @@ -1601,6 +1646,18 @@ class TestGrok43StaleCacheGuard: assert ctx == 256_000, f"{slug} should stay 256000, got {ctx}" +class TestMuseSparkStaleCacheGuard: + """Muse Spark (1M window per OpenRouter live metadata) had no catalog + entry, so older builds persisted the 256K default fallback. The cache + guard must flag that stale value and keep correct/probed values.""" + + def test_stale_muse_spark_detected_by_generic_guard(self): + from agent.model_metadata import _stale_pre_catalog_cache_entry + for slug in ("muse-spark-1.3", "meta/muse-spark-1.3-contributor", "muse-spark-1.2-contributor"): + assert _stale_pre_catalog_cache_entry(slug, 256_000), slug + assert not _stale_pre_catalog_cache_entry(slug, 1_048_576), slug + + class TestGrok46StaleCacheGuard: """Pre-catalog builds resolved grok-4.6 via the generic 'grok-4' catch-all (256,000) and persisted it before the 500K catalog entry existed. diff --git a/tests/agent/test_models_dev_meta_mapping.py b/tests/agent/test_models_dev_meta_mapping.py index 01dc09256e..a4c110cf5a 100644 --- a/tests/agent/test_models_dev_meta_mapping.py +++ b/tests/agent/test_models_dev_meta_mapping.py @@ -1,4 +1,4 @@ -"""Meta Model API maps to the models.dev 'meta' provider id (context/pricing).""" +"""Muse Spark hosts map to the right models.dev provider id (context/pricing).""" from agent.models_dev import PROVIDER_TO_MODELS_DEV @@ -6,3 +6,8 @@ from agent.models_dev import PROVIDER_TO_MODELS_DEV def test_meta_ai_maps_to_meta(): assert PROVIDER_TO_MODELS_DEV.get("meta-ai") == "meta" assert PROVIDER_TO_MODELS_DEV.get("meta") == "meta" + + +def test_opencode_free_maps_to_zen_catalog(): + # The free tier is served by the Zen relay, whose models.dev id is "opencode". + assert PROVIDER_TO_MODELS_DEV.get("opencode-free") == "opencode" diff --git a/tests/agent/test_opencode_session_affinity.py b/tests/agent/test_opencode_session_affinity.py new file mode 100644 index 0000000000..b2b6cf2536 --- /dev/null +++ b/tests/agent/test_opencode_session_affinity.py @@ -0,0 +1,62 @@ +"""x-opencode-session rides on every OpenCode request, on every transport.""" + +from __future__ import annotations + +import pytest + +from agent import auxiliary_client as aux +from agent.chat_completion_helpers import build_api_kwargs +from run_agent import AIAgent + +_MSGS = [{"role": "user", "content": "hi"}] + + +def _agent(provider, model, base_url, api_mode=None): + agent = AIAgent( + api_key="test-key", + base_url=base_url, + model=model, + provider=provider, + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + session_id="sess-affinity-1", + ) + if api_mode: + agent.api_mode = api_mode + agent._transport = None + agent._anthropic_base_url = base_url + return agent + + +@pytest.mark.parametrize( + "provider, model, base_url, api_mode", + [ + ("opencode-go", "glm-5", "https://opencode.ai/zen/go/v1", None), # chat_completions + ("opencode-go", "gpt-5.6-luna", "https://opencode.ai/zen/go/v1", None), # codex_responses + ("opencode-go", "minimax-m2.7", "https://opencode.ai/zen/go/v1", "anthropic_messages"), + ("opencode-free", "laguna-s-2.1-free", "https://opencode.ai/zen/v1", None), + ("custom", "glm-5", "https://opencode.ai/zen/go/v1", None), # URL-only detection + ], +) +def test_main_turn_sends_stable_session_header_on_every_transport(provider, model, base_url, api_mode): + agent = _agent(provider, model, base_url, api_mode) + first = build_api_kwargs(agent, _MSGS)["extra_headers"]["x-opencode-session"] + second = build_api_kwargs(agent, _MSGS)["extra_headers"]["x-opencode-session"] + assert first == second == "sess-affinity-1" + + other = _agent("openrouter", "anthropic/claude-sonnet-4.6", "https://openrouter.ai/api/v1") + assert "x-opencode-session" not in (build_api_kwargs(other, _MSGS).get("extra_headers") or {}) + + +def test_auxiliary_calls_share_the_main_turn_session_key(): + token = aux.set_runtime_main( + "opencode-go", "glm-5", base_url="https://opencode.ai/zen/go/v1", session_id="sess-affinity-1" + ) + try: + kwargs = aux._build_call_kwargs("opencode-go", "glm-5", _MSGS, base_url="https://opencode.ai/zen/go/v1") + assert kwargs["extra_headers"]["x-opencode-session"] == "sess-affinity-1" + other = aux._build_call_kwargs("openrouter", "x", _MSGS, base_url="https://openrouter.ai/api/v1") + assert "x-opencode-session" not in (other.get("extra_headers") or {}) + finally: + aux._RUNTIME_MAIN_CONTEXT.reset(token) diff --git a/tests/agent/test_outbound_stale_vision.py b/tests/agent/test_outbound_stale_vision.py new file mode 100644 index 0000000000..1cab4234a0 --- /dev/null +++ b/tests/agent/test_outbound_stale_vision.py @@ -0,0 +1,119 @@ +"""Send-path eviction of stale vision_analyze / screenshot tool payloads. + +Issue #89296: compression only retires older image-bearing tool results when +prune/compress fires, so OpenAI-style screenshots are re-serialized on every +later turn until a 413. ``evict_stale_outbound_tool_images`` is the +unconditional per-call chokepoint. +""" + +from __future__ import annotations + +from agent.agent_runtime_helpers import sanitize_api_messages +from agent.context_compressor import ( + _MAX_KEEP_TOOL_IMAGES, + _tool_content_has_images, + evict_stale_outbound_tool_images, +) + + +def _image_tool(i: int, *, blob: str = "A" * 80) -> list[dict]: + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": f"call_{i}", + "type": "function", + "function": { + "name": "vision_analyze", + "arguments": f'{{"image_url":"shot{i}.png"}}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": f"call_{i}", + "content": [ + {"type": "text", "text": f"Image attached natively shot {i}"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{blob}{i}"}, + }, + ], + }, + ] + + +def _history_with_screenshots(n: int) -> list[dict]: + msgs: list[dict] = [{"role": "user", "content": "look at these"}] + for i in range(n): + msgs.extend(_image_tool(i)) + msgs.append({"role": "user", "content": "compare them"}) + return msgs + + +def _image_bearing_tool_ids(messages: list[dict]) -> list[str]: + return [ + m["tool_call_id"] + for m in messages + if m.get("role") == "tool" and _tool_content_has_images(m.get("content")) + ] + + +class TestOutboundStaleVisionEviction: + def test_sanitize_alone_keeps_every_screenshot(self): + """The previous send chokepoint does not close #89296 by itself.""" + history = _history_with_screenshots(5) + sanitized = sanitize_api_messages(history) + assert _image_bearing_tool_ids(sanitized) == [f"call_{i}" for i in range(5)] + + def test_eviction_keeps_only_newest_window(self): + history = _history_with_screenshots(5) + outbound = sanitize_api_messages(history) + pruned = evict_stale_outbound_tool_images(outbound) + assert pruned == 5 - _MAX_KEEP_TOOL_IMAGES + kept = _image_bearing_tool_ids(outbound) + assert kept == [f"call_{i}" for i in range(5 - _MAX_KEEP_TOOL_IMAGES, 5)] + + oldest = next(m for m in outbound if m.get("tool_call_id") == "call_0") + assert isinstance(oldest["content"], list) + assert not _tool_content_has_images(oldest["content"]) + assert any( + isinstance(part, dict) + and part.get("type") == "text" + and "screenshot removed" in str(part.get("text", "")) + for part in oldest["content"] + ) + + def test_does_not_rewrite_persisted_history(self): + from agent.conversation_loop import _clone_message_for_send + + history = _history_with_screenshots(5) + outbound = [_clone_message_for_send(m) for m in history] + evict_stale_outbound_tool_images(outbound) + assert _image_bearing_tool_ids(history) == [f"call_{i}" for i in range(5)] + assert _image_bearing_tool_ids(outbound) == [ + f"call_{i}" for i in range(5 - _MAX_KEEP_TOOL_IMAGES, 5) + ] + + def test_user_uploads_are_not_evicted(self): + history = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,USERUPLOAD"}, + }, + ], + } + ] + for i in range(_MAX_KEEP_TOOL_IMAGES + 2): + history.extend(_image_tool(i)) + outbound = sanitize_api_messages(history) + evict_stale_outbound_tool_images(outbound) + user = next(m for m in outbound if m.get("role") == "user") + assert user["content"][1]["image_url"]["url"].endswith("USERUPLOAD") diff --git a/tests/agent/test_outbound_webhooks.py b/tests/agent/test_outbound_webhooks.py index 29e05a4cdb..39c8a25221 100644 --- a/tests/agent/test_outbound_webhooks.py +++ b/tests/agent/test_outbound_webhooks.py @@ -301,6 +301,23 @@ class TestPayload: assert payload["delivery_id"] == "did_1234" assert payload["timestamp"].endswith("Z") + def test_profile_field_reflects_bound_profile_home(self, tmp_path, monkeypatch): + """Receivers behind a multiplexed gateway need to know which profile + fired (#92674): ``profile`` follows the bound home at fire time.""" + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + profile_home = tmp_path / "profiles" / "b" + profile_home.mkdir(parents=True) + token = set_hermes_home_override(profile_home) + try: + body = outbound_webhooks._serialize_payload("on_session_end", {}, "did_1") + finally: + reset_hermes_home_override(token) + assert json.loads(body)["profile"] == "b" + body = outbound_webhooks._serialize_payload("on_session_end", {}, "did_2") + assert json.loads(body)["profile"] == "default" + def test_unserialisable_values_stringified(self): body = outbound_webhooks._serialize_payload( "on_session_end", {"weird": object()}, "did_1" @@ -345,6 +362,46 @@ class TestRegistration: assert len(http_server.captured) == 1 +class TestForceReloadHomeScoping: + """Force-reloading one profile's plugin manager must restore that + profile's own outbound webhook and leave it firing exactly once — + the mirror of the shell-hook force-reload symmetry fix (#92682 + review: outbound webhooks were the "same symptom class... after a + supported lifecycle transition instead of initial startup"). + """ + + def test_force_reload_restores_webhook_and_fires_once( + self, monkeypatch, http_server, + ): + from hermes_cli import plugins + + cfg = _cfg({"url": _url(http_server), "events": ["on_session_end"]}) + monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg) + + monkeypatch.setenv("HERMES_HOME", "/tmp/profile-b-webhook") + mgr_b = plugins.PluginManager() + plugins._plugin_manager = mgr_b + outbound_webhooks.register_from_config(cfg) + assert len(mgr_b._hooks.get("on_session_end", [])) == 1 + + # Force-reload: unload() wipes _hooks (config-owned webhook + # callbacks included, same as the ledger-driven plugin sweep), so + # without the fix the idempotence key alone would survive and a + # later register_from_config() call would see it and skip + # re-wiring — leaving the webhook silently inert. + mgr_b.unload() + assert mgr_b._hooks.get("on_session_end", []) == [] + + outbound_webhooks.re_register_config_hooks() + assert len(mgr_b._hooks.get("on_session_end", [])) == 1 + + plugins.get_plugin_manager().invoke_hook( + "on_session_end", session_id="s1", + ) + assert outbound_webhooks.flush() + assert len(http_server.captured) == 1 + + # ── E2E delivery against a real HTTP server ────────────────────────────── diff --git a/tests/agent/test_periodic_scheduler.py b/tests/agent/test_periodic_scheduler.py new file mode 100644 index 0000000000..e1a2b3765a --- /dev/null +++ b/tests/agent/test_periodic_scheduler.py @@ -0,0 +1,92 @@ +"""agent/periodic_scheduler: one shared thread runs every periodic timer.""" + +import threading +import time + +from agent import periodic_scheduler +from agent.periodic_scheduler import PeriodicScheduler, schedule + + +def _wait_until(pred, timeout=3.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(0.005) + return pred() + + +def test_two_intervals_fire_proportionally_and_cancel_stops_one(): + sched = PeriodicScheduler() + fast, slow = [], [] + h_fast = sched.schedule(lambda: fast.append(time.monotonic()), 0.01) + h_slow = sched.schedule(lambda: slow.append(time.monotonic()), 0.05) + + assert _wait_until(lambda: len(slow) >= 3) + assert len(fast) > len(slow) # 5x interval ratio -> clearly more fast ticks + # Both ran on this scheduler's single thread, not on new threads. + before = threading.active_count() + sched.schedule(lambda: None, 0.01).cancel() + assert threading.active_count() == before + assert sched._thread is not None and sched._thread.is_alive() + + h_fast.cancel() + n_fast = len(fast) + time.sleep(0.1) + assert len(fast) == n_fast, "cancelled callback kept firing" + assert len(slow) > 3, "sibling callback stopped when another was cancelled" + h_slow.cancel() + + +def test_raising_callback_is_rescheduled_and_does_not_kill_sibling(): + sched = PeriodicScheduler() + boom, ok = [], [] + + def raises(): + boom.append(1) + raise RuntimeError("bad callback") + + h1 = sched.schedule(raises, 0.01) + h2 = sched.schedule(lambda: ok.append(1), 0.01) + assert _wait_until(lambda: len(boom) >= 3 and len(ok) >= 3) + h1.cancel() + h2.cancel() + + +def test_returning_false_stops_callback_and_cancel_wait_joins_inflight(): + sched = PeriodicScheduler() + calls = [] + sched.schedule(lambda: (calls.append(1), False)[1], 0.01) + assert _wait_until(lambda: len(calls) == 1) + time.sleep(0.05) + assert calls == [1] + + entered = threading.Event() + release = threading.Event() + + def blocking(): + entered.set() + release.wait(2.0) + + h = sched.schedule(blocking, 0.01) + assert entered.wait(2.0) + threading.Timer(0.05, release.set).start() + t0 = time.monotonic() + h.cancel(wait=2.0) # returns once the in-flight run finished + assert release.is_set() + assert time.monotonic() - t0 < 1.5 + + +def test_module_level_schedule_uses_shared_default(): + hits = [] + h = schedule(lambda: hits.append(1), 0.01) + assert _wait_until(lambda: hits) + h.cancel() + thread = periodic_scheduler._DEFAULT._thread + assert thread is not None and thread.name == "hermes-periodic-scheduler" + # Scheduling more timers on the shared default adds no OS threads. + before = threading.active_count() + handles = [schedule(lambda: None, 0.01) for _ in range(20)] + assert threading.active_count() == before + for handle in handles: + handle.cancel() diff --git a/tests/agent/test_phantom_tool_references.py b/tests/agent/test_phantom_tool_references.py index 045f356198..836522827a 100644 --- a/tests/agent/test_phantom_tool_references.py +++ b/tests/agent/test_phantom_tool_references.py @@ -65,8 +65,8 @@ class TestCodingBriefTodoGating: return prefix[0] def test_todo_kept_when_tool_available(self): - brief = self._brief({"todo", "terminal", "read_file"}) - assert "Track multi-step work with `todo`" in brief + brief = self._brief({"todo_list", "terminal", "read_file"}) + assert "Track multi-step work with `todo_list`" in brief def test_todo_dropped_when_tool_missing(self): brief = self._brief({"terminal", "read_file"}) @@ -76,7 +76,7 @@ class TestCodingBriefTodoGating: def test_unknown_toolset_keeps_full_brief(self): brief = self._brief(None) - assert "Track multi-step work with `todo`" in brief + assert "Track multi-step work with `todo_list`" in brief class TestEssentialSkillsUndisableable: diff --git a/tests/agent/test_proactive_prune_rearm_threshold.py b/tests/agent/test_proactive_prune_rearm_threshold.py new file mode 100644 index 0000000000..9d49eb5626 --- /dev/null +++ b/tests/agent/test_proactive_prune_rearm_threshold.py @@ -0,0 +1,227 @@ +"""Proactive-prune rearm must not lock out an over-threshold session (#101889). + +``_proactive_prune_rearm_tokens`` is armed from a message-bodies-only estimate, +but the provider bills the system prompt and tool schemas too. On a schema-heavy +session the message-only estimate can sit permanently just below the rearm mark +while the real request rides *above* ``threshold_tokens`` — the prune declines +every iteration, full compression never gets there, and nothing is logged. The +session then grows until the provider rejects the request. + +Pinned here as invariants (no frozen config literals): the gates are evaluated +against this compressor's own ``threshold_tokens`` / ``proactive_prune_tokens``. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List +from unittest.mock import patch + +from agent.context_compressor import ContextCompressor, _estimate_msg_budget_tokens + +LARGE_WINDOW = 1_000_000 + + +def _compressor(**kw: Any) -> ContextCompressor: + defaults = dict( + model="test", + quiet_mode=True, + threshold_percent=0.50, + protect_first_n=2, + protect_last_n=4, + proactive_prune_tokens=48_000, + proactive_prune_min_result_chars=8_000, + ) + defaults.update(kw) + with patch( + "agent.context_compressor.get_model_context_length", + return_value=LARGE_WINDOW, + ): + return ContextCompressor(**defaults) + + +def _history(n_pairs: int = 8, big: int = 9_000) -> List[Dict[str, Any]]: + msgs: List[Dict[str, Any]] = [{"role": "system", "content": "sys"}] + for i in range(n_pairs): + cid = f"call_{i}" + msgs.append({ + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": cid, + "type": "function", + "function": {"name": "terminal", "arguments": '{"cmd":"ls"}'}, + }], + }) + msgs.append({ + "role": "tool", + "tool_call_id": cid, + "content": chr(65 + i) * big if i < 3 else "ok", + }) + return msgs + + +def _park_rearm_just_above_messages( + compressor: ContextCompressor, messages: List[Dict[str, Any]] +) -> int: + """Reproduce the reporter's state: message-only estimate stuck 913 tokens + below the rearm mark (schema overhead makes up the rest of the request).""" + before = sum(_estimate_msg_budget_tokens(m) for m in messages) + compressor._proactive_prune_rearm_tokens = before + 913 + assert before < compressor._proactive_prune_rearm_tokens + return before + + +def _over_threshold_warnings(caplog) -> list: + return [ + r for r in caplog.records + if r.levelno >= logging.WARNING + and "over the compression threshold" in r.getMessage() + ] + + +def test_billed_basis_over_threshold_defeats_message_only_rearm_lockout() -> None: + """Over ``threshold_tokens`` on the provider-billed basis, the rearm gate + must not short-circuit the prune on the message-only estimate alone.""" + c = _compressor() + msgs = _history() + _park_rearm_just_above_messages(c, msgs) + billed = c.threshold_tokens + 1 # provider says: over threshold, now + + scans: List[int] = [] + # Stand in for the real multi-pass scan: a NEW list whose old tool outputs + # are reclaimed, so the (untouched) reclaim gate can commit it. + reclaimed = [dict(m) for m in msgs] + for m in reclaimed[:-2]: + if m.get("role") == "tool": + m["content"] = "[pruned]" + + def _scan(*args: Any, **kwargs: Any) -> tuple[List[Dict[str, Any]], int]: + scans.append(1) + return reclaimed, 3 + + with patch.object(c, "_prune_old_tool_results", _scan): + result, pruned = c.prune_tool_results_only(msgs, current_tokens=billed) + + assert scans, "rearm gate short-circuited on the message-only estimate" + assert pruned == 3 + assert result is not msgs + + +def test_message_only_rearm_still_holds_below_threshold() -> None: + """Prompt-cache hysteresis is intact while the real request is under the + compression threshold — the rearm bypass is an overflow escape hatch only.""" + c = _compressor() + msgs = _history() + _park_rearm_just_above_messages(c, msgs) + under = c.threshold_tokens - 1 + assert under >= c.proactive_prune_tokens # above the prune trigger + + with patch.object( + c, + "_prune_old_tool_results", + side_effect=AssertionError("scan must not run below threshold"), + ): + result, pruned = c.prune_tool_results_only(msgs, current_tokens=under) + + assert result is msgs + assert pruned == 0 + + +def test_no_op_below_the_prune_trigger() -> None: + """Under ``proactive_prune_tokens`` nothing is reclaimed, rearm or not — + the bypass must not turn into over-pruning of small sessions.""" + c = _compressor() + msgs = _history() + c.on_session_reset() # fully rearmed; only the trigger gates + + with patch.object( + c, + "_prune_old_tool_results", + side_effect=AssertionError("scan must not run below the trigger"), + ): + result, pruned = c.prune_tool_results_only( + msgs, current_tokens=c.proactive_prune_tokens - 1 + ) + + assert result is msgs + assert pruned == 0 + + +def test_over_threshold_reclamation_no_op_warns_once(caplog) -> None: + """A session riding above the threshold with every reclamation path + declining must be distinguishable in the log — and must not spam the same + reason on every tool iteration.""" + # Reclaim floor above anything this transcript can free: the scan runs, + # finds candidates, and the commit gate rejects it — a silent no-op today. + c = _compressor(proactive_prune_min_reclaim_tokens=10_000_000) + msgs = _history() + billed = c.threshold_tokens + 5_000 + + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + result, pruned = c.prune_tool_results_only(msgs, current_tokens=billed) + assert (result, pruned) == (msgs, 0) + + warnings = _over_threshold_warnings(caplog) + assert warnings, "over-threshold reclamation no-op was silent" + + # Same state on the next tool iteration: deduped, not re-logged. + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + c.prune_tool_results_only(msgs, current_tokens=billed) + assert len(_over_threshold_warnings(caplog)) == len(warnings) + + +def test_under_threshold_no_op_is_not_warned(caplog) -> None: + """Ordinary hysteresis below the threshold stays quiet.""" + c = _compressor(proactive_prune_min_reclaim_tokens=10_000_000) + msgs = _history() + + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + result, pruned = c.prune_tool_results_only( + msgs, current_tokens=c.threshold_tokens - 1 + ) + + assert (result, pruned) == (msgs, 0) + assert not _over_threshold_warnings(caplog) + + +def test_lockout_warns_again_after_rearm_reset(caplog) -> None: + """A full compaction (or session rebind / model recalibration) zeroes the + rearm mark. An identical lockout afterwards must warn again — the dedup key + must not outlive the reclamation that should have cleared it.""" + c = _compressor(proactive_prune_min_reclaim_tokens=10_000_000) + msgs = _history() + billed = c.threshold_tokens + 5_000 + + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + c.prune_tool_results_only(msgs, current_tokens=billed) + assert len(_over_threshold_warnings(caplog)) == 1 + # Same state, same key (reason, rearm=0): deduped. + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + c.prune_tool_results_only(msgs, current_tokens=billed) + assert len(_over_threshold_warnings(caplog)) == 1 + + # A public rearm boundary (same helper as compress(), on_session_end, + # bind_session_state and update_model): pins the wiring, not just the body. + c.on_session_reset() + assert c._proactive_prune_rearm_tokens == 0 + + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + c.prune_tool_results_only(msgs, current_tokens=billed) + assert len(_over_threshold_warnings(caplog)) == 2, ( + "lockout after a rearm reset was deduped against the stale key" + ) + + +def test_dropping_under_threshold_clears_dedup_key(caplog) -> None: + """Back under threshold (e.g. compaction elsewhere shrank the request), the + key is released so the next over-threshold lockout is reported.""" + c = _compressor(proactive_prune_min_reclaim_tokens=10_000_000) + msgs = _history() + billed = c.threshold_tokens + 5_000 + + with caplog.at_level(logging.WARNING, logger="agent.context_compressor"): + c.prune_tool_results_only(msgs, current_tokens=billed) + c.prune_tool_results_only(msgs, current_tokens=c.threshold_tokens - 1) + c.prune_tool_results_only(msgs, current_tokens=billed) + assert len(_over_threshold_warnings(caplog)) == 2 diff --git a/tests/agent/test_proactive_tool_result_pruning.py b/tests/agent/test_proactive_tool_result_pruning.py index bbb01e4b16..2a963f7f01 100644 --- a/tests/agent/test_proactive_tool_result_pruning.py +++ b/tests/agent/test_proactive_tool_result_pruning.py @@ -130,7 +130,12 @@ def test_rearms_only_after_reclaimed_token_runway(): _tool_msg("call_9", "ok"), ] assert sum(map(_estimate_msg_budget_tokens, grown)) < rearm_tokens - blocked, n2 = c.prune_tool_results_only(grown, current_tokens=1_000_000) + # Below the full-compression threshold, where the runway is pure + # prompt-cache hysteresis. (Above it the runway is bypassed on the + # provider-billed reading instead — see + # tests/agent/test_proactive_prune_rearm_threshold.py, #101889.) + _under_threshold = c.threshold_tokens - 1 + blocked, n2 = c.prune_tool_results_only(grown, current_tokens=_under_threshold) assert n2 == 0 assert blocked is grown assert len(_tool_by_id(blocked, "call_6")["content"]) == 9000 @@ -139,7 +144,7 @@ def test_rearms_only_after_reclaimed_token_runway(): missing = rearm_tokens - sum(map(_estimate_msg_budget_tokens, grown)) regrown = grown + [{"role": "user", "content": "x" * (missing * 4)}] assert sum(map(_estimate_msg_budget_tokens, regrown)) >= rearm_tokens - rearmed, n3 = c.prune_tool_results_only(regrown, current_tokens=1_000_000) + rearmed, n3 = c.prune_tool_results_only(regrown, current_tokens=_under_threshold) assert n3 >= 2 assert rearmed is not regrown diff --git a/tests/agent/test_prompt_builder.py b/tests/agent/test_prompt_builder.py index 0ec2dadb01..4331c2bc9d 100644 --- a/tests/agent/test_prompt_builder.py +++ b/tests/agent/test_prompt_builder.py @@ -5,6 +5,8 @@ import importlib import logging import os import sys +import time +from pathlib import Path import pytest @@ -68,7 +70,12 @@ class TestGuidanceConstants: assert "declarative facts" in MEMORY_GUIDANCE assert "imperative phrasing" in MEMORY_GUIDANCE assert "stale within a week" in MEMORY_GUIDANCE - assert "Save proactively" in MEMORY_GUIDANCE # positive posture leads + # Skills are the default home for task-learned knowledge (incl. the + # user's preferences/corrections for that work); memory is the narrow + # every-session exception. The routing rule must LEAD, not trail. + assert MEMORY_GUIDANCE.index("Skills come first") < MEMORY_GUIDANCE.index("Memory is the narrow exception") + assert "preferences and corrections" in MEMORY_GUIDANCE + assert "Save proactively" not in MEMORY_GUIDANCE assert "workflows belong" in MEMORY_GUIDANCE # The category/SKIP curricula must NOT be re-taught here. assert "PR numbers" not in MEMORY_GUIDANCE @@ -820,6 +827,126 @@ class TestEnvironmentHints: assert "Linux 6.8.0" in line assert "root" in line + def test_probe_remote_backend_tears_down_its_sandbox(self, monkeypatch): + """THE BUG: the probe leaked a second, permanently idle sandbox. + + ``_probe_remote_backend`` spins up an environment with + ``task_id="prompt-backend-probe"`` purely to run one ``uname``. Container + backends default to ``container_persistent`` / + ``docker_persist_across_processes``, so that throwaway sandbox stayed up + for the whole process lifetime *next to* the agent's own ``default`` + sandbox — one wasted idle container per profile, forever. The probe owns + that environment, so it must tear it down. + """ + import agent.prompt_builder as _pb + + monkeypatch.setenv("TERMINAL_ENV", "docker") + _pb._clear_backend_probe_cache() + + cleaned = {} + + class _FakeEnv: + def execute(self, cmd, timeout=None): + return { + "returncode": 0, + "output": ( + "os=Linux\nkernel=6.8.0\nhome=/root\n" + "cwd=/workspace\nuser=root\n" + ), + } + + def cleanup(self, *, force_remove=False): + cleaned["force_remove"] = force_remove + + import tools.terminal_tool as _tt + monkeypatch.setattr(_tt, "_create_environment", lambda **kw: _FakeEnv()) + + assert _pb._probe_remote_backend("docker") is not None + # force_remove=True: persist mode would otherwise leave it running. + assert cleaned == {"force_remove": True} + + def test_probe_remote_backend_tears_down_sandbox_on_failure(self, monkeypatch): + """Teardown must also run when the probe command blows up — a flaky + backend would otherwise leak the container the probe just created.""" + import agent.prompt_builder as _pb + + monkeypatch.setenv("TERMINAL_ENV", "docker") + _pb._clear_backend_probe_cache() + + cleaned = [] + + class _ExplodingEnv: + def execute(self, cmd, timeout=None): + raise RuntimeError("backend went away") + + def cleanup(self, *, force_remove=False): + cleaned.append(force_remove) + + import tools.terminal_tool as _tt + monkeypatch.setattr(_tt, "_create_environment", lambda **kw: _ExplodingEnv()) + + assert _pb._probe_remote_backend("docker") is None + assert cleaned == [True] + + def test_probe_remote_backend_tolerates_kwargless_cleanup(self, monkeypatch): + """Backends that inherit the base ``cleanup(self)`` take no kwargs; the + probe must use the bare call instead of dying on TypeError.""" + import agent.prompt_builder as _pb + + monkeypatch.setenv("TERMINAL_ENV", "singularity") + _pb._clear_backend_probe_cache() + + calls = [] + + class _LegacyEnv: + def execute(self, cmd, timeout=None): + return { + "returncode": 0, + "output": ( + "os=Linux\nkernel=6.8.0\nhome=/home/u\n" + "cwd=/home/u\nuser=u\n" + ), + } + + def cleanup(self): + calls.append("bare") + + import tools.terminal_tool as _tt + monkeypatch.setattr(_tt, "_create_environment", lambda **kw: _LegacyEnv()) + + assert _pb._probe_remote_backend("singularity") is not None + assert calls == ["bare"] + + def test_probe_remote_backend_does_not_tear_down_ssh(self, monkeypatch): + """SSH has no task-scoped sandbox: its cleanup() closes a ControlMaster + socket shared with the agent's real environment, so the probe must + leave it alone (nothing leaks — ControlPersist expires the master).""" + import agent.prompt_builder as _pb + + monkeypatch.setenv("TERMINAL_ENV", "ssh") + _pb._clear_backend_probe_cache() + + calls = [] + + class _SharedSshEnv: + def execute(self, cmd, timeout=None): + return { + "returncode": 0, + "output": ( + "os=Linux\nkernel=6.8.0\nhome=/home/u\n" + "cwd=/home/u\nuser=u\n" + ), + } + + def cleanup(self): + calls.append("cleanup") + + import tools.terminal_tool as _tt + monkeypatch.setattr(_tt, "_create_environment", lambda **kw: _SharedSshEnv()) + + assert _pb._probe_remote_backend("ssh") is not None + assert calls == [] + def test_environment_hint_from_env_var_is_appended(self, monkeypatch): """HERMES_ENVIRONMENT_HINT lets an embedder describe the runtime env.""" @@ -986,6 +1113,13 @@ class TestExecutionGuidanceModels: for fam in ("deepseek", "kimi", "qwen", "glm", "minimax", "mimo", "mistral"): assert fam in EXECUTION_GUIDANCE_MODELS + def test_muse_spark_gets_both_guidance_blocks(self): + # Muse Spark closes the turn after a chat-only response on defaults + # (#96550) — it needs tool-use enforcement AND execution guidance. + from agent.prompt_builder import EXECUTION_GUIDANCE_MODELS + assert any(p in "meta/muse-spark-1.3-contributor" for p in TOOL_USE_ENFORCEMENT_MODELS) + assert any(p in "meta/muse-spark-1.3-contributor" for p in EXECUTION_GUIDANCE_MODELS) + def test_excludes_google_and_claude(self): # Gemini/Gemma get GOOGLE_MODEL_OPERATIONAL_GUIDANCE instead; # Claude doesn't exhibit the targeted failure modes. @@ -1020,3 +1154,40 @@ class TestParallelToolCallGuidance: # ========================================================================= + + +class TestContextFileReadTimeout: + def test_slow_hermes_md_is_skipped_and_agents_md_still_loads(self, tmp_path, monkeypatch, caplog): + (tmp_path / ".git").mkdir() + (tmp_path / ".hermes.md").write_text("Hermes project rules.") + (tmp_path / "AGENTS.md").write_text("Agent fallback rules.") + # Patch the module object build_context_files_prompt actually closes + # over: an earlier test re-imports agent.prompt_builder, so the + # sys.modules entry can be a different module object. + pb_mod = sys.modules[build_context_files_prompt.__module__] + monkeypatch.setattr(pb_mod, "_get_context_file_read_timeout", lambda: 0.05) + + original_read_text = Path.read_text + + def slow_read_text(self, *args, **kwargs): + if self.name == ".hermes.md": + time.sleep(0.6) + return original_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", slow_read_text) + + start = time.monotonic() + with caplog.at_level(logging.WARNING, logger=pb_mod.__name__): + result = build_context_files_prompt(cwd=str(tmp_path)) + elapsed = time.monotonic() - start + + assert elapsed < 0.4, f"context load blocked for {elapsed:.2f}s" + assert "Agent fallback rules" in result + assert "Hermes project rules" not in result + assert "timed out" in caplog.text.lower() + + def test_read_errors_still_propagate_to_caller(self, tmp_path): + from agent.prompt_builder import _read_text_with_timeout + + with pytest.raises(FileNotFoundError): + _read_text_with_timeout(tmp_path / "missing.md", timeout=1.0) diff --git a/tests/agent/test_provider_client_seam.py b/tests/agent/test_provider_client_seam.py new file mode 100644 index 0000000000..2514977b0c --- /dev/null +++ b/tests/agent/test_provider_client_seam.py @@ -0,0 +1,104 @@ +"""A provider profile can supply its own client — the registration seam. + +A provider whose wire protocol is not OpenAI-over-HTTP (the ACP subprocess +shims) supplies its client via ``ProviderProfile.create_client()``, from inside +or outside the tree. These tests pin the hook through the real entry point +(``create_openai_client``), its failure isolation, and the capability flags +that let such a client opt out of the auxiliary transport/async wrappers. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +import providers as _providers +from providers.base import ProviderProfile + + +class _FakeClient: + HERMES_SKIP_TRANSPORT_WRAP = True + HERMES_SKIP_ASYNC_WRAP = True + api_key = "k" + base_url = "acp://seam-test" + + def __init__(self, **kwargs): + self.kwargs = kwargs + + +class _SeamProfile(ProviderProfile): + def create_client(self, **kwargs): + return _FakeClient(**kwargs) + + +class _ExplodingProfile(ProviderProfile): + def create_client(self, **kwargs): + raise RuntimeError("plugin is broken") + + +@pytest.fixture +def registered(): + """Register profiles for one test and restore the registry afterwards.""" + _providers._discover_providers() + snapshot = (dict(_providers._REGISTRY), dict(_providers._ALIASES), _providers._PROVIDER_LIST_CACHE) + yield _providers.register_provider + _providers._REGISTRY.clear() + _providers._REGISTRY.update(snapshot[0]) + _providers._ALIASES.clear() + _providers._ALIASES.update(snapshot[1]) + _providers._PROVIDER_LIST_CACHE = snapshot[2] + + +def _agent(provider: str = ""): + return SimpleNamespace( + provider=provider, + _client_log_context=lambda: "", + _build_keepalive_http_client=lambda *a, **k: None, + ) + + +def _build(provider, base_url): + from agent.agent_runtime_helpers import create_openai_client + + return create_openai_client(_agent(provider), {"api_key": "k", "base_url": base_url}, reason="t", shared=False) + + +def test_an_out_of_tree_profile_supplies_the_client_through_create_openai_client(registered): + from openai import OpenAI + + registered(_SeamProfile(name="seam-test", aliases=("seam",), base_url="acp://seam-test")) + # By name, by alias, and by base_url prefix when no provider name is set. + assert isinstance(_build("seam-test", "acp://seam-test"), _FakeClient) + assert isinstance(_build("seam", "acp://seam-test"), _FakeClient) + assert isinstance(_build("", "acp://seam-test/x"), _FakeClient) + # No hook → the standard client, untouched by the seam. + assert isinstance(_build("openai-api", "https://api.example/v1"), OpenAI) + + +def test_a_broken_plugin_falls_through_instead_of_taking_the_turn_down(registered): + from agent.agent_runtime_helpers import _provider_supplied_client + + registered(_ExplodingProfile(name="seam-boom", base_url="acp://seam-boom")) + assert _provider_supplied_client(_agent("seam-boom"), {"api_key": "k"}) is None + + +def test_copilot_acp_still_gets_its_acp_client_via_its_profile(): + from agent.copilot_acp_client import CopilotACPClient + + assert isinstance(_build("copilot-acp", "acp://copilot"), CopilotACPClient) + assert isinstance(_build("", "acp://copilot"), CopilotACPClient) + + +def test_skip_flags_replace_the_isinstance_checks_for_in_and_out_of_tree_clients(): + from agent.auxiliary_client import _maybe_wrap_anthropic, _to_async_client + from agent.copilot_acp_client import CopilotACPClient + from agent.gemini_native_adapter import GeminiNativeClient + + assert CopilotACPClient.HERMES_SKIP_TRANSPORT_WRAP and CopilotACPClient.HERMES_SKIP_ASYNC_WRAP + assert GeminiNativeClient.HERMES_SKIP_TRANSPORT_WRAP + assert not getattr(GeminiNativeClient, "HERMES_SKIP_ASYNC_WRAP", False) + + client = _FakeClient() # never imported by auxiliary_client + assert _maybe_wrap_anthropic(client, "m", "k", "acp://seam-test") is client + assert _to_async_client(client, "m")[0] is client diff --git a/tests/agent/test_refine_snapshot_isolation.py b/tests/agent/test_refine_snapshot_isolation.py new file mode 100644 index 0000000000..234beb1993 --- /dev/null +++ b/tests/agent/test_refine_snapshot_isolation.py @@ -0,0 +1,101 @@ +"""Every review path hands the fork a snapshot that cannot alias the live transcript. + +``AIAgent._spawn_background_review`` is the single chokepoint the automatic +post-turn review, the idle-queue deferral and both explicit ``/refine`` entry +points (CLI mixin + gateway slash command) go through; it clones the snapshot +structurally there. A shallow ``list()`` would share the nested +``tool_calls`` / ``content`` containers with the persisted history, so the +fork's in-place transcript sanitization would rewrite the parent's messages +(#100795). These tests drive the real /refine handlers into the real +chokepoint and capture what reaches the spawn. +""" + +import threading +from unittest.mock import MagicMock + +import pytest + + +def _agent_with_real_chokepoint(): + """MagicMock agent whose _spawn_background_review is the REAL method. + + Everything below the chokepoint (thread spawn) is captured at + ``_spawn_background_review_now`` so no fork actually runs. + """ + from run_agent import AIAgent + + agent = MagicMock() + agent.valid_tool_names = {"memory"} + agent._delegate_depth = 0 + agent._spawn_background_review = AIAgent._spawn_background_review.__get__(agent) + return agent + + +def _nested_history(): + return [ + {"role": "user", "content": [{"type": "text", "text": "ask"}]}, + { + "role": "assistant", + "content": "ok", + "tool_calls": [{ + "id": "call-1", + "function": {"name": "read_file", "arguments": '{"path":"x"}'}, + }], + }, + ] + + +def _assert_isolated(live, snapshot): + assert snapshot == live # same shape/bytes … + for live_msg, snap_msg in zip(live, snapshot): + assert snap_msg is not live_msg # … but no shared containers + for key in ("content", "tool_calls"): + if isinstance(live_msg.get(key), (dict, list)): + assert snap_msg[key] is not live_msg[key] + # Mutating the snapshot the way the fork's sanitizers do must not leak. + snapshot[0]["content"][0]["text"] = "mutated" + snapshot[1]["tool_calls"][0]["function"]["arguments"] = "{}" + assert live[0]["content"][0]["text"] == "ask" + assert live[1]["tool_calls"][0]["function"]["arguments"] == '{"path":"x"}' + + +def test_cli_refine_snapshot_does_not_alias_live_history(monkeypatch): + from hermes_cli.cli_commands_mixin import CLICommandsMixin + + monkeypatch.setattr("cli._cprint", lambda *a, **k: None, raising=False) + agent = _agent_with_real_chokepoint() + cli = object.__new__(CLICommandsMixin) + cli.agent = agent + cli.conversation_history = _nested_history() + + cli._handle_refine_command("/refine") + + agent._spawn_background_review_now.assert_called_once() + snapshot = agent._spawn_background_review_now.call_args.kwargs["messages_snapshot"] + _assert_isolated(cli.conversation_history, snapshot) + + +@pytest.mark.asyncio +async def test_gateway_refine_snapshot_does_not_alias_live_history(): + from gateway.run import GatewayRunner + + key = "agent:main:test:dm:1" + agent = _agent_with_real_chokepoint() + agent._session_messages = _nested_history() + + runner = object.__new__(GatewayRunner) + runner._running_agents = {} + runner._agent_cache = {key: agent} + runner._agent_cache_lock = threading.Lock() + runner._session_key_for_source = lambda source: key + + event = MagicMock() + event.source = object() + event.get_command_args.return_value = "" + + out = await runner._handle_refine_command(event) + + assert out.startswith("⚗") + agent._spawn_background_review_now.assert_called_once() + snapshot = agent._spawn_background_review_now.call_args.kwargs["messages_snapshot"] + _assert_isolated(agent._session_messages, snapshot) diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py index f091fb1222..a31553af88 100644 --- a/tests/agent/test_relay_llm.py +++ b/tests/agent/test_relay_llm.py @@ -259,6 +259,141 @@ def test_provider_request_overlays_interceptor_added_extra_body(): assert provider_request["extra_body"] == {"prompt_cache_retention": "24h"} +@pytest.mark.parametrize( + "api_mode", + ["chat_completions", "codex_responses", "anthropic_messages"], +) +def test_provider_request_maps_headers_for_supported_sdk_modes(api_mode): + original = {"model": "test-model"} + relay_request_body = relay_llm._relay_request_body( + original, + {"api_mode": api_mode}, + ) + + provider_request = relay_llm._provider_request( + original, + SimpleNamespace( + content=relay_request_body, + headers={ + "traceparent": ( + "00-11111111111111111111111111111111-" + "2222222222222222-01" + ) + }, + ), + relay_request_body=relay_request_body, + codec_baseline_body=dict(relay_request_body), + metadata={"api_mode": api_mode}, + ) + + assert provider_request["extra_headers"] == { + "traceparent": ( + "00-11111111111111111111111111111111-2222222222222222-01" + ) + } + + +def test_provider_request_preserves_custom_headers_for_native_transport(): + original = {"payload": "provider-native"} + + provider_request = relay_llm._provider_request( + original, + SimpleNamespace( + content=original, + headers={ + "traceparent": ( + "00-11111111111111111111111111111111-" + "2222222222222222-01" + ), + "x-custom-route": "private", + }, + ), + relay_request_body=original, + codec_baseline_body=dict(original), + metadata={"api_mode": "strict_native"}, + ) + + assert provider_request["extra_headers"] == { + "x-custom-route": "private" + } + + +def test_provider_request_traces_custom_transport_with_header_capability(): + original = { + "payload": "provider-native", + "extra_headers": {"authorization": "Bearer provider-token"}, + } + traceparent = ( + "00-11111111111111111111111111111111-2222222222222222-01" + ) + + provider_request = relay_llm._provider_request( + original, + SimpleNamespace( + content=original, + headers={"traceparent": traceparent}, + ), + relay_request_body=original, + codec_baseline_body=dict(original), + metadata={"api_mode": "custom"}, + ) + + assert provider_request["extra_headers"] == { + "authorization": "Bearer provider-token", + "traceparent": traceparent, + } + + +def test_managed_request_does_not_add_sdk_headers_to_strict_callback(relay_turn): + del relay_turn + observed = [] + + def strict_transport(*, payload): + observed.append(payload) + return {"content": payload} + + result = relay_llm.execute( + {"payload": "provider-native"}, + lambda request: strict_transport(**request), + session_id="session-1", + name="strict-native", + model_name="strict-model", + metadata={ + "api_mode": "bedrock_converse", + "api_request_id": "strict-native-request", + }, + ) + + assert observed == ["provider-native"] + assert result == {"content": "provider-native"} + + +def test_managed_stream_does_not_add_sdk_headers_to_strict_callback(relay_turn): + del relay_turn + observed = [] + chunks = [{"delta": "provider-native"}] + + def strict_transport(*, payload): + observed.append(payload) + return iter(chunks) + + stream = relay_llm.stream( + {"payload": "provider-native"}, + lambda request: strict_transport(**request), + session_id="session-1", + name="strict-native", + model_name="strict-model", + finalizer=lambda: {"content": "provider-native"}, + metadata={ + "api_mode": "bedrock_converse", + "api_request_id": "strict-native-stream", + }, + ) + + assert list(stream) == chunks + assert observed == ["provider-native"] + + def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn): relay, turn = relay_turn captured_requests = [] @@ -355,9 +490,15 @@ def test_stream_uses_rewritten_request_and_post_intercept_chunks(relay_turn): relay.intercepts.deregister_llm_request("hermes-test-request") assert captured_requests[0]["temperature"] == 0.25 - assert captured_requests[0]["extra_headers"] == { - "authorization": "Bearer provider-token" - } + headers = captured_requests[0]["extra_headers"] + assert headers["authorization"] == "Bearer provider-token" + version, trace_id, parent_id, flags = headers["traceparent"].split("-") + assert version == "00" + assert len(trace_id) == 32 + assert len(parent_id) == 16 + assert flags == "01" + int(trace_id, 16) + int(parent_id, 16) assert chunks[0].choices[0].delta.content == "HELLO" assert stream.output_modified is True assert turn.logical_llm_calls == {} diff --git a/tests/agent/test_relay_runtime_plugins.py b/tests/agent/test_relay_runtime_plugins.py index 4cd1c03721..b8f44c64de 100644 --- a/tests/agent/test_relay_runtime_plugins.py +++ b/tests/agent/test_relay_runtime_plugins.py @@ -1049,7 +1049,7 @@ mode = "overwrite" assert not (atof_dir / "events.jsonl").exists() -def test_real_binding_layers_project_config_after_explicit_opt_in( +def test_real_binding_ignores_project_config_with_explicit_opt_in( tmp_path, monkeypatch, ): @@ -1061,7 +1061,8 @@ def test_real_binding_layers_project_config_after_explicit_opt_in( working_directory = project_root / "workspace" config_directory = project_root / ".nemo-relay" selected_directory = tmp_path / "selected-config" - atof_dir = tmp_path / "atof" + project_atof_dir = tmp_path / "project-atof" + selected_atof_dir = tmp_path / "selected-atof" working_directory.mkdir(parents=True) config_directory.mkdir() selected_directory.mkdir() @@ -1081,14 +1082,35 @@ enabled = true [[components.config.atof.sinks]] type = "file" -output_directory = "{atof_dir.as_posix()}" +output_directory = "{project_atof_dir}" filename = "events.jsonl" mode = "overwrite" """.strip(), encoding="utf-8", ) selected_config = selected_directory / "plugins.toml" - selected_config.write_text("version = 1", encoding="utf-8") + selected_config.write_text( + f""" +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 4 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +output_directory = "{selected_atof_dir}" +filename = "events.jsonl" +mode = "overwrite" +""".strip(), + encoding="utf-8", + ) xdg_config_home = tmp_path / "xdg" xdg_config_home.mkdir() monkeypatch.chdir(working_directory) @@ -1102,12 +1124,13 @@ mode = "overwrite" host = relay_runtime.RelayRuntime(relay=relay, profile_key="profile") try: assert host.managed_execution_enabled() - host.ensure_session({"session_id": "native-layered-plugins"}) + host.ensure_session({"session_id": "native-explicit-plugins"}) finally: host.shutdown() relay_runtime._reset_for_tests() - assert (atof_dir / "events.jsonl").is_file() + assert (selected_atof_dir / "events.jsonl").is_file() + assert not (project_atof_dir / "events.jsonl").exists() def test_real_binding_keeps_two_profile_trajectories_separate_in_shared_exporters( diff --git a/tests/agent/test_relay_tools.py b/tests/agent/test_relay_tools.py index 22dec2fe52..760aa7335b 100644 --- a/tests/agent/test_relay_tools.py +++ b/tests/agent/test_relay_tools.py @@ -78,7 +78,10 @@ def test_request_rewrite_reaches_authorized_callback_once(relay_turn): async def wrap_execution(_name, args, next_call): result = await next_call(args) - return relay.ToolExecutionInterceptOutcome({**result, "wrapped": True}) + return relay.ToolExecutionInterceptOutcome( + {**result.result, "wrapped": True}, + annotation={"audit": "annotation-canary"}, + ) relay.intercepts.register_tool_request( "hermes-test-tool-request", 1, False, rewrite_request @@ -102,10 +105,33 @@ def test_request_rewrite_reaches_authorized_callback_once(relay_turn): assert observed_args == {"path": "/approved/path"} assert isinstance(result, str) assert json.loads(result) == {"ok": True, "wrapped": True} + assert "annotation-canary" not in result +def test_tool_call_id_uses_canonical_relay_argument(relay_turn, monkeypatch): + relay = relay_turn + captured = {} + async def capture_execute(_name, args, callback, **kwargs): + captured.update(kwargs) + result = callback(args) + assert isinstance(result, relay.ToolExecutionResult) + return result + monkeypatch.setattr(relay.tools, "execute", capture_execute) + original_result = {"ok": True} + + result, observed_args = relay_tools.execute( + "write_file", + {"path": "/tmp/output"}, + lambda _args: original_result, + session_id="session-1", + tool_call_id="call-42", + ) + + assert result is original_result + assert observed_args == {"path": "/tmp/output"} + assert captured["tool_call_id"] == "call-42" def test_tool_error_is_preserved_from_relay_wrapper_suffix(relay_turn, monkeypatch): @@ -135,8 +161,3 @@ def test_tool_error_is_preserved_from_relay_wrapper_suffix(relay_turn, monkeypat ) assert caught.value is tool_error - - - - - diff --git a/tests/agent/test_secret_scope.py b/tests/agent/test_secret_scope.py index 7e73f12dbc..5a42f842d8 100644 --- a/tests/agent/test_secret_scope.py +++ b/tests/agent/test_secret_scope.py @@ -347,3 +347,34 @@ class TestRelayRoutingStampGlobals: ss.set_multiplex_active(False) for name in self.AUTH_VARS: assert not ss._is_global_env(name), name + + +class TestSecretScopeAcrossExecutorThreads: + """Multiplexed profile state must reach pool workers (see #95119). + + The context-compression timeout fence runs auxiliary LLM calls in a + daemon thread pool. Bundled CPython runtime builds omit + ``ThreadPoolExecutor``'s context propagation, so the profile secret + scope was absent in the worker and ``get_secret`` failed closed with + ``UnscopedSecretError``, silently degrading compression to lossy + deterministic summaries. ``DaemonThreadPoolExecutor.submit`` restores + stdlib context semantics; these tests lock that in. + """ + + def test_scoped_read_works_in_daemon_pool_worker(self, monkeypatch): + from tools.daemon_pool import DaemonThreadPoolExecutor + + monkeypatch.setenv("SURPLUS_API_KEY", "env-key") + ss.set_multiplex_active(True) + token = ss.set_secret_scope({"SURPLUS_API_KEY": "scope-key"}) + pool = DaemonThreadPoolExecutor(max_workers=1) + try: + # The scope (authoritative under multiplex) must reach the worker. + seen = pool.submit(ss.get_secret, "SURPLUS_API_KEY").result(timeout=10) + assert seen == "scope-key" + # A scoped miss must still not borrow the (cross-profile) env value. + monkeypatch.setenv("OPENAI_API_KEY", "env-leak") + assert pool.submit(ss.get_secret, "OPENAI_API_KEY").result(timeout=10) is None + finally: + pool.shutdown(wait=True) + ss.reset_secret_scope(token) diff --git a/tests/agent/test_shared_http_transport.py b/tests/agent/test_shared_http_transport.py new file mode 100644 index 0000000000..d0c541723a --- /dev/null +++ b/tests/agent/test_shared_http_transport.py @@ -0,0 +1,185 @@ +"""Keepalive httpx clients share one HTTPTransport per (verify, proxy) identity. + +Every AIAgent (and every delegated child) gets its own ``httpx.Client`` — the +#10933 contract that closing one client must never poison the next. What is +shared underneath is the connection pool + SSL context, so a fan-out of N +children no longer holds N TLS socket sets to the same provider. +""" + +import ssl +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import certifi +import httpx +import pytest + +from agent import process_bootstrap +from agent.agent_runtime_helpers import _iter_pool_sockets, force_close_tcp_sockets +from agent.process_bootstrap import build_keepalive_http_client + + +@pytest.fixture +def no_proxy_env(monkeypatch): + for name in ( + "HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", + "https_proxy", "http_proxy", "all_proxy", "NO_PROXY", "no_proxy", + ): + monkeypatch.delenv(name, raising=False) + process_bootstrap.close_shared_transports() + yield + process_bootstrap.close_shared_transports() + + +class _Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" # keep-alive so pooled connections persist + + def do_GET(self): # noqa: N802 + body = b"ok" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + +@pytest.fixture +def local_server(): + server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +def _inner(client, scheme="https://"): + mount = next(t for pat, t in client._mounts.items() if str(pat.pattern) == scheme) + return mount._inner + + +def test_same_identity_clients_share_transport_but_not_client(no_proxy_env): + a = build_keepalive_http_client("https://api.example.com/v1") + b = build_keepalive_http_client("https://api.example.com/v1") + assert isinstance(a, httpx.Client) and isinstance(b, httpx.Client) + assert a is not b + assert _inner(a) is _inner(b) + assert _inner(a, "http://") is _inner(b, "http://") + # The per-client view is distinct, so each client has its own close state. + assert a._mounts is not b._mounts + a.close() + b.close() + + +def test_closing_one_client_leaves_sibling_functional(no_proxy_env, local_server): + a = build_keepalive_http_client(local_server) + b = build_keepalive_http_client(local_server) + assert _inner(a, "http://") is _inner(b, "http://") + assert a.get(local_server + "/x").status_code == 200 + a.close() + assert a.is_closed + # #10933 shape: the shared pool must still serve the surviving client and + # any successor client built after the close. + assert b.get(local_server + "/y").status_code == 200 + c = build_keepalive_http_client(local_server) + assert _inner(c, "http://") is _inner(b, "http://") + assert c.get(local_server + "/z").status_code == 200 + with pytest.raises(RuntimeError): + a.get(local_server + "/closed") + b.close() + c.close() + + +def test_pool_survives_client_close(no_proxy_env, local_server): + a = build_keepalive_http_client(local_server) + a.get(local_server + "/warm") + pool = _inner(a, "http://")._pool + before = len(pool.connections) + assert before >= 1 + a.close() + assert len(pool.connections) == before, "client close must not drain the shared pool" + + +def test_different_verify_or_proxy_get_different_transports(no_proxy_env, monkeypatch): + default = build_keepalive_http_client("https://api.example.com/v1") + insecure = build_keepalive_http_client("https://api.example.com/v1", verify=False) + ctx = ssl.create_default_context(cafile=certifi.where()) + with_ctx = build_keepalive_http_client("https://api.example.com/v1", verify=ctx) + with_ctx2 = build_keepalive_http_client("https://api.example.com/v1", verify=ctx) + codex = build_keepalive_http_client("https://chatgpt.com/backend-api/codex") + assert _inner(default) is not _inner(insecure) + assert _inner(default) is not _inner(with_ctx) + assert _inner(with_ctx) is _inner(with_ctx2) + assert _inner(with_ctx)._pool._ssl_context is ctx + assert _inner(insecure)._pool._ssl_context.check_hostname is False + # Codex cloud gets the happy-eyeballs backend, so it can't share a pool. + assert _inner(codex) is not _inner(default) + assert isinstance( + _inner(codex)._pool._network_backend, process_bootstrap._HappyEyeballsSyncBackend + ) + for c in (default, insecure, with_ctx, with_ctx2, codex): + c.close() + + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:3128") + proxied = build_keepalive_http_client("https://api.example.com/v1") + # Proxy clients keep httpx's own per-client proxy transport (unshared). + assert all( + type(t).__name__ != "_SharedTransport" for t in proxied._mounts.values() if t + ) + proxied.close() + + +def test_async_clients_are_not_shared(no_proxy_env): + a = build_keepalive_http_client("https://api.example.com/v1", async_mode=True) + b = build_keepalive_http_client("https://api.example.com/v1", async_mode=True) + assert isinstance(a, httpx.AsyncClient) + ta = [t for t in a._mounts.values() if t is not None] + tb = [t for t in b._mounts.values() if t is not None] + assert all(isinstance(t, httpx.AsyncHTTPTransport) for t in ta + tb) + assert not {id(t) for t in ta} & {id(t) for t in tb} + + +def test_force_close_only_touches_owning_clients_inflight_sockets(no_proxy_env, local_server): + """A stranger-thread abort on client A must not shut down client B's + idle/in-flight connections that live on the same shared pool.""" + a = build_keepalive_http_client(local_server) + b = build_keepalive_http_client(local_server) + b.get(local_server + "/warm") # idle keepalive connection on the shared pool + pool = _inner(a, "http://")._pool + assert pool.connections + # A has nothing in flight: nothing of A's may be touched. + assert list(_iter_pool_sockets(a)) == [] + assert force_close_tcp_sockets(a) == 0 + # B's idle connection is still healthy. + assert b.get(local_server + "/again").status_code == 200 + + # Now hold a B stream open and confirm A's abort still sees zero sockets + # while B's abort sees exactly its own. + with b.stream("GET", local_server + "/stream") as resp: + assert resp.status_code == 200 + assert list(_iter_pool_sockets(a)) == [] + assert len(list(_iter_pool_sockets(b))) == 1 + a.close() + b.close() + + +def test_shared_transport_cache_is_bounded(no_proxy_env, monkeypatch): + monkeypatch.setattr(process_bootstrap, "_SHARED_TRANSPORTS_MAX", 2) + clients = [ + build_keepalive_http_client("https://api.example.com/v1", verify=False), + build_keepalive_http_client("https://api.example.com/v1"), + ] + assert len(process_bootstrap._SHARED_TRANSPORTS) == 2 + ctx = ssl.create_default_context() + extra = build_keepalive_http_client("https://api.example.com/v1", verify=ctx) + assert len(process_bootstrap._SHARED_TRANSPORTS) == 2 + # Past the cap the caller still gets a working (private) transport. + assert _inner(extra)._pool._ssl_context is ctx + for c in clients + [extra]: + c.close() + assert process_bootstrap.close_shared_transports() == 2 diff --git a/tests/agent/test_skill_commands.py b/tests/agent/test_skill_commands.py index 691e4c1853..dd7d4a14a4 100644 --- a/tests/agent/test_skill_commands.py +++ b/tests/agent/test_skill_commands.py @@ -255,6 +255,41 @@ class TestScanSkillCommands: assert "/b-only" in profile_b_commands assert "/a-only" not in profile_b_commands + def test_get_skill_commands_scans_profile_skills_dir_not_frozen_import_dir(self, tmp_path): + """Under a profile home override the scan must read /skills/, + not the launch home's import-time ``SKILLS_DIR`` (#67277): a + multiplexed webhook routed to profile B otherwise sees default's skills. + Deliberately does NOT patch ``tools.skills_tool.SKILLS_DIR``. + """ + import agent.skill_commands as sc_mod + from agent.skill_commands import build_skill_invocation_message, get_skill_commands + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + profile_b = tmp_path / "profiles" / "b" + _make_skill(profile_b / "skills", "b-only", body="Body of b-only.") + (profile_b / "config.yaml").write_text("{}\n") + + with ( + patch.object(sc_mod, "_skill_commands", {}), + patch.object(sc_mod, "_skill_commands_platform", None), + patch.object(sc_mod, "_skill_commands_home", None), + ): + token = set_hermes_home_override(profile_b) + try: + commands = dict(get_skill_commands()) + assert "/b-only" in commands + # Frozen SKILLS_DIR (the launch home) must not leak in. + launch_dir = str(skills_tool_module._SKILLS_DIR_AT_IMPORT) + assert not any( + info["skill_dir"].startswith(launch_dir) for info in commands.values() + ) + # And the absolute skill_dir round-trips through skill_view + # (normalize_skill_lookup_name must use the same live root). + msg = build_skill_invocation_message("/b-only", user_instruction="go") + finally: + reset_hermes_home_override(token) + assert msg is not None and "Body of b-only." in msg + def test_get_skill_commands_rescans_when_leaving_platform_scope(self, tmp_path, monkeypatch): """Returning to no-platform-scope (CLI / cron / RL) after a gateway session must rescan so the unfiltered view is repopulated (#14536). diff --git a/tests/agent/test_stall_guards.py b/tests/agent/test_stall_guards.py index b79ba55c5b..013f8b83f8 100644 --- a/tests/agent/test_stall_guards.py +++ b/tests/agent/test_stall_guards.py @@ -92,7 +92,7 @@ def test_arg_canonicalization_ignores_key_order(): def test_allowlisted_pollers_never_fire(): c = ToolCallGuardrailController() - for tool in ("process", "vendor_get_result", "job_poll"): + for tool in ("process_manage", "vendor_get_result", "job_poll"): for _ in range(STALL_GUARD_IDENTICAL_CALL_THRESHOLD + 2): assert c.observe_identical_call(tool, {"id": "j1"}, "Generating") is None diff --git a/tests/agent/test_subagent_progress.py b/tests/agent/test_subagent_progress.py index 4ec939780b..8fc656ac06 100644 --- a/tests/agent/test_subagent_progress.py +++ b/tests/agent/test_subagent_progress.py @@ -132,11 +132,12 @@ class TestBuildChildProgressCallback: parent._delegate_spinner = spinner parent.tool_progress_callback = None - # task_index=0 in a batch of 3 → prefix "[1]" + # task_index=0 in a batch of 3 → prefix "[1/3]" (batch slot; a + # delegation batch tag is prepended when the id is known) cb0 = _build_child_progress_callback(0, "test goal", parent, task_count=3) cb0("tool.started", "web_search", "test", {}) output = buf.getvalue() - assert "[1]" in output + assert "[1/3]" in output # task_index=2 in a batch of 3 → prefix "[3]" buf.truncate(0) @@ -144,7 +145,7 @@ class TestBuildChildProgressCallback: cb2 = _build_child_progress_callback(2, "test goal", parent, task_count=3) cb2("tool.started", "web_search", "test", {}) output = buf.getvalue() - assert "[3]" in output + assert "[3/3]" in output diff --git a/tests/agent/test_subdirectory_hints.py b/tests/agent/test_subdirectory_hints.py index e310aed62e..a686258287 100644 --- a/tests/agent/test_subdirectory_hints.py +++ b/tests/agent/test_subdirectory_hints.py @@ -1,9 +1,12 @@ """Tests for progressive subdirectory hint discovery.""" +import time + import pytest from pathlib import Path from unittest.mock import patch +from agent.search_policy import SEARCH_PRUNE_DIR_NAMES from agent.subdirectory_hints import SubdirectoryHintTracker @@ -116,6 +119,40 @@ class TestSubdirectoryHintTracker: + def test_timeout_skips_slow_hint_files(self, project, monkeypatch, caplog): + """Slow hint reads time out instead of blocking the turn.""" + backend = project / "backend" + (backend / "AGENTS.md").write_text("Backend-specific instructions") + import sys + + from agent import subdirectory_hints as sh_mod + + # Patch the module object the hint tracker's helper closes over. + pb_mod = sys.modules[sh_mod._read_text_with_timeout.__module__] + monkeypatch.setattr(pb_mod, "_get_context_file_read_timeout", lambda: 0.05) + + original_read_text = Path.read_text + + def slow_read_text(self, *args, **kwargs): + if self.name.lower() == "agents.md" and self.parent == backend: + time.sleep(0.6) + return original_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", slow_read_text) + + tracker = SubdirectoryHintTracker(working_dir=str(project)) + start = time.monotonic() + with caplog.at_level("WARNING", logger="agent.prompt_builder"): + result = tracker.check_tool_call( + "read_file", {"path": str(project / "backend" / "src" / "main.py")} + ) + elapsed = time.monotonic() - start + + assert elapsed < 0.4, f"hint load blocked for {elapsed:.2f}s" + assert result is None + assert "timed out" in caplog.text.lower() + + class TestPermissionErrorHandling: """Regression tests for PermissionError in filesystem checks (ref #6214).""" @@ -246,7 +283,7 @@ class TestExcludedDirectories: @pytest.mark.parametrize( "excluded", - ["backups", "node_modules", ".git", "venv", "site-packages", ".Trash", "vendor"], + sorted(SEARCH_PRUNE_DIR_NAMES), ) def test_excluded_directory_skipped(self, tmp_path, excluded): target = tmp_path / excluded / "snapshot" diff --git a/tests/agent/test_summarize_tool_result_type_safety.py b/tests/agent/test_summarize_tool_result_type_safety.py index 2899c9be27..f05cca0c0c 100644 --- a/tests/agent/test_summarize_tool_result_type_safety.py +++ b/tests/agent/test_summarize_tool_result_type_safety.py @@ -112,7 +112,7 @@ class TestBackstopWrapper: "terminal", "read_file", "write_file", "search_files", "patch", "browser_navigate", "web_search", "web_extract", "delegate_task", "execute_code", "skill_view", "vision_analyze", "memory", - "cronjob", "process", "totally_unknown_tool", + "cronjob_manage", "process_manage", "totally_unknown_tool", ] keys = ["command", "path", "content", "pattern", "url", "query", "urls", "goal", "code", "name", "question", "action", @@ -151,12 +151,12 @@ class TestDisplayPreviewTypeSafety: def test_process_preview_non_string_data(self): from agent.display import build_tool_preview result = build_tool_preview( - "process", {"action": "submit", "session_id": "abc", "data": 42} + "process_manage", {"action": "submit", "session_id": "abc", "data": 42} ) assert result == 'submit abc "42"' def test_process_preview_none_action(self): from agent.display import build_tool_preview - result = build_tool_preview("process", {"action": None, "session_id": "abc"}) + result = build_tool_preview("process_manage", {"action": None, "session_id": "abc"}) assert isinstance(result, str) diff --git a/tests/agent/test_tool_guardrails.py b/tests/agent/test_tool_guardrails.py index dbeb2d9d3f..63ae5debd3 100644 --- a/tests/agent/test_tool_guardrails.py +++ b/tests/agent/test_tool_guardrails.py @@ -33,6 +33,18 @@ def test_tool_call_signature_hashes_canonical_nested_unicode_args_without_exposi assert "☤" not in json.dumps(metadata) +def test_default_config_is_soft_warning_only_with_hard_stop_disabled(): + cfg = ToolCallGuardrailConfig() + + assert cfg.warnings_enabled is True + assert cfg.hard_stop_enabled is False + assert cfg.non_interactive_hard_stop_enabled is True + assert cfg.exact_failure_warn_after == 2 + assert cfg.same_tool_failure_warn_after == 3 + assert cfg.no_progress_warn_after == 2 + assert cfg.exact_failure_block_after == 5 + assert cfg.same_tool_failure_halt_after == 8 + assert cfg.no_progress_block_after == 5 def test_config_parses_nested_warn_and_hard_stop_thresholds(): @@ -63,6 +75,29 @@ def test_config_parses_nested_warn_and_hard_stop_thresholds(): assert cfg.no_progress_block_after == 8 +def test_gateway_platform_defaults_to_hard_stop_without_changing_interactive_defaults(): + interactive_configs = [ + ToolCallGuardrailConfig.from_mapping({}, platform=platform) + for platform in ("cli", "tui", "desktop", "acp") + ] + telegram_cfg = ToolCallGuardrailConfig.from_mapping({}, platform="telegram") + cron_cfg = ToolCallGuardrailConfig.from_mapping({}, platform="cron") + + assert all(cfg.hard_stop_enabled is False for cfg in interactive_configs) + assert telegram_cfg.hard_stop_enabled is True + assert cron_cfg.hard_stop_enabled is True + + +def test_non_interactive_hard_stop_can_be_disabled_explicitly(): + cfg = ToolCallGuardrailConfig.from_mapping( + {"non_interactive_hard_stop_enabled": False}, + platform="telegram", + ) + + assert cfg.hard_stop_enabled is False + assert cfg.non_interactive_hard_stop_enabled is False + + def test_default_repeated_identical_failed_call_warns_without_blocking(): controller = ToolCallGuardrailController() args = {"query": "same"} @@ -119,6 +154,41 @@ def test_hard_stop_enabled_blocks_repeated_exact_failure_before_next_execution() +def test_skill_read_tools_are_idempotent_and_block_repeated_identical_success_output(): + cases = [ + ( + "skill_view", + {"name": "gui-agent-ml-operations"}, + '{"success":true,"name":"gui-agent-ml-operations","content":"same"}', + ), + ( + "skills_list", + {"category": "mlops"}, + '{"success":true,"skills":[{"name":"gui-agent-ml-operations"}]}', + ), + ] + + for tool_name, args, result in cases: + controller = ToolCallGuardrailController( + ToolCallGuardrailConfig( + hard_stop_enabled=True, + no_progress_warn_after=2, + no_progress_block_after=2, + ) + ) + + assert controller.before_call(tool_name, args).action == "allow" + assert controller.after_call(tool_name, args, result, failed=False).action == "allow" + assert controller.before_call(tool_name, args).action == "allow" + warn = controller.after_call(tool_name, args, result, failed=False) + assert warn.action == "warn" + assert warn.code == "idempotent_no_progress_warning" + + blocked = controller.before_call(tool_name, args) + assert blocked.action == "block" + assert blocked.code == "idempotent_no_progress_block" + + def test_mutating_or_unknown_tools_are_not_blocked_for_repeated_identical_success_output_by_default(): controller = ToolCallGuardrailController( ToolCallGuardrailConfig(no_progress_warn_after=2, no_progress_block_after=2) @@ -131,6 +201,49 @@ def test_mutating_or_unknown_tools_are_not_blocked_for_repeated_identical_succes assert controller.after_call("custom_tool", {"x": 1}, "ok", failed=False).action == "allow" +def test_identical_call_streak_halts_any_tool_when_hard_stop_enabled(): + # #89069 / #100849 bundle: a model replaying the same SUCCESSFUL + # terminal/skill_view call with a byte-identical result is not covered by + # the idempotent_tools no-progress block. The consecutive-identical + # streak (observe_call) is tool-agnostic; under hard_stop it must halt. + controller = ToolCallGuardrailController( + ToolCallGuardrailConfig(hard_stop_enabled=True, no_progress_block_after=5) + ) + args = {"command": "hermes config get memory.provider"} + for i in range(1, 5): + controller.after_call("terminal", args, "local\n", failed=False) + controller.observe_call("terminal", args, "local\n", failed=False) + assert controller.halt_decision is None, f"halted early at {i}" + + controller.after_call("terminal", args, "local\n", failed=False) + controller.observe_call("terminal", args, "local\n", failed=False) + halt = controller.halt_decision + assert halt is not None and halt.should_halt + assert halt.code == "identical_call_streak_halt" + assert halt.tool_name == "terminal" and halt.count == 5 + + +def test_identical_call_streak_never_halts_when_hard_stop_disabled_or_for_pollers(): + soft = ToolCallGuardrailController( + ToolCallGuardrailConfig(hard_stop_enabled=False, no_progress_block_after=2) + ) + for _ in range(6): + soft.observe_call("terminal", {"command": "ls"}, "a\nb\n", failed=False) + assert soft.halt_decision is None # notice-only in interactive sessions + + hard = ToolCallGuardrailController( + ToolCallGuardrailConfig(hard_stop_enabled=True, no_progress_block_after=2) + ) + for _ in range(6): + hard.observe_call("process_manage", {"action": "poll", "session_id": "p1"}, "running", failed=False) + assert hard.halt_decision is None # an unchanged poll is legitimate progress + + # A changed result resets the streak. + for i in range(6): + hard.observe_call("terminal", {"command": "date"}, f"t{i}", failed=False) + assert hard.halt_decision is None + + @@ -177,3 +290,89 @@ def test_web_search_cap_blocks_after_limit_regardless_of_hard_stop(): + + +# ── Legitimate flows must survive hard stops (Teknium, Sep 2026) ──────────── +# Hard stops default ON for unattended platforms. These pin the flows that +# must NEVER be cut off there: edit -> re-run loops, diagnostic sweeps of +# distinct red commands, and browser retry-after-action — while the pure +# replay (same call, nothing changed between attempts) is still stopped. + +_HARD = lambda: ToolCallGuardrailController( # noqa: E731 + ToolCallGuardrailConfig(hard_stop_enabled=True) +) +_PYTEST = {"command": "pytest tests/test_x.py -q"} +_RED = '{"output": "1 failed", "exit_code": 1}' + + +def _run_red(c, args=_PYTEST): + assert c.before_call("terminal", args).allows_execution + return c.after_call("terminal", args, _RED, failed=True) + + +def test_fix_retest_loop_is_never_hard_stopped(): + c = _HARD() + for i in range(12): + d = _run_red(c) + assert not d.should_halt, f"halted on red run {i + 1}" + # the model edits between runs — a landed mutation is progress + c.after_call("patch", {"path": "x.py", "old_string": "a", "new_string": f"b{i}"}, + '{"success": true, "diff": "..."}', failed=False) + assert c.halt_decision is None + assert c.before_call("terminal", _PYTEST).allows_execution + + +def test_pure_replay_with_no_intervening_change_is_still_blocked(): + c = _HARD() + for _ in range(5): + _run_red(c) + d = c.before_call("terminal", _PYTEST) + assert d.action == "block" and d.code == "repeated_exact_failure_block" + + +def test_intervening_mutation_resets_the_replay_streak_only_once(): + # 4 reds, one edit, then 4 reds with NO edit: the second run of 4 is a + # fresh streak, and the 5th unchanged retry after it is blocked. + c = _HARD() + for _ in range(4): + _run_red(c) + c.after_call("write_file", {"path": "x.py", "content": "y"}, '{"bytes_written": 1}', failed=False) + for _ in range(5): + assert c.before_call("terminal", _PYTEST).allows_execution + c.after_call("terminal", _PYTEST, _RED, failed=True) + assert c.before_call("terminal", _PYTEST).action == "block" + + +def test_distinct_failing_terminal_commands_warn_but_never_halt(): + # A diagnostic sweep: grep with no matches, missing binaries, red builds. + c = _HARD() + for i in range(12): + args = {"command": f"grep -q needle{i} haystack.txt"} + d = c.after_call("terminal", args, _RED, failed=True) + assert not d.should_halt, f"same_tool halt on distinct command #{i + 1}" + assert c.halt_decision is None + # ...while a non-tolerant tool failing 8 distinct ways still halts. + c2 = _HARD() + last = None + for i in range(8): + last = c2.after_call("send_message", {"to": f"u{i}"}, '{"error": "no route"}', failed=True) + assert last.should_halt and last.code == "same_tool_failure_halt" + + +def test_browser_retry_after_action_is_not_a_replay(): + c = _HARD() + nav = {"url": "https://example.test/app"} + for _ in range(8): + assert c.before_call("browser_navigate", nav).allows_execution + c.after_call("browser_navigate", nav, '{"error": "timeout"}', failed=True) + c.after_call("browser_click", {"selector": "#retry"}, '{"ok": true}', failed=False) + assert c.halt_decision is None + + +def test_supervised_task_platforms_keep_warning_only_default(): + for platform in ("subagent", "api_server", "cli"): + cfg = ToolCallGuardrailConfig.from_mapping({}, platform=platform) + assert cfg.hard_stop_enabled is False, platform + for platform in ("telegram", "discord", "cron", "kanban"): + cfg = ToolCallGuardrailConfig.from_mapping({}, platform=platform) + assert cfg.hard_stop_enabled is True, platform diff --git a/tests/agent/test_turn_retry_state.py b/tests/agent/test_turn_retry_state.py index a182f17795..2b7cb6b56b 100644 --- a/tests/agent/test_turn_retry_state.py +++ b/tests/agent/test_turn_retry_state.py @@ -26,6 +26,7 @@ EXPECTED_FIELDS = { "native_compaction_reject_retry_attempted", "image_shrink_retry_attempted", "multimodal_tool_content_retry_attempted", + "reasoning_mandatory_retry_attempted", "oauth_1m_beta_retry_attempted", "llama_cpp_grammar_retry_attempted", "primary_recovery_attempted", diff --git a/tests/agent/test_usage_pricing.py b/tests/agent/test_usage_pricing.py index 1b6862aac6..4d4e8a1ea8 100644 --- a/tests/agent/test_usage_pricing.py +++ b/tests/agent/test_usage_pricing.py @@ -105,6 +105,48 @@ def test_deepseek_v4_pro_pricing_entry_exists(): assert float(entry.cache_read_cost_per_million) == 0.003625 +def test_bundled_pricing_skips_endpoint_metadata(monkeypatch): + """An exact bundled price must not block on the provider's /models API.""" + monkeypatch.setattr( + "agent.usage_pricing.fetch_endpoint_model_metadata", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("endpoint metadata should not be fetched") + ), + ) + + entry = get_pricing_entry( + "deepseek-chat", + provider="deepseek", + base_url="https://api.deepseek.com/v1", + ) + + assert entry is not None + assert entry.source == "official_docs_snapshot" + + +def test_unknown_model_falls_back_to_endpoint_metadata(monkeypatch): + """Models absent from the bundled table still use endpoint pricing.""" + monkeypatch.setattr( + "agent.usage_pricing.fetch_endpoint_model_metadata", + lambda *_args, **_kwargs: { + "deepseek-future": { + "pricing": {"prompt": "0.000001", "completion": "0.000002"} + } + }, + ) + + entry = get_pricing_entry( + "deepseek-future", + provider="deepseek", + base_url="https://api.deepseek.com/v1", + ) + + assert entry is not None + assert entry.source == "provider_models_api" + assert entry.input_cost_per_million == Decimal("1") + assert entry.output_cost_per_million == Decimal("2") + + def test_deepseek_deprecated_aliases_price_as_v4_flash(): diff --git a/tests/ci/test_classify_changes.py b/tests/ci/test_classify_changes.py index 4130a0fd9a..8b20a3e69d 100644 --- a/tests/ci/test_classify_changes.py +++ b/tests/ci/test_classify_changes.py @@ -42,13 +42,14 @@ DEFAULT = { "npm_lock": True, "installer": True, "bootstrap": True, + "desktop_updater": True, "rust": True, "mcp_catalog": False, "ci_review": True, } -def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_lock=False, npm_lock=False, installer=False, bootstrap=False, rust=False, mcp_catalog=False, docker_meta=False, ci_review=False, python_prod=None, nix=None, docker=None) -> dict[str, bool]: +def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_lock=False, npm_lock=False, installer=False, bootstrap=False, desktop_updater=False, rust=False, mcp_catalog=False, docker_meta=False, ci_review=False, python_prod=None, nix=None, docker=None) -> dict[str, bool]: # python_prod tracks python except for tests-only diffs; default it to # python so the majority of cases don't need to spell it out. # @@ -71,6 +72,7 @@ def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_ "npm_lock": npm_lock, "installer": installer, "bootstrap": bootstrap, + "desktop_updater": desktop_updater, "rust": rust, "mcp_catalog": mcp_catalog, "ci_review": ci_review, @@ -80,7 +82,9 @@ def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_ CASES = { "docs-only → nothing heavy": (["README.md", "docs/guide.md"], _lanes()), "python source → python": (["run_agent.py"], _lanes(python=True, scan=True)), - "dep manifest → python": (["pyproject.toml"], _lanes(python=True, scan=True, deps=True, uv_lock=True)), + # pyproject.toml declares the pytest markers the OS lanes select on, so it + # also re-arms the desktop_updater integration tests (fail-open). + "dep manifest → python": (["pyproject.toml"], _lanes(python=True, scan=True, deps=True, uv_lock=True, desktop_updater=True)), "uv.lock → python": (["uv.lock"], _lanes(python=True, uv_lock=True)), "ts package → frontend": (["apps/desktop/src/app.tsx"], _lanes(frontend=True)), "ui-tui → frontend": (["ui-tui/src/entry.ts"], _lanes(frontend=True)), @@ -143,6 +147,23 @@ CASES = { _lanes(python=True, installer=True), ), "python source alone → no installer lane": (["run_agent.py"], _lanes(python=True, scan=True)), + # The Windows desktop-update hand-off is a PowerShell integration surface: + # its tests spawn the real script and poll its loopback server. They run + # when the script, the Electron side that launches it, or their own test + # files change — not on every hermes_state.py PR. + "windows.ps1 → desktop_updater": ( + ["scripts/desktop-update/windows.ps1"], + _lanes(python=True, desktop_updater=True), + ), + "desktop-update test → desktop_updater": ( + ["tests/test_desktop_update_windows_progress.py"], + _lanes(python=True, python_prod=False, scan=True, desktop_updater=True), + ), + "updater-process.ts → desktop_updater": ( + ["apps/desktop/electron/updater-process.ts"], + _lanes(frontend=True, desktop_updater=True), + ), + "python source alone → no desktop_updater lane": (["hermes_state.py"], _lanes(python=True, scan=True)), # `.rs` lives under apps/, so it matches `frontend` too. That lane builds # TypeScript and cannot notice a Rust error — before `rust` existed it was # the ONLY lane a Rust change ran, and the crate's tests never executed. @@ -170,9 +191,15 @@ CASES = { # tests-only diffs: pytest lanes stay ON, product jobs (Desktop E2E, # Docker) gate on python_prod and skip. "tests-only → python without python_prod": ( - ["tests/agent/test_foo.py", "tests/conftest.py"], + ["tests/agent/test_foo.py"], _lanes(python=True, python_prod=False, scan=True), ), + # conftest.py owns the _OS_MARKS skip logic, so it re-arms the + # desktop_updater integration tests too (fail-open). + "conftest → python + desktop_updater": ( + ["tests/conftest.py"], + _lanes(python=True, python_prod=False, scan=True, desktop_updater=True), + ), "tests + prod source → both lanes": ( ["tests/agent/test_foo.py", "agent/x.py"], _lanes(python=True, scan=True), diff --git a/tests/cli/test_cli_clarify_batch.py b/tests/cli/test_cli_clarify_batch.py index c839780bbb..a19575956c 100644 --- a/tests/cli/test_cli_clarify_batch.py +++ b/tests/cli/test_cli_clarify_batch.py @@ -341,3 +341,33 @@ class TestClarifyBatchNavigation: thread.join(timeout=2) assert result["value"] == {"answers": {"q0": "red", "q1": "small"}} + + +class TestClarifyBellOnPrompt: + """display.bell_on_prompt rings BEL when a clarify modal opens; off is silent.""" + + @staticmethod + def _run_clarify(bell_on_prompt): + import io + + cli = _make_cli_stub() + cli.bell_on_prompt = bell_on_prompt + out = io.StringIO() + with patch("cli.sys.stdout", out), patch( + "tools.clarify_gateway.resolve_clarify_timeout", return_value=60 + ): + thread = threading.Thread( + target=cli._clarify_callback, args=("Color?", ["red", "blue"]), daemon=True + ) + thread.start() + deadline = time.time() + 2 + while cli._clarify_state is None and time.time() < deadline: + time.sleep(0.01) + assert cli._clarify_state is not None + cli._clarify_state["response_queue"].put("red") + thread.join(timeout=2) + return out.getvalue() + + def test_bell_on_prompt_rings_and_off_is_silent(self): + assert "\a" in self._run_clarify(True) + assert "\a" not in self._run_clarify(False) diff --git a/tests/cli/test_fast_command.py b/tests/cli/test_fast_command.py index 87a6b2689c..203dfe329d 100644 --- a/tests/cli/test_fast_command.py +++ b/tests/cli/test_fast_command.py @@ -159,8 +159,8 @@ class TestFastModeRouting(unittest.TestCase): stub = SimpleNamespace( model="gpt-5.4", api_key="primary-key", - base_url="https://openrouter.ai/api/v1", - provider="openrouter", + base_url="https://api.openai.com/v1", + provider="openai", api_mode="chat_completions", acp_command=None, acp_args=[], @@ -171,11 +171,16 @@ class TestFastModeRouting(unittest.TestCase): route = cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi") # Provider should NOT have changed - assert route["runtime"]["provider"] == "openrouter" + assert route["runtime"]["provider"] == "openai" assert route["runtime"]["api_mode"] == "chat_completions" # But request_overrides should be set assert route["request_overrides"] == {"service_tier": "priority"} + # Proxied routes (OpenRouter etc.) strip/400 on the param — never sent. + stub.base_url = "https://openrouter.ai/api/v1" + stub.provider = "openrouter" + assert cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi")["request_overrides"] is None + def test_turn_route_keeps_primary_runtime_when_model_has_no_fast_backend(self): cli_mod = _import_cli() stub = SimpleNamespace( @@ -202,28 +207,36 @@ class TestAnthropicFastMode(unittest.TestCase): def test_anthropic_opus_supported(self): from hermes_cli.models import model_supports_fast_mode + # Per the live fast-mode docs: Opus 4.8 + Opus 5, Claude API only. # Native Anthropic format (hyphens) - assert model_supports_fast_mode("claude-opus-4-6") is True + assert model_supports_fast_mode("claude-opus-4-8") is True # OpenRouter format (dots) - assert model_supports_fast_mode("claude-opus-4.6") is True + assert model_supports_fast_mode("claude-opus-4.8") is True # With vendor prefix - assert model_supports_fast_mode("anthropic/claude-opus-4-6") is True - assert model_supports_fast_mode("anthropic/claude-opus-4.6") is True + assert model_supports_fast_mode("anthropic/claude-opus-4-8") is True + assert model_supports_fast_mode("anthropic/claude-opus-4.8") is True + assert model_supports_fast_mode("claude-opus-5") is True + assert model_supports_fast_mode("anthropic/claude-opus-5") is True - def test_anthropic_non_opus46_models_excluded(self): - """The speed=fast parameter is gated to Opus 4.6 — others excluded. + def test_anthropic_unsupported_models_excluded(self): + """The speed=fast parameter is gated to Opus 4.8 / Opus 5. - Per https://platform.claude.com/docs/en/build-with-claude/fast-mode, - sending speed=fast to Opus 4.7, Sonnet, or Haiku returns HTTP 400. - Opus 4.8 uses a separate ``…-fast`` model id, not this parameter. + Per https://platform.claude.com/docs/en/build-with-claude/fast-mode: + Opus 4.6 LOST fast mode 2026-06-29 (the param is silently ignored — + standard speed at standard billing — so a toggle would do nothing); + Opus 4.7 hard-400s; Sonnet/Haiku never had it; dedicated ``…-fast`` + ids select fast inference via the model field, not the parameter. """ from hermes_cli.models import model_supports_fast_mode assert model_supports_fast_mode("claude-sonnet-4-6") is False assert model_supports_fast_mode("claude-sonnet-4.6") is False assert model_supports_fast_mode("claude-haiku-4-5") is False + assert model_supports_fast_mode("claude-opus-4-6") is False + assert model_supports_fast_mode("claude-opus-4.6") is False assert model_supports_fast_mode("claude-opus-4-7") is False - assert model_supports_fast_mode("claude-opus-4-8") is False + assert model_supports_fast_mode("claude-opus-4-8-fast") is False + assert model_supports_fast_mode("anthropic/claude-opus-4.8-fast") is False assert model_supports_fast_mode("anthropic/claude-sonnet-4.6") is False assert model_supports_fast_mode("anthropic/claude-opus-4-7") is False @@ -232,10 +245,10 @@ class TestAnthropicFastMode(unittest.TestCase): def test_resolve_overrides_returns_speed_for_anthropic(self): from hermes_cli.models import resolve_fast_mode_overrides - result = resolve_fast_mode_overrides("claude-opus-4-6") + result = resolve_fast_mode_overrides("claude-opus-4-8") assert result == {"speed": "fast"} - result = resolve_fast_mode_overrides("anthropic/claude-opus-4.6") + result = resolve_fast_mode_overrides("anthropic/claude-opus-4.8") assert result == {"speed": "fast"} @@ -243,7 +256,7 @@ class TestAnthropicFastMode(unittest.TestCase): def test_fast_command_hidden_for_anthropic_sonnet(self): - """Sonnet doesn't support fast mode (Opus 4.6 only) — /fast must be hidden.""" + """Sonnet doesn't support fast mode (Opus 4.8/5 only) — /fast must be hidden.""" cli_mod = _import_cli() stub = SimpleNamespace( provider="anthropic", requested_provider="anthropic", @@ -257,7 +270,7 @@ class TestAnthropicFastMode(unittest.TestCase): """Anthropic models should get speed:'fast' override, not service_tier.""" cli_mod = _import_cli() stub = SimpleNamespace( - model="claude-opus-4-6", + model="claude-opus-4-8", api_key="sk-ant-test", base_url="https://api.anthropic.com", provider="anthropic", @@ -281,7 +294,7 @@ class TestAnthropicFastModeAdapter(unittest.TestCase): from agent.anthropic_adapter import build_anthropic_kwargs, _FAST_MODE_BETA kwargs = build_anthropic_kwargs( - model="claude-opus-4-6", + model="claude-opus-4-8", messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], tools=None, max_tokens=None, @@ -297,7 +310,7 @@ class TestAnthropicFastModeAdapter(unittest.TestCase): from agent.anthropic_adapter import build_anthropic_kwargs kwargs = build_anthropic_kwargs( - model="claude-opus-4-6", + model="claude-opus-4-8", messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], tools=None, max_tokens=None, @@ -312,7 +325,7 @@ class TestAnthropicFastModeAdapter(unittest.TestCase): from agent.anthropic_adapter import build_anthropic_kwargs kwargs = build_anthropic_kwargs( - model="claude-opus-4-6", + model="claude-opus-4-8", messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], tools=None, max_tokens=None, diff --git a/tests/cli/test_personality_none.py b/tests/cli/test_personality_none.py index ba4847607c..5a8752122c 100644 --- a/tests/cli/test_personality_none.py +++ b/tests/cli/test_personality_none.py @@ -87,7 +87,6 @@ class TestGatewayPersonalityNone: def _make_runner(self, personalities=None): from gateway.run import GatewayRunner runner = GatewayRunner.__new__(GatewayRunner) - runner._ephemeral_system_prompt = "You are kawaii~" runner.config = { "agent": { "personalities": personalities or {"helpful": "You are helpful."} @@ -125,7 +124,9 @@ class TestGatewayPersonalityNone: saved = yaml.safe_load(config_file.read_text()) assert saved["agent"]["system_prompt"] == "manual forever" assert saved.get("display", {}).get("personality", None) == "" - assert runner._ephemeral_system_prompt == "manual forever" + # The next turn re-resolves from config (no in-memory snapshot). + with p1, p2: + assert runner._get_system_prompt_for_channel(None, "c") == "manual forever" @pytest.mark.asyncio async def test_set_persists_display_personality_not_system_prompt(self, tmp_path): @@ -147,7 +148,8 @@ class TestGatewayPersonalityNone: saved = yaml.safe_load(config_file.read_text()) assert saved["agent"]["system_prompt"] == "manual forever" assert saved["display"]["personality"] == "helpful" - assert runner._ephemeral_system_prompt == "You are helpful." + with p1, p2: + assert runner._get_system_prompt_for_channel(None, "c") == "You are helpful." assert "helpful" in result.lower() @pytest.mark.asyncio diff --git a/tests/cli/test_resume_display.py b/tests/cli/test_resume_display.py index 3c8755c31b..e1c8fb2e5b 100644 --- a/tests/cli/test_resume_display.py +++ b/tests/cli/test_resume_display.py @@ -323,6 +323,27 @@ class TestPreloadResumedSession: assert "safe resume limit is 20000" in output.getvalue() mock_db.get_resume_conversations.assert_not_called() + def test_tip_only_guard_goes_through_the_shared_resume_guard(self): + """The mid-setup path loads only the tip, so it asks the ONE resume + guard for a tip-only bound instead of borrowing the export guard.""" + from hermes_state import SessionResumeTooLargeError + + cli = _make_cli(resume="deep-lineage") + cli.session_id = "deep-lineage" + mock_db = MagicMock() + guard = MagicMock(return_value=666) + mock_db.assert_resume_safe = guard + cli._session_db = mock_db + + assert cli._resume_history_limit_error(tip_only=True) is None + guard.assert_called_once_with("deep-lineage", tip_only=True) + + guard.side_effect = SessionResumeTooLargeError( + 20_001, 20_000, scope="in its tip segment" + ) + error = cli._resume_history_limit_error(tip_only=True) + assert error and "in its tip segment" in error + # ── Tests for _handle_resume_command recap display ─────────────────── diff --git a/tests/computer_use/test_cua_wayland_env.py b/tests/computer_use/test_cua_wayland_env.py new file mode 100644 index 0000000000..df06e132d6 --- /dev/null +++ b/tests/computer_use/test_cua_wayland_env.py @@ -0,0 +1,22 @@ +from unittest.mock import patch + +from tools.computer_use import cua_backend + + +_VAR = "CUA_DRIVER_RS_ENABLE_WAYLAND" + + +def _child_env(base_env, native_wayland): + config = {"computer_use": {"native_wayland": native_wayland}} + with patch("hermes_cli.config.load_config", return_value=config), \ + patch.object(cua_backend.sys, "platform", "linux"): + return cua_backend.cua_driver_child_env(base_env) + + +def test_configured_native_wayland_reaches_linux_wayland_child(): + assert _child_env({"WAYLAND_DISPLAY": "wayland-1"}, True)[_VAR] == "1" + + +def test_native_wayland_not_injected_without_wayland_display_or_opt_in(): + assert _VAR not in _child_env({"DISPLAY": ":0"}, True) + assert _VAR not in _child_env({"WAYLAND_DISPLAY": "wayland-1"}, False) diff --git a/tests/cron/test_cron_drift_alert_once.py b/tests/cron/test_cron_drift_alert_once.py index a6014f2591..c3e57eab29 100644 --- a/tests/cron/test_cron_drift_alert_once.py +++ b/tests/cron/test_cron_drift_alert_once.py @@ -48,7 +48,7 @@ def _tick(job, tmp_path, current_provider, deliveries): """Run one run_one_job tick with the provider resolution pinned.""" fake_db = MagicMock() - def fake_deliver(job, content, adapters=None, loop=None): + def fake_deliver(job, content, adapters=None, loop=None, **kwargs): deliveries.append(content) return None @@ -129,7 +129,7 @@ class TestDriftAlertOnce: job = _job(provider_snapshot=None, drift_alerted=True) deliveries = [] - def fake_deliver(jb, content, adapters=None, loop=None): + def fake_deliver(jb, content, adapters=None, loop=None, **kwargs): deliveries.append(content) return None diff --git a/tests/cron/test_cron_failure_deliver.py b/tests/cron/test_cron_failure_deliver.py new file mode 100644 index 0000000000..88e6fee244 --- /dev/null +++ b/tests/cron/test_cron_failure_deliver.py @@ -0,0 +1,462 @@ +"""Per-job ``failure_deliver`` routing (NS-788). + +A job's FAILURE notices (run failed, escaped scheduler exception, drift-skip / +blocked-config alerts) resolve their delivery targets from ``failure_deliver`` +when the job sets it, falling back to ``deliver`` when unset — so existing +jobs behave byte-identically. ``failure_deliver: local`` is structural silence +for failures: nothing is sent, but state (last_status, run history, output +file) is still recorded. Success-path delivery never reads ``failure_deliver``. + +The grammar is exactly the ``deliver`` grammar — same normalization, same +validation — reused, not duplicated. +""" + +import json + +import pytest + +import cron.scheduler as s +from cron.scheduler import _resolve_delivery_targets + + +@pytest.fixture +def cron_env(tmp_path, monkeypatch): + """Isolated cron environment with temp HERMES_HOME.""" + hermes_home = tmp_path / ".hermes" + hermes_home.mkdir() + (hermes_home / "cron").mkdir() + (hermes_home / "cron" / "output").mkdir() + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + import cron.jobs as jobs_mod + monkeypatch.setattr(jobs_mod, "HERMES_DIR", hermes_home) + monkeypatch.setattr(jobs_mod, "CRON_DIR", hermes_home / "cron") + monkeypatch.setattr(jobs_mod, "JOBS_FILE", hermes_home / "cron" / "jobs.json") + monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", hermes_home / "cron" / "output") + + return hermes_home + + +@pytest.fixture +def run_env(monkeypatch, tmp_path): + """Drive run_one_job with the REAL delivery path down to a fake sender. + + Bookkeeping primitives are stubbed (recorded), but _deliver_result and + _resolve_delivery_targets are the genuine articles — the send that would + leave the process is captured at the platform-registry sender seam, + exactly where a real slack delivery exits. + """ + home = tmp_path / "hermes-home" + home.mkdir() + (home / "config.yaml").write_text( + "platforms:\n slack:\n enabled: true\n token: xoxb-test\n" + ) + monkeypatch.setenv("HERMES_HOME", str(home)) + + send_calls = [] + + async def fake_sender(pconfig, chat_id, message, *, thread_id=None, + media_files=None, force_document=False, caption=None): + send_calls.append({"chat_id": chat_id, "message": message}) + return {"success": True, "chat_id": chat_id, "message_id": "1.2"} + + import gateway.platform_registry as reg + import hermes_cli.plugins as hp + + entry = reg.platform_registry.get("slack") + if entry is None: + hp.discover_plugins() + entry = reg.platform_registry.get("slack") + if entry is None: + pytest.skip("slack platform entry not registered") + monkeypatch.setattr(entry, "standalone_sender_fn", fake_sender) + monkeypatch.setattr(hp, "discover_plugins", lambda *a, **k: None) + + state = {"send": send_calls, "marked": [], "saved": [], "finished": []} + + monkeypatch.setattr(s, "create_execution", lambda *_a, **_kw: {"id": "exec-t"}) + monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) + monkeypatch.setattr( + s, "save_job_output", + lambda jid, out: state["saved"].append(jid) or f"/tmp/{jid}.txt", + ) + monkeypatch.setattr( + s, "mark_job_run", + lambda *a, **kw: state["marked"].append((a, kw)) or True, + ) + monkeypatch.setattr( + s, "finish_execution", + lambda *a, **kw: state["finished"].append((a, kw)), + ) + # No durable incident store in play: never acked, no id. + monkeypatch.setattr( + s, "_upsert_incident_for_failure", lambda *_a, **_kw: (False, None) + ) + monkeypatch.setattr(s, "load_config", lambda: {}) + return state + + +def _failing_run_job(error="provider exploded"): + def _fake(job, **_kw): + return (False, "raw output", "", error) + return _fake + + +def _succeeding_run_job(final="all good, here is the brief"): + def _fake(job, **_kw): + return (True, "raw output", final, None) + return _fake + + +class TestFailureDeliverRouting: + def test_failure_without_failure_deliver_goes_to_deliver_targets( + self, run_env, monkeypatch + ): + """(a) Unset failure_deliver = today's behavior: failure summary to + the job's deliver targets.""" + monkeypatch.setattr(s, "run_job", _failing_run_job()) + + s.run_one_job({"id": "j1", "name": "scout", "deliver": "slack:D0MAIN"}) + + assert [c["chat_id"] for c in run_env["send"]] == ["D0MAIN"] + assert "failed" in run_env["send"][0]["message"].lower() + + def test_failure_deliver_local_is_silent_but_state_is_recorded( + self, run_env, monkeypatch + ): + """(b) failure_deliver: local — no delivery leaves the process, but + the run is still saved and marked failed.""" + monkeypatch.setattr(s, "run_job", _failing_run_job()) + + s.run_one_job({ + "id": "j2", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "local", + }) + + assert run_env["send"] == [] + # State recording is untouched by the silence. + assert run_env["saved"] == ["j2"] + assert len(run_env["marked"]) == 1 + args, _kw = run_env["marked"][0] + assert args[0] == "j2" and args[1] is False + assert "provider exploded" in args[2] + + def test_failure_deliver_explicit_target_wins_over_deliver( + self, run_env, monkeypatch + ): + """(c) failure_deliver set to a different target: the failure notice + goes THERE, and nothing goes to the deliver target.""" + monkeypatch.setattr(s, "run_job", _failing_run_job()) + + s.run_one_job({ + "id": "j3", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS", + }) + + assert [c["chat_id"] for c in run_env["send"]] == ["D0ALERTS"] + assert "failed" in run_env["send"][0]["message"].lower() + + def test_success_ignores_failure_deliver(self, run_env, monkeypatch): + """(d) Success output still goes to deliver — failure_deliver is + never consulted on the success path.""" + monkeypatch.setattr(s, "run_job", _succeeding_run_job()) + + ok = s.run_one_job({ + "id": "j4", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS", + }) + + assert ok is True + assert [c["chat_id"] for c in run_env["send"]] == ["D0MAIN"] + assert "all good, here is the brief" in run_env["send"][0]["message"] + + +class TestEscapedExceptionPath: + """The scheduler-layer exception handler is the second failure-delivery + site — it must honor failure_deliver identically.""" + + def _raise_run_job(self, monkeypatch): + monkeypatch.setattr( + s, "run_job", + lambda *_a, **_kw: (_ for _ in ()).throw( + RuntimeError("cannot import name X") + ), + ) + + def test_escaped_failure_honors_failure_deliver_target( + self, run_env, monkeypatch + ): + self._raise_run_job(monkeypatch) + + ok = s.run_one_job({ + "id": "j5", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS", + }) + + assert ok is False + assert [c["chat_id"] for c in run_env["send"]] == ["D0ALERTS"] + + def test_escaped_failure_with_failure_deliver_local_is_silent( + self, run_env, monkeypatch + ): + self._raise_run_job(monkeypatch) + + ok = s.run_one_job({ + "id": "j6", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "local", + }) + + assert ok is False + assert run_env["send"] == [] + # Failure is still recorded. + assert len(run_env["marked"]) == 1 + args, _kw = run_env["marked"][0] + assert args[1] is False and "cannot import name X" in args[2] + + +class TestResolutionGrammar: + """(e) failure_deliver shares deliver's exact value grammar — the same + normalization/expansion path, not a parallel one.""" + + def test_for_failure_resolves_failure_deliver_value(self): + job = {"deliver": "local", "failure_deliver": "slack:D0ALERTS"} + targets = _resolve_delivery_targets(job, for_failure=True) + assert [(t["platform"], t["chat_id"]) for t in targets] == [ + ("slack", "D0ALERTS") + ] + + def test_for_failure_falls_back_to_deliver_when_unset(self): + job = {"deliver": "slack:D0MAIN"} + targets = _resolve_delivery_targets(job, for_failure=True) + assert [(t["platform"], t["chat_id"]) for t in targets] == [ + ("slack", "D0MAIN") + ] + + def test_success_resolution_never_reads_failure_deliver(self): + job = {"deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS"} + targets = _resolve_delivery_targets(job) + assert [(t["platform"], t["chat_id"]) for t in targets] == [ + ("slack", "D0MAIN") + ] + + def test_local_yields_zero_failure_targets(self): + job = {"deliver": "slack:D0MAIN", "failure_deliver": "local"} + assert _resolve_delivery_targets(job, for_failure=True) == [] + + def test_comma_list_and_thread_grammar(self): + """The comma-combine + platform:chat:thread forms deliver's grammar + supports work identically for failure_deliver.""" + job = { + "deliver": "local", + "failure_deliver": "slack:D0ALERTS,telegram:-1001:17", + } + targets = _resolve_delivery_targets(job, for_failure=True) + assert [(t["platform"], t["chat_id"], t.get("thread_id")) for t in targets] == [ + ("slack", "D0ALERTS", None), + ("telegram", "-1001", "17"), + ] + + def test_legacy_list_value_is_flattened_like_deliver(self): + """Same list/tuple tolerance _normalize_deliver_value grants deliver.""" + job = {"deliver": "local", "failure_deliver": ["slack:D0ALERTS"]} + targets = _resolve_delivery_targets(job, for_failure=True) + assert [(t["platform"], t["chat_id"]) for t in targets] == [ + ("slack", "D0ALERTS") + ] + + +class TestToolSurface: + """cronjob(action=create/update) accepts failure_deliver with deliver's + validation — reusing the same normalize/validate helpers.""" + + def test_create_stores_failure_deliver(self, cron_env): + from tools.cronjob_tools import cronjob + from cron.jobs import get_job + + result = json.loads(cronjob( + action="create", + prompt="scan", + schedule="every 1h", + deliver="slack:D0MAIN", + failure_deliver="local", + )) + assert result["success"] is True + assert get_job(result["job_id"])["failure_deliver"] == "local" + + def test_create_without_failure_deliver_does_not_persist_the_key(self, cron_env): + """Existing-job byte-identity: the field only exists when set.""" + from tools.cronjob_tools import cronjob + from cron.jobs import get_job + + result = json.loads(cronjob( + action="create", prompt="scan", schedule="every 1h", + )) + assert result["success"] is True + assert "failure_deliver" not in get_job(result["job_id"]) + + def test_create_flattens_list_value_like_deliver(self, cron_env): + from tools.cronjob_tools import cronjob + from cron.jobs import get_job + + result = json.loads(cronjob( + action="create", + prompt="scan", + schedule="every 1h", + failure_deliver=["slack", "telegram"], + )) + assert result["success"] is True + assert get_job(result["job_id"])["failure_deliver"] == "slack,telegram" + + def test_create_rejects_bad_bot_chat_profile_same_as_deliver(self, cron_env): + from tools.cronjob_tools import cronjob + + via_failure = json.loads(cronjob( + action="create", prompt="scan", schedule="every 1h", + failure_deliver="bot-chat:no-such-profile-xyz", + )) + via_deliver = json.loads(cronjob( + action="create", prompt="scan", schedule="every 1h", + deliver="bot-chat:no-such-profile-xyz", + )) + assert via_failure["success"] is False + assert via_deliver["success"] is False + # Same validator, same message. + assert via_failure["error"] == via_deliver["error"] + + def test_update_sets_and_clears_failure_deliver(self, cron_env): + from cron.jobs import create_job, get_job + from tools.cronjob_tools import cronjob + + job = create_job(prompt="scan", schedule="every 1h") + result = json.loads(cronjob( + action="update", job_id=job["id"], failure_deliver="slack:D0ALERTS", + )) + assert result["success"] is True + assert get_job(job["id"])["failure_deliver"] == "slack:D0ALERTS" + + # '' clears — job falls back to deliver on failures again. + result = json.loads(cronjob( + action="update", job_id=job["id"], failure_deliver="", + )) + assert result["success"] is True + assert not get_job(job["id"]).get("failure_deliver") + + +class TestOutcomeBookkeeping: + """Review finding B1 (NS-788): delivery bookkeeping — outcome + classification, unresolved-origin, incident 'alerted' marking — must + read the SAME lane the notice was actually routed through, or the + execution history and incident store record lies (silenced failures + logged 'delivered'; delivered failures logged 'not_configured').""" + + @staticmethod + def _outcome(state): + assert state["finished"], "finish_execution never called" + _a, kw = state["finished"][-1] + return kw.get("delivery_outcome") + + def test_fd_local_failure_records_suppressed_not_delivered( + self, run_env, monkeypatch + ): + alerted = [] + monkeypatch.setattr(s, "_mark_incident_alerted", alerted.append) + monkeypatch.setattr(s, "run_job", _failing_run_job()) + + s.run_one_job({ + "id": "b1a", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "local", + }) + + assert run_env["send"] == [] + assert self._outcome(run_env) == "suppressed" + assert alerted == [], "silenced failure must NOT mark incident alerted" + + def test_fd_explicit_target_failure_records_delivered( + self, run_env, monkeypatch + ): + """deliver=origin (unresolvable) + failure_deliver=explicit target: + the notice IS delivered — outcome must say so, not 'not_configured'.""" + alerted = [] + monkeypatch.setattr(s, "_mark_incident_alerted", alerted.append) + monkeypatch.setattr( + s, "_upsert_incident_for_failure", lambda *_a, **_kw: (False, "inc-b1") + ) + monkeypatch.setattr(s, "run_job", _failing_run_job()) + + s.run_one_job({ + "id": "b1b", "name": "scout", + "deliver": "origin", "failure_deliver": "slack:D0OPS", + }) + + assert [c["chat_id"] for c in run_env["send"]] == ["D0OPS"] + assert self._outcome(run_env) == "delivered" + assert alerted == ["inc-b1"], "delivered failure ping must mark incident alerted" + + def test_success_outcome_still_reads_deliver_lane(self, run_env, monkeypatch): + """Success bookkeeping is untouched: fd set, success delivers to + deliver and records 'delivered'.""" + monkeypatch.setattr(s, "run_job", _succeeding_run_job()) + + s.run_one_job({ + "id": "b1c", "name": "scout", + "deliver": "slack:D0MAIN", "failure_deliver": "local", + }) + + assert [c["chat_id"] for c in run_env["send"]] == ["D0MAIN"] + assert self._outcome(run_env) == "delivered" + + +class TestPreflightAndDashboardLanes: + """Follow-up (salvage): the failure lane is validated everywhere the + deliver lane is — preflight config checks and the dashboard update + normalizer — so a typo'd failure target is caught before a failure + needs it.""" + + def test_preflight_blocks_unknown_failure_platform(self, monkeypatch): + """A bogus failure_deliver platform blocks at preflight, exactly + like a bogus deliver platform would.""" + monkeypatch.setattr(s, "_is_known_delivery_platform", lambda _p: False) + err = s._preflight_check_delivery({ + "id": "p1", "deliver": "local", + "failure_deliver": "nonexistent-platform:C1", + }) + assert err is not None and "not a known" in err + + def test_preflight_failure_deliver_local_adds_no_platforms(self): + """failure_deliver: local adds nothing to check — a deliver=local + job with suppressed failures stays zero-cost at preflight.""" + assert s._preflight_check_delivery({ + "id": "p2", "deliver": "local", "failure_deliver": "local", + }) is None + + def test_preflight_duplicate_lane_not_checked_twice(self, monkeypatch): + """failure_deliver equal to deliver must not double-check (or + double-report) the same platform.""" + seen = [] + + def _known(p): + seen.append(p) + return False + + monkeypatch.setattr(s, "_is_known_delivery_platform", _known) + s._preflight_check_delivery({ + "id": "p3", "deliver": "ghost:C1", "failure_deliver": "ghost:C1", + }) + assert seen == ["ghost"] + + def test_dashboard_update_normalizes_failure_deliver(self, tmp_path): + """The dashboard update lane normalizes failure_deliver like + deliver: text stripped, empty clears (None) instead of + coalescing to a target.""" + from hermes_cli.web_server import _normalize_dashboard_cron_updates + + out = _normalize_dashboard_cron_updates( + {"failure_deliver": " slack:D0ALERTS "}, tmp_path + ) + assert out["failure_deliver"] == "slack:D0ALERTS" + + cleared = _normalize_dashboard_cron_updates( + {"failure_deliver": ""}, tmp_path + ) + assert cleared["failure_deliver"] is None diff --git a/tests/cron/test_cron_incidents.py b/tests/cron/test_cron_incidents.py index c495506336..22ec1c1d95 100644 --- a/tests/cron/test_cron_incidents.py +++ b/tests/cron/test_cron_incidents.py @@ -46,7 +46,7 @@ def _tick_failing(job, tmp_path, deliveries, error="boom unrelated"): harness so the incident gating is exercised through the real scheduler.""" fake_db = MagicMock() - def fake_deliver(jb, content, adapters=None, loop=None): + def fake_deliver(jb, content, adapters=None, loop=None, **kwargs): deliveries.append(content) return None diff --git a/tests/cron/test_cron_live_delivery_confirmation.py b/tests/cron/test_cron_live_delivery_confirmation.py new file mode 100644 index 0000000000..b2822edc24 --- /dev/null +++ b/tests/cron/test_cron_live_delivery_confirmation.py @@ -0,0 +1,404 @@ +"""Live-adapter delivery confirmation for cron (#77763). + +A ``no_agent`` job fired, the scheduler logged +``delivered to telegram: via live adapter``, and the user received +nothing — no message row, no delivery obligation. The log line was not +evidence of a send: + +* the silence-narration filter returns ``{"success": True, "delivered": False}`` + (a successful *drop*), and the normalization block read only ``success``; +* an empty payload skipped the send entirely and still fell into the + "delivered" branch; +* the log line named the chat but not the lane, so a wrong-thread delivery and + a phantom one look identical after the fact. + +These tests pin the confirmation contract: positive evidence, honest logging, +and fail-closed on nothing-to-send. +""" + +import asyncio +import logging +from concurrent.futures import Future +from unittest.mock import MagicMock, patch + +import pytest + +from cron import scheduler as sched +from cron.scheduler import _confirm_adapter_delivery, _deliver_result +from gateway.config import Platform, PlatformConfig + + +# --------------------------------------------------------------------------- +# _confirm_adapter_delivery: the contract in isolation +# --------------------------------------------------------------------------- + +class _SendResult: + """Minimal stand-in for an adapter SendResult.""" + + def __init__(self, success=True, message_id=None, raw_response=None, **extra): + self.success = success + self.message_id = message_id + self.raw_response = raw_response + for key, value in extra.items(): + setattr(self, key, value) + + +class TestConfirmAdapterDelivery: + def test_none_is_not_delivered(self): + assert _confirm_adapter_delivery(None, "j1") is False + + def test_missing_success_is_not_delivered(self): + assert _confirm_adapter_delivery(object(), "j1") is False + assert _confirm_adapter_delivery({"message_id": 7}, "j1") is False + + def test_explicit_failure_is_not_delivered(self): + assert _confirm_adapter_delivery(_SendResult(success=False), "j1") is False + assert _confirm_adapter_delivery({"success": False}, "j1") is False + + def test_filtered_dict_is_not_delivered(self): + """The exact silence-filter shape: a successful DROP is not a delivery.""" + filtered = {"success": True, "filtered": "silence_narration", "delivered": False} + assert _confirm_adapter_delivery(filtered, "j1") is False + + def test_delivered_false_on_an_object_is_not_delivered(self): + result = _SendResult(success=True, message_id=42, delivered=False) + assert _confirm_adapter_delivery(result, "j1") is False + + def test_positive_evidence_is_delivered_without_warning(self, caplog): + with caplog.at_level(logging.WARNING, logger="cron.scheduler"): + assert _confirm_adapter_delivery(_SendResult(message_id=1234), "j1") is True + assert "UNVERIFIED" not in caplog.text + + def test_raw_response_alone_counts_as_evidence(self, caplog): + with caplog.at_level(logging.WARNING, logger="cron.scheduler"): + result = _SendResult(raw_response={"ok": True}) + assert _confirm_adapter_delivery(result, "j1") is True + assert "UNVERIFIED" not in caplog.text + + def test_evidence_free_success_is_accepted_but_warned(self, caplog): + """Not proof of failure either — accept it, but say so in the log.""" + with caplog.at_level(logging.WARNING, logger="cron.scheduler"): + assert _confirm_adapter_delivery(_SendResult(), "92e639af907f") is True + assert "UNVERIFIED" in caplog.text + assert "92e639af907f" in caplog.text + + def test_evidence_free_success_dict_is_accepted_but_warned(self, caplog): + with caplog.at_level(logging.WARNING, logger="cron.scheduler"): + assert _confirm_adapter_delivery({"success": True}, "j1") is True + assert "UNVERIFIED" in caplog.text + + +# --------------------------------------------------------------------------- +# _deliver_result: the live lane end to end +# --------------------------------------------------------------------------- + +CHAT_ID = "-1001234567890" + + +def _job(thread_id=None): + origin = {"platform": "telegram", "chat_id": CHAT_ID} + if thread_id is not None: + origin["thread_id"] = thread_id + return { + "id": "92e639af907f", + "name": "Ghost Delivery", + "deliver": "origin", + "origin": origin, + } + + +def _gateway_config(relay=False): + config = MagicMock() + platforms = {Platform.TELEGRAM: PlatformConfig(enabled=True)} + if relay: + platforms[Platform.RELAY] = PlatformConfig(enabled=True) + config.platforms = platforms + config.get_home_channel = lambda p: None + return config + + +def _adapters(relay=False): + adapter = MagicMock() + if relay: + adapter.fronts_platform = lambda p: p == Platform.TELEGRAM + return {Platform.RELAY: adapter} + return {Platform.TELEGRAM: adapter} + + +RECORDED_VERIFICATION = [] + + +def _record_verification(job, unverified_targets): + RECORDED_VERIFICATION.append((job["id"], list(unverified_targets))) + + +def _run(job, content, send_result, relay=False, standalone_result=None, cron_cfg=None): + """Drive ``_deliver_result`` over the live lane with a stubbed router. + + Returns ``(error, router_calls, standalone_calls)``. ``cron_cfg`` extends + the ``cron:`` section handed to the scheduler (default: unwrapped output). + """ + loop = MagicMock() + loop.is_running.return_value = True + + def fake_run_coro(coro, _loop): + future = Future() + try: + future.set_result(asyncio.run(coro)) + except BaseException as e: # noqa: BLE001 + future.set_exception(e) + return future + + router_calls = [] + standalone_calls = [] + RECORDED_VERIFICATION.clear() + + router = MagicMock() + + async def _deliver_to_platform(target, text, metadata): + router_calls.append({"target": target, "text": text, "metadata": metadata}) + return send_result + + router._deliver_to_platform = _deliver_to_platform + + async def _fake_send_to_platform(platform, pconfig, chat_id, text, **kwargs): + standalone_calls.append({"chat_id": chat_id, "text": text, "kwargs": kwargs}) + return standalone_result if standalone_result is not None else {} + + with patch("gateway.config.load_gateway_config", return_value=_gateway_config(relay)), \ + patch("cron.scheduler.load_config", + return_value={"cron": {"wrap_response": False, **(cron_cfg or {})}}), \ + patch("cron.scheduler._record_delivery_verification", side_effect=_record_verification), \ + patch("gateway.delivery.DeliveryRouter", return_value=router), \ + patch("tools.send_message_tool._send_to_platform", _fake_send_to_platform), \ + patch("asyncio.run_coroutine_threadsafe", side_effect=fake_run_coro): + error = _deliver_result(job, content, adapters=_adapters(relay), loop=loop) + return error, router_calls, standalone_calls + + +class TestFilteredResultIsNotDelivered: + FILTERED = {"success": True, "filtered": "silence_narration", "delivered": False} + + def test_filtered_dict_does_not_log_a_live_delivery(self, caplog): + with caplog.at_level(logging.INFO, logger="cron.scheduler"): + _, router_calls, standalone_calls = _run(_job(), "...", self.FILTERED) + + assert len(router_calls) == 1 # the live send was attempted + assert "via live adapter" not in caplog.text # but never claimed as delivered + assert len(standalone_calls) == 1 # fell back instead of lying + + def test_filtered_dict_fails_closed_on_the_relay_lane(self): + """Relay owns the destination, so there is no fallback — report it.""" + error, _, standalone_calls = _run(_job(), "...", self.FILTERED, relay=True) + + assert error is not None + assert "unconfirmed result" in error + assert "silence_narration" in error # names the filter, not "unknown" + assert standalone_calls == [] + + def test_confirmed_send_result_still_delivers(self, caplog): + with caplog.at_level(logging.INFO, logger="cron.scheduler"): + error, router_calls, standalone_calls = _run( + _job(), "Nightly report.", _SendResult(message_id=1234), + ) + + assert error is None + assert len(router_calls) == 1 + assert standalone_calls == [] + assert "via live adapter" in caplog.text + + +class TestEmptyPayloadFailsClosed: + def test_empty_payload_never_reaches_the_adapter(self, caplog): + with caplog.at_level(logging.INFO, logger="cron.scheduler"): + _, router_calls, _ = _run(_job(), " ", _SendResult(message_id=1)) + + assert router_calls == [] # nothing was sent + assert "via live adapter" not in caplog.text # and nothing was claimed + assert "empty text and no media" in caplog.text + + def test_empty_payload_never_reaches_the_standalone_sender(self, caplog): + """The native fallback must not re-open the hole the live lane closed. + + Telegram's adapter returns ``SendResult(success=True)`` for empty + content without an API call, so an unguarded fallback would log a + standalone "delivered" for the same phantom payload (#77763). + """ + with caplog.at_level(logging.INFO, logger="cron.scheduler"): + error, router_calls, standalone_calls = _run( + _job(), " ", _SendResult(message_id=1), + ) + + assert router_calls == [] + assert standalone_calls == [] # _send_to_platform never called + assert error is not None + assert "standalone send skipped (empty text and no media)" in error + assert "delivered to" not in caplog.text + + def test_empty_payload_is_reported_on_the_relay_lane(self): + error, router_calls, _ = _run(_job(), "", _SendResult(message_id=1), relay=True) + + assert router_calls == [] + assert error is not None + assert "live adapter send skipped (empty text and no media)" in error + + +class TestDeliveredLogNamesTheLane: + def test_log_includes_thread_and_message_id(self, caplog): + with caplog.at_level(logging.INFO, logger="cron.scheduler"): + error, _, _ = _run( + _job(thread_id="99"), "Nightly report.", _SendResult(message_id=1234), + ) + + assert error is None + assert "via live adapter thread=99 message_id=1234" in caplog.text + + def test_log_uses_a_dash_when_the_lane_is_unknown(self, caplog): + """No thread and an evidence-free result must still be attributable.""" + with caplog.at_level(logging.INFO, logger="cron.scheduler"): + error, _, _ = _run(_job(), "Nightly report.", _SendResult()) + + assert error is None + assert "via live adapter thread=- message_id=-" in caplog.text + assert "UNVERIFIED" in caplog.text + + +class TestLiveDeliveryIsAFinalNotification: + """Cron output is a final user-visible delivery, not a progress send. + + Telegram's adapter defaults to ``_notifications_mode = "important"`` and + sends with ``disable_notification=True`` unless ``metadata["notify"]`` is + set — so a cron brief without the marker lands silently, which users + report as "never delivered" (#77763 thread, #58258 typing bubble). The + marker must ride both the text route and the media route, in every + Telegram routing mode. + """ + + def test_text_route_metadata_carries_notify(self): + _, router_calls, _ = _run(_job(), "Nightly report.", _SendResult(message_id=1)) + assert len(router_calls) == 1 + metadata = router_calls[0]["metadata"] + assert metadata["job_id"] == "92e639af907f" + assert metadata["notify"] is True + + def test_forum_topic_route_keeps_thread_and_notify(self): + _, router_calls, _ = _run( + _job(thread_id="99"), "Nightly report.", _SendResult(message_id=1), + ) + metadata = router_calls[0]["metadata"] + assert metadata["thread_id"] == "99" + assert metadata["notify"] is True + + def test_media_route_metadata_carries_notify(self, tmp_path): + media = tmp_path / "report.png" + media.write_bytes(b"\x89PNG\r\n\x1a\n") + sent = [] + + def fake_send_media(adapter, chat_id, media_files, metadata, loop, job, platform=None): + sent.append({"media": list(media_files), "metadata": metadata}) + return [] + + with patch("cron.scheduler._send_media_via_adapter", side_effect=fake_send_media), \ + patch("gateway.platforms.base.BasePlatformAdapter.filter_media_delivery_paths", + side_effect=lambda files: files): + error, router_calls, _ = _run( + _job(), f"Nightly report.\nMEDIA:{media}", _SendResult(message_id=1), + ) + + assert error is None + assert len(router_calls) == 1 + assert len(sent) == 1 + assert sent[0]["metadata"]["notify"] is True + + +class TestNotifyIsConfigurable: + """``cron.delivery.notify`` (config.yaml) gates the notify marker. + + The current behaviour (push notification) stays the default; only an + explicit ``false`` restores silent deliveries. The knob rides both the + text route and the media route so the two never disagree. + """ + + def test_default_is_notify(self): + _, router_calls, _ = _run(_job(), "Nightly report.", _SendResult(message_id=1)) + assert router_calls[0]["metadata"]["notify"] is True + + def test_explicit_false_disables_notify_on_text_route(self): + _, router_calls, _ = _run( + _job(thread_id="99"), "Nightly report.", _SendResult(message_id=1), + cron_cfg={"delivery": {"notify": False}}, + ) + metadata = router_calls[0]["metadata"] + assert metadata["notify"] is False + assert metadata["thread_id"] == "99" # routing untouched + + def test_explicit_false_disables_notify_on_media_route(self, tmp_path): + media = tmp_path / "report.png" + media.write_bytes(b"\x89PNG\r\n\x1a\n") + sent = [] + + def fake_send_media(adapter, chat_id, media_files, metadata, loop, job, platform=None): + sent.append(metadata) + return [] + + with patch("cron.scheduler._send_media_via_adapter", side_effect=fake_send_media), \ + patch("gateway.platforms.base.BasePlatformAdapter.filter_media_delivery_paths", + side_effect=lambda files: files): + _run( + _job(), f"Nightly report.\nMEDIA:{media}", _SendResult(message_id=1), + cron_cfg={"delivery": {"notify": False}}, + ) + assert sent[0]["notify"] is False + + @pytest.mark.parametrize("cron_cfg", [ + {"delivery": None}, # `delivery:` with no body parses to null + {"delivery": "yes"}, # malformed scalar + {"delivery": {"notify": None}}, # `notify:` with no value + ]) + def test_malformed_section_keeps_the_default(self, cron_cfg): + _, router_calls, _ = _run(_job(), "Nightly report.", _SendResult(message_id=1), cron_cfg=cron_cfg) + assert router_calls[0]["metadata"]["notify"] is True + + def test_default_config_ships_notify_true(self): + from hermes_cli.config_defaults import DEFAULT_CONFIG + + assert DEFAULT_CONFIG["cron"]["delivery"]["notify"] is True + + +class TestUnverifiedDeliveryIsRecordedOnTheJob: + """An evidence-free ack is accepted, but the state must reach the job + record (and from there ``hermes cron list`` / ``cron doctor``), not only a + WARNING log line.""" + + def test_evidence_free_ack_records_the_target(self): + error, _, _ = _run(_job(), "Nightly report.", _SendResult()) + assert error is None + assert RECORDED_VERIFICATION == [("92e639af907f", [f"telegram:{CHAT_ID}"])] + + def test_positive_evidence_clears_the_marker(self): + error, _, _ = _run(_job(), "Nightly report.", _SendResult(message_id=1234)) + assert error is None + assert RECORDED_VERIFICATION == [("92e639af907f", [])] + + def test_recorder_skips_the_write_when_nothing_changed(self): + with patch("cron.jobs.update_job") as update_job: + sched._record_delivery_verification({"id": "j1", "last_delivery_unverified": None}, []) + update_job.assert_not_called() + sched._record_delivery_verification({"id": "j1", "last_delivery_unverified": None}, ["slack:C1"]) + update_job.assert_called_once_with("j1", {"last_delivery_unverified": ["slack:C1"]}) + + def test_recorder_clears_a_stale_marker(self): + with patch("cron.jobs.update_job") as update_job: + sched._record_delivery_verification({"id": "j1", "last_delivery_unverified": ["slack:C1"]}, []) + update_job.assert_called_once_with("j1", {"last_delivery_unverified": None}) + + def test_tool_listing_exposes_the_field(self): + from tools.cronjob_tools import _format_job + + assert _format_job({"id": "j1", "name": "n", "prompt": "p", + "last_delivery_unverified": ["slack:C1"]})["last_delivery_unverified"] == ["slack:C1"] + + +def test_scheduler_module_exposes_the_confirmation_helper(): + """Guard the import surface the delivery block depends on.""" + assert callable(sched._confirm_adapter_delivery) diff --git a/tests/cron/test_cron_multiplex_desktop_ticker_scope.py b/tests/cron/test_cron_multiplex_desktop_ticker_scope.py new file mode 100644 index 0000000000..a01aa0ae3c --- /dev/null +++ b/tests/cron/test_cron_multiplex_desktop_ticker_scope.py @@ -0,0 +1,139 @@ +"""Regression tests for #100489 — desktop multiplex ticker must not deliver a +secondary profile's cron output through the default profile's identity. + +Two halves: + +1. ``_deliver_result``'s standalone fallback pool (taken when the caller has a + RUNNING event loop — the desktop dashboard shape) spawns a fresh thread that + did not inherit the profile ContextVars; it must run inside a copy of the + active context so the sender reads THIS profile's home + secrets. +2. The desktop ticker must stand down, per tick, for a profile whose OWN + gateway is running — that gateway ticks it with live adapters, and racing it + on the tick lock lets the adapter-less desktop ticker deliver standalone. +""" +import asyncio +import threading +from unittest.mock import patch + + + +def test_standalone_fallback_pool_keeps_profile_scope(tmp_path, monkeypatch): + from agent.secret_scope import ( + get_secret, + set_multiplex_active, + set_secret_scope, + ) + from hermes_constants import get_hermes_home, set_hermes_home_override + import cron.scheduler as sched + import tools.send_message_tool as smt + + default_home = tmp_path / "default" + sec_home = tmp_path / "profiles" / "ops" + for home in (default_home, sec_home): + (home / "cron").mkdir(parents=True) + (home / "config.yaml").write_text("platforms:\n telegram:\n enabled: true\n") + monkeypatch.setenv("HERMES_HOME", str(default_home)) + monkeypatch.setenv("TELEGRAM_BOT_TOKEN", "DEFAULT-TOKEN") + set_multiplex_active(True) + + seen = {} + + async def fake_send(platform, pconfig, chat_id, message, **kwargs): + seen["home"] = str(get_hermes_home()) + seen["token"] = get_secret("TELEGRAM_BOT_TOKEN", None) + return {"success": True, "message_id": "1"} + + job = {"id": "j1", "name": "probe", "deliver": "telegram:12345", "schedule": {"kind": "cron"}} + + async def _inside_running_loop(): + # Emulate the multiplex ticker's per-profile scope on the caller. + set_hermes_home_override(str(sec_home)) + set_secret_scope({"TELEGRAM_BOT_TOKEN": "OPS-TOKEN"}) + return sched._deliver_result(job, "hello", adapters={}, loop=None) + + try: + with patch.object(smt, "_send_to_platform", fake_send): + err = asyncio.run(_inside_running_loop()) + finally: + set_multiplex_active(False) + + assert err is None, err + assert seen["home"] == str(sec_home.resolve()) + assert seen["token"] == "OPS-TOKEN" + + +def test_multiplex_ticker_profile_gate_skips_rejected_profile(tmp_path): + from cron.scheduler_provider import InProcessCronScheduler + from hermes_constants import get_hermes_home + + own_gateway = tmp_path / "own-gateway" + orphan = tmp_path / "orphan" + for home in (own_gateway, orphan): + (home / "cron").mkdir(parents=True) + + stop = threading.Event() + ticked: list[str] = [] + + def _tick(*args, **kwargs): + ticked.append(str(get_hermes_home())) + if len(ticked) >= 3: + stop.set() + return 0 + + provider = InProcessCronScheduler() + with patch("cron.scheduler.tick", side_effect=_tick): + thread = threading.Thread( + target=provider.start, + args=(stop,), + kwargs={ + "interval": 0, + "profile_homes": [("own-gateway", own_gateway), ("orphan", orphan)], + "profile_gate": lambda name, home: name != "own-gateway", + }, + daemon=True, + ) + thread.start() + thread.join(timeout=5) + stop.set() + thread.join(timeout=5) + + assert not thread.is_alive() + assert set(ticked) == {str(orphan)} + # The gated profile gets no tick-loop success marker either: its own + # gateway owns that status surface. + assert not (own_gateway / "cron" / "ticker_last_success").exists() + assert (orphan / "cron" / "ticker_last_success").exists() + + +def test_desktop_ticker_gates_on_profile_gateway_running(tmp_path, monkeypatch): + """The desktop ticker wires the gate to ``_check_gateway_running``.""" + from hermes_cli import web_server + + homes = [("default", tmp_path / "default"), ("ops", tmp_path / "ops")] + monkeypatch.setattr( + "hermes_cli.profiles.profiles_to_serve", lambda multiplex=False: list(homes) + ) + monkeypatch.setattr( + "hermes_cli.profiles._check_gateway_running", lambda home: home.name == "ops" + ) + captured = {} + + class _Provider: + name = "fake" + + def start(self, stop_event, **kwargs): + captured.update(kwargs) + + from cron import scheduler_provider as sp + + monkeypatch.setattr(web_server, "resolve_cron_scheduler", lambda: _Provider(), raising=False) + monkeypatch.setattr(sp, "resolve_cron_scheduler", lambda: _Provider()) + monkeypatch.setattr(sp, "InProcessCronScheduler", _Provider) + monkeypatch.setattr("hermes_logging.enable_profile_log_routing", lambda homes: None) + + web_server._start_desktop_cron_ticker(threading.Event(), interval=0) + + gate = captured.get("profile_gate") + assert gate is not None, "desktop ticker did not install a profile gate" + assert gate("default", tmp_path / "default") is True + assert gate("ops", tmp_path / "ops") is False diff --git a/tests/cron/test_cron_multiplex_shared_route_delivery.py b/tests/cron/test_cron_multiplex_shared_route_delivery.py new file mode 100644 index 0000000000..92aa6b4f7b --- /dev/null +++ b/tests/cron/test_cron_multiplex_shared_route_delivery.py @@ -0,0 +1,112 @@ +"""Regression tests for #101113 — a credentialless satellite profile under +``gateway.profile_routes`` delivers cron output through the PRIMARY adapter +for exactly the targets the primary routes to it, and fails closed otherwise. + +The multiplex ticker hands such a profile a ``SharedRouteAdapters`` view over +the primary adapter map; ``_deliver_result`` resolves a transport from it per +target using the same ``ProfileRoute.matches`` predicate as inbound routing. +""" +import asyncio +from concurrent.futures import Future +from unittest.mock import MagicMock, patch + +import yaml + +from cron.scheduler import ( + SharedRouteAdapters, + _deliver_result, + _primary_profile_routes_for_current_home, +) +from gateway.config import Platform, PlatformConfig +from hermes_constants import reset_hermes_home_override, set_hermes_home_override + +PRIMARY_YAML = { + "gateway": { + "multiplex_profiles": True, + "profile_routes": [ + {"name": "fit", "platform": "discord", "chat_id": "1543065293755256852", "profile": "fitness"}, + {"name": "off", "platform": "discord", "chat_id": "999", "profile": "fitness", "enabled": False}, + {"name": "other", "platform": "discord", "chat_id": "777", "profile": "other"}, + ], + } +} + + +def _job(chat_id: str) -> dict: + return {"id": "a7ae1520356c", "name": "brief", "deliver": f"discord:{chat_id}"} + + +def _run(job, adapters): + """Drive ``_deliver_result`` with a live loop and a real DeliveryRouter.""" + loop = MagicMock() + loop.is_running.return_value = True + + def fake_run_coro(coro, _loop): + future = Future() + future.set_result(asyncio.run(coro)) + return future + + standalone = [] + + async def _fake_send_to_platform(platform, pconfig, chat_id, text, **kwargs): + standalone.append(chat_id) + return {"success": False, "error": "DISCORD_BOT_TOKEN is not set"} + + config = MagicMock() + config.platforms = {Platform.DISCORD: PlatformConfig(enabled=True)} + config.get_home_channel = lambda p: None + with patch("gateway.config.load_gateway_config", return_value=config), \ + patch("cron.scheduler.load_config", return_value={"cron": {"wrap_response": False}}), \ + patch("tools.send_message_tool._send_to_platform", _fake_send_to_platform), \ + patch("asyncio.run_coroutine_threadsafe", side_effect=fake_run_coro): + error = _deliver_result(job, "hello", adapters=adapters, loop=loop) + return error, standalone + + +def _primary_adapter(): + adapter = MagicMock() + adapter.sent = [] + + async def send(chat_id, content, metadata=None): + adapter.sent.append(chat_id) + return {"success": True, "message_id": "m1"} + + adapter.send = send + return adapter + + +def test_satellite_routes_exact_target_through_primary_adapter(tmp_path, monkeypatch): + root = tmp_path / "root" + fitness_home = root / "profiles" / "fitness" + fitness_home.mkdir(parents=True) + (root / "config.yaml").write_text(yaml.safe_dump(PRIMARY_YAML), encoding="utf-8") + monkeypatch.setattr("hermes_constants.get_default_hermes_root", lambda: root) + primary = _primary_adapter() + + token = set_hermes_home_override(str(fitness_home)) + try: + shared = SharedRouteAdapters( + {Platform.DISCORD: primary}, _primary_profile_routes_for_current_home() + ) + # exact enabled route → primary adapter sends, no standalone attempt + error, standalone = _run(_job("1543065293755256852"), shared) + assert error is None, error + assert primary.sent == ["1543065293755256852"] + assert standalone == [] + + # unmatched chat, disabled route, route for another profile → the + # primary bot is NEVER used; delivery stays on the satellite's own + # (credentialless) standalone path and reports its failure. + for chat in ("424242", "999", "777"): + primary.sent.clear() + error, standalone = _run(_job(chat), shared) + assert error is not None and "DISCORD_BOT_TOKEN" in error + assert primary.sent == [] + assert standalone == [chat] + finally: + reset_hermes_home_override(token) + + +def test_shared_view_is_falsy_without_routes_or_primary_adapters(): + assert not SharedRouteAdapters({}, []) + assert SharedRouteAdapters({Platform.DISCORD: object()}, []).get(Platform.DISCORD) is None diff --git a/tests/cron/test_cron_reasoning_effort.py b/tests/cron/test_cron_reasoning_effort.py index cea6d11230..4f45549a05 100644 --- a/tests/cron/test_cron_reasoning_effort.py +++ b/tests/cron/test_cron_reasoning_effort.py @@ -194,7 +194,7 @@ class TestCronjobToolReasoningEffort: def _tool_handler(self): import tools.cronjob_tools as mod - return mod.registry._tools["cronjob"].handler + return mod.registry._tools["cronjob_manage"].handler def test_schema_does_not_expose_reasoning_effort(self): """Policy pin: the model-facing surface must NOT offer the diff --git a/tests/cron/test_cron_timezone_migration_catchup.py b/tests/cron/test_cron_timezone_migration_catchup.py new file mode 100644 index 0000000000..ef4cf4a108 --- /dev/null +++ b/tests/cron/test_cron_timezone_migration_catchup.py @@ -0,0 +1,213 @@ +"""Timezone-migration silent misfire on the cron fire path. + +Production incident: after upgrading from a build that scheduled in UTC to +one that honours the profile timezone (Europe/Brussels), daily cron jobs +stopped running. Their ``jobs.json`` rows still held pre-migration instants +like ``2026-09-02T04:00:00+00:00`` for expr ``0 4 * * *``. ``_ensure_aware`` +normalizes that to ``06:00+02``, which ``0 4 * * *`` excludes, so the +stale-expression guard (#93049) classified it as a direct ``jobs.json`` edit, +logged exactly that, and re-anchored to tomorrow WITHOUT firing — the due +occurrence vanished with no failure anywhere. + +The fix classifies the mismatch instead of assuming an edit: an instant whose +own wall clock is a legal occurrence, and which only left the lattice because +normalization changed its offset, is a representation migration and fires. + +These exercise the real store against a temp ``HERMES_HOME`` (no mocks) per +the E2E-over-mocks discipline for file-touching code. +""" + +from __future__ import annotations + +from datetime import datetime + +import pytest + + +@pytest.fixture +def temp_home(tmp_path, monkeypatch): + """Isolated HERMES_HOME so jobs.json doesn't touch the real store.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + yield tmp_path + + +@pytest.fixture(autouse=True) +def _reset_migration_counters(monkeypatch): + """Module-level telemetry counters must not leak between tests.""" + from cron import jobs as J + + monkeypatch.setattr(J, "_timezone_migration_catchups", 0) + monkeypatch.setattr(J, "_timezone_migration_catchups_recent", []) + yield + + +# Europe/Brussels is +02:00 on this date; the legacy row was written by a +# build that scheduled everything at the UTC offset. +_BRUSSELS_NOW = datetime.fromisoformat("2026-09-02T06:05:00+02:00") +_LEGACY_UTC_NEXT_RUN = "2026-09-02T04:00:00+00:00" +_DAILY_0400 = "0 4 * * *" + + +def _write_cron_job(expr: str, next_run_at: str, name: str = "t") -> str: + """Persist a cron job with a pinned next_run_at (the legacy-row shape).""" + from cron.jobs import create_job, load_jobs, save_jobs + + job = create_job(prompt="x", schedule="every 5m", name=name) + jobs = load_jobs() + for j in jobs: + if j["id"] == job["id"]: + j["schedule"] = {"kind": "cron", "expr": expr} + j["next_run_at"] = next_run_at + save_jobs(jobs) + return job["id"] + + +def test_legacy_utc_offset_next_run_still_fires(temp_home, monkeypatch): + """The incident case: a pre-migration +00:00 instant for a Brussels + ``0 4 * * *`` job must fire its due occurrence, not be re-anchored away.""" + from cron.jobs import get_due_jobs, get_timezone_migration_catchup_stats + + monkeypatch.setattr("cron.jobs._hermes_now", lambda: _BRUSSELS_NOW) + jid = _write_cron_job(_DAILY_0400, _LEGACY_UTC_NEXT_RUN) + + due = get_due_jobs() + + assert jid in [j["id"] for j in due] + stats = get_timezone_migration_catchup_stats() + assert stats["timezone_migration_catchups"] == 1 + record = stats["recent"][0] + assert record["job_id"] == jid + assert record["expr"] == _DAILY_0400 + assert record["stored_next_run_at"] == _LEGACY_UTC_NEXT_RUN + assert record["normalized_next_run_at"] == "2026-09-02T06:00:00+02:00" + + +def test_legacy_offset_catchup_fires_at_most_once(temp_home, monkeypatch): + """The catch-up run is a single fire: once the scheduler advances the + job, the legacy instant is gone and a second scan finds nothing due.""" + from cron.jobs import advance_next_run, get_due_jobs, get_job + + monkeypatch.setattr("cron.jobs._hermes_now", lambda: _BRUSSELS_NOW) + jid = _write_cron_job(_DAILY_0400, _LEGACY_UTC_NEXT_RUN) + + assert jid in [j["id"] for j in get_due_jobs()] + assert advance_next_run(jid) is True + + # Re-anchored to tomorrow's occurrence, expressed in the configured zone. + assert get_job(jid)["next_run_at"] == "2026-09-03T04:00:00+02:00" + assert [j["id"] for j in get_due_jobs() if j["id"] == jid] == [] + + +def test_genuine_expr_edit_still_reanchors_without_firing(temp_home, monkeypatch): + """#93049 protection intact: a stale instant in the CURRENT offset (no + representation change) is still treated as an edit and does not fire.""" + from cron.jobs import get_due_jobs, get_job, get_timezone_migration_catchup_stats + + monkeypatch.setattr("cron.jobs._hermes_now", lambda: _BRUSSELS_NOW) + # Stored at the configured offset, but the expr was edited to 09:00. + jid = _write_cron_job("0 9 * * *", "2026-09-02T04:00:00+02:00") + + due = get_due_jobs() + + assert [j["id"] for j in due if j["id"] == jid] == [] + assert get_job(jid)["next_run_at"] == "2026-09-02T09:00:00+02:00" + assert ( + get_timezone_migration_catchup_stats()["timezone_migration_catchups"] == 0 + ) + + +def test_expr_edit_on_a_legacy_offset_row_still_does_not_fire(temp_home, monkeypatch): + """A legacy +00:00 row whose expr was ALSO edited must not fire: the + stored wall clock is not an occurrence of the new expression either, so + the migration escape hatch does not open.""" + from cron.jobs import get_due_jobs, get_job, get_timezone_migration_catchup_stats + + monkeypatch.setattr("cron.jobs._hermes_now", lambda: _BRUSSELS_NOW) + jid = _write_cron_job("0 9 * * *", _LEGACY_UTC_NEXT_RUN) + + due = get_due_jobs() + + assert [j["id"] for j in due if j["id"] == jid] == [] + assert get_job(jid)["next_run_at"] == "2026-09-02T09:00:00+02:00" + assert ( + get_timezone_migration_catchup_stats()["timezone_migration_catchups"] == 0 + ) + + +def test_future_local_wall_clock_is_left_scheduled(temp_home, monkeypatch): + """A legacy row whose normalized instant has not arrived yet is simply + not due — no catch-up, no re-anchor, no telemetry.""" + from cron.jobs import get_due_jobs, get_job, get_timezone_migration_catchup_stats + + before_due = datetime.fromisoformat("2026-09-02T05:00:00+02:00") + monkeypatch.setattr("cron.jobs._hermes_now", lambda: before_due) + jid = _write_cron_job(_DAILY_0400, _LEGACY_UTC_NEXT_RUN) + + due = get_due_jobs() + + assert [j["id"] for j in due if j["id"] == jid] == [] + assert get_job(jid)["next_run_at"] == _LEGACY_UTC_NEXT_RUN + assert ( + get_timezone_migration_catchup_stats()["timezone_migration_catchups"] == 0 + ) + + +def test_future_stored_wall_clock_still_takes_the_offset_repair_path( + temp_home, monkeypatch +): + """#28934 regression: a westward TZ move (+10 -> +02) that makes a still- + future wall clock look due recomputes rather than firing early, and is + NOT reclassified as a migration catch-up.""" + from cron.jobs import get_due_jobs, get_job, get_timezone_migration_catchup_stats + + scan_time = datetime.fromisoformat("2026-09-02T14:00:00+02:00") + monkeypatch.setattr("cron.jobs._hermes_now", lambda: scan_time) + jid = _write_cron_job("0 21 * * *", "2026-09-02T21:00:00+10:00") + + due = get_due_jobs() + + assert [j["id"] for j in due if j["id"] == jid] == [] + assert get_job(jid)["next_run_at"] == "2026-09-02T21:00:00+02:00" + assert ( + get_timezone_migration_catchup_stats()["timezone_migration_catchups"] == 0 + ) + + +def test_classifier_separates_migration_from_edit(temp_home): + """Unit-level: the three classifications the fire path branches on.""" + from cron.jobs import ( + STALE_CRON_EXPR_EDIT, + STALE_CRON_MATCH, + STALE_CRON_TIMEZONE_MIGRATION, + _classify_stale_cron_next_run, + ) + + daily = {"kind": "cron", "expr": _DAILY_0400} + raw_legacy = datetime.fromisoformat(_LEGACY_UTC_NEXT_RUN) + normalized = datetime.fromisoformat("2026-09-02T06:00:00+02:00") + on_lattice = datetime.fromisoformat("2026-09-02T04:00:00+02:00") + + # Stored instant already occurs under the current expression. + assert ( + _classify_stale_cron_next_run(daily, on_lattice, on_lattice) + == STALE_CRON_MATCH + ) + # Only the offset representation changed. + assert ( + _classify_stale_cron_next_run(daily, raw_legacy, normalized) + == STALE_CRON_TIMEZONE_MIGRATION + ) + # Wall clock never moved, so a mismatch can only be a schedule edit. + assert ( + _classify_stale_cron_next_run( + {"kind": "cron", "expr": "0 9 * * *"}, on_lattice, on_lattice + ) + == STALE_CRON_EXPR_EDIT + ) + # Wall clock moved, but the stored wall clock is not an occurrence either. + assert ( + _classify_stale_cron_next_run( + {"kind": "cron", "expr": "0 9 * * *"}, raw_legacy, normalized + ) + == STALE_CRON_EXPR_EDIT + ) diff --git a/tests/cron/test_delivery_queue.py b/tests/cron/test_delivery_queue.py new file mode 100644 index 0000000000..b4c56edfaf --- /dev/null +++ b/tests/cron/test_delivery_queue.py @@ -0,0 +1,212 @@ +"""Durable at-most-once delivery handoff for restart-safe cron workers.""" + +from __future__ import annotations + +import sqlite3 +from unittest.mock import Mock + +import pytest + + +def test_pending_delivery_is_claimed_and_sent_once(tmp_path, monkeypatch): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + queue.enqueue("exec-1", {"id": "job-1"}, "brief") + send = Mock(return_value=None) + + assert queue.drain(send) == 1 + assert queue.drain(send) == 0 + send.assert_called_once_with({"id": "job-1"}, "brief", False) + status = queue.get_status("exec-1") + assert status["status"] == "delivered" + assert status["job_json"] == "{}" + assert status["content"] == "" + + +def test_terminal_delivery_retention_is_bounded(tmp_path, monkeypatch): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + monkeypatch.setattr(queue, "MAX_TERMINAL_DELIVERIES", 2, raising=False) + for index in range(4): + execution_id = f"exec-{index}" + queue.enqueue(execution_id, {"id": f"job-{index}"}, f"brief-{index}") + assert queue.claim_next()["execution_id"] == execution_id + assert queue._finish(execution_id, error=None) + + # Pruning may discard verbose outcome rows, but never the durable + # idempotency tombstone for an execution that could be replayed later. + pruned = queue.get_status("exec-0") + assert pruned is not None + assert pruned["status"] == "delivered" + assert queue.get_status("exec-1")["status"] == "delivered" + assert queue.get_status("exec-2")["status"] == "delivered" + assert queue.get_status("exec-3")["status"] == "delivered" + + # Pruning may discard verbose outcome rows, but never the durable + # idempotency tombstone for an execution that could be replayed later. + queue.enqueue("exec-0", {"id": "job-replayed"}, "duplicate brief") + send = Mock(return_value=None) + assert queue.drain(send) == 0 + send.assert_not_called() + + +def test_failure_delivery_lane_survives_durable_handoff(tmp_path, monkeypatch): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + queue.enqueue( + "exec-failure", + {"id": "job-failure", "failure_deliver": "local"}, + "failed", + for_failure=True, + ) + send = Mock(return_value=None) + + assert queue.drain(send) == 1 + send.assert_called_once_with( + {"id": "job-failure", "failure_deliver": "local"}, + "failed", + True, + ) + + +def test_legacy_queue_schema_adds_failure_lane_before_enqueue(tmp_path, monkeypatch): + import cron.delivery_queue as queue + + db = tmp_path / "deliveries.db" + with sqlite3.connect(db) as conn: + conn.execute( + """CREATE TABLE deliveries ( + execution_id TEXT PRIMARY KEY, + job_json TEXT NOT NULL, + content TEXT NOT NULL, + status TEXT NOT NULL, + owner_process_id TEXT, + owner_pid INTEGER, + owner_started_at INTEGER, + created_at TEXT NOT NULL, + finished_at TEXT, + error TEXT + )""" + ) + monkeypatch.setattr(queue, "DELIVERY_DB", db) + + queue.enqueue( + "exec-migrated", + {"id": "job-migrated"}, + "failed", + for_failure=True, + ) + + assert queue.get_status("exec-migrated")["for_failure"] == 1 + + +def test_wait_timeout_marks_inflight_delivery_unknown_without_retry( + tmp_path, monkeypatch +): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + queue.enqueue("exec-inflight", {"id": "job-inflight"}, "result") + assert queue.claim_next() is not None + + error = queue.enqueue_and_wait( + "exec-inflight", {"id": "job-inflight"}, "result", timeout=0 + ) + + assert error is not None + assert "outcome is unknown" in error + status = queue.get_status("exec-inflight") + assert status is not None + assert status["status"] == "unknown" + send = Mock(return_value=None) + assert queue.drain(send) == 0 + send.assert_not_called() + + +def test_dead_delivery_owner_becomes_unknown_and_is_not_retried( + tmp_path, monkeypatch +): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + queue.enqueue("exec-1", {"id": "job-1"}, "brief") + assert queue.claim_next() is not None + monkeypatch.setattr(queue, "_PROCESS_ID", "replacement-gateway") + monkeypatch.setattr(queue, "_owner_is_live", lambda _pid, _started: False) + + assert queue.recover_abandoned() == 1 + send = Mock() + assert queue.drain(send) == 0 + send.assert_not_called() + assert queue.get_status("exec-1")["status"] == "unknown" + + +def test_delivery_failure_is_terminal_not_retried_and_redacted( + tmp_path, monkeypatch +): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + queue.enqueue("exec-1", {"id": "job-1"}, "brief") + send = Mock(return_value="request failed: https://example.test/?token=TOKEN123") + + assert queue.drain(send) == 1 + assert queue.drain(send) == 0 + assert send.call_count == 1 + status = queue.get_status("exec-1") + assert status is not None + assert status["status"] == "failed" + assert "TOKEN123" not in status["error"] + assert "token=***" in status["error"] + + +def test_wait_timeout_leaves_unclaimed_delivery_queued_for_next_gateway( + tmp_path, monkeypatch +): + """A row nobody claimed was never attempted: it is not uncertain, so a + gateway outage longer than the worker's wait budget must not lose it.""" + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + job = {"id": "job-3", "deliver": "origin"} + + error = queue.enqueue_and_wait("exec-3", job, "result", timeout=0) + + # Deferred, not failed: the worker must not record delivery_failed. + assert error is None + status = queue.get_status("exec-3") + assert status is not None + assert status["status"] == "pending" + send = Mock(return_value=None) + assert queue.drain(send) == 1 + send.assert_called_once_with(job, "result", False) + assert queue.get_status("exec-3")["status"] == "delivered" + + +def test_same_gateway_recovers_terminalization_failure_without_resending( + tmp_path, monkeypatch +): + import cron.delivery_queue as queue + + monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") + queue.enqueue("exec-4", {"id": "job-4"}, "result") + send = Mock(return_value=None) + original_finish = queue._finish + monkeypatch.setattr( + queue, + "_finish", + Mock(side_effect=OSError("database temporarily unavailable")), + ) + + with pytest.raises(OSError, match="temporarily unavailable"): + queue.drain(send) + + monkeypatch.setattr(queue, "_finish", original_finish) + assert queue.drain(send) == 0 + send.assert_called_once() + status = queue.get_status("exec-4") + assert status["status"] == "unknown" + assert "not retried" in status["error"] diff --git a/tests/cron/test_execution_ledger.py b/tests/cron/test_execution_ledger.py index ffe39a5164..d68259ae4b 100644 --- a/tests/cron/test_execution_ledger.py +++ b/tests/cron/test_execution_ledger.py @@ -39,6 +39,102 @@ def test_execution_transitions_are_durable(monkeypatch, tmp_path): assert persisted == [completed] +def test_execution_can_be_loaded_by_exact_attempt_id(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + first = executions.create_execution("same-job", source="builtin") + second = executions.create_execution("same-job", source="builtin") + + assert executions.get_execution(first["id"]) == first + assert executions.get_execution(second["id"]) == second + assert executions.get_execution("missing") is None + + +def test_fresh_external_handoff_is_not_recovered_before_worker_adopts( + monkeypatch, tmp_path +): + executions = _point_ledger(monkeypatch, tmp_path) + record = executions.create_execution("handoff-job", source="builtin") + assert executions.mark_execution_handoff_pending(record["id"]) is not None + + monkeypatch.setattr(executions, "_PROCESS_ID", "replacement-gateway") + monkeypatch.setattr(executions, "_owner_is_live", lambda _pid, _started: False) + + assert executions.recover_interrupted_executions() == 0 + assert executions.get_execution(record["id"])["status"] == "claimed" + adopted = executions.adopt_claimed_execution(record["id"]) + assert adopted["status"] == "running" + assert adopted["handoff_pending"] == 0 + + +def test_stale_external_handoff_is_recovered_unknown(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + record = executions.create_execution("handoff-job", source="builtin") + pending = executions.mark_execution_handoff_pending(record["id"]) + + monkeypatch.setattr(executions, "_PROCESS_ID", "replacement-gateway") + monkeypatch.setattr(executions, "_owner_is_live", lambda _pid, _started: False) + monkeypatch.setattr( + executions.time, + "time", + lambda: pending["handoff_started_at"] + + executions.HANDOFF_ADOPTION_GRACE_SECONDS + + 1, + ) + + assert executions.recover_interrupted_executions() == 1 + recovered = executions.get_execution(record["id"]) + assert recovered["status"] == "unknown" + assert recovered["handoff_pending"] == 0 + + +def test_recovery_does_not_overwrite_concurrent_worker_adoption(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + record = executions.create_execution("adoption-race", source="builtin") + pending = executions.mark_execution_handoff_pending(record["id"]) + assert pending is not None + monkeypatch.setattr(executions, "_PROCESS_ID", "replacement-scheduler") + monkeypatch.setattr( + executions.time, + "time", + lambda: pending["handoff_started_at"] + + executions.HANDOFF_ADOPTION_GRACE_SECONDS + + 1, + ) + + def adopt_while_liveness_is_checked(_pid, _started_at): + monkeypatch.setattr(executions, "_PROCESS_ID", "external-worker") + monkeypatch.setattr(executions.os, "getpid", lambda: 4242) + monkeypatch.setattr(executions, "_process_start_time", lambda _pid: 9876) + assert executions.adopt_claimed_execution(record["id"]) is not None + return False + + monkeypatch.setattr(executions, "_owner_is_live", adopt_while_liveness_is_checked) + + assert executions.recover_interrupted_executions() == 0 + current = executions.get_execution(record["id"]) + assert current is not None + assert current["status"] == "running" + assert current["process_id"] == "external-worker" + assert current["pid"] == 4242 + + +def test_foreign_process_cannot_start_or_finish_execution(monkeypatch, tmp_path): + executions = _point_ledger(monkeypatch, tmp_path) + record = executions.create_execution("owner-fence", source="builtin") + original_process_id = executions._PROCESS_ID + original_pid = record["pid"] + + monkeypatch.setattr(executions, "_PROCESS_ID", "foreign-process") + monkeypatch.setattr(executions.os, "getpid", lambda: original_pid + 1) + assert executions.mark_execution_running(record["id"]) is None + assert executions.finish_execution(record["id"], success=True) is None + + monkeypatch.setattr(executions, "_PROCESS_ID", original_process_id) + monkeypatch.setattr(executions.os, "getpid", lambda: original_pid) + assert executions.mark_execution_running(record["id"]) is not None + assert executions.finish_execution(record["id"], success=True) is not None + + def test_execution_ledger_follows_the_current_profile_home(monkeypatch, tmp_path): import cron.executions as executions @@ -83,6 +179,24 @@ def test_retention_bounds_terminal_history_but_preserves_inflight(monkeypatch, t assert executions.latest_execution("live")["status"] == "running" +def test_recently_finished_long_running_execution_survives_retention( + monkeypatch, tmp_path +): + executions = _point_ledger(monkeypatch, tmp_path) + monkeypatch.setattr(executions, "MAX_TERMINAL_EXECUTIONS", 1) + long_running = executions.create_execution("long-running", source="builtin") + assert executions.mark_execution_running(long_running["id"]) is not None + newer = executions.create_execution("newer", source="builtin") + assert executions.finish_execution(newer["id"], success=True) is not None + + finished = executions.finish_execution(long_running["id"], success=True) + + assert finished is not None + assert finished["status"] == "completed" + assert executions.get_execution(long_running["id"])["status"] == "completed" + assert executions.get_execution(newer["id"]) is None + + def test_corrupt_store_fails_closed_without_overwrite(monkeypatch, tmp_path): executions = _point_ledger(monkeypatch, tmp_path) executions.EXECUTIONS_FILE.parent.mkdir(parents=True) @@ -221,7 +335,7 @@ def test_run_one_job_records_running_then_terminal(monkeypatch): monkeypatch.setattr( scheduler, "mark_execution_running", - lambda execution_id: events.append(("running", execution_id)), + lambda execution_id: events.append(("running", execution_id)) or {}, raising=False, ) monkeypatch.setattr( diff --git a/tests/cron/test_jobs.py b/tests/cron/test_jobs.py index 6a3492f390..f993928890 100644 --- a/tests/cron/test_jobs.py +++ b/tests/cron/test_jobs.py @@ -645,6 +645,8 @@ class TestMarkJobRun: assert updated is not None assert updated["state"] == "completed" assert updated["last_delivery_error"] == "platform 'telegram' not configured" + # A terminal completion that never reached the user is not a success. + assert updated["last_status"] == "delivery_failed" def test_completed_oneshot_visible_in_list(self, tmp_cron_dir): """list_jobs(include_disabled=True) surfaces the completed record.""" @@ -654,6 +656,7 @@ class TestMarkJobRun: assert job["id"] in listed assert listed[job["id"]]["state"] == "completed" assert listed[job["id"]]["last_delivery_error"] == "send failed: 502" + assert listed[job["id"]]["last_status"] == "delivery_failed" # Default (enabled-only) listing hides it, matching paused/disabled jobs. assert job["id"] not in {j["id"] for j in list_jobs()} @@ -672,13 +675,53 @@ class TestMarkJobRun: assert updated["last_error"] == "timeout" def test_delivery_error_tracked_separately(self, tmp_cron_dir): - """Agent succeeds but delivery fails — both tracked independently.""" + """Agent succeeds but delivery fails — surfaced, not hidden behind ok. + + Regression guard for #83993: recording ``last_status="ok"`` made a run + the user never received look like a quiet success everywhere that keys + off "ok". The agent error stays independent of the delivery error, and + the delivery failure is not an agent failure (no streak). + """ job = create_job(prompt="Report", schedule="every 1h") - mark_job_run(job["id"], success=True, delivery_error="platform 'telegram' not configured") + mark_job_run(job["id"], success=True, delivery_error="send failed: 502") updated = get_job(job["id"]) - assert updated["last_status"] == "ok" + assert updated["last_status"] == "delivery_failed" assert updated["last_error"] is None - assert updated["last_delivery_error"] == "platform 'telegram' not configured" + assert updated["last_delivery_error"] == "send failed: 502" + assert updated["failure_streak"] == 0 + + def test_success_without_delivery_error_stays_ok(self, tmp_cron_dir): + """A fully successful run is still plain "ok".""" + job = create_job(prompt="Report", schedule="every 1h") + mark_job_run(job["id"], success=True) + assert get_job(job["id"])["last_status"] == "ok" + # An empty delivery error is no error at all. + mark_job_run(job["id"], success=True, delivery_error="") + assert get_job(job["id"])["last_status"] == "ok" + + def test_agent_failure_still_error_with_delivery_error(self, tmp_cron_dir): + """An agent failure outranks delivery: still "error", still a streak.""" + job = create_job(prompt="Report", schedule="every 1h") + mark_job_run( + job["id"], success=False, error="timeout", + delivery_error="send failed: 502", + ) + updated = get_job(job["id"]) + assert updated["last_status"] == "error" + assert updated["last_error"] == "timeout" + assert updated["failure_streak"] == 1 + + def test_explicit_status_override_wins_over_delivery_failed(self, tmp_cron_dir): + """An explicit terminal status (T1-26 blocked_config) still wins.""" + job = create_job(prompt="Report", schedule="every 1h") + mark_job_run( + job["id"], success=True, + delivery_error="send failed: 502", + status="blocked_config", + ) + updated = get_job(job["id"]) + assert updated["last_status"] == "blocked_config" + assert updated["last_delivery_error"] == "send failed: 502" def test_failure_streak_increments_and_resets(self, tmp_cron_dir): """failure_streak counts consecutive agent failures; success resets.""" diff --git a/tests/cron/test_lifecycle_guard_budget.py b/tests/cron/test_lifecycle_guard_budget.py new file mode 100644 index 0000000000..181461744f --- /dev/null +++ b/tests/cron/test_lifecycle_guard_budget.py @@ -0,0 +1,241 @@ +"""Whole-walk work budget for the gateway lifecycle guard (#78398). + +The per-file byte cap and recursion depth bound one read, not the walk. These +tests pin the shared budget that bounds the whole referenced-script walk and +is charged *before* any text reaches ``shlex``. + +Budget constants are monkeypatched to tiny values so the tests are fast and +deterministic; ``_LifecycleScanBudget`` reads them at construction time. +""" + +from __future__ import annotations + +import pytest + +import cron.lifecycle_guard as lifecycle_guard + +guard = lifecycle_guard.contains_gateway_lifecycle_command_or_referenced_script + + +def _explode(*_args, **_kwargs): + raise AssertionError("over-budget text reached shlex") + + +# --- root command (depth 0) ----------------------------------------------- + + +def test_root_byte_limit_allows_exact_and_rejects_plus_one(monkeypatch): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 8) + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 8) + + assert guard("x" * 8) is False + + monkeypatch.setattr(lifecycle_guard.shlex, "shlex", _explode) + assert guard("x" * 9) is True + + +def test_root_line_limit_allows_exact_and_rejects_plus_one(monkeypatch): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINES", 2) + + assert guard("one\ntwo") is False + assert guard("one\ntwo\nthree") is True + + +def test_single_giant_line_rejected_before_shlex(monkeypatch): + """One enormous token is the quadratic shlex case.""" + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 8) + monkeypatch.setattr(lifecycle_guard.shlex, "shlex", _explode) + + assert guard("xxxxxxxxx\necho ok") is True + + +def test_root_budget_counts_utf8_bytes(monkeypatch): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 4) + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 4) + + assert guard("éé") is False + assert guard("ééé") is True + + +def test_exhaustion_is_logged_at_warning(monkeypatch, caplog): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 4) + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 4) + + with caplog.at_level("WARNING", logger=lifecycle_guard.logger.name): + assert guard("echo hello") is True + assert "budget exhausted" in caplog.text + + +def test_lifecycle_scan_root_within_budget_is_not_a_verdict(monkeypatch): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 8) + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 8) + + assert lifecycle_guard.lifecycle_scan_root_within_budget("x" * 8) is True + assert lifecycle_guard.lifecycle_scan_root_within_budget("x" * 9) is False + + +# --- referenced-script walk ------------------------------------------------ + + +def test_unique_path_budget_bounds_reads_and_fails_closed(monkeypatch, tmp_path): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_PATHS", 2) + for i in range(3): + (tmp_path / f"s{i}.sh").write_text("echo ok\n", encoding="utf-8") + + two = " && ".join(f"bash {tmp_path}/s{i}.sh" for i in range(2)) + three = " && ".join(f"bash {tmp_path}/s{i}.sh" for i in range(3)) + + assert guard(two) is False + assert guard(three) is True + + +def test_repeated_path_does_not_spend_unique_path_budget(monkeypatch, tmp_path): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_PATHS", 1) + script = tmp_path / "s.sh" + script.write_text("echo ok\n", encoding="utf-8") + + assert guard(f"bash {script} && bash {script} && sh {script}") is False + + +def test_remote_read_budget_charged_before_remote_read(monkeypatch): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_REMOTE_READS", 1) + reads: list[str] = [] + + def remote(path: str): + reads.append(path) + return "echo ok\n" + + assert ( + guard( + "bash /remote/a.sh && bash /remote/b.sh", + read_remote_script=remote, + ) + is True + ) + assert reads == ["/remote/a.sh"] + + +def test_cumulative_text_budget_bounds_recursive_scan(monkeypatch, tmp_path): + """Two scripts individually under the per-file cap exceed the walk cap. + + Relative references keep the root command short so the budget arithmetic + is about the scripts, not the tmp_path length.""" + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 48) + (tmp_path / "a.sh").write_text("echo " + "a" * 10 + "\n", encoding="utf-8") # 16 + (tmp_path / "b.sh").write_text("echo " + "b" * 10 + "\n", encoding="utf-8") # 16 + cwd = str(tmp_path) + + # 9 (root) + 16 fits in 48; 19 (root) + 16 + 16 does not → fail closed. + assert guard("bash a.sh", cwd=cwd) is False + assert guard("bash a.sh;bash b.sh", cwd=cwd) is True + + +def test_referenced_read_is_capped_at_remaining_budget(monkeypatch, tmp_path): + """A file bigger than what the walk can still afford is never read whole: + the read helper receives the remaining budget as its cap.""" + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 64) + (tmp_path / "big.sh").write_text("echo " + "x" * 200 + "\n", encoding="utf-8") + + caps: list = [] + original = lifecycle_guard._read_referenced_script + + def spy(path, *, max_bytes=None): + caps.append(max_bytes) + return original(path, max_bytes=max_bytes) + + monkeypatch.setattr(lifecycle_guard, "_read_referenced_script", spy) + + root = "bash big.sh" + assert guard(root, cwd=str(tmp_path)) is True + assert caps == [64 - len(root)] + + +def test_remote_script_sanitizer_honours_remaining_budget(): + text, unsafe = lifecycle_guard._sanitize_remote_script_text( + "echo ok\n", max_bytes=4 + ) + assert (text, unsafe) == (None, True) + text, unsafe = lifecycle_guard._sanitize_remote_script_text( + "echo ok\n", max_bytes=8 + ) + assert (text, unsafe) == ("echo ok\n", False) + + +def test_line_budget_fails_closed_before_tokenizing_every_line( + monkeypatch, tmp_path +): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINES", 4) + script = tmp_path / "many.sh" + script.write_text("echo ok\n" * 10, encoding="utf-8") + + lexers = 0 + real_shlex = lifecycle_guard.shlex.shlex + + def counting(*args, **kwargs): + nonlocal lexers + lexers += 1 + return real_shlex(*args, **kwargs) + + monkeypatch.setattr(lifecycle_guard.shlex, "shlex", counting) + root = f"bash {script}" + assert guard(root) is True + # Only the one-line root was tokenized (a handful of lexers across the + # direct scans); the 10-line script never was. + assert 0 < lexers < 10 + + +# --- scheduler entry point -------------------------------------------------- + + +def test_check_gateway_lifecycle_shell_script_budget(monkeypatch, tmp_path): + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 8) + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 8) + script = tmp_path / "long-line.sh" + + script.write_text("x" * 7, encoding="utf-8") + lifecycle_guard.check_gateway_lifecycle("", str(script)) + + script.write_text("x" * 9, encoding="utf-8") + with pytest.raises(lifecycle_guard.GatewayLifecycleBlocked): + lifecycle_guard.check_gateway_lifecycle("", str(script)) + + +def test_check_gateway_lifecycle_python_path_charges_masker(monkeypatch, tmp_path): + """The .py branch's data-exemption masker tokenizes too, so it is budgeted + and fails closed before shlex on an over-budget line.""" + monkeypatch.setattr(lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 16) + + small = tmp_path / "small.py" + small.write_text("x = 1\n", encoding="utf-8") + lifecycle_guard.check_gateway_lifecycle("run report", str(small)) + + monkeypatch.setattr(lifecycle_guard.shlex, "shlex", _explode) + long_line = tmp_path / "long.py" + long_line.write_text("x = 1\n" + "y" * 40 + "\n", encoding="utf-8") + with pytest.raises(lifecycle_guard.GatewayLifecycleBlocked): + lifecycle_guard.check_gateway_lifecycle("run report", str(long_line)) + + +# --- no regression on realistic benign graphs ------------------------------ + + +def test_default_budget_admits_a_wide_benign_wrapper_graph(tmp_path): + """Issue #78398's shape: one wrapper invoking 200 small legitimate scripts + must still be allowed under the DEFAULT limits (an earlier fail-closed + attempt with a 64-path cap blocked exactly this).""" + children = [] + for i in range(200): + child = tmp_path / f"c{i}.sh" + child.write_text("echo step && ls -la /tmp\n" * 20, encoding="utf-8") + children.append(child) + hub = tmp_path / "hub.sh" + hub.write_text("".join(f"bash {c}\n" for c in children), encoding="utf-8") + + assert guard(f"bash {hub}") is False + + # ...and a lifecycle command hidden behind the 200 benign scripts is still + # found: the budget bounds work, it does not stop the walk early. + evil = tmp_path / "evil.sh" + evil.write_text("hermes gateway restart\n", encoding="utf-8") + hub.write_text(hub.read_text() + f"bash {evil}\n", encoding="utf-8") + assert guard(f"bash {hub}") is True diff --git a/tests/cron/test_notepad.py b/tests/cron/test_notepad.py index 9d140e5099..e91c1ac139 100644 --- a/tests/cron/test_notepad.py +++ b/tests/cron/test_notepad.py @@ -8,6 +8,7 @@ use the notepad, and the `hermes cron notepad` CLI handler. from __future__ import annotations import argparse +import importlib import sys from pathlib import Path @@ -103,6 +104,33 @@ class TestNotepadCrud: assert not notepad.NOTEPAD_FILE.exists() +class TestNotepadProfileIsolation: + def test_profile_override_routes_writes_to_current_home(self, tmp_path): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + import cron.notepad as notepad_mod + + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + + import_token = set_hermes_home_override(profile_a) + try: + importlib.reload(notepad_mod) + finally: + reset_hermes_home_override(import_token) + + runtime_token = set_hermes_home_override(profile_b) + try: + notepad_mod.set_note("job-1", "cursor", "page=7") + finally: + reset_hermes_home_override(runtime_token) + + assert (profile_b / "cron" / "notepad.db").exists() + assert not (profile_a / "cron" / "notepad.db").exists() + + class TestJobRemovalCleanup: def test_remove_job_clears_notepad(self, cron_env, notepad): """remove_job must clear the job's notepad rows — without this, diff --git a/tests/cron/test_parallel_pool.py b/tests/cron/test_parallel_pool.py index b159f265d7..9853dbc229 100644 --- a/tests/cron/test_parallel_pool.py +++ b/tests/cron/test_parallel_pool.py @@ -187,7 +187,7 @@ class TestRunningJobGuard: if job_id == "healthy-job" else None, ) - monkeypatch.setattr(sched, "mark_execution_running", lambda *_a, **_kw: None) + monkeypatch.setattr(sched, "mark_execution_running", lambda *_a, **_kw: {}) monkeypatch.setattr(sched, "heartbeat_fire_claim", lambda *_a, **_kw: True) n = sched.tick(verbose=False) diff --git a/tests/cron/test_preflight_config.py b/tests/cron/test_preflight_config.py index 0a12721d8b..4e56b3d6e9 100644 --- a/tests/cron/test_preflight_config.py +++ b/tests/cron/test_preflight_config.py @@ -129,7 +129,7 @@ class TestMissingProviderKeyBlocks: job = _job() deliveries = [] - def fake_deliver(job, content, adapters=None, loop=None): + def fake_deliver(job, content, adapters=None, loop=None, **kwargs): deliveries.append(content) return None @@ -233,7 +233,7 @@ class TestOptOut: job = _job() deliveries = [] - def fake_deliver(job, content, adapters=None, loop=None): + def fake_deliver(job, content, adapters=None, loop=None, **kwargs): deliveries.append(content) return None diff --git a/tests/cron/test_restart_safe_worker.py b/tests/cron/test_restart_safe_worker.py new file mode 100644 index 0000000000..1be7e30654 --- /dev/null +++ b/tests/cron/test_restart_safe_worker.py @@ -0,0 +1,603 @@ +"""Restart-safe cron worker handoff and ownership contracts.""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def execution_ledger(tmp_path, monkeypatch): + import cron.executions as executions + + monkeypatch.setattr(executions, "EXECUTIONS_FILE", tmp_path / "executions.db") + return executions + + +def test_execution_owner_moves_to_external_worker_before_running( + execution_ledger, monkeypatch +): + record = execution_ledger.create_execution("job-1", source="builtin") + assert execution_ledger.mark_execution_handoff_pending(record["id"]) is not None + monkeypatch.setattr(execution_ledger.os, "getpid", lambda: 4242) + monkeypatch.setattr(execution_ledger, "_process_start_time", lambda pid: 9876) + + adopted = execution_ledger.adopt_claimed_execution(record["id"]) + + assert adopted is not None + assert adopted["pid"] == 4242 + assert adopted["process_started_at"] == 9876 + assert adopted["status"] == "running" + assert execution_ledger.adopt_claimed_execution(record["id"]) is None + assert execution_ledger.mark_execution_running(record["id"]) is None + + +def test_external_worker_cannot_adopt_execution_without_handoff_fence( + execution_ledger, monkeypatch +): + record = execution_ledger.create_execution("job-unfenced", source="builtin") + monkeypatch.setattr(execution_ledger.os, "getpid", lambda: 4242) + monkeypatch.setattr(execution_ledger, "_process_start_time", lambda _pid: 9876) + + assert execution_ledger.adopt_claimed_execution(record["id"]) is None + assert execution_ledger.get_execution(record["id"])["status"] == "claimed" + + +def test_genuine_external_worker_crash_is_recovered_unknown( + execution_ledger, monkeypatch +): + record = execution_ledger.create_execution("job-crash", source="builtin") + assert execution_ledger.mark_execution_handoff_pending(record["id"]) is not None + script = ( + "import os\n" + "from pathlib import Path\n" + "import cron.executions as executions\n" + f"executions.EXECUTIONS_FILE = Path({str(execution_ledger.EXECUTIONS_FILE)!r})\n" + f"assert executions.adopt_claimed_execution({record['id']!r}) is not None\n" + "os._exit(9)\n" + ) + + crashed = subprocess.run([sys.executable, "-c", script], check=False) + assert crashed.returncode == 9 + + monkeypatch.setattr(execution_ledger, "_PROCESS_ID", "replacement-scheduler") + assert execution_ledger.recover_interrupted_executions() == 1 + recovered = execution_ledger.latest_execution("job-crash") + assert recovered["status"] == "unknown" + assert "whether side effects ran is unknown" in recovered["error"] + + +@pytest.mark.linux_only +def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch): + import tools.process_registry as process_registry + + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + monkeypatch.setenv("INVOCATION_ID", "managed-service") + monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False) + + with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"): + process_registry.restart_safe_gateway_child_argv( + ["python", "worker.py"], unit_suffix="cron-job-1" + ) + + +def test_restart_safe_gateway_child_is_unchanged_outside_managed_gateway(monkeypatch): + import tools.process_registry as process_registry + + command = ["python", "worker.py"] + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False) + + assert process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1" + ) is command + + +def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch): + import tools.process_registry as process_registry + + command = ["python", "worker.py"] + probe = Mock(side_effect=AssertionError("systemd probe ran off Linux")) + monkeypatch.setattr(process_registry, "_IS_LINUX", False) + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", probe) + monkeypatch.setenv("INVOCATION_ID", "managed-service") + + assert process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1" + ) is command + probe.assert_not_called() + + +def test_external_worker_adopts_execution_and_runs_payload_once( + tmp_path, monkeypatch +): + import cron.scheduler as scheduler + + payload = tmp_path / "payload.json" + ack = tmp_path / "ready.json" + payload.write_text( + json.dumps({ + "job": {"id": "job-1", "execution_id": "exec-1"}, + "profile_home": str(tmp_path / "profile"), + }), + encoding="utf-8", + ) + from hermes_constants import get_hermes_home + + observed_homes = [] + adopted = Mock( + side_effect=lambda execution_id: ( + observed_homes.append(get_hermes_home().resolve()) + or {"id": execution_id, "status": "running"} + ) + ) + run = Mock( + side_effect=lambda *_args, **_kwargs: ( + observed_homes.append(get_hermes_home().resolve()) or True + ) + ) + monkeypatch.setattr("cron.executions.adopt_claimed_execution", adopted) + monkeypatch.setattr(scheduler, "run_one_job", run) + + assert scheduler._run_external_worker_payload(payload, ack) is True + + adopted.assert_called_once_with("exec-1") + run.assert_called_once() + assert run.call_args.args[0]["id"] == "job-1" + expected_home = (tmp_path / "profile").resolve() + assert observed_homes == [expected_home, expected_home] + assert ack.exists() + assert not payload.exists() + + +def test_external_worker_refuses_to_run_without_durable_ownership( + tmp_path, monkeypatch +): + import cron.scheduler as scheduler + + payload = tmp_path / "payload.json" + ack = tmp_path / "ready.json" + payload.write_text( + json.dumps({ + "job": {"id": "job-1", "execution_id": "exec-1"}, + "profile_home": str(tmp_path / "profile"), + }), + encoding="utf-8", + ) + monkeypatch.setattr("cron.executions.adopt_claimed_execution", lambda _id: None) + run = Mock() + monkeypatch.setattr(scheduler, "run_one_job", run) + + assert scheduler._run_external_worker_payload(payload, ack) is False + + run.assert_not_called() + assert not ack.exists() + + +def test_launch_external_worker_uses_restart_safe_scope_and_acknowledges( + tmp_path, monkeypatch +): + import cron.scheduler as scheduler + + job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"} + monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path) + wrapped_commands = [] + + def wrap(command, *, unit_suffix): + wrapped_commands.append((command, unit_suffix)) + return ["scope", "--", *command] + + monkeypatch.setattr( + "tools.process_registry.restart_safe_gateway_child_argv", wrap + ) + + class FakeProcess: + returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + if self.returncode is None: + raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout) + return self.returncode + + spawned = [] + + payloads = [] + + def popen(command, **kwargs): + spawned.append((command, kwargs)) + payload_index = command.index("--external-worker-file") + 1 + payloads.append(json.loads(Path(command[payload_index]).read_text())) + ack_index = command.index("--ack-file") + 1 + Path(command[ack_index]).write_text( + json.dumps({"pid": 4321, "execution_id": "exec-1"}), + encoding="utf-8", + ) + return FakeProcess() + + handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1}) + monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff) + monkeypatch.setattr(scheduler.subprocess, "Popen", popen) + observed_statuses = iter( + [ + {"id": "exec-1", "status": "running"}, + {"id": "exec-1", "status": "completed"}, + ] + ) + get = Mock(side_effect=lambda _execution_id: next(observed_statuses)) + monkeypatch.setattr(scheduler, "get_execution", get) + monkeypatch.setenv("ANTHROPIC_API_KEY", "should-not-cross-profile") + from agent.secret_scope import set_multiplex_active + + set_multiplex_active(True) + try: + assert scheduler._launch_external_cron_worker(job) is True + finally: + set_multiplex_active(False) + assert wrapped_commands[0][1] == "cron-job-1-exec-exec-1" + assert spawned[0][0][0:2] == ["scope", "--"] + assert spawned[0][1]["start_new_session"] is True + assert "ANTHROPIC_API_KEY" not in spawned[0][1]["env"] + handoff.assert_called_once_with("exec-1") + assert get.call_count == 2 + assert payloads[0]["multiplex_active"] is True + # Once the attempt is terminal the parent reaps its own handoff artifacts. + assert not (tmp_path / "cron/external-workers/exec-1.json").exists() + + +def test_external_worker_exit_rechecks_exact_execution_before_failure(monkeypatch): + import cron.scheduler as scheduler + + statuses = iter( + [ + {"id": "exec-1", "status": "running"}, + {"id": "exec-1", "status": "completed"}, + ] + ) + get = Mock(side_effect=lambda _execution_id: next(statuses)) + monkeypatch.setattr(scheduler, "get_execution", get, raising=False) + process = Mock() + process.poll.return_value = 0 + process.wait.return_value = 0 + + assert scheduler._wait_for_external_cron_worker( + process, execution_id="exec-1" + ) is True + assert get.call_count == 2 + + +def test_external_worker_crash_recovers_uncertain_attempt(monkeypatch): + import cron.scheduler as scheduler + + statuses = iter( + [ + {"id": "exec-1", "status": "running"}, + {"id": "exec-1", "status": "unknown"}, + ] + ) + get = Mock(side_effect=lambda _execution_id: next(statuses)) + recover = Mock(return_value=1) + monkeypatch.setattr(scheduler, "get_execution", get) + monkeypatch.setattr( + scheduler, "recover_interrupted_executions", recover, raising=False + ) + process = Mock() + process.poll.return_value = 9 + process.wait.return_value = 9 + + assert scheduler._wait_for_external_cron_worker( + process, execution_id="exec-1" + ) is True + recover.assert_called_once_with() + assert get.call_count == 2 + + +def test_launch_external_worker_stays_in_process_outside_managed_gateway( + monkeypatch, +): + import cron.scheduler as scheduler + + command_calls = [] + + def unchanged(command, *, unit_suffix): + command_calls.append((command, unit_suffix)) + return command + + monkeypatch.setattr( + "tools.process_registry.restart_safe_gateway_child_argv", unchanged + ) + popen = Mock() + monkeypatch.setattr(scheduler.subprocess, "Popen", popen) + + assert scheduler._launch_external_cron_worker( + {"id": "job-1", "execution_id": "exec-1"} + ) is False + assert command_calls + popen.assert_not_called() + + +def test_shared_run_path_hands_gateway_fire_to_external_worker(monkeypatch): + import cron.scheduler as scheduler + + launch = Mock(return_value=True) + run = Mock(side_effect=AssertionError("agent ran inside gateway")) + monkeypatch.setattr(scheduler, "_launch_external_cron_worker", launch) + monkeypatch.setattr(scheduler, "run_job", run) + job = {"id": "job-1", "execution_id": "exec-1"} + + assert scheduler.run_one_job(job, adapters={"discord": object()}) is True + + launch.assert_called_once_with(job) + run.assert_not_called() + + +def test_shutdown_does_not_interrupt_restart_safe_waiter(): + import cron.scheduler as scheduler + + job_id = "external-waiter" + scheduler._running_job_ids.add(job_id) + scheduler._restart_safe_waiter_job_ids.add(job_id) + try: + assert scheduler.mark_running_jobs_interrupted("gateway restart") == [] + assert job_id not in scheduler._interrupted_job_ids + finally: + scheduler._restart_safe_waiter_job_ids.discard(job_id) + scheduler._running_job_ids.discard(job_id) + scheduler._interrupted_job_ids.discard(job_id) + + +def test_worker_delivery_queue_is_keyed_by_the_delivering_jobs_own_execution( + monkeypatch, tmp_path +): + """A nested in-process dispatch inside a worker (e.g. a script running + ``hermes cron run ``) must not queue under the OUTER execution id.""" + import cron.scheduler as scheduler + + queued = [] + monkeypatch.setattr( + "cron.delivery_queue.enqueue_and_wait", + lambda execution_id, job, content, **kw: ( + queued.append(execution_id) or "queued-marker" + ), + ) + monkeypatch.setattr( + scheduler, + "_resolve_delivery_targets", + lambda job, for_failure=False: [{"platform": "telegram", "chat_id": "123"}], + ) + + def _standalone(*_args, **_kwargs): + raise RuntimeError("standalone path reached") + + # First call the standalone (non-queue) path makes after the guard; the + # failure is reported as the delivery error string. + monkeypatch.setattr("gateway.config.load_gateway_config", _standalone) + monkeypatch.setenv("_HERMES_CRON_EXTERNAL_WORKER", "exec-outer") + + # Own attempt: routed through the durable queue. + assert scheduler._deliver_result( + {"id": "job-1", "execution_id": "exec-outer", "deliver": "telegram:123"}, + "done", + adapters=None, + loop=None, + ) == "queued-marker" + assert queued == ["exec-outer"] + + # A different job's attempt: must NOT be queued under exec-outer; it falls + # through to the standalone path. + error = scheduler._deliver_result( + {"id": "job-2", "execution_id": "exec-inner", "deliver": "telegram:123"}, + "done", + adapters=None, + loop=None, + ) + assert error == "failed to load gateway config: standalone path reached" + assert queued == ["exec-outer"] + + +def test_gateway_tool_run_without_adapter_objects_hands_off(monkeypatch): + import cron.scheduler as scheduler + + created = Mock(return_value={"id": "exec-tool"}) + launch = Mock(return_value=True) + run = Mock(side_effect=AssertionError("agent ran inside gateway")) + monkeypatch.setattr(scheduler, "create_execution", created) + monkeypatch.setattr(scheduler, "_launch_external_cron_worker", launch) + monkeypatch.setattr(scheduler, "run_job", run) + job = {"id": "tool-job"} + + assert scheduler.run_one_job(job, adapters=None) is True + + created.assert_called_once_with("tool-job", source="direct") + assert job["execution_id"] == "exec-tool" + launch.assert_called_once_with(job) + run.assert_not_called() + + +def test_shared_run_path_creates_execution_before_managed_handoff(monkeypatch): + import cron.scheduler as scheduler + + created = Mock(return_value={"id": "exec-new"}) + launch = Mock(return_value=True) + monkeypatch.setattr(scheduler, "create_execution", created) + monkeypatch.setattr(scheduler, "_launch_external_cron_worker", launch) + job = {"id": "manual-job"} + + assert scheduler.run_one_job(job, adapters={"discord": object()}) is True + + created.assert_called_once_with("manual-job", source="direct") + assert job["execution_id"] == "exec-new" + launch.assert_called_once_with(job) + + +def test_lost_execution_start_cas_prevents_side_effects(monkeypatch): + import cron.scheduler as scheduler + + run = Mock(side_effect=AssertionError("side effect ran without ownership")) + monkeypatch.setattr(scheduler, "claim_dispatch", lambda _job_id: True) + monkeypatch.setattr(scheduler, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr(scheduler, "run_job", run) + + assert scheduler.run_one_job( + {"id": "job-1", "execution_id": "exec-1"}, adapters=None + ) is True + run.assert_not_called() + + +@pytest.mark.linux_only +@pytest.mark.live_system_guard_bypass +def test_managed_gateway_restart_preserves_active_worker_and_single_side_effect( + tmp_path, monkeypatch +): + import cron.delivery_queue as delivery_queue + import cron.executions as executions + import cron.scheduler as scheduler + from cron.jobs import create_job, use_cron_store + from gateway.config import Platform, PlatformConfig + from gateway.status import _pid_exists + from tools import process_registry + + if not process_registry._systemd_run_user_scope_available(): + pytest.skip("systemd-run --user --scope is unavailable on this host") + + home = tmp_path / "profile" + scripts_dir = home / "scripts" + scripts_dir.mkdir(parents=True) + started = tmp_path / "started" + release = tmp_path / "release" + side_effect = tmp_path / "side-effect" + probe = scripts_dir / "restart_probe.py" + probe.write_text( + "import pathlib, time\n" + f"started = pathlib.Path({str(started)!r})\n" + f"release = pathlib.Path({str(release)!r})\n" + f"side_effect = pathlib.Path({str(side_effect)!r})\n" + "started.write_text('started')\n" + "deadline = time.monotonic() + 15\n" + "while not release.exists() and time.monotonic() < deadline:\n" + " time.sleep(0.05)\n" + "if not release.exists():\n" + " raise SystemExit('release timeout')\n" + "with side_effect.open('a') as handle:\n" + " handle.write('once\\n')\n" + "print('completed')\n", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_HOME", str(home)) + with use_cron_store(home): + job = create_job( + prompt=None, + schedule="every 1h", + name="restart probe", + script=probe.name, + no_agent=True, + deliver="telegram:123", + ) + payload = tmp_path / "job.json" + launched = tmp_path / "launched.json" + payload.write_text(json.dumps(job), encoding="utf-8") + + sent = [] + adapter = Mock() + + async def send(_chat_id, content, metadata=None): + sent.append((content, metadata)) + return {"success": True, "message_id": "restart-delivery-1"} + + adapter.send = send + gateway_config = Mock() + gateway_config.platforms = { + Platform.TELEGRAM: PlatformConfig(enabled=True), + } + gateway_config.get_home_channel = lambda _platform: None + monkeypatch.setattr( + "gateway.config.load_gateway_config", lambda: gateway_config + ) + monkeypatch.setattr( + scheduler, "load_config", lambda: {"cron": {"wrap_response": False}} + ) + replacement_loop = asyncio.new_event_loop() + replacement_thread = threading.Thread( + target=replacement_loop.run_forever, + daemon=True, + ) + replacement_thread.start() + deadline = time.monotonic() + 2 + while not replacement_loop.is_running() and time.monotonic() < deadline: + time.sleep(0.01) + assert replacement_loop.is_running() + + harness = ( + "import json, os, pathlib, time\n" + f"os.environ['HERMES_HOME'] = {str(home)!r}\n" + "os.environ['INVOCATION_ID'] = 'restart-fixture'\n" + "from cron import scheduler\n" + "from tools import process_registry\n" + "process_registry._is_supervised_gateway_process = lambda: True\n" + f"job = json.loads(pathlib.Path({str(payload)!r}).read_text())\n" + "if not scheduler.run_one_job(job, adapters=None, loop=None):\n" + " raise SystemExit('worker was not isolated')\n" + f"pathlib.Path({str(launched)!r}).write_text('returned')\n" + ) + parent = subprocess.Popen([sys.executable, "-c", harness]) + worker_pid = None + try: + deadline = time.monotonic() + 10 + current = None + while time.monotonic() < deadline: + if parent.poll() is not None: + pytest.fail(f"gateway fixture exited early with {parent.returncode}") + current = executions.latest_execution(job["id"]) + if started.exists() and current and current.get("pid") != os.getpid(): + break + time.sleep(0.05) + assert started.exists() + assert current is not None + execution = current + worker_pid = int(current["pid"]) + assert not launched.exists(), "handoff returned before execution completed" + + # Replacing a managed gateway kills its old process tree. The active + # cron owner must remain in its transient scope and keep the same PID. + parent.terminate() + parent.wait(timeout=5) + assert _pid_exists(worker_pid) + + release.write_text("go", encoding="utf-8") + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + row = delivery_queue.get_status(execution["id"]) + if row and row["status"] == "pending": + scheduler.drain_delivery_queue( + {Platform.TELEGRAM: adapter}, replacement_loop + ) + current = executions.latest_execution(job["id"]) + if current and current["status"] == "completed": + break + time.sleep(0.05) + assert executions.latest_execution(job["id"])["status"] == "completed" + assert side_effect.read_text(encoding="utf-8").splitlines() == ["once"] + assert delivery_queue.get_status(execution["id"])["status"] == "delivered" + assert len(sent) == 1 + assert "completed" in sent[0][0] + finally: + replacement_loop.call_soon_threadsafe(replacement_loop.stop) + replacement_thread.join(timeout=2) + replacement_loop.close() + if parent.poll() is None: + parent.terminate() + parent.wait(timeout=5) + if worker_pid is not None and _pid_exists(worker_pid): + os.kill(worker_pid, signal.SIGKILL) diff --git a/tests/cron/test_run_one_job.py b/tests/cron/test_run_one_job.py index a3b8bb425a..9b241de1b6 100644 --- a/tests/cron/test_run_one_job.py +++ b/tests/cron/test_run_one_job.py @@ -29,7 +29,7 @@ def _patch_pipeline(monkeypatch, *, success=True, output="out", final="final res calls.append(("save", jid)) return f"/tmp/{jid}.txt" - def fake_deliver(job, content, adapters=None, loop=None): + def fake_deliver(job, content, adapters=None, loop=None, **kwargs): calls.append(("deliver", job["id"])) return None @@ -88,7 +88,7 @@ def test_run_one_job_exception_delivers_failure_alert(monkeypatch): s, "create_execution", lambda *_a, **_kw: {"id": "exec-j3"} ) monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) - monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) monkeypatch.setattr( s, "run_job", @@ -141,7 +141,7 @@ def test_run_one_job_exception_records_failure_alert_delivery_error(monkeypatch) s, "create_execution", lambda *_a, **_kw: {"id": "exec-j4"} ) monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) - monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) monkeypatch.setattr( s, "run_job", @@ -165,7 +165,7 @@ def _patch_escaped_failure(monkeypatch, delivered, *, exec_id, err): """Make run_job raise, and capture what the escape handler delivers.""" monkeypatch.setattr(s, "create_execution", lambda *_a, **_kw: {"id": exec_id}) monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) - monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) monkeypatch.setattr( s, "run_job", @@ -246,7 +246,7 @@ def test_run_one_job_exception_after_delivery_does_not_redeliver(monkeypatch): s, "create_execution", lambda *_a, **_kw: {"id": "exec-j5"} ) monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) - monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) monkeypatch.setattr( s, "run_job", @@ -288,7 +288,7 @@ def test_run_one_job_keyboard_interrupt_skips_delivery_and_reraises(monkeypatch) s, "create_execution", lambda *_a, **_kw: {"id": "exec-j6"} ) monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True) - monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: None) + monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {}) monkeypatch.setattr( s, "run_job", diff --git a/tests/cron/test_scheduler_provider.py b/tests/cron/test_scheduler_provider.py index 12ac73560c..6bf710e4ed 100644 --- a/tests/cron/test_scheduler_provider.py +++ b/tests/cron/test_scheduler_provider.py @@ -786,3 +786,109 @@ def test_multiplex_missing_secondary_does_not_fall_back_to_shared(tmp_path): assert default_ad is shared assert sec_ad is not shared assert not sec_ad + + +def test_multiplex_ticker_isolates_profile_failures(tmp_path): + """A failing profile's tick must not skip healthy siblings in the same + cycle, nor darken their status (#74878).""" + from cron.jobs import get_ticker_last_error, record_ticker_error, use_cron_store + from cron.scheduler_provider import InProcessCronScheduler + from hermes_constants import get_hermes_home + + failing_home = tmp_path / "failing" + healthy_home = tmp_path / "healthy" + for home in (failing_home, healthy_home): + (home / "cron").mkdir(parents=True) + with use_cron_store(home): + record_ticker_error("RuntimeError: stale failure") + + stop = threading.Event() + tick_homes: list[str] = [] + + def _tick(*args, **kwargs): + home = str(get_hermes_home()) + tick_homes.append(home) + if home == str(failing_home): + raise RuntimeError("profile-local failure") + stop.set() + return 0 + + provider = InProcessCronScheduler() + with patch("cron.scheduler.tick", side_effect=_tick): + thread = threading.Thread( + target=provider.start, + args=(stop,), + kwargs={ + "interval": 0, + "profile_homes": [("failing", failing_home), ("healthy", healthy_home)], + }, + daemon=True, + ) + thread.start() + thread.join(timeout=5) + stop.set() + thread.join(timeout=5) + + assert not thread.is_alive() + assert str(healthy_home) in tick_homes, "healthy sibling was skipped" + assert not (failing_home / "cron" / "ticker_last_success").exists() + assert (healthy_home / "cron" / "ticker_last_success").exists() + with use_cron_store(failing_home): + assert get_ticker_last_error() == "RuntimeError: profile-local failure" + with use_cron_store(healthy_home): + assert get_ticker_last_error() is None + + +def test_multiplex_recovery_isolates_profile_failures(tmp_path): + """A startup-recovery error in one profile's ledger must not kill the + ticker thread before it ever ticks (#74878).""" + import sqlite3 + + from cron.scheduler_provider import InProcessCronScheduler + from hermes_constants import get_hermes_home + + failing_home = tmp_path / "failing" + healthy_home = tmp_path / "healthy" + for home in (failing_home, healthy_home): + (home / "cron").mkdir(parents=True) + + stop = threading.Event() + recovery_homes: list[str] = [] + tick_homes: list[str] = [] + + def _recover(): + home = str(get_hermes_home()) + recovery_homes.append(home) + if home == str(failing_home): + raise sqlite3.OperationalError("unable to open database file") + return 0 + + def _tick(*args, **kwargs): + tick_homes.append(str(get_hermes_home())) + if len(tick_homes) >= 2: + stop.set() + return 0 + + provider = InProcessCronScheduler() + with ( + patch.object(provider, "recover_interrupted", side_effect=_recover), + patch("cron.scheduler.tick", side_effect=_tick), + ): + thread = threading.Thread( + target=provider.start, + args=(stop,), + kwargs={ + "interval": 0, + "profile_homes": [("failing", failing_home), ("healthy", healthy_home)], + }, + daemon=True, + ) + thread.start() + thread.join(timeout=5) + stop.set() + thread.join(timeout=5) + + assert not thread.is_alive() + assert recovery_homes == [str(failing_home), str(healthy_home)] + # The failing profile stays in rotation: its ledger may still hold jobs. + assert set(tick_homes) == {str(failing_home), str(healthy_home)} diff --git a/tests/cron/test_script_claim_heartbeat.py b/tests/cron/test_script_claim_heartbeat.py index 7b001d8bdb..9cd09960a9 100644 --- a/tests/cron/test_script_claim_heartbeat.py +++ b/tests/cron/test_script_claim_heartbeat.py @@ -447,7 +447,7 @@ def test_lost_fire_claim_stops_stale_delivery(monkeypatch): monkeypatch.setattr(scheduler, "heartbeat_fire_claim", _heartbeat) monkeypatch.setattr(scheduler, "run_job", _run_job) monkeypatch.setattr(scheduler, "claim_dispatch", lambda job_id: True) - monkeypatch.setattr(scheduler, "mark_execution_running", lambda execution_id: None) + monkeypatch.setattr(scheduler, "mark_execution_running", lambda execution_id: {}) monkeypatch.setattr(scheduler, "finish_execution", lambda *args, **kwargs: None) save_output = MagicMock() deliver_result = MagicMock() @@ -627,7 +627,7 @@ def test_terminal_owner_cas_failure_marks_ledger_ownership_lost(monkeypatch): finish = MagicMock() monkeypatch.setattr(scheduler, "heartbeat_fire_claim", lambda *args, **kwargs: True) monkeypatch.setattr(scheduler, "claim_dispatch", lambda *_args, **_kwargs: True) - monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_args: None) + monkeypatch.setattr(scheduler, "mark_execution_running", lambda *_args: {}) monkeypatch.setattr( scheduler, "run_job", diff --git a/tests/cron/test_suggestions.py b/tests/cron/test_suggestions.py index 605686f52c..3abaf54d31 100644 --- a/tests/cron/test_suggestions.py +++ b/tests/cron/test_suggestions.py @@ -19,7 +19,6 @@ def store(tmp_path, monkeypatch): home = tmp_path / ".hermes" home.mkdir() monkeypatch.setenv("HERMES_HOME", str(home)) - # Reload so module-level CRON_DIR/SUGGESTIONS_FILE pick up the temp home. import hermes_constants importlib.reload(hermes_constants) import cron.suggestions as s @@ -38,6 +37,51 @@ def _add(store, key="k1", title="Test", source="catalog", schedule="0 9 * * *"): class TestStore: + def test_explicit_file_override_wins_over_profile_home(self, tmp_path, monkeypatch): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + import cron.suggestions as suggestions_mod + + explicit_file = tmp_path / "explicit" / "suggestions.json" + profile_home = tmp_path / "profile" + monkeypatch.setattr(suggestions_mod, "SUGGESTIONS_FILE", explicit_file) + + token = set_hermes_home_override(profile_home) + try: + _add(suggestions_mod, key="explicit-file") + finally: + reset_hermes_home_override(token) + + assert explicit_file.exists() + assert not (profile_home / "cron" / "suggestions.json").exists() + + def test_profile_override_routes_writes_to_current_home(self, tmp_path): + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + import cron.suggestions as suggestions_mod + + profile_a = tmp_path / "profile-a" + profile_b = tmp_path / "profile-b" + + import_token = set_hermes_home_override(profile_a) + try: + importlib.reload(suggestions_mod) + finally: + reset_hermes_home_override(import_token) + + runtime_token = set_hermes_home_override(profile_b) + try: + _add(suggestions_mod, key="profile-b") + finally: + reset_hermes_home_override(runtime_token) + + assert (profile_b / "cron" / "suggestions.json").exists() + assert not (profile_a / "cron" / "suggestions.json").exists() + def test_add_and_list_pending(self, store): rec = _add(store) assert rec is not None diff --git a/tests/gateway/feishu_helpers.py b/tests/gateway/feishu_helpers.py index ae8a4bfc37..f9b7822a21 100644 --- a/tests/gateway/feishu_helpers.py +++ b/tests/gateway/feishu_helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import threading from types import SimpleNamespace from typing import Any, Optional @@ -34,6 +35,7 @@ def make_adapter_skeleton( allow_bots: str = "none", require_mention: bool = True, group_policy: str = "allowlist", + allow_all_dm: bool = False, ) -> Any: from plugins.platforms.feishu.adapter import FeishuAdapter @@ -48,6 +50,7 @@ def make_adapter_skeleton( adapter._default_group_policy = group_policy adapter._allowed_group_users = frozenset() adapter._allow_bots = allow_bots + adapter._allow_all_dm = allow_all_dm adapter._require_mention = require_mention return adapter @@ -57,6 +60,7 @@ def install_dedup_state(adapter: Any, seen: Optional[dict] = None) -> None: adapter._seen_message_order = list((seen or {}).keys()) adapter._dedup_cache_size = 100 adapter._dedup_lock = threading.Lock() + adapter._dedup_persist_lock = asyncio.Lock() adapter._dedup_state_path = None adapter._persist_seen_message_ids = lambda: None diff --git a/tests/gateway/relay/test_relay_passthrough.py b/tests/gateway/relay/test_relay_passthrough.py index 2150e9bf0b..a8e27e4335 100644 --- a/tests/gateway/relay/test_relay_passthrough.py +++ b/tests/gateway/relay/test_relay_passthrough.py @@ -44,7 +44,7 @@ def adapter(): return RelayAdapter(PlatformConfig(), _desc(), transport=StubConnector(_desc())) -def _interaction_forward(payload: dict) -> PassthroughForward: +def _interaction_forward(payload: dict, *, profile: str | None = None) -> PassthroughForward: body = json.dumps(payload).encode("utf-8") return PassthroughForward( platform="discord", @@ -53,6 +53,7 @@ def _interaction_forward(payload: dict) -> PassthroughForward: path="/interactions/discord/appShared", headers=[("content-type", "application/json")], body=body, + profile=profile, ) @@ -75,6 +76,27 @@ def test_passthrough_from_wire_byte_preserves_body(): assert fwd.headers == [("content-type", "application/json")] +def test_passthrough_from_wire_stamps_routed_profile(): + """A connector-routed profile on the wire frame lands on PassthroughForward. + + Mirrors _event_from_wire's profile stamping for the ``inbound`` frame + (#60586) — the passthrough plane needs the same carry-through so a + Team-Gateway's Discord interactions route to the same profile a plain + message would. + """ + wire = { + "platform": "discord", + "botId": "appShared", + "method": "POST", + "path": "/interactions/discord/appShared", + "headers": [], + "bodyB64": "", + "profile": "reviewer", + } + fwd = _passthrough_from_wire(wire) + assert fwd.profile == "reviewer" + + @pytest.mark.asyncio async def test_connect_wires_passthrough_handler_over_ws(adapter): """connect() registers the passthrough handler on the transport so a @@ -137,6 +159,40 @@ async def test_discord_interaction_routes_through_handle_message(adapter, monkey assert adapter._platform_by_chat.get("chan-9") == "discord" +@pytest.mark.asyncio +async def test_discord_interaction_stamps_routed_profile(adapter, monkeypatch): + """A connector-routed profile on the passthrough forward lands on the + resulting event's SessionSource, the same way it does for a plain relayed + message (#60586) — so a Team-Gateway's Discord slash-command/button/modal + routes to the same profile a plain message would, instead of always + falling back to agent:main.""" + await adapter.connect() + stub = adapter._transport + + seen = [] + + async def fake_handle(event): + seen.append(event) + + monkeypatch.setattr(adapter, "handle_message", fake_handle) + + fwd = _interaction_forward( + { + "id": "interaction-2", + "type": 2, # APPLICATION_COMMAND + "channel_id": "chan-9", + "guild_id": "guild-7", + "data": {"name": "summarize"}, + "member": {"user": {"id": "user-3", "username": "ben"}}, + }, + profile="reviewer", + ) + await stub.push_passthrough(fwd, buffer_id=None) + + assert len(seen) == 1 + assert seen[0].source.profile == "reviewer" + + @pytest.mark.asyncio async def test_application_command_subcommand_nesting_renders_names_then_values( adapter, monkeypatch diff --git a/tests/gateway/test_42039_duplicate_user_message.py b/tests/gateway/test_42039_duplicate_user_message.py index 13a73181f6..3ddc30d90e 100644 --- a/tests/gateway/test_42039_duplicate_user_message.py +++ b/tests/gateway/test_42039_duplicate_user_message.py @@ -24,7 +24,7 @@ import pytest import gateway.run as gateway_run from gateway.config import GatewayConfig, Platform from gateway.platforms.base import MessageEvent -from gateway.session import SessionEntry, SessionSource +from gateway.session import SessionEntry, SessionSource, TranscriptReadError def _bootstrap(monkeypatch, tmp_path): @@ -185,6 +185,24 @@ async def test_not_new_messages_skip_db_when_agent_has_session_db( ) +@pytest.mark.asyncio +async def test_transcript_read_failure_stops_turn_before_agent_or_append( + monkeypatch, tmp_path +): + runner = _bootstrap(monkeypatch, tmp_path) + runner.session_store.load_transcript.side_effect = TranscriptReadError("sess-dedup") + runner._run_agent = AsyncMock() + + response = await runner._handle_message_with_agent( + _event(), _source(), "agent:main:telegram:group:-1001:12345", 1 + ) + + assert "history is temporarily unavailable" in response + assert "not processed" in response + runner._run_agent.assert_not_awaited() + runner.session_store.append_to_transcript.assert_not_called() + + # ── Post-stream MEDIA delivery keeps prior-turn deduplication ────────── diff --git a/tests/gateway/test_64674_multiplex_primary_token_scope.py b/tests/gateway/test_64674_multiplex_primary_token_scope.py index 44398aec19..b11fedb177 100644 --- a/tests/gateway/test_64674_multiplex_primary_token_scope.py +++ b/tests/gateway/test_64674_multiplex_primary_token_scope.py @@ -119,6 +119,60 @@ class TestPlatformHasBotCredential: Platform.TELEGRAM, PlatformConfig(enabled=True, token=None) ) is False + def test_matrix_password_login_is_a_credential(self): + """Matrix password auth has no .token but is fully reconnectable. + + MATRIX_USER_ID + MATRIX_PASSWORD with no MATRIX_ACCESS_TOKEN is a + supported setup (build_config puts it on extra). Treating it as + credential-less evicted it from the reconnect queue on the first + transient failure, so a momentary DNS blip took Matrix down until + the gateway was restarted by hand. + """ + from gateway.run import _platform_has_bot_credential + + cfg = PlatformConfig(enabled=True) + cfg.extra = { + "homeserver": "https://matrix.example.org", + "user_id": "@bot:matrix.example.org", + "password": "hunter2", + } + assert _platform_has_bot_credential(Platform.MATRIX, cfg) is True + + @pytest.mark.parametrize( + "extra", + [ + {}, + {"homeserver": "https://matrix.example.org", "password": "hunter2"}, + {"user_id": "@bot:matrix.example.org", "password": "hunter2"}, + {"homeserver": "https://matrix.example.org", "user_id": "@bot:m.example.org"}, + {"homeserver": " ", "user_id": " ", "password": " "}, + ], + ids=["empty", "no-user-id", "no-homeserver", "no-password", "blank"], + ) + def test_matrix_incomplete_password_config_still_dropped(self, extra, monkeypatch): + """An incomplete Matrix config can never connect — keep evicting it. + + Guards the #64674 intent, and specifically pins "read extra, not the + environment". A fully-populated MATRIX_* environment is set here on + purpose: on a real host those vars are present (build_config exports + them, and importing gateway.run loads ~/.hermes/.env), so an + implementation that falls back to os.getenv would report every Matrix + config as credentialed and never evict anything. + + conftest sandboxes HERMES_HOME and scrubs MATRIX_* from the + environment, so without these explicit setenv calls this test would + pass against an env-reading implementation and guard nothing. + """ + from gateway.run import _platform_has_bot_credential + + monkeypatch.setenv("MATRIX_HOMESERVER", "https://env.example.org") + monkeypatch.setenv("MATRIX_USER_ID", "@envbot:env.example.org") + monkeypatch.setenv("MATRIX_PASSWORD", "env-password") + + cfg = PlatformConfig(enabled=True) + cfg.extra = dict(extra) + assert _platform_has_bot_credential(Platform.MATRIX, cfg) is False + class TestPrimaryStartupSkipsEmptyTokenUnderMultiplex: @pytest.mark.asyncio diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index ecb49cd7a8..56e298dc06 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -611,7 +611,9 @@ class TestDisconnectedAgentReap: adapter._active_run_agents["run_x"] = agent request = MagicMock() + request.headers = {} request.match_info = {"run_id": "run_x"} + adapter._run_owners["run_x"] = adapter._run_idempotency_scope(request) resp = await adapter._handle_stop_run(request) assert resp.status == 200 diff --git a/tests/gateway/test_api_server_runs.py b/tests/gateway/test_api_server_runs.py index 0f9f573ac4..8d68f919c9 100644 --- a/tests/gateway/test_api_server_runs.py +++ b/tests/gateway/test_api_server_runs.py @@ -22,6 +22,7 @@ from aiohttp.test_utils import TestClient, TestServer from gateway.config import PlatformConfig from gateway.platforms.api_server import ( APIServerAdapter, + _api_request_profile, _approval_event_choices, cors_middleware, security_headers_middleware, @@ -68,6 +69,13 @@ def _make_adapter(api_key: str = "") -> APIServerAdapter: return adapter +def _claim_run(adapter: APIServerAdapter, run_id: str) -> None: + """Stamp *run_id* as owned by the unprefixed (default) request scope.""" + request = MagicMock() + request.headers = {} + adapter._run_owners[run_id] = adapter._run_idempotency_scope(request) + + def _create_runs_app(adapter: APIServerAdapter) -> web.Application: """Create an aiohttp app with /v1/runs routes registered.""" mws = [mw for mw in (cors_middleware, security_headers_middleware) if mw is not None] @@ -468,6 +476,7 @@ class TestSteerRun: adapter._active_run_agents["run_123"] = agent adapter._run_streams["run_123"] = queue adapter._set_run_status("run_123", "running") + _claim_run(adapter, "run_123") async with TestClient(TestServer(app)) as cli: resp = await cli.post("/v1/runs/run_123/steer", json={"input": "tighten the ending"}) @@ -500,6 +509,7 @@ class TestSteerRun: async def test_steer_inactive_run_returns_409(self, adapter): app = _create_runs_app(adapter) adapter._set_run_status("run_done", "completed") + _claim_run(adapter, "run_done") async with TestClient(TestServer(app)) as cli: resp = await cli.post("/v1/runs/run_done/steer", json={"input": "hello"}) @@ -515,6 +525,7 @@ class TestSteerRun: agent.steer.return_value = True adapter._active_run_agents["run_123"] = agent adapter._set_run_status("run_123", "running") + _claim_run(adapter, "run_123") async with TestClient(TestServer(app)) as cli: resp = await cli.post("/v1/runs/run_123/steer", json={"input": ""}) @@ -681,6 +692,92 @@ class TestRunLifecycleSweep: mock_agent.interrupt.assert_called_once_with("Stop requested via API") +# --------------------------------------------------------------------------- +# Run ownership across served profiles (#93689 / #90415) +# --------------------------------------------------------------------------- + + +class TestRunOwnershipAcrossProfiles: + """Every served profile holds a valid key under multiplex; only the + creating profile may see or control a run.""" + + KEYS = {"victim": "sk-victim-profile-key-0001", "attacker": "sk-attacker-profile-key-01"} + + @classmethod + def _profile_app(cls, adapter: APIServerAdapter) -> web.Application: + """Runs routes behind a stand-in for the /p// middleware: + the routed profile arrives in ``X-Test-Profile`` and each profile + authenticates with its own key, as under gateway.multiplex_profiles.""" + + @web.middleware + async def stamp_profile(request, handler): + token = _api_request_profile.set(request.headers.get("X-Test-Profile")) + try: + return await handler(request) + finally: + _api_request_profile.reset(token) + + adapter._expected_api_key = lambda: cls.KEYS.get(_api_request_profile.get(), "") + app = _create_runs_app(adapter) + app.middlewares.append(stamp_profile) + app.router.add_post( + "/api/sessions/{session_id}/chat/stream", adapter._handle_session_chat_stream + ) + return app + + @pytest.mark.asyncio + async def test_unstamped_run_state_fails_closed(self, adapter): + """Run state with no owner stamp is nobody's — not everybody's.""" + app = _create_runs_app(adapter) + adapter._active_run_agents["run_unstamped"] = MagicMock() + adapter._set_run_status("run_unstamped", "running") + + async with TestClient(TestServer(app)) as cli: + get_resp = await cli.get("/v1/runs/run_unstamped") + stop_resp = await cli.post("/v1/runs/run_unstamped/stop") + + assert (get_resp.status, stop_resp.status) == (404, 404) + + @pytest.mark.asyncio + async def test_session_chat_stream_run_is_owned_by_creating_profile(self, adapter): + """The session-chat-stream run mint claims ownership like /v1/runs does.""" + app = self._profile_app(adapter) + victim = {"X-Test-Profile": "victim", "Authorization": f"Bearer {self.KEYS['victim']}"} + attacker = {"X-Test-Profile": "attacker", "Authorization": f"Bearer {self.KEYS['attacker']}"} + gate = asyncio.Event() + + async def slow_run_agent(**kwargs): + await gate.wait() + return {"final_response": "ok"}, {} + + async with TestClient(TestServer(app)) as cli: + with ( + patch.object(adapter, "_get_existing_session_or_404", new=AsyncMock(return_value=({"id": "s1"}, None))), + patch.object(adapter, "_conversation_history_for_session", new=AsyncMock(return_value=[])), + patch.object(adapter, "_run_agent", new=slow_run_agent), + ): + stream = await cli.post( + "/api/sessions/s1/chat/stream", json={"message": "hi"}, headers=victim + ) + await stream.content.readline() + (run_id,) = list(adapter._run_statuses) + assert run_id in adapter._run_owners + + foreign_get = await cli.get(f"/v1/runs/{run_id}", headers=attacker) + foreign_stop = await cli.post(f"/v1/runs/{run_id}/stop", headers=attacker) + own_get = await cli.get(f"/v1/runs/{run_id}", headers=victim) + assert (foreign_get.status, foreign_stop.status, own_get.status) == (404, 404, 200) + + gate.set() + await stream.text() + + # The owner outlives the terminal status and goes with the last surface. + assert run_id in adapter._run_owners + adapter._run_statuses.pop(run_id) + adapter._release_run_owner_if_forgotten(run_id) + assert run_id not in adapter._run_owners + + # --------------------------------------------------------------------------- # POST /v1/runs/{run_id}/stop — interrupt a running agent # --------------------------------------------------------------------------- diff --git a/tests/gateway/test_api_server_toolset.py b/tests/gateway/test_api_server_toolset.py index fb9fe9176b..debdbbfb52 100644 --- a/tests/gateway/test_api_server_toolset.py +++ b/tests/gateway/test_api_server_toolset.py @@ -17,11 +17,11 @@ class TestHermesApiServerToolset: def test_toolset_includes_core_tools(self): tools = resolve_toolset("hermes-api-server") expected = [ - "terminal", "process", + "terminal", "process_manage", "read_file", "write_file", "patch", "search_files", "vision_analyze", "image_generate", "execute_code", "delegate_task", - "todo", "memory", "session_search", "cronjob", + "todo_list", "memory", "session_search", "cronjob_manage", ] for tool in expected: assert tool in tools, f"Missing expected tool: {tool}" diff --git a/tests/gateway/test_async_media_cache.py b/tests/gateway/test_async_media_cache.py new file mode 100644 index 0000000000..1745d9fbe6 --- /dev/null +++ b/tests/gateway/test_async_media_cache.py @@ -0,0 +1,99 @@ +import asyncio +import threading +from pathlib import Path + +import pytest + +import gateway.platforms.base as base + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("async_name", "sync_name", "args"), + [ + ("cache_image_from_bytes_async", "cache_image_from_bytes", (b"data", ".png")), + ("cache_audio_from_bytes_async", "cache_audio_from_bytes", (b"data", ".ogg")), + ("cache_video_from_bytes_async", "cache_video_from_bytes", (b"data", ".mp4")), + ( + "cache_document_from_bytes_async", + "cache_document_from_bytes", + (b"data", "report.pdf"), + ), + ], +) +async def test_async_cache_wrappers_keep_event_loop_responsive( + monkeypatch, async_name, sync_name, args +): + loop_thread = threading.get_ident() + cache_started = threading.Event() + release_cache = threading.Event() + observed = {} + + def blocking_cache(*call_args): + observed["thread"] = threading.get_ident() + observed["args"] = call_args + cache_started.set() + observed["ticker_ran_during_cache"] = release_cache.wait(timeout=1) + return "cached" + + monkeypatch.setattr(base, sync_name, blocking_cache) + + async def ticker(): + while not cache_started.is_set(): + await asyncio.sleep(0) + release_cache.set() + + ticker_task = asyncio.create_task(ticker()) + result = await getattr(base, async_name)(*args) + await ticker_task + + assert result == "cached" + assert observed["args"] == args + assert observed["thread"] != loop_thread + assert observed["ticker_ran_during_cache"] is True + + +@pytest.mark.asyncio +async def test_async_cache_wrapper_propagates_validation_errors(monkeypatch): + def reject_image(data, ext): + raise ValueError("invalid image") + + monkeypatch.setattr(base, "cache_image_from_bytes", reject_image) + + with pytest.raises(ValueError, match="invalid image"): + await base.cache_image_from_bytes_async(b"not-an-image", ".png") + + +@pytest.mark.asyncio +async def test_cache_media_bytes_async_runs_off_loop_and_forwards_kwargs(monkeypatch): + loop_thread = threading.get_ident() + observed = {} + + def fake_cache_media_bytes(data, *, filename="", mime_type="", default_kind=None): + observed["thread"] = threading.get_ident() + observed["call"] = (data, filename, mime_type, default_kind) + return "cached-media" + + monkeypatch.setattr(base, "cache_media_bytes", fake_cache_media_bytes) + + result = await base.cache_media_bytes_async( + b"payload", filename="report.pdf", mime_type="application/pdf", default_kind="document" + ) + + assert result == "cached-media" + assert observed["call"] == (b"payload", "report.pdf", "application/pdf", "document") + assert observed["thread"] != loop_thread + + +@pytest.mark.asyncio +async def test_async_cache_wrapper_uses_active_profile_home(monkeypatch, tmp_path): + profile_home = tmp_path / "profile" + monkeypatch.setenv("HERMES_HOME", str(profile_home)) + + cached = await base.cache_image_from_bytes_async( + b"\x89PNG\r\n\x1a\nminimal", ".png" + ) + + cached_path = Path(cached) + assert cached_path.parent == profile_home / "cache" / "images" + assert cached_path.read_bytes() == b"\x89PNG\r\n\x1a\nminimal" diff --git a/tests/gateway/test_bluebubbles.py b/tests/gateway/test_bluebubbles.py index 695a26297d..7d95190cd0 100644 --- a/tests/gateway/test_bluebubbles.py +++ b/tests/gateway/test_bluebubbles.py @@ -259,13 +259,13 @@ class TestBlueBubblesAttachmentDownload: cached_path = None - def mock_cache_image(data, ext): + async def mock_cache_image(data, ext): nonlocal cached_path cached_path = f"/tmp/test_image{ext}" return cached_path monkeypatch.setattr( - "gateway.platforms.bluebubbles.cache_image_from_bytes", + "gateway.platforms.bluebubbles.cache_image_from_bytes_async", mock_cache_image, ) diff --git a/tests/gateway/test_buzz_adapter.py b/tests/gateway/test_buzz_adapter.py index 18587187f0..bee8af29e3 100644 --- a/tests/gateway/test_buzz_adapter.py +++ b/tests/gateway/test_buzz_adapter.py @@ -1134,8 +1134,8 @@ class TestInboundAttachments: ) monkeypatch.setattr( _buzz_mod, - "cache_media_bytes", - MagicMock(side_effect=OSError(36, "File name too long")), + "cache_media_bytes_async", + AsyncMock(side_effect=OSError(36, "File name too long")), ) adapter = _make_adapter() diff --git a/tests/gateway/test_choice_picker.py b/tests/gateway/test_choice_picker.py index a2c9a52961..c8e6712ec0 100644 --- a/tests/gateway/test_choice_picker.py +++ b/tests/gateway/test_choice_picker.py @@ -126,7 +126,7 @@ class TestFastChoicePicker: assert result is None values = [c["value"] for c in adapter.calls[0]["choices"]] - assert values == ["fast", "normal"] + assert values == ["fast", "normal", "auto", "cold"] @pytest.mark.asyncio async def test_fast_picker_selection_is_session_scoped(self, tmp_path, monkeypatch): diff --git a/tests/gateway/test_codex_hygiene_compaction.py b/tests/gateway/test_codex_hygiene_compaction.py index 71dd907f3b..7fa795d694 100644 --- a/tests/gateway/test_codex_hygiene_compaction.py +++ b/tests/gateway/test_codex_hygiene_compaction.py @@ -336,3 +336,46 @@ def test_manual_compress_without_live_thread_reports_honestly(): host._compress_codex_app_server_session("tg:123", "sess-1") ) assert "Nothing to compact" in reply + + +# --------------------------------------------------------------------------- +# Multiplexed gateway: the hygiene worker must see the caller's ContextVars +# (profile secret scope / HERMES_HOME override). A bare run_in_executor worker +# starts with an EMPTY Context, so get_secret(_API_KEY) inside the +# summary path fails closed and every hygiene compaction degrades to a lossy +# truncation (#100849 bundle). +# --------------------------------------------------------------------------- + +def test_hygiene_worker_inherits_caller_contextvars(tmp_path): + import contextvars + import threading + + marker = contextvars.ContextVar("hygiene_scope_marker", default=None) + seen = {} + + class ScopeProbeAgent(LiveCodexAgent): + def _compress_context(self, messages, system_message, **kwargs): + seen["value"] = marker.get() + seen["thread"] = threading.current_thread().name + return super()._compress_context(messages, system_message, **kwargs) + + agent = ScopeProbeAgent(mode="hermes") + key = "tg:ctx" + gw, _db = _gateway(tmp_path, key, agent) + + async def _scoped(): + token = marker.set("profile-scope") + try: + return await run_codex_hygiene_compaction( + gw, key, agent.session_id, auto_mode="hermes", + history=_history(), approx_tokens=345_000, timeout_seconds=30.0, + ) + finally: + marker.reset(token) + + assert asyncio.run(_scoped()) == "compacted" + assert seen["thread"] != "MainThread", "compaction must still run off-loop" + assert seen["value"] == "profile-scope", ( + "hygiene worker lost the caller's ContextVars — under multiplex_profiles " + "this is the UnscopedSecretError / lossy-truncation regression" + ) diff --git a/tests/gateway/test_compaction_heartbeat_gateway_filter.py b/tests/gateway/test_compaction_heartbeat_gateway_filter.py new file mode 100644 index 0000000000..1689157162 --- /dev/null +++ b/tests/gateway/test_compaction_heartbeat_gateway_filter.py @@ -0,0 +1,36 @@ +"""The compaction heartbeat (#98371) must be classified like COMPACTION_STATUS. + +Chat platforms suppress routine compression chatter unless +``compression.progress_notices`` is enabled; a heartbeat that slipped past +that gate would post a bubble per tick to Telegram/Discord. The TUI gateway, +by contrast, must keep receiving it so idle-turn watchdogs see progress. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +from agent.conversation_compression import ( + COMPACTION_HEARTBEAT_STATUS, + COMPACTION_STATUS, + is_compaction_progress_status, +) +from gateway.run import _prepare_gateway_status_message + + +def _telegram(): + return SimpleNamespace(value="telegram") + + +def test_heartbeat_is_compaction_progress_for_tui_retagging(): + assert is_compaction_progress_status(COMPACTION_HEARTBEAT_STATUS) + + +def test_heartbeat_suppressed_on_chat_platforms_by_default(): + with patch("gateway.run._gateway_compression_progress_notices_enabled", return_value=False): + assert _prepare_gateway_status_message(_telegram(), "lifecycle", COMPACTION_STATUS) is None + assert _prepare_gateway_status_message(_telegram(), "lifecycle", COMPACTION_HEARTBEAT_STATUS) is None + + +def test_heartbeat_passes_when_progress_notices_enabled(): + with patch("gateway.run._gateway_compression_progress_notices_enabled", return_value=True): + assert _prepare_gateway_status_message(_telegram(), "lifecycle", COMPACTION_HEARTBEAT_STATUS) == COMPACTION_HEARTBEAT_STATUS diff --git a/tests/gateway/test_config.py b/tests/gateway/test_config.py index 2e4285f68f..480e26d48c 100644 --- a/tests/gateway/test_config.py +++ b/tests/gateway/test_config.py @@ -1409,3 +1409,58 @@ class TestApiServerEnvOverride: assert config.platforms[Platform.API_SERVER].enabled is False # The key is still wired through for the shared listener. assert config.platforms[Platform.API_SERVER].extra.get("key") == api_server_key + + +class TestWebhookEnvOverride: + def test_env_key_does_not_reenable_explicitly_disabled_webhook(self): + """An explicit ``platforms.webhook.enabled: false`` must survive + _apply_env_overrides() even when WEBHOOK_ENABLED is truthy in the env. + + Regression (#85637): _apply_env_overrides() force-set + webhook.enabled = True whenever WEBHOOK_ENABLED was truthy. In + multiplex mode a secondary profile pins ``webhook.enabled: false`` so + it shares the default profile's listener instead of binding its own + port, but it still inherits the process-level WEBHOOK_ENABLED + (or carries one in its own .env). The unconditional re-enable + flipped it back on and tripped the MultiplexConfigError check. + + The fix honors the explicit disable, flagged by ``_enabled_explicit`` + in the platform's extra (set when the config.yaml pins enabled). + The MSGRAPH_WEBHOOK branch shares the shape and the fix. + """ + config = GatewayConfig( + platforms={ + Platform.WEBHOOK: PlatformConfig( + enabled=False, + extra={"_enabled_explicit": True}, + ), + Platform.MSGRAPH_WEBHOOK: PlatformConfig( + enabled=False, + extra={"_enabled_explicit": True}, + ), + }, + ) + + with patch.dict( + os.environ, + { + "WEBHOOK_ENABLED": "true", + "WEBHOOK_PORT": "9999", + "WEBHOOK_SECRET": "shared-secret", + "MSGRAPH_WEBHOOK_ENABLED": "true", + "MSGRAPH_WEBHOOK_PORT": "9998", + }, + clear=True, + ): + _apply_env_overrides(config) + + # Explicit disable wins over the env-var presence. + assert config.platforms[Platform.WEBHOOK].enabled is False + assert config.platforms[Platform.MSGRAPH_WEBHOOK].enabled is False + assert config.platforms[Platform.MSGRAPH_WEBHOOK].extra.get("port") == 9998 + # Port/secret are still wired through for the shared listener. + assert config.platforms[Platform.WEBHOOK].extra.get("port") == 9999 + assert ( + config.platforms[Platform.WEBHOOK].extra.get("secret") + == "shared-secret" + ) diff --git a/tests/gateway/test_cron_delivery_housekeeping.py b/tests/gateway/test_cron_delivery_housekeeping.py new file mode 100644 index 0000000000..1228b04772 --- /dev/null +++ b/tests/gateway/test_cron_delivery_housekeeping.py @@ -0,0 +1,157 @@ +"""Gateway-independent draining of restart-safe cron deliveries.""" + +from contextlib import contextmanager +from types import SimpleNamespace + +import cron.scheduler as scheduler +import gateway.run as gateway_run + + +class _OneTickStopEvent: + def __init__(self): + self.waited = False + + def is_set(self): + return self.waited + + def wait(self, timeout=None): + self.waited = True + return True + + +def test_gateway_housekeeping_drains_cron_delivery_with_live_adapters(monkeypatch): + adapters = {"discord": object()} + loop = object() + calls = [] + monkeypatch.setattr( + scheduler, + "drain_delivery_queue", + lambda live_adapters, live_loop: calls.append((live_adapters, live_loop)), + raising=False, + ) + + gateway_run._start_gateway_housekeeping( + _OneTickStopEvent(), adapters=adapters, loop=loop, interval=0 + ) + + assert calls == [(adapters, loop)] + + +def test_gateway_housekeeping_drains_cron_delivery_without_connected_adapters(monkeypatch): + adapters = {} + loop = object() + calls = [] + monkeypatch.setattr( + scheduler, + "drain_delivery_queue", + lambda live_adapters, live_loop: calls.append((live_adapters, live_loop)), + raising=False, + ) + + gateway_run._start_gateway_housekeeping( + _OneTickStopEvent(), adapters=adapters, loop=loop, interval=0 + ) + + assert calls == [(adapters, loop)] + + +def test_multiplex_housekeeping_scopes_primary_and_drains_each_profile( + tmp_path, monkeypatch +): + root_adapters = {} + secondary_adapters = {} + runner = SimpleNamespace( + config=SimpleNamespace(multiplex_profiles=True), + adapters=root_adapters, + _profile_adapters={"secondary": secondary_adapters}, + ) + root_home = tmp_path / "root" + secondary_home = tmp_path / "secondary" + calls = [] + + monkeypatch.setattr(gateway_run, "get_hermes_home", lambda: root_home) + + monkeypatch.setattr( + gateway_run, + "_handoff_watch_scopes", + lambda _runner: [(None, None), ("secondary", secondary_home)], + ) + + @contextmanager + def fake_scope(home): + calls.append(("scope", home)) + yield + + monkeypatch.setattr(gateway_run, "_profile_runtime_scope", fake_scope) + monkeypatch.setattr( + scheduler, + "drain_delivery_queue", + lambda adapters, loop: calls.append(("drain", adapters)), + ) + + gateway_run._start_gateway_housekeeping( + _OneTickStopEvent(), + adapters=root_adapters, + loop=object(), + interval=0, + runner=runner, + ) + + assert calls == [ + ("scope", root_home), + ("drain", root_adapters), + ("scope", secondary_home), + ("drain", secondary_adapters), + ] + + +def test_multiplex_housekeeping_uses_primary_routes_for_credentialless_satellite( + tmp_path, monkeypatch +): + root_adapters = {"slack": object()} + secondary_home = tmp_path / "secondary" + runner = SimpleNamespace( + config=SimpleNamespace(multiplex_profiles=True), + adapters=root_adapters, + _profile_adapters={"secondary": {}}, + ) + calls = [] + routed = object() + + monkeypatch.setattr( + gateway_run, + "_handoff_watch_scopes", + lambda _runner: [(None, None), ("secondary", secondary_home)], + ) + + @contextmanager + def fake_scope(_home): + yield + + class FakeSharedRouteAdapters: + def __new__(cls, adapters, routes): + calls.append(("routed", adapters, routes)) + return routed + + monkeypatch.setattr(gateway_run, "_profile_runtime_scope", fake_scope) + monkeypatch.setattr(scheduler, "SharedRouteAdapters", FakeSharedRouteAdapters) + monkeypatch.setattr( + scheduler, + "_primary_profile_routes_for_current_home", + lambda: ["route-to-secondary"], + ) + monkeypatch.setattr( + scheduler, + "drain_delivery_queue", + lambda adapters, _loop: calls.append(("drain", adapters)), + ) + + gateway_run._drain_restart_safe_cron_deliveries( + root_adapters, object(), runner + ) + + assert calls == [ + ("drain", root_adapters), + ("routed", root_adapters, ["route-to-secondary"]), + ("drain", routed), + ] diff --git a/tests/gateway/test_cron_interrupt_notification.py b/tests/gateway/test_cron_interrupt_notification.py index bde4738c20..f157e17478 100644 --- a/tests/gateway/test_cron_interrupt_notification.py +++ b/tests/gateway/test_cron_interrupt_notification.py @@ -120,6 +120,37 @@ class TestNotifyInterruptedCronJobs: assert sent == 0 assert adapter.sent == [] + @pytest.mark.asyncio + async def test_failure_deliver_local_suppresses_interrupt_notice(self): + """Interrupted notices are failure-category engine status (NS-788): + a job with failure_deliver='local' opted out of failure pings, and + the shutdown notice must honor that. Real target resolution — no + _resolve_delivery_targets patch — so the failure_deliver override is + actually exercised.""" + runner, adapter = make_restart_runner() + _bind_notifier(runner) + job = dict(_telegram_job(), failure_deliver="local") + + with patch("cron.jobs.get_job", return_value=job): + sent = await runner._notify_interrupted_cron_jobs([job["id"]]) + + assert sent == 0 + assert adapter.sent == [] + + @pytest.mark.asyncio + async def test_failure_deliver_unset_notice_reaches_deliver_target(self): + """Control for the suppress test: same job without failure_deliver, + same real resolution path — the notice goes to the deliver target.""" + runner, adapter = make_restart_runner() + _bind_notifier(runner) + job = _telegram_job() + + with patch("cron.jobs.get_job", return_value=job): + sent = await runner._notify_interrupted_cron_jobs([job["id"]]) + + assert sent == 1 + assert adapter.sent_calls[0][0] == "123456" + @pytest.mark.asyncio async def test_empty_job_list_is_a_noop(self): runner, adapter = make_restart_runner() diff --git a/tests/gateway/test_delivery_silence_filter.py b/tests/gateway/test_delivery_silence_filter.py index 1013e4bc75..11b7ba3296 100644 --- a/tests/gateway/test_delivery_silence_filter.py +++ b/tests/gateway/test_delivery_silence_filter.py @@ -124,6 +124,50 @@ async def test_env_override_enables_filter_over_config(tmp_path, monkeypatch): assert result["filtered"] == "silence_narration" +# --- Cron artifacts are exempt ---------------------------------------------- +# +# The filter exists to stop bot-to-bot mirror loops of *model chatter*. Cron +# output is an artifact: a job that legitimately emits "..." (a quiet script, +# a terse digest) has no loop partner, and dropping it while returning +# {"success": True} produced a cron the scheduler logged as delivered and the +# user never received (#77763). Cron sends carry job_id in metadata. + + +@pytest.mark.asyncio +async def test_cron_job_id_metadata_bypasses_the_filter(tmp_path, monkeypatch): + monkeypatch.setattr("gateway.delivery.get_hermes_home", lambda: tmp_path) + monkeypatch.delenv("HERMES_FILTER_SILENCE_NARRATION", raising=False) + adapter = RecordingAdapter() + router = DeliveryRouter(GatewayConfig(), adapters={Platform.DISCORD: adapter}) + target = DeliveryTarget.parse("discord:99887766") + + result = await router._deliver_to_platform( + target, "*(silent)*", metadata={"job_id": "92e639af907f"}, + ) + + assert len(adapter.calls) == 1 + assert adapter.calls[0]["content"] == "*(silent)*" + assert result.get("filtered") is None + assert result.get("delivered") is not False + + +@pytest.mark.asyncio +async def test_non_cron_metadata_still_filters(tmp_path, monkeypatch): + """The exemption keys on job_id alone — everything else is unchanged.""" + monkeypatch.setattr("gateway.delivery.get_hermes_home", lambda: tmp_path) + monkeypatch.delenv("HERMES_FILTER_SILENCE_NARRATION", raising=False) + adapter = RecordingAdapter() + router = DeliveryRouter(GatewayConfig(), adapters={Platform.DISCORD: adapter}) + target = DeliveryTarget.parse("discord:99887766") + + result = await router._deliver_to_platform( + target, "*(silent)*", metadata={"thread_id": "42", "user_id": "u1"}, + ) + + assert adapter.calls == [] + assert result["filtered"] == "silence_narration" + + # --- Config round-trip ------------------------------------------------------ diff --git a/tests/gateway/test_discord_attachment_download.py b/tests/gateway/test_discord_attachment_download.py index a97632aa15..5b9575ee91 100644 --- a/tests/gateway/test_discord_attachment_download.py +++ b/tests/gateway/test_discord_attachment_download.py @@ -132,8 +132,8 @@ class TestCacheDiscordImage: att = _make_attachment_with_read(b"forbidden") with patch( - "plugins.platforms.discord.adapter.cache_image_from_bytes", - side_effect=ValueError("not a valid image"), + "plugins.platforms.discord.adapter.cache_image_from_bytes_async", + new=AsyncMock(side_effect=ValueError("not a valid image")), ), patch( "plugins.platforms.discord.adapter.cache_image_from_url", new_callable=AsyncMock, @@ -156,8 +156,8 @@ class TestCacheDiscordAudio: att = _make_attachment_with_read(_OGG_BYTES) with patch( - "plugins.platforms.discord.adapter.cache_audio_from_bytes", - return_value="/tmp/voice.ogg", + "plugins.platforms.discord.adapter.cache_audio_from_bytes_async", + new=AsyncMock(return_value="/tmp/voice.ogg"), ) as mock_bytes, patch( "plugins.platforms.discord.adapter.cache_audio_from_url", new_callable=AsyncMock, @@ -165,7 +165,7 @@ class TestCacheDiscordAudio: result = await adapter._cache_discord_audio(att, ".ogg") assert result == "/tmp/voice.ogg" - mock_bytes.assert_called_once_with(_OGG_BYTES, ext=".ogg") + mock_bytes.assert_awaited_once_with(_OGG_BYTES, ext=".ogg") mock_url.assert_not_called() @@ -215,8 +215,8 @@ class TestHandleMessageUsesAuthenticatedRead: adapter.handle_message = AsyncMock() with patch( - "plugins.platforms.discord.adapter.cache_image_from_bytes", - return_value="/tmp/img_from_read.png", + "plugins.platforms.discord.adapter.cache_image_from_bytes_async", + new=AsyncMock(return_value="/tmp/img_from_read.png"), ), patch( "plugins.platforms.discord.adapter.cache_image_from_url", new_callable=AsyncMock, diff --git a/tests/gateway/test_discord_free_response.py b/tests/gateway/test_discord_free_response.py index fdba6fea2a..fc58b982a3 100644 --- a/tests/gateway/test_discord_free_response.py +++ b/tests/gateway/test_discord_free_response.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import sys import pytest @@ -378,7 +378,7 @@ async def test_fetch_channel_context_skips_self_improvement_boundary_message(ada ], channel_id=123, ) - adapter._nonconversational_messages.mark_many(["9"]) + await adapter._nonconversational_messages.mark_many(["9"]) result = await adapter._fetch_channel_context(channel, before=make_message(channel=channel, content="trigger")) @@ -827,3 +827,58 @@ async def test_discord_reply_in_free_channel_triggers_backfill(adapter, monkeypa ) +class TestNonConversationalTrackerOffload: + """atomic_json_write() calls os.fsync(), which blocks until the write + reaches stable storage. mark_many() runs on the event loop from both + DiscordAdapter.send() and send_update_prompt(), so the persist step + must be offloaded to a thread — mirrors + test_directory_write_runs_off_event_loop_thread in + test_channel_directory.py for the same #83906 bug class. + """ + + @pytest.mark.asyncio + async def test_mark_many_persist_runs_off_event_loop_thread(self): + import threading + + tracker = discord_platform._DiscordNonConversationalMessageTracker() + loop_thread = threading.get_ident() + write_threads = [] + + def fake_write(path, data, *args, **kwargs): + write_threads.append(threading.get_ident()) + + with patch.object(discord_platform, "atomic_json_write", side_effect=fake_write): + await tracker.mark_many(["999"]) + + assert "999" in tracker + assert write_threads + assert all(tid != loop_thread for tid in write_threads) + + @pytest.mark.asyncio + async def test_concurrent_mark_many_persists_land_in_order(self): + """Two in-flight mark_many() calls (send() racing a history fetch) must + not let an older snapshot overwrite a newer one on disk.""" + import asyncio as _asyncio + import time + + tracker = discord_platform._DiscordNonConversationalMessageTracker() + tracker._ids = {} + writes = [] + calls = [0] + + def slow_first_write(path, data, *args, **kwargs): + idx = calls[0] + calls[0] += 1 + if idx == 0: + time.sleep(0.05) + writes.append(list(data)) + + with patch.object(discord_platform, "atomic_json_write", side_effect=slow_first_write): + first = _asyncio.create_task(tracker.mark_many(["1"])) + await _asyncio.sleep(0.005) + second = _asyncio.create_task(tracker.mark_many(["2"])) + await _asyncio.gather(first, second) + + assert sorted(writes[-1]) == ["1", "2"] + + diff --git a/tests/gateway/test_discord_slash_commands.py b/tests/gateway/test_discord_slash_commands.py index e3e5a39ff5..4f602a8768 100644 --- a/tests/gateway/test_discord_slash_commands.py +++ b/tests/gateway/test_discord_slash_commands.py @@ -602,3 +602,43 @@ def test_register_skill_command_payload_fits_discord_8kb_limit(adapter): ) + + +# ------------------------------------------------------------------ +# _build_slash_event — guild/parent ids reach profile_routes (#69178, #91633) +# ------------------------------------------------------------------ + + +def test_build_slash_event_routes_guild_profile_like_messages(adapter, monkeypatch): + """A guild-keyed profile route must match a native slash command exactly + as it matches a regular message: build_source needs guild_id (and the + thread's parent_chat_id) or the route never fires and /new resets the + default profile's session instead of the routed one.""" + from gateway import run as gateway_run + from gateway.config import GatewayConfig + from gateway.profile_routing import ProfileRoute + + runner = object.__new__(gateway_run.GatewayRunner) + runner.config = GatewayConfig( + multiplex_profiles=True, + profile_routes=[ProfileRoute(name="work", profile="work", platform="discord", guild_id="1")], + ) + monkeypatch.setattr(gateway_run, "_multiplex_profile_homes", lambda _cfg: [("work", None)]) + adapter.gateway_runner = runner + user = SimpleNamespace(display_name="Jezza", id=42) + + channel_event = adapter._build_slash_event( + SimpleNamespace(channel=SimpleNamespace(id=200, name="general", guild=SimpleNamespace(id=1, name="G"), topic=None), + channel_id=200, guild_id=1, user=user), + "/new", + ) + thread_event = adapter._build_slash_event( + SimpleNamespace(channel=_FakeThreadChannel(channel_id=555), channel_id=555, guild_id=None, user=user), + "/status", + ) + + assert channel_event.source.guild_id == "1" + assert channel_event.source.profile == "work" + assert thread_event.source.guild_id == "1" + assert thread_event.source.parent_chat_id == "100" + assert thread_event.source.profile == "work" diff --git a/tests/gateway/test_email_robustness.py b/tests/gateway/test_email_robustness.py index c1266196c6..5df979fb78 100644 --- a/tests/gateway/test_email_robustness.py +++ b/tests/gateway/test_email_robustness.py @@ -77,5 +77,39 @@ class TestMessageIdDomain(unittest.TestCase): self.assertEqual(adapter._message_id_domain(), "localhost") +class TestTransportSecurity(unittest.TestCase): + """platforms.email.extra.imap_security / smtp_security select the transport (#99641).""" + + def _adapter(self, **extra): + from gateway.config import PlatformConfig + + with patch.dict(os.environ, { + "EMAIL_ADDRESS": "hermes@test.com", "EMAIL_PASSWORD": "secret", + "EMAIL_IMAP_HOST": "127.0.0.1", "EMAIL_IMAP_PORT": "1143", + "EMAIL_SMTP_HOST": "127.0.0.1", "EMAIL_SMTP_PORT": "1025", + }, clear=True): + from plugins.platforms.email.adapter import EmailAdapter + + return EmailAdapter(PlatformConfig(enabled=True, extra=extra)) + + def test_starttls_builds_plain_imap_then_upgrades(self): + adapter = self._adapter(imap_security="starttls", imap_tls_verify=False) + imap = MagicMock() + with patch("imaplib.IMAP4", return_value=imap) as imap_cls, \ + patch("imaplib.IMAP4_SSL") as imap_ssl_cls: + self.assertIs(adapter._connect_imap(), imap) + imap_cls.assert_called_once_with("127.0.0.1", 1143, timeout=30) + imap_ssl_cls.assert_not_called() + imap.starttls.assert_called_once() + + def test_unknown_mode_falls_back_to_secure_default(self): + adapter = self._adapter(imap_security="bogus", smtp_security="bogus") + self.assertEqual(adapter._imap_security, "tls") + self.assertEqual(adapter._smtp_security, "starttls") # port 1025 != 465 + # verification stays ON unless explicitly opted out + self.assertTrue(adapter._imap_tls_verify) + self.assertTrue(adapter._smtp_tls_verify) + + if __name__ == "__main__": unittest.main() diff --git a/tests/gateway/test_env_override_explicit_disable_48820.py b/tests/gateway/test_env_override_explicit_disable_48820.py new file mode 100644 index 0000000000..1d8452c5a0 --- /dev/null +++ b/tests/gateway/test_env_override_explicit_disable_48820.py @@ -0,0 +1,199 @@ +"""Regression tests for #48820 Bug 2: an explicit ``platforms..enabled: false`` +in config.yaml must survive ``_apply_env_overrides`` when that platform's +credentials are present in the environment. + +Before the fix, twelve credential-presence branches (weixin, whatsapp_cloud, +homeassistant, email, sms, dingtalk, feishu, wecom, wecom_callback, bluebubbles, +qqbot, yuanbao) force-set ``enabled = True`` unconditionally, while Telegram / +Discord / Slack routed through ``_enable_from_env`` and honored the +``_enabled_explicit`` marker. These tests drive the real ``load_gateway_config`` +against a temp HERMES_HOME — real YAML I/O, no mocks of the code under test. +""" + +import logging + +import pytest + +from gateway import config as gateway_config +from gateway.config import Platform, load_gateway_config + + +# platform -> env credentials that trigger its env-enable branch +CRED_ENV = { + "weixin": { + "WEIXIN_TOKEN": "wx_9f8e7d6c5b4a3f2e1d0c9b8a7f6e5d4c3b2a1f0e", + "WEIXIN_ACCOUNT_ID": "acct_12345", + }, + "whatsapp_cloud": { + "WHATSAPP_CLOUD_PHONE_NUMBER_ID": "1234567890", + "WHATSAPP_CLOUD_ACCESS_TOKEN": "EAAB-test-access-token", + }, + "homeassistant": {"HASS_TOKEN": "hass-long-lived-token"}, + "email": { + "EMAIL_ADDRESS": "bot@example.com", + "EMAIL_PASSWORD": "app-password", + "EMAIL_IMAP_HOST": "imap.example.com", + "EMAIL_SMTP_HOST": "smtp.example.com", + }, + "sms": {"TWILIO_ACCOUNT_SID": "ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}, + "dingtalk": {"DINGTALK_CLIENT_ID": "ding-id", "DINGTALK_CLIENT_SECRET": "ding-secret"}, + "feishu": {"FEISHU_APP_ID": "cli_feishu", "FEISHU_APP_SECRET": "feishu-secret"}, + "wecom": {"WECOM_BOT_ID": "wecom-bot", "WECOM_SECRET": "wecom-secret"}, + "wecom_callback": { + "WECOM_CALLBACK_CORP_ID": "corp-id", + "WECOM_CALLBACK_CORP_SECRET": "corp-secret", + }, + "bluebubbles": { + "BLUEBUBBLES_SERVER_URL": "http://127.0.0.1:1234", + "BLUEBUBBLES_PASSWORD": "bb-password", + }, + "qqbot": {"QQ_APP_ID": "qq-app", "QQ_CLIENT_SECRET": "qq-secret"}, + "yuanbao": {"YUANBAO_APP_ID": "yb-app", "YUANBAO_APP_SECRET": "yb-secret"}, + # control: the pattern that always honored the explicit disable + "telegram": {"TELEGRAM_BOT_TOKEN": "123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11"}, +} + +_PLATFORM_ENV_PREFIXES = ( + "TELEGRAM_", "DISCORD_", "SLACK_", "WEIXIN_", "WHATSAPP_", "HASS_", "EMAIL_", + "TWILIO_", "DINGTALK_", "FEISHU_", "WECOM_", "BLUEBUBBLES_", "QQ_", "QQBOT_", + "YUANBAO_", "GATEWAY_RELAY", "SIGNAL_", "MATTERMOST_", "MATRIX_", +) + + +def _isolate(monkeypatch, tmp_path, env): + import os + + for key in list(os.environ): + if key.startswith(_PLATFORM_ENV_PREFIXES): + monkeypatch.delenv(key, raising=False) + hermes_home = tmp_path / ".hermes" + hermes_home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + for k, v in env.items(): + monkeypatch.setenv(k, v) + return hermes_home + + +@pytest.mark.parametrize("platform", sorted(CRED_ENV)) +def test_yaml_explicit_disable_survives_env_credentials(platform, tmp_path, monkeypatch): + """``platforms..enabled: false`` + credentials in env -> stays disabled.""" + hermes_home = _isolate(monkeypatch, tmp_path, CRED_ENV[platform]) + (hermes_home / "config.yaml").write_text( + f"platforms:\n {platform}:\n enabled: false\n", encoding="utf-8" + ) + + config = load_gateway_config() + + cfg = config.platforms.get(Platform(platform)) + assert cfg is not None + assert cfg.enabled is False, ( + f"{platform}: env credentials re-enabled a platform the user explicitly " + "disabled in config.yaml (#48820 Bug 2)" + ) + + +@pytest.mark.parametrize("platform", sorted(CRED_ENV)) +def test_env_credentials_still_enable_without_yaml_opinion(platform, tmp_path, monkeypatch): + """No ``enabled`` key in YAML + credentials in env -> env-only setup still works.""" + hermes_home = _isolate(monkeypatch, tmp_path, CRED_ENV[platform]) + (hermes_home / "config.yaml").write_text("platforms: {}\n", encoding="utf-8") + + config = load_gateway_config() + + cfg = config.platforms.get(Platform(platform)) + assert cfg is not None and cfg.enabled is True, ( + f"{platform}: env-only configuration must still enable the platform" + ) + + +def test_env_credentials_still_populate_extra_when_yaml_disables(tmp_path, monkeypatch): + """The disable only gates ``enabled``; credentials are still wired through + (mirrors the Slack/API-server contract so send-only tooling keeps working).""" + hermes_home = _isolate(monkeypatch, tmp_path, CRED_ENV["weixin"]) + (hermes_home / "config.yaml").write_text( + "platforms:\n weixin:\n enabled: false\n", encoding="utf-8" + ) + + config = load_gateway_config() + + cfg = config.platforms[Platform.WEIXIN] + assert cfg.enabled is False + assert cfg.token == CRED_ENV["weixin"]["WEIXIN_TOKEN"] + assert cfg.extra.get("account_id") == "acct_12345" + # marker never leaks out of config load + assert "_enabled_explicit" not in cfg.extra + + +@pytest.fixture() +def _fresh_warn_dedup(monkeypatch): + """The explicit-disable notice is one-time per process; start each test clean.""" + monkeypatch.setattr(gateway_config, "_EXPLICIT_DISABLE_WARNED", set()) + + +@pytest.mark.usefixtures("_fresh_warn_dedup") +@pytest.mark.parametrize("platform", sorted(CRED_ENV)) +def test_explicit_disable_with_env_credentials_warns_once(platform, tmp_path, monkeypatch, caplog): + """Users who relied on 'creds in .env = platform on' must be told why it went + dark: one WARNING naming the platform, the winning config key, and the env + credential(s) — emitted once per process, not on every config reload.""" + hermes_home = _isolate(monkeypatch, tmp_path, CRED_ENV[platform]) + (hermes_home / "config.yaml").write_text( + f"platforms:\n {platform}:\n enabled: false\n", encoding="utf-8" + ) + + with caplog.at_level(logging.WARNING, logger="gateway.config"): + load_gateway_config() + load_gateway_config() # reload: must not repeat + + hits = [ + r for r in caplog.records + if r.levelno == logging.WARNING and f"platforms.{platform}.enabled: false" in r.getMessage() + ] + assert len(hits) == 1, [r.getMessage() for r in caplog.records] + msg = hits[0].getMessage() + assert f"Platform '{platform}'" in msg + for env_name in CRED_ENV[platform]: + assert env_name in msg + assert f"platforms.{platform}.enabled: true" in msg # the remedy + + +@pytest.mark.usefixtures("_fresh_warn_dedup") +def test_no_warning_when_yaml_has_no_opinion_or_is_enabled(tmp_path, monkeypatch, caplog): + hermes_home = _isolate(monkeypatch, tmp_path, {**CRED_ENV["weixin"], **CRED_ENV["homeassistant"]}) + (hermes_home / "config.yaml").write_text( + "platforms:\n homeassistant:\n enabled: true\n", encoding="utf-8" + ) + + with caplog.at_level(logging.WARNING, logger="gateway.config"): + config = load_gateway_config() + + assert config.platforms[Platform.WEIXIN].enabled is True + assert config.platforms[Platform.HOMEASSISTANT].enabled is True + assert not [r for r in caplog.records if "explicitly disabled" in r.getMessage()] + + +@pytest.mark.usefixtures("_fresh_warn_dedup") +def test_no_warning_when_disabled_and_no_env_credentials(tmp_path, monkeypatch, caplog): + """The notice is about credentials being IGNORED; a plain disable is silent.""" + hermes_home = _isolate(monkeypatch, tmp_path, {}) + (hermes_home / "config.yaml").write_text( + "platforms:\n weixin:\n enabled: false\n", encoding="utf-8" + ) + + with caplog.at_level(logging.WARNING, logger="gateway.config"): + config = load_gateway_config() + + assert config.platforms[Platform.WEIXIN].enabled is False + assert not [r for r in caplog.records if "explicitly disabled" in r.getMessage()] + + +def test_every_env_enable_branch_is_named_for_the_warning(): + """Each platform routed through ``_enable_from_env`` needs a credential + entry so the WARNING can name what is being ignored.""" + import inspect, re + + src = inspect.getsource(gateway_config._apply_env_overrides) + routed = {Platform[name] for name in re.findall(r"_enable_from_env\(Platform\.([A-Z_]+)\)", src)} + routed.add(Platform.SLACK) # Slack has its own inline copy of the logic + missing = {p.value for p in routed} - {p.value for p in gateway_config._ENV_ENABLE_CREDENTIALS} + assert not missing, f"platforms without a credential entry for the explicit-disable warning: {missing}" diff --git a/tests/gateway/test_fast_command.py b/tests/gateway/test_fast_command.py index c714b76e84..b8792ecce4 100644 --- a/tests/gateway/test_fast_command.py +++ b/tests/gateway/test_fast_command.py @@ -109,8 +109,8 @@ def test_turn_route_injects_priority_processing_without_changing_runtime(): runner._service_tier = "priority" runtime_kwargs = { "api_key": "***", - "base_url": "https://openrouter.ai/api/v1", - "provider": "openrouter", + "base_url": "https://api.openai.com/v1", + "provider": "openai", "api_mode": "chat_completions", "command": None, "args": [], @@ -119,10 +119,15 @@ def test_turn_route_injects_priority_processing_without_changing_runtime(): route = gateway_run.GatewayRunner._resolve_turn_agent_config(runner, "hi", "gpt-5.4", runtime_kwargs) - assert route["runtime"]["provider"] == "openrouter" + assert route["runtime"]["provider"] == "openai" assert route["runtime"]["api_mode"] == "chat_completions" assert route["request_overrides"] == {"service_tier": "priority"} + # Proxied routes never receive the param (OpenRouter strips it / others 400). + runtime_kwargs.update(base_url="https://openrouter.ai/api/v1", provider="openrouter") + route = gateway_run.GatewayRunner._resolve_turn_agent_config(runner, "hi", "gpt-5.4", runtime_kwargs) + assert route["request_overrides"] == {} + @pytest.mark.asyncio async def test_handle_fast_command_global_flag_persists_config(monkeypatch, tmp_path): diff --git a/tests/gateway/test_feishu.py b/tests/gateway/test_feishu.py index a834a45e58..4120b376cf 100644 --- a/tests/gateway/test_feishu.py +++ b/tests/gateway/test_feishu.py @@ -1109,8 +1109,8 @@ class TestAdapterBehavior(unittest.TestCase): side_effect=lambda **_kwargs: _FakeAsyncClient(), ): with patch( - "plugins.platforms.feishu.adapter.cache_document_from_bytes", - return_value="/tmp/cached-doc.bin", + "plugins.platforms.feishu.adapter.cache_document_from_bytes_async", + new=AsyncMock(return_value="/tmp/cached-doc.bin"), ): return await adapter._download_remote_document( "https://example.com/doc.bin", @@ -1190,9 +1190,9 @@ class TestAdapterBehavior(unittest.TestCase): with tempfile.TemporaryDirectory() as temp_home: with patch.dict(os.environ, {"HERMES_HOME": temp_home}, clear=False): first = FeishuAdapter(PlatformConfig()) - self.assertFalse(first._is_duplicate("om_same")) + self.assertFalse(asyncio.run(first._is_duplicate("om_same"))) second = FeishuAdapter(PlatformConfig()) - self.assertTrue(second._is_duplicate("om_same")) + self.assertTrue(asyncio.run(second._is_duplicate("om_same"))) @patch.dict(os.environ, _SYS_ENV, clear=True) @@ -1627,7 +1627,7 @@ class TestDedupTTL(unittest.TestCase): with patch.object(adapter, "_persist_seen_message_ids"): adapter._seen_message_ids = {"om_dup": time.time()} adapter._seen_message_order = ["om_dup"] - self.assertTrue(adapter._is_duplicate("om_dup")) + self.assertTrue(asyncio.run(adapter._is_duplicate("om_dup"))) @patch.dict(os.environ, _SYS_ENV, clear=True) @@ -1661,6 +1661,61 @@ class TestDedupTTL(unittest.TestCase): assert "om_bad_str" not in adapter._seen_message_ids assert "om_bad_null" not in adapter._seen_message_ids + @patch.dict(os.environ, {}, clear=True) + def test_persist_on_new_message_runs_off_event_loop_thread(self): + """atomic_json_write() calls os.fsync(), which blocks until the write + reaches stable storage. _is_duplicate() runs on the event loop for + every inbound message (_handle_message_event_data), so the persist + step must be offloaded to a thread — mirrors + test_directory_write_runs_off_event_loop_thread in + test_channel_directory.py for the same #83906 bug class.""" + import threading + from gateway.config import PlatformConfig + from plugins.platforms.feishu.adapter import FeishuAdapter + + adapter = FeishuAdapter(PlatformConfig()) + loop_thread = threading.get_ident() + write_threads = [] + + def fake_write(path, data, *args, **kwargs): + write_threads.append(threading.get_ident()) + + with patch("plugins.platforms.feishu.adapter.atomic_json_write", side_effect=fake_write): + is_dup = asyncio.run(adapter._is_duplicate("om_new")) + + self.assertFalse(is_dup) + self.assertTrue(write_threads) + self.assertTrue(all(tid != loop_thread for tid in write_threads)) + + @patch.dict(os.environ, {}, clear=True) + def test_concurrent_dedup_persists_land_in_order(self): + """Two in-flight _is_duplicate() calls (two chats) must not let an + older seen-ids snapshot overwrite a newer one on disk.""" + from gateway.config import PlatformConfig + from plugins.platforms.feishu.adapter import FeishuAdapter + + adapter = FeishuAdapter(PlatformConfig()) + writes = [] + calls = [0] + + def slow_first_write(path, data, *args, **kwargs): + idx = calls[0] + calls[0] += 1 + if idx == 0: + time.sleep(0.05) + writes.append(sorted(data["message_ids"])) + + async def run(): + first = asyncio.create_task(adapter._is_duplicate("om_a")) + await asyncio.sleep(0.005) + second = asyncio.create_task(adapter._is_duplicate("om_b")) + await asyncio.gather(first, second) + + with patch("plugins.platforms.feishu.adapter.atomic_json_write", side_effect=slow_first_write): + asyncio.run(run()) + + self.assertEqual(writes[-1], ["om_a", "om_b"]) + class TestGroupMentionAtAll(unittest.TestCase): """Tests for @_all (Feishu @everyone) group mention routing.""" diff --git a/tests/gateway/test_feishu_bot_admission.py b/tests/gateway/test_feishu_bot_admission.py index 09157c757e..b396e90d29 100644 --- a/tests/gateway/test_feishu_bot_admission.py +++ b/tests/gateway/test_feishu_bot_admission.py @@ -566,3 +566,77 @@ def test_handle_message_event_data_forwards_sender_when_admitted(): assert captured.get("sender_id") is sender.sender_id assert captured.get("is_bot") is True assert captured.get("message_id") == "om_bot_ok" + + +# --- Profile-scoped admission config (#86905) ------------------------------- + + +def test_dm_admission_config_resolves_from_profile_scope_under_multiplex(tmp_path, monkeypatch): + """os.environ holds the DEFAULT profile's admission view; a secondary + profile's .env must govern its own adapter — Feishu open_ids are + app-scoped, so the default allow-list can never match the role app's + senders, and the role profile's allow-all flag must be honored.""" + import agent.secret_scope as ss + from plugins.platforms.feishu.adapter import FeishuAdapter + + monkeypatch.setenv("FEISHU_APP_ID", "cli_default") + monkeypatch.setenv("FEISHU_APP_SECRET", "secret_default") + monkeypatch.setenv("FEISHU_ALLOWED_USERS", "ou_default_view") + monkeypatch.setenv("FEISHU_ALLOW_BOTS", "all") + monkeypatch.delenv("GATEWAY_ALLOW_ALL_USERS", raising=False) + monkeypatch.delenv("FEISHU_ALLOW_ALL_USERS", raising=False) + (tmp_path / ".env").write_text( + "FEISHU_APP_ID=cli_role\nFEISHU_APP_SECRET=secret_role\n" + "FEISHU_ALLOWED_USERS=ou_role_view\n", + encoding="utf-8", + ) + + ss.set_multiplex_active(True) + tok = ss.set_secret_scope(ss.build_profile_secret_scope(tmp_path)) + try: + settings = FeishuAdapter._load_settings(extra={}) + finally: + ss.reset_secret_scope(tok) + (tmp_path / ".env").write_text( + "FEISHU_APP_ID=cli_role\nFEISHU_APP_SECRET=secret_role\nGATEWAY_ALLOW_ALL_USERS=true\n", + encoding="utf-8", + ) + tok = ss.set_secret_scope(ss.build_profile_secret_scope(tmp_path)) + try: + allow_all = FeishuAdapter._load_settings(extra={}) + finally: + ss.reset_secret_scope(tok) + ss.set_multiplex_active(False) + + assert settings.app_id == "cli_role" + assert settings.allowed_group_users == frozenset({"ou_role_view"}) + assert settings.allow_bots == "none" # default's "all" must not leak in + assert settings.allow_all_dm is False + + # _admit runs on the WS thread with no scope: the snapshot must carry. + adapter = object.__new__(FeishuAdapter) + adapter._apply_settings(settings) + assert adapter._admit(make_sender(open_id="ou_role_view"), make_message(chat_type="p2p")) is None + assert adapter._admit(make_sender(open_id="ou_default_view"), make_message(chat_type="p2p")) == "dm_policy_rejected" + + assert allow_all.allow_all_dm is True + adapter = object.__new__(FeishuAdapter) + adapter._apply_settings(allow_all) + assert adapter._admit(make_sender(open_id="ou_anyone"), make_message(chat_type="p2p")) is None + + +def test_dm_admission_config_falls_back_to_os_environ_when_unscoped(monkeypatch): + """Single-profile behavior unchanged: process env still configures DMs.""" + from plugins.platforms.feishu.adapter import FeishuAdapter + + monkeypatch.setenv("FEISHU_APP_ID", "cli_test") + monkeypatch.setenv("FEISHU_APP_SECRET", "secret_test") + monkeypatch.setenv("GATEWAY_ALLOW_ALL_USERS", "true") + monkeypatch.setenv("FEISHU_ALLOWED_USERS", "ou_a,ou_b") + + settings = FeishuAdapter._load_settings(extra={}) + assert settings.allow_all_dm is True + assert settings.allowed_group_users == frozenset({"ou_a", "ou_b"}) + adapter = object.__new__(FeishuAdapter) + adapter._apply_settings(settings) + assert adapter._admit(make_sender(open_id="ou_anyone"), make_message(chat_type="p2p")) is None diff --git a/tests/gateway/test_feishu_meeting_invite.py b/tests/gateway/test_feishu_meeting_invite.py index 47ce7472d0..d8e4725a64 100644 --- a/tests/gateway/test_feishu_meeting_invite.py +++ b/tests/gateway/test_feishu_meeting_invite.py @@ -76,7 +76,7 @@ class _Adapter: self.dedup_keys = [] self.profile_requests = [] - def _is_duplicate(self, key): + async def _is_duplicate(self, key): self.dedup_keys.append(key) return self.duplicate @@ -166,6 +166,19 @@ class TestMeetingInviteHandler(unittest.TestCase): self.assertIn("You have been invited to join a meeting: 赵磊的视频会议", event.text) self.assertNotIn("{'open_id'", event.text) + def test_duplicate_event_is_dropped_without_routing(self): + """_is_duplicate() is async on the real FeishuAdapter (dedup persist + is offloaded off the event loop); the dedup check here must await + it — a missing await would leave an un-awaited coroutine, which is + always truthy, and drop every event as a false duplicate.""" + adapter = _Adapter(duplicate=True) + + self._run(handle_meeting_invited_event(adapter, _make_payload())) + + self.assertEqual(adapter.dedup_keys, ["vc_invite:evt_1"]) + self.assertEqual(adapter.events, []) + self.assertEqual(adapter.profile_requests, []) + class TestMeetingInviteSendRouting(unittest.TestCase): def _run(self, coro): diff --git a/tests/gateway/test_feishu_ws_multiplex_isolation.py b/tests/gateway/test_feishu_ws_multiplex_isolation.py new file mode 100644 index 0000000000..ca7d514cc3 --- /dev/null +++ b/tests/gateway/test_feishu_ws_multiplex_isolation.py @@ -0,0 +1,167 @@ +"""Multiplex isolation for the lark_oapi WS client (issue #73779). + +``lark_oapi.ws.client`` keeps the loop used by ``Client.start()`` in a +module-level global and Hermes monkey-patches ``websockets.connect`` on the +shared module. With N profile WS threads the globals were last-write-wins: +"Future attached to a different loop" crashes or a client bound to a +sibling's loop that never hears anything again. +""" + +import asyncio +import sys +import threading +import types +from types import SimpleNamespace +from unittest.mock import MagicMock + +from plugins.platforms.feishu import adapter as feishu_adapter + + +def _inject_fake_lark_module(monkeypatch, connect=None): + """Make ``import lark_oapi.ws.client`` resolve to a module with the SDK's + global layout (``loop`` + ``websockets.connect``).""" + if connect is None: + connect = MagicMock(name="real-connect") + lark = types.ModuleType("lark_oapi") + lark_ws = types.ModuleType("lark_oapi.ws") + client_mod = types.ModuleType("lark_oapi.ws.client") + client_mod.loop = SimpleNamespace(name="sdk-default-loop") + client_mod.websockets = SimpleNamespace(connect=connect) + lark.ws = lark_ws + lark_ws.client = client_mod + monkeypatch.setitem(sys.modules, "lark_oapi", lark) + monkeypatch.setitem(sys.modules, "lark_oapi.ws", lark_ws) + monkeypatch.setitem(sys.modules, "lark_oapi.ws.client", client_mod) + monkeypatch.setattr(feishu_adapter, "_WS_ISOLATION_INSTALLED", False) + return client_mod + + +def _adapter_stub(**overrides): + stub = SimpleNamespace( + _ws_thread_loop=None, + _ws_reconnect_nonce=None, + _ws_reconnect_interval=None, + _ws_ping_interval=None, + _ws_ping_timeout=None, + ) + for key, value in overrides.items(): + setattr(stub, key, value) + return stub + + +def test_two_concurrent_clients_each_use_their_own_loop_and_overrides(monkeypatch): + """Two profiles start() concurrently through the module global: each must + run on its own loop, and websockets.connect must receive only the + calling profile's ping overrides. On main both are last-write-wins.""" + real_connect = MagicMock(name="real-connect") + client_mod = _inject_fake_lark_module(monkeypatch, connect=real_connect) + + results = {} + barrier = threading.Barrier(2) + + class FakeClient: + def __init__(self, name): + self._name = name + + def start(self): + barrier.wait(timeout=10) # both threads past the global "assign" + + async def probe(): + await asyncio.sleep(0.02) + return id(asyncio.get_running_loop()) + + results[self._name] = client_mod.loop.run_until_complete(probe()) + client_mod.websockets.connect(f"wss://{self._name}") + + pings = {"p0": 10, "p1": 20} + + def run(name): + feishu_adapter._run_official_feishu_ws_client( + FakeClient(name), _adapter_stub(_ws_ping_interval=pings[name]) + ) + + threads = [threading.Thread(target=run, args=(f"p{i}",)) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=15) + assert not t.is_alive() + + assert results["p0"] != results["p1"] + calls = {c.args[0]: c.kwargs for c in real_connect.call_args_list} + assert calls == {"wss://p0": {"ping_interval": 10}, "wss://p1": {"ping_interval": 20}} + # Thread-local registrations are cleared for the pooled executor thread. + assert getattr(feishu_adapter._ws_isolation_state, "loop", None) is None + assert getattr(feishu_adapter._ws_isolation_state, "connect_kwargs", None) is None + + +def _supervisor_stub(): + stub = SimpleNamespace( + _running=True, + _ws_future=None, + _ws_client=object(), + _ws_restart_backoff=0.01, + connect_calls=0, + connect_should_fail=0, + ) + + async def _connect_websocket(): + stub.connect_calls += 1 + if stub.connect_should_fail > 0: + stub.connect_should_fail -= 1 + raise RuntimeError("simulated restart failure") + fut = asyncio.get_running_loop().create_future() + fut.set_result(None) # new thread dies immediately too + stub._ws_future = fut + + stub._connect_websocket = _connect_websocket + return stub + + +def test_supervisor_restarts_a_dead_ws_thread_with_backoff(): + """A dead WS thread used to leave the profile silently deaf (the future + was awaited only by disconnect()). The supervisor must rebuild the client + and survive a failed restart without hot-looping.""" + + async def scenario(): + stub = _supervisor_stub() + stub.connect_should_fail = 1 + fut = asyncio.get_running_loop().create_future() + fut.set_result(None) # the WS "thread" is already dead + stub._ws_future = fut + + task = asyncio.ensure_future( + feishu_adapter.FeishuAdapter._supervise_websocket_thread(stub) + ) + for _ in range(300): + await asyncio.sleep(0.01) + if stub.connect_calls >= 2: + break + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + return stub.connect_calls + + assert asyncio.run(scenario()) == 2 # failed restart, then a successful one + + +def test_supervisor_stops_when_disconnect_nils_the_client(): + async def scenario(): + stub = _supervisor_stub() + fut = asyncio.get_running_loop().create_future() # thread "alive" + stub._ws_future = fut + + task = asyncio.ensure_future( + feishu_adapter.FeishuAdapter._supervise_websocket_thread(stub) + ) + await asyncio.sleep(0.01) + stub._ws_client = None # deliberate disconnect ... + fut.set_result(None) # ... then the thread exits + await asyncio.wait_for(asyncio.shield(task), timeout=2.0) + return stub, task + + stub, task = asyncio.run(scenario()) + assert task.done() + assert stub.connect_calls == 0 diff --git a/tests/gateway/test_fifo_overflow_rescue.py b/tests/gateway/test_fifo_overflow_rescue.py new file mode 100644 index 0000000000..e1f3efd110 --- /dev/null +++ b/tests/gateway/test_fifo_overflow_rescue.py @@ -0,0 +1,159 @@ +"""Regression tests for #99882: FIFO overflow orphan rescue. + +When a follow-up is demoted to /queue during compression-in-flight, +it lands in SessionState.conversation.queued_events (overflow) with +the current turn's event occupying adapter._pending_messages[session_key] +(slot). After the slot's turn completes, _promote_queued_event moves +the overflow head into the slot. When that drain never runs — the +busy window ended through an exit that skipped the promotion site +(/stop, turn exception, generation bump) — the overflow is silently +orphaned: never dispatched, never persisted, never logged. + +The rescue in GatewayRunner._rescue_orphaned_overflow pops the oldest +orphan for the caller to run as the current turn and stages the next +orphan in the slot, so FIFO order (#28503) holds and nothing runs twice. +""" + +from unittest.mock import MagicMock + +from gateway.platforms.base import ( + BasePlatformAdapter, + MessageEvent, + MessageType, + Platform, + PlatformConfig, +) +from gateway.run import GatewayRunner + + +class _StubAdapter(BasePlatformAdapter): + def __init__(self): + super().__init__(PlatformConfig(enabled=True, token="test"), Platform.TELEGRAM) + + async def connect(self, *, is_reconnect: bool = False) -> bool: + return True + + async def disconnect(self) -> None: + self._mark_disconnected() + + async def send(self, chat_id, content, reply_to=None, metadata=None): + from gateway.platforms.base import SendResult + + return SendResult(success=True, message_id="msg-1") + + async def get_chat_info(self, chat_id): + return {"id": chat_id, "type": "dm"} + + +def _text_event(text: str, msg_id: str) -> MessageEvent: + return MessageEvent( + text=text, + message_type=MessageType.TEXT, + source=MagicMock(chat_id="123", platform=Platform.TELEGRAM, profile=None), + message_id=msg_id, + ) + + +def _runner() -> GatewayRunner: + runner = GatewayRunner.__new__(GatewayRunner) + runner._queued_events = {} + return runner + + +class TestRescueOrphanedOverflow: + def test_single_orphan_is_returned_and_removed_from_both_stores(self): + runner = _runner() + adapter = _StubAdapter() + session_key = "telegram:user:1" + runner._session_state(session_key).conversation.queued_events.append( + _text_event("orphan-1", "o1") + ) + assert session_key not in adapter._pending_messages + + rescued = runner._rescue_orphaned_overflow(session_key, adapter) + + assert rescued is not None and rescued.text == "orphan-1" + # The rescued event runs as the current turn, so it must NOT also + # sit in the slot — the post-turn drain would run it a second time. + assert session_key not in adapter._pending_messages + assert runner._session_state(session_key).conversation.queued_events == [] + + def test_two_orphans_return_oldest_and_stage_next_in_slot(self): + runner = _runner() + adapter = _StubAdapter() + session_key = "telegram:user:1b" + runner._session_state(session_key).conversation.queued_events.extend( + [_text_event("orphan-1", "o1"), _text_event("orphan-2", "o2")] + ) + + rescued = runner._rescue_orphaned_overflow(session_key, adapter) + + assert rescued is not None and rescued.text == "orphan-1" + # Slot now holds the NEXT orphan so the drain continues the chain. + assert adapter._pending_messages[session_key].text == "orphan-2" + assert runner._session_state(session_key).conversation.queued_events == [] + + def test_noop_when_slot_occupied(self): + runner = _runner() + adapter = _StubAdapter() + session_key = "telegram:user:2" + runner._session_state(session_key).conversation.queued_events.append( + _text_event("orphan", "o1") + ) + adapter._pending_messages[session_key] = _text_event("busy-slot", "slot") + + rescued = runner._rescue_orphaned_overflow(session_key, adapter) + + assert rescued is None + assert adapter._pending_messages[session_key].text == "busy-slot" + assert len(runner._session_state(session_key).conversation.queued_events) == 1 + + def test_noop_when_no_overflow(self): + runner = _runner() + adapter = _StubAdapter() + session_key = "telegram:user:3" + + rescued = runner._rescue_orphaned_overflow(session_key, adapter) + + assert rescued is None + assert session_key not in adapter._pending_messages + + def test_fifo_order_preserved_across_rescue_and_new_message(self): + """Oldest orphan runs first, new arrival last — FIFO (#28503). + + Mirrors the idle-arrival call site: rescue → _enqueue_fifo(new). + """ + runner = _runner() + adapter = _StubAdapter() + session_key = "telegram:user:4" + runner._session_state(session_key).conversation.queued_events.extend( + [_text_event("orphan-1", "o1"), _text_event("orphan-2", "o2")] + ) + + rescued = runner._rescue_orphaned_overflow(session_key, adapter) + assert rescued is not None and rescued.text == "orphan-1" + runner._enqueue_fifo(session_key, _text_event("new-msg", "new1"), adapter) + + # Drain order after this turn: slot (orphan-2), then overflow (new-msg) + assert adapter._pending_messages[session_key].text == "orphan-2" + overflow_texts = [ + e.text for e in runner._session_state(session_key).conversation.queued_events + ] + assert overflow_texts == ["new-msg"] + + def test_single_orphan_then_new_message_lands_in_slot(self): + """With one orphan the slot is free after rescue, so the incoming + message must go to the slot (not overflow) or the drain never sees it.""" + runner = _runner() + adapter = _StubAdapter() + session_key = "telegram:user:5" + runner._session_state(session_key).conversation.queued_events.append( + _text_event("orphan-1", "o1") + ) + + rescued = runner._rescue_orphaned_overflow(session_key, adapter) + assert rescued is not None and rescued.text == "orphan-1" + runner._enqueue_fifo(session_key, _text_event("new-msg", "new1"), adapter) + + assert adapter._pending_messages[session_key].text == "new-msg" + assert runner._session_state(session_key).conversation.queued_events == [] diff --git a/tests/gateway/test_gateway_trust_env.py b/tests/gateway/test_gateway_trust_env.py new file mode 100644 index 0000000000..78965ee66b --- /dev/null +++ b/tests/gateway/test_gateway_trust_env.py @@ -0,0 +1,47 @@ +"""gateway.trust_env — one config key controls aiohttp proxy-env honoring at every adapter site (#48820).""" +import re +from pathlib import Path + +import pytest + +from gateway.platforms import base as gw_base + +REPO = Path(__file__).resolve().parents[2] +_ADAPTER_FILES = sorted( + list((REPO / "gateway" / "platforms").rglob("*.py")) + + list((REPO / "plugins" / "platforms").rglob("*.py")) +) + + +def _write_config(tmp_path, monkeypatch, body: str) -> None: + # load_config caches on (path, mtime) — a fresh tmp HERMES_HOME per test is a fresh cache key. + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text(body) + + +@pytest.mark.parametrize( + "yaml_body, expected", + [("gateway:\n trust_env: false\n", False), ("gateway:\n trust_env: true\n", True), ("{}\n", True)], +) +def test_gateway_trust_env_reads_config(tmp_path, monkeypatch, yaml_body, expected): + """gateway.trust_env in config.yaml drives the shared helper; absent → True (default).""" + _write_config(tmp_path, monkeypatch, yaml_body) + assert gw_base.gateway_trust_env() is expected + # The generic-proxy discovery path is gated by the same knob; explicit per-platform vars are not. + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:7890") + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + assert (gw_base.resolve_proxy_url() is not None) is expected + monkeypatch.setenv("X_PLATFORM_PROXY", "http://127.0.0.1:1080") + assert gw_base.resolve_proxy_url("X_PLATFORM_PROXY") == "http://127.0.0.1:1080" + + +def test_no_bare_trust_env_literal_in_adapters(): + """Every aiohttp session in gateway/ + plugins/platforms/ must go through gateway_trust_env().""" + bare = re.compile(r"trust_env\s*=\s*(True|False)\b") + offenders = [] + for path in _ADAPTER_FILES: + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if bare.search(line) and "httpx" not in line: + offenders.append(f"{path.relative_to(REPO)}:{lineno}: {line.strip()}") + assert not offenders, "hard-coded aiohttp trust_env literal(s); use gateway_trust_env():\n" + "\n".join(offenders) diff --git a/tests/gateway/test_goal_gate_access.py b/tests/gateway/test_goal_gate_access.py new file mode 100644 index 0000000000..80f449b212 --- /dev/null +++ b/tests/gateway/test_goal_gate_access.py @@ -0,0 +1,131 @@ +"""Gateway authorization tests for shell-backed goal quality gates.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.session import SessionSource + + +def _source(user_id: str = "user-1", *, chat_type: str = "dm") -> SessionSource: + return SessionSource( + platform=Platform.DISCORD, + user_id=user_id, + chat_id=f"{chat_type}-1", + chat_type=chat_type, + ) + + +def _runner(*, admins=(), group_admins=()): + from gateway.run import GatewayRunner + + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig( + platforms={ + Platform.DISCORD: PlatformConfig( + enabled=True, + extra={ + "allow_admin_from": list(admins), + "group_allow_admin_from": list(group_admins), + }, + ) + } + ) + manager = MagicMock() + manager.add_gate.return_value = SimpleNamespace( + command="touch /tmp/host-in-the-shell", + max_retries=3, + timeout_seconds=300, + ) + runner._get_goal_manager_for_event = AsyncMock( + return_value=(manager, SimpleNamespace(session_id="session-1")) + ) + return runner, manager + + +def _event(user_id: str = "user-1", *, chat_type: str = "dm"): + event = MagicMock() + event.source = _source(user_id, chat_type=chat_type) + event.get_command_args.return_value = ( + "gate add touch /tmp/host-in-the-shell" + ) + return event + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("admins", "user_id"), + [ + ((), "user-1"), + (("admin-1",), "user-1"), + ], +) +async def test_gateway_gate_add_requires_explicit_admin(admins, user_id): + """Allowed chat users must not turn /goal into an unrestricted host shell.""" + from gateway.run import GatewayRunner + + runner, manager = _runner(admins=admins) + + result = await GatewayRunner._handle_goal_command(runner, _event(user_id)) + + assert "explicitly configured gateway admin" in result + manager.add_gate.assert_not_called() + + +@pytest.mark.asyncio +async def test_gateway_explicit_admin_can_add_goal_gate(): + """The fix preserves the documented quality-gate capability for operators.""" + from gateway.run import GatewayRunner + + runner, manager = _runner(admins=("admin-1",)) + + result = await GatewayRunner._handle_goal_command(runner, _event("admin-1")) + + assert "Gate added" in result + manager.add_gate.assert_called_once_with( + "touch /tmp/host-in-the-shell" + ) + + +@pytest.mark.asyncio +async def test_gateway_group_gate_add_uses_group_admin_scope(): + """DM admin status must not silently grant host-shell access in groups.""" + from gateway.run import GatewayRunner + + runner, manager = _runner( + admins=("dm-admin",), + group_admins=("group-admin",), + ) + + denied = await GatewayRunner._handle_goal_command( + runner, + _event("dm-admin", chat_type="group"), + ) + allowed = await GatewayRunner._handle_goal_command( + runner, + _event("group-admin", chat_type="group"), + ) + + assert "explicitly configured gateway admin" in denied + assert "Gate added" in allowed + manager.add_gate.assert_called_once_with( + "touch /tmp/host-in-the-shell" + ) + + +@pytest.mark.asyncio +async def test_gateway_non_admin_can_still_list_goal_gates(): + """Non-admins retain read-only visibility into the active goal's gates.""" + from gateway.run import GatewayRunner + + runner, manager = _runner(admins=("admin-1",)) + manager.render_gates.return_value = "- 1. $ scripts/run_tests.sh" + event = _event("user-1") + event.get_command_args.return_value = "gate list" + + result = await GatewayRunner._handle_goal_command(runner, event) + + assert result == "- 1. $ scripts/run_tests.sh" + manager.render_gates.assert_called_once_with() diff --git a/tests/gateway/test_google_chat.py b/tests/gateway/test_google_chat.py index 19aa5163e6..c9f1f1584f 100644 --- a/tests/gateway/test_google_chat.py +++ b/tests/gateway/test_google_chat.py @@ -270,6 +270,61 @@ class TestEnvConfigLoading: cfg = load_gateway_config() assert _GC not in cfg.platforms + def test_multiplex_scoped_profile_never_borrows_process_env( + self, monkeypatch, tmp_path + ): + """Under multiplex a scoped profile sees ONLY its own Google Chat + settings, and the ADC branch fails closed instead of authenticating + as the default profile's service account (#73439).""" + from agent.secret_scope import ( + build_profile_secret_scope, + set_multiplex_active, + set_secret_scope, + ) + + self._clean_env(monkeypatch) + monkeypatch.setenv("GOOGLE_CHAT_PROJECT_ID", "default-proj") + monkeypatch.setenv("GOOGLE_CHAT_SUBSCRIPTION_NAME", "default-sub") + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/secrets/default.json") + monkeypatch.setenv("GOOGLE_CHAT_BOOTSTRAP_SPACES", "spaces/DEFAULT") + profile_home = tmp_path / "beta" + profile_home.mkdir() + (profile_home / ".env").write_text( + "GOOGLE_CHAT_PROJECT_ID=beta-proj\nGOOGLE_CHAT_SUBSCRIPTION_NAME=beta-sub\n" + ) + set_multiplex_active(True) + token = set_secret_scope(build_profile_secret_scope(profile_home)) + try: + seed = _gc_mod._env_enablement() or {} + beta = GoogleChatAdapter( + PlatformConfig(enabled=True, extra={"project_id": "beta-proj", "subscription_name": "beta-sub"}) + ) + with pytest.raises(ValueError, match="ADC skipped"): + beta._load_sa_credentials() + finally: + from agent.secret_scope import reset_secret_scope + + reset_secret_scope(token) + set_multiplex_active(False) + assert seed["project_id"] == "beta-proj" + assert "service_account_json" not in seed + assert beta._bootstrap_spaces == "" + + def test_multiplex_default_profile_constructs_unscoped(self, monkeypatch): + """The default profile's adapter is built OUTSIDE any scope while + multiplex is active (gateway startup/reconnect); it must keep reading + its own process env instead of raising UnscopedSecretError.""" + from agent.secret_scope import set_multiplex_active + + self._clean_env(monkeypatch) + monkeypatch.setenv("GOOGLE_CHAT_BOOTSTRAP_SPACES", "spaces/DEFAULT") + set_multiplex_active(True) + try: + default = GoogleChatAdapter(_base_config()) + finally: + set_multiplex_active(False) + assert default._bootstrap_spaces == "spaces/DEFAULT" + # =========================================================================== # Pure helpers @@ -1281,9 +1336,9 @@ class TestAttachmentSSRFGuard: monkeypatch.setattr(asyncio, "to_thread", _fake_to_thread) from plugins.platforms.google_chat import adapter as gc_mod monkeypatch.setattr( - gc_mod, "cache_document_from_bytes", - lambda data, ext=None, filename=None: str(tmp_path / "out.pdf"), - raising=False, + gc_mod, + "cache_document_from_bytes_async", + AsyncMock(return_value=str(tmp_path / "out.pdf")), ) path, mime = await adapter._download_attachment(attachment) diff --git a/tests/gateway/test_handoff_secondary_profile_adapter.py b/tests/gateway/test_handoff_secondary_profile_adapter.py index e637b293c1..b65e75bfd6 100644 --- a/tests/gateway/test_handoff_secondary_profile_adapter.py +++ b/tests/gateway/test_handoff_secondary_profile_adapter.py @@ -167,6 +167,34 @@ async def test_default_profile_handoff_keeps_primary_adapter(monkeypatch): assert used["home_chat_id"] == "1111" +@pytest.mark.asyncio +async def test_secondary_profile_config_load_failure_fails_closed(monkeypatch): + """A secondary profile whose config cannot load must fail the handoff. + + Falling back to the primary's config delivers through the right bot to + the WRONG chat (the primary's home channel) and reports completed. + """ + runner, _ = _make_multiplex_runner() + used = {} + + def _boom(): + raise RuntimeError("config.yaml exploded") + + monkeypatch.setattr( + "gateway.run.resolve_delivery_transport", _spy_transport_factory(used), + ) + monkeypatch.setattr("gateway.run.load_gateway_config", _boom) + + with pytest.raises(RuntimeError, match="could not load config"): + await runner._process_handoff( + {"id": "cli-session", "title": "work", "handoff_platform": "telegram"}, + profile_name="medicina", + ) + assert used == {}, ( + "nothing may be delivered when the profile config fails to load" + ) + + @pytest.mark.asyncio async def test_secondary_profile_without_live_adapters_fails_loudly(monkeypatch): """Never silently fall back to the primary's bot — that ships to the wrong chat. diff --git a/tests/gateway/test_handoff_watcher_multiprofile.py b/tests/gateway/test_handoff_watcher_multiprofile.py index 08bcf95710..bea32bc7bc 100644 --- a/tests/gateway/test_handoff_watcher_multiprofile.py +++ b/tests/gateway/test_handoff_watcher_multiprofile.py @@ -15,6 +15,7 @@ These tests pin the two halves of the fix: """ import asyncio +import threading import types from pathlib import Path @@ -111,14 +112,14 @@ async def test_watcher_enters_profile_scope_for_each_home(monkeypatch): def __init__(self, home): self.home = home - def __enter__(self): + async def __aenter__(self): entered.append(self.home) return self - def __exit__(self, *exc): + async def __aexit__(self, *exc): return False - monkeypatch.setattr(run, "_profile_runtime_scope", _SpyScope) + monkeypatch.setattr(run, "_async_profile_runtime_scope", _SpyScope) async def _no_sleep(_seconds): return None @@ -159,6 +160,68 @@ async def test_watcher_enters_profile_scope_for_each_home(monkeypatch): assert db.polls == 3, "root + both profiles polled once each per tick" +@pytest.mark.asyncio +async def test_slow_profile_secret_load_does_not_block_event_loop(monkeypatch, tmp_path): + """A slow profile ``.env`` read must not stall unrelated loop work.""" + profile_home = tmp_path / "profiles" / "slow" + profile_home.mkdir(parents=True) + monkeypatch.setattr( + run, + "_handoff_watch_scopes", + lambda _runner: [(None, None), ("slow", profile_home)], + ) + + from agent import secret_scope + + load_started = threading.Event() + ticker_progressed = threading.Event() + ticker_progressed_while_loading = [] + + def _slow_build(_home): + load_started.set() + ticker_progressed_while_loading.append( + ticker_progressed.wait(timeout=2) + ) + return {} + + monkeypatch.setattr(secret_scope, "build_profile_secret_scope", _slow_build) + + class _DB: + async def list_pending_handoffs(self): + return [] + + fake = types.SimpleNamespace( + _session_db=_DB(), + _running=False, + ) + + async def _process_handoff(_row, _profile_name=None): + return None + + fake._process_handoff = _process_handoff + + real_sleep = asyncio.sleep + + async def _skip_initial_delay(seconds): + await real_sleep(0 if seconds == 5 else seconds) + + monkeypatch.setattr(run.asyncio, "sleep", _skip_initial_delay) + async def _ticker(): + assert await asyncio.to_thread(load_started.wait, 5) + ticker_progressed.set() + + watcher = asyncio.create_task( + run.GatewayRunner._handoff_watcher(fake, interval=0.0) + ) + ticker = asyncio.create_task(_ticker()) + await asyncio.wait_for(asyncio.gather(watcher, ticker), timeout=5) + + assert ticker_progressed_while_loading == [True], ( + "profile secret loading blocked the asyncio event loop until the " + "filesystem operation completed" + ) + + @pytest.mark.asyncio async def test_each_scope_resolves_its_own_store_and_profile(monkeypatch): """The whole point: a DIFFERENT ``state.db`` per scope, and the profile @@ -183,15 +246,15 @@ async def test_each_scope_resolves_its_own_store_and_profile(monkeypatch): def __init__(self, home): self.home = home - def __enter__(self): + async def __aenter__(self): active["home"] = self.home return self - def __exit__(self, *exc): + async def __aexit__(self, *exc): active["home"] = None return False - monkeypatch.setattr(run, "_profile_runtime_scope", _SpyScope) + monkeypatch.setattr(run, "_async_profile_runtime_scope", _SpyScope) async def _no_sleep(_seconds): return None diff --git a/tests/gateway/test_handoff_watcher_resilience.py b/tests/gateway/test_handoff_watcher_resilience.py index cfe672b784..b7b4e25d73 100644 --- a/tests/gateway/test_handoff_watcher_resilience.py +++ b/tests/gateway/test_handoff_watcher_resilience.py @@ -219,13 +219,13 @@ async def test_reclaim_runs_per_profile_store(monkeypatch): def __init__(self, home): self.home = home - def __enter__(self): + async def __aenter__(self): return self - def __exit__(self, *exc): + async def __aexit__(self, *exc): return False - monkeypatch.setattr(run, "_profile_runtime_scope", _Scope) + monkeypatch.setattr(run, "_async_profile_runtime_scope", _Scope) async def _no_sleep(_seconds): return None diff --git a/tests/gateway/test_hygiene_deferred_work_drain.py b/tests/gateway/test_hygiene_deferred_work_drain.py new file mode 100644 index 0000000000..51d85a7fab --- /dev/null +++ b/tests/gateway/test_hygiene_deferred_work_drain.py @@ -0,0 +1,150 @@ +"""Shutdown accounting for executor work detached by hygiene timeouts.""" + +import asyncio +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from gateway.run import run_codex_hygiene_compaction +from tests.gateway.restart_test_helpers import make_restart_runner + + +@pytest.mark.asyncio +async def test_deferred_worker_remains_active_until_executor_future_finishes(): + runner, _adapter = make_restart_runner() + runner._cleanup_agent_resources_off_loop = AsyncMock() + worker = asyncio.get_running_loop().create_future() + agent = MagicMock() + + runner._defer_agent_cleanup_until_future_done( + worker, + agent, + context="test hygiene timeout", + ) + + assert runner._active_deferred_agent_worker_count() == 1 + assert runner._active_work_count() == 1 + + worker.set_result(([], None)) + await asyncio.gather(*runner._deferred_agent_cleanup_tasks) + + assert runner._active_deferred_agent_worker_count() == 0 + runner._cleanup_agent_resources_off_loop.assert_awaited_once_with( + agent, + context="test hygiene timeout", + ) + + +@pytest.mark.asyncio +async def test_timed_out_codex_hygiene_worker_remains_visible_to_shutdown(): + runner, _adapter = make_restart_runner() + started = threading.Event() + release = threading.Event() + + class BlockingCodexAgent: + _codex_session = object() + context_compressor = SimpleNamespace(compression_count=0) + + def _compress_context(self, *_args, **_kwargs): + started.set() + release.wait(timeout=5.0) + + agent = BlockingCodexAgent() + runner._agent_cache = {"tg:123": (agent, 0.0)} + runner._agent_cache_lock = None + + outcome = await run_codex_hygiene_compaction( + runner, + "tg:123", + "sess-1", + auto_mode="hermes", + history=[{"role": "user", "content": "hello"}], + approx_tokens=100, + timeout_seconds=0.01, + failure_cooldown_seconds=-1.0, + ) + + assert started.is_set() + assert outcome == "failed:timeout" + assert runner._active_deferred_agent_worker_count() == 1 + with patch("gateway.run.request_hard_interrupt") as interrupt: + runner._interrupt_running_agents("gateway shutdown") + interrupt.assert_called_once_with(agent, "gateway shutdown") + + release.set() + for _ in range(100): + if runner._active_deferred_agent_worker_count() == 0: + break + await asyncio.sleep(0.01) + assert runner._active_deferred_agent_worker_count() == 0 + + +@pytest.mark.asyncio +async def test_shutdown_drain_waits_for_deferred_hygiene_worker(): + runner, _adapter = make_restart_runner() + worker = asyncio.get_running_loop().create_future() + runner._deferred_agent_workers = {worker: MagicMock()} + + async def finish_worker(): + await asyncio.sleep(0.12) + worker.set_result(([], None)) + + finisher = asyncio.create_task(finish_worker()) + _snapshot, timed_out = await runner._drain_active_agents(2.0) + await finisher + + assert timed_out is False + assert _snapshot == {} + + +@pytest.mark.asyncio +async def test_deferred_hygiene_worker_times_out_and_receives_interrupt(): + runner, _adapter = make_restart_runner() + worker = asyncio.get_running_loop().create_future() + agent = MagicMock() + runner._deferred_agent_workers = {worker: agent} + + _snapshot, timed_out = await runner._drain_active_agents(0.01) + + assert timed_out is True + assert _snapshot == {} + with patch("gateway.run.request_hard_interrupt") as interrupt: + runner._interrupt_running_agents("gateway shutdown") + interrupt.assert_called_once_with(agent, "gateway shutdown") + + worker.cancel() + + +@pytest.mark.asyncio +async def test_stop_interrupts_deferred_worker_before_teardown(): + runner, adapter = make_restart_runner() + runner._restart_drain_timeout = 0.01 + worker = asyncio.get_running_loop().create_future() + + class DeferredAgent: + def __init__(self): + self.interrupts = [] + + def hard_interrupt(self, reason): + self.interrupts.append(reason) + if not worker.done(): + worker.set_result(([], None)) + + agent = DeferredAgent() + runner._deferred_agent_workers = {worker: agent} + adapter.disconnect = AsyncMock() + + with ( + patch("gateway.status.remove_pid_file"), + patch("gateway.status.write_runtime_status"), + patch("cron.scheduler.mark_job_run"), + patch("tools.process_registry.process_registry.kill_all", return_value=0), + patch("tools.terminal_tool.cleanup_all_environments"), + patch("tools.browser_tool.cleanup_all_browsers"), + ): + await runner.stop() + + assert agent.interrupts == ["Gateway shutting down"] + assert worker.done() diff --git a/tests/gateway/test_irc_adapter.py b/tests/gateway/test_irc_adapter.py index f08cf73614..e703f5e1fd 100644 --- a/tests/gateway/test_irc_adapter.py +++ b/tests/gateway/test_irc_adapter.py @@ -18,6 +18,8 @@ check_requirements = _irc_mod.check_requirements validate_config = _irc_mod.validate_config register = _irc_mod.register _standalone_send = _irc_mod._standalone_send +is_connected = _irc_mod.is_connected +_env_enablement = _irc_mod._env_enablement class TestIRCProtocolHelpers: @@ -406,3 +408,96 @@ class TestIRCStandaloneSend: assert "registration" in result["error"].lower() or "timeout" in result["error"].lower() +# --------------------------------------------------------------------------- +# Multiplex secondary-profile scope +# --------------------------------------------------------------------------- +# +# __init__'s server/port/nickname/channel/use_tls, check_requirements/ +# validate_config/is_connected's server/channel, and _env_enablement's +# server/channel/port/nickname/use_tls/home_channel, all previously read raw +# os.getenv unconditionally (only IRC_SERVER_PASSWORD/IRC_NICKSERV_PASSWORD +# were already scoped). Under multiplex, os.environ holds the DEFAULT +# profile's YAML-to-env bridge output -- a secondary profile with its own +# (different or absent) IRC config would silently connect to the default +# profile's server/channel, or (for _env_enablement) get auto-enabled using +# the default's channel as its cron home_channel -- a real message- +# misdelivery risk, not just cosmetic. Mirrors the LINE/Buzz/SimpleX fix for +# #98738. + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + tokens = [] + + def install(scope=None): + from agent.secret_scope import set_multiplex_active, set_secret_scope + + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + return tokens[-1] + + yield install + + from agent.secret_scope import reset_secret_scope, set_multiplex_active + + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("IRC_SERVER", "default.example.net") + monkeypatch.setenv("IRC_CHANNEL", "#default") + monkeypatch.setenv("IRC_PORT", "6667") + monkeypatch.setenv("IRC_NICKNAME", "default-bot") + monkeypatch.setenv("IRC_USE_TLS", "false") + + +class TestMultiplexProfileScope: + + def test_secondary_extra_wins_over_default_profile_env( + self, multiplex_scope, default_profile_env + ): + """The secondary profile's own config.yaml extra is authoritative, + not the default profile's bridged server/channel/port/nick/tls.""" + from gateway.config import PlatformConfig + + multiplex_scope() + cfg = PlatformConfig( + enabled=True, + extra={ + "server": "profile.example.net", + "channel": "#profile", + "port": 6697, + "nickname": "profile-bot", + "use_tls": True, + }, + ) + adapter = IRCAdapter(cfg) + assert adapter.server == "profile.example.net" + assert adapter.channel == "#profile" + assert adapter.port == 6697 + assert adapter.nickname == "profile-bot" + assert adapter.use_tls is True + + def test_secondary_missing_keys_fail_closed( + self, multiplex_scope, default_profile_env + ): + """Keys absent from the profile's own scope must NOT borrow the + default profile's bridged env values -- that would silently connect + the secondary profile's bot to the wrong IRC server/channel.""" + from gateway.config import PlatformConfig + + multiplex_scope() + adapter = IRCAdapter(PlatformConfig(enabled=True, extra={})) + assert adapter.server == "" + assert adapter.channel == "" + assert adapter.port == 6697 # falls through to the hardcoded default + assert adapter.nickname == "hermes-bot" + assert adapter.use_tls is True # extra.get("use_tls", True) default + # Nor may the registry auto-enable IRC for this profile off the default's channel. + assert _env_enablement() is None + assert is_connected(PlatformConfig(enabled=True, extra={})) is False + diff --git a/tests/gateway/test_line_plugin.py b/tests/gateway/test_line_plugin.py index e59bd8286e..5a6386a7be 100644 --- a/tests/gateway/test_line_plugin.py +++ b/tests/gateway/test_line_plugin.py @@ -205,10 +205,14 @@ class TestInboundMedia: return adapter.handle_message.await_args.args[0] def test_image_message_uses_photo_type_and_image_mime(self, adapter): - with patch.object(_line, "cache_image_from_bytes", return_value="/cache/image.jpg") as cache: + with patch.object( + _line, + "cache_image_from_bytes_async", + new=AsyncMock(return_value="/cache/image.jpg"), + ) as cache: asyncio.run(adapter._handle_message_event(self._event("image"))) - cache.assert_called_once_with(b"line-bytes", ext=".jpg") + cache.assert_awaited_once_with(b"line-bytes", ext=".jpg") event = self._captured_event(adapter) assert event.message_type is _line.MessageType.PHOTO assert event.media_urls == ["/cache/image.jpg"] diff --git a/tests/gateway/test_loop_command.py b/tests/gateway/test_loop_command.py index 7e8caee8fc..19c9c88520 100644 --- a/tests/gateway/test_loop_command.py +++ b/tests/gateway/test_loop_command.py @@ -1,7 +1,10 @@ """Gateway /loop command tests — dispatch, routing capture, mid-run guard.""" +import asyncio import logging +import threading import time +from types import SimpleNamespace from unittest.mock import AsyncMock, Mock import pytest @@ -255,3 +258,147 @@ async def test_post_turn_session_resolution_failure_is_logged(loop_env, caplog): ) assert "post-turn session resolution failed: store unavailable" in caplog.text + + +@pytest.mark.asyncio +async def test_loop_wakeup_watcher_keeps_event_loop_responsive_under_writer_lock(loop_env): + """The wakeup scan's SessionDB calls (list_active_loops / fire_tick / + complete_tick) must run off the loop thread. A slow writer holding the + SessionDB writer lock used to block the whole gateway event loop for the + duration of the hold (#92413).""" + runner = _make_runner() + runner._running = True + runner._running_agents = {} + runner.adapters = {} + + # Persist an active loop that is due now, routed to a platform with no + # adapter (so the scan exits after list_active_loops(), before fire_tick). + await GatewayRunner._handle_loop_command(runner, _make_event("/loop 5m poll CI")) + state = loops.load_loop("sid-gateway-loop") + state.next_due_at = time.time() - 1 + loops.save_loop("sid-gateway-loop", state) + + db = loops._get_session_db() + hold_s = 0.6 + released = threading.Event() + + def _hold_writer_lock(): + with db._lock: + time.sleep(hold_s) + released.set() + + # Force the read path onto the writer lock (non-WAL degradation) so the + # test binds the offload regardless of the host SQLite's WAL support. + db._wal_active = False + + # One scan only: patch asyncio.sleep inside the watcher to stop the loop + # after the first iteration. + orig_sleep = asyncio.sleep + calls = {"n": 0} + + async def _one_pass_sleep(delay): + calls["n"] += 1 + if calls["n"] >= 2: # first call is the 5s connect grace, second ends the scan + runner._running = False + return await orig_sleep(0) + + holder = threading.Thread(target=_hold_writer_lock) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(asyncio, "sleep", _one_pass_sleep) + holder.start() + time.sleep(0.05) # ensure the lock is held before the scan starts + watcher = asyncio.ensure_future(GatewayRunner._loop_wakeup_watcher(runner, interval=0)) + + # Heartbeat coroutine: measures the longest gap between loop turns + # while the watcher is (supposedly) blocked in the executor. + gaps = [] + last = time.monotonic() + while not watcher.done(): + await orig_sleep(0.01) + now = time.monotonic() + gaps.append(now - last) + last = now + await watcher + holder.join() + + assert released.is_set() + # If the DB call ran on the loop thread, one heartbeat gap would be ~hold_s. + assert max(gaps) < hold_s / 2, f"event loop stalled for {max(gaps):.3f}s" + + +@pytest.mark.asyncio +async def test_loop_wakeup_watcher_runs_every_sessiondb_call_off_loop_thread(loop_env): + """Full wakeup path (slash-command loop, adapter present): list_active_loops, + fire_tick and complete_tick must each execute on an executor thread, never + on the event-loop thread (#92413).""" + runner = _make_runner() + runner._running = True + runner._running_agents = {} + + class _Adapter: + handled = [] + + async def handle_message(self, event): + self.handled.append(event.text) + + runner.adapters = {Platform.DISCORD: _Adapter()} + runner._build_process_event_source = lambda evt: SimpleNamespace( + platform=Platform.DISCORD, chat_id=evt["chat_id"], chat_type=evt["chat_type"], + thread_id=evt["thread_id"] or None, user_id=evt["user_id"], user_name=evt["user_name"], + ) + runner._session_key_for_source = lambda source: "agent:main:discord:channel:chat-loop" + + # A slash-command loop that is due now. + await GatewayRunner._handle_loop_command(runner, _make_event("/loop 5m /status")) + state = loops.load_loop("sid-gateway-loop") + state.next_due_at = time.time() - 1 + loops.save_loop("sid-gateway-loop", state) + + loop_thread = threading.current_thread() + on_loop_calls = [] + seen_calls = [] + + def _record(name): + # Positive count guards against a future import hoist in run.py that + # would bypass these wrappers and leave the off-loop assertion vacuous. + seen_calls.append(name) + if threading.current_thread() is loop_thread: + on_loop_calls.append(name) + + real_list = loops.list_active_loops + real_fire = loops.LoopManager.fire_tick + real_complete = loops.LoopManager.complete_tick + + def _list_active_loops(*a, **k): + _record("list_active_loops") + return real_list(*a, **k) + + def _fire_tick(self): + _record("fire_tick") + return real_fire(self) + + def _complete_tick(self, last_response): + _record("complete_tick") + return real_complete(self, last_response) + + orig_sleep = asyncio.sleep + calls = {"n": 0} + + async def _one_pass_sleep(delay): + calls["n"] += 1 + if calls["n"] >= 2: + runner._running = False + return await orig_sleep(0) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(loops, "list_active_loops", _list_active_loops) + mp.setattr(loops.LoopManager, "fire_tick", _fire_tick) + mp.setattr(loops.LoopManager, "complete_tick", _complete_tick) + mp.setattr(asyncio, "sleep", _one_pass_sleep) + await GatewayRunner._loop_wakeup_watcher(runner, interval=0) + + assert _Adapter.handled == ["/status"], _Adapter.handled + assert seen_calls == ["list_active_loops", "fire_tick", "complete_tick"], seen_calls + assert on_loop_calls == [], f"SessionDB calls ran on the event-loop thread: {on_loop_calls}" + # complete_tick ran (slash-command loops complete immediately). + assert loops.load_loop("sid-gateway-loop").ticks_fired == 1 diff --git a/tests/gateway/test_loop_liveness_watchdog.py b/tests/gateway/test_loop_liveness_watchdog.py index ae07106b27..d763fbc461 100644 --- a/tests/gateway/test_loop_liveness_watchdog.py +++ b/tests/gateway/test_loop_liveness_watchdog.py @@ -3,8 +3,11 @@ from __future__ import annotations import asyncio +import json +import os import pathlib import inspect +import tempfile import threading import time from unittest.mock import MagicMock, patch @@ -403,3 +406,100 @@ def test_loop_scheduling_witness_is_served_by_the_loop_itself(): assert "await asyncio.start_unix_server(" in body, ( "the loop-scheduling witness socket is not armed by the loop task" ) + + +def test_windows_tcp_witness_arms_and_publishes_port(): + """On non-POSIX platforms the witness must arm over TCP loopback. + + ``asyncio.start_unix_server`` does not exist on Windows (no AF_UNIX + event-loop support), so the producer arm fell into the broad except and + recorded ``loop_tick_socket=False`` — every stale-file probe then + classified UNKNOWN forever, disabling the wedge interlock on Windows + entirely. The TCP loopback witness restores the same contract: armed by + the loop task (an awaited ``asyncio.start_server`` is structurally + loop-owned exactly like the Unix variant), answered only while the loop + dispatches, port published in the heartbeat payload. + """ + if os.name == "posix": + pytest.skip("TCP loopback witness is the non-POSIX arm") + + async def scenario() -> tuple[dict, bool]: + task = asyncio.create_task( + loop_heartbeat_forever(interval_s=1.0, home=tmp_home) + ) + try: + deadline = time.monotonic() + 5.0 + payload = None + while time.monotonic() < deadline: + hb = tmp_home.joinpath(*("state", "gateway.heartbeat")) + if hb.exists(): + try: + payload = json.loads(hb.read_text(encoding="utf-8")) + except Exception: + payload = None + if payload and payload.get("loop_tick_tcp_port"): + break + await asyncio.sleep(0.02) + assert payload is not None, "heartbeat never appeared" + assert payload.get("loop_tick_socket") is True, ( + "witness reported unarmed on a platform where the TCP arm " + "must work" + ) + port = int(payload["loop_tick_tcp_port"]) + assert 0 < port <= 65535, "published port out of range" + + # Probe from a worker thread so the blocking connect/recv never + # stalls the very loop we are witnessing (an external process + # probes from its own loop/thread — reproduce that shape). + from hermes_cli.gateway import _probe_loop_tick_tcp + + result_box: dict[str, object] = {} + + def _probe() -> None: + result_box["r"] = _probe_loop_tick_tcp(port, timeout=2.0) + + worker = threading.Thread(target=_probe) + worker.start() + while worker.is_alive(): + await asyncio.sleep(0.05) + worker.join() + return payload, bool(result_box.get("r") is True) + finally: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + with tempfile.TemporaryDirectory(prefix="lw-tcp-") as raw: + tmp_home = pathlib.Path(raw) + payload, answered = asyncio.run(scenario()) + assert answered, ( + "the loop-tick TCP witness did not answer a probe while the loop " + "was dispatching — the two-witness interlock would misclassify " + "this gateway as UNKNOWN" + ) + + +def test_windows_tcp_witness_arms_on_loop_task_source_shape(): + """The TCP arm must be awaited by the loop task, never thread-owned. + + Structural companion to ``test_loop_scheduling_witness_is_served_by_the_ + loop_itself``: the same property that makes the Unix socket an honest + witness (a coroutine cannot run inside a thread) must hold for the TCP + loopback arm, or a wedged loop could keep answering pings and the + interlock would be void on Windows. + """ + src = pathlib.Path( + inspect.getsourcefile(loop_heartbeat_forever) or "" + ).read_text() + body = src[src.index("async def loop_heartbeat_forever("):] + body = body[: body.index("\ndef ") if "\ndef " in body else len(body)] + assert "await asyncio.start_server(" in body, ( + "the TCP loop-scheduling witness is not armed by the loop task" + ) + # The Unix arm must stay gated to POSIX-only code paths so the missing + # attribute can never raise on Windows again. + assert 'os.name == "posix"' in body, ( + "the AF_UNIX witness arm is not gated to POSIX platforms" + ) diff --git a/tests/gateway/test_matrix_crypto_store_per_profile.py b/tests/gateway/test_matrix_crypto_store_per_profile.py new file mode 100644 index 0000000000..3705689260 --- /dev/null +++ b/tests/gateway/test_matrix_crypto_store_per_profile.py @@ -0,0 +1,47 @@ +"""Matrix crypto store must be pinned per profile at connect(), not at import. + +Under ``gateway.multiplex_profiles`` one process imports +``plugins.platforms.matrix.adapter`` once; the old module-level +``_STORE_DIR``/``_CRYPTO_DB_PATH`` resolved against the root HERMES_HOME at +import time, so every profile's adapter opened the SAME crypto.db and inbound +E2EE failed with "no session found" (#89168). ``connect()`` calls +``_resolve_store_dir()`` inside ``_profile_runtime_scope`` (context-local +HERMES_HOME), so resolving there -- and caching on the instance -- gives each +profile its own store. Exercised via ``_resolve_store_dir`` directly so the +test needs no mautrix install. +""" +from gateway.config import PlatformConfig +from hermes_constants import reset_hermes_home_override, set_hermes_home_override +from plugins.platforms.matrix import adapter as matrix_adapter + + +def _make_adapter() -> matrix_adapter.MatrixAdapter: + return matrix_adapter.MatrixAdapter( + PlatformConfig( + enabled=True, + token="syt_test_token", + extra={"homeserver": "https://matrix.example.org", "user_id": "@bot:example.org"}, + ) + ) + + +def test_store_dir_pinned_to_each_profile_home(tmp_path): + """Two profiles resolving in one process get two stores, and each + adapter keeps reporting its own store after the scope is gone.""" + stores = {} + for profile in ("accountant", "engineering-lead"): + home = tmp_path / "profiles" / profile + home.mkdir(parents=True) + adapter = _make_adapter() + token = set_hermes_home_override(str(home)) + try: + adapter._resolve_store_dir().mkdir(parents=True, exist_ok=True) + finally: + reset_hermes_home_override(token) + # Cached on the instance: correct even when read outside the scope. + path = adapter.get_diagnostics()["e2ee"]["crypto_store_path"] + assert path.startswith(str(home)), f"store not profile-scoped: {path}" + assert adapter._store_dir.is_dir() + stores[profile] = path + + assert stores["accountant"] != stores["engineering-lead"] diff --git a/tests/gateway/test_mattermost.py b/tests/gateway/test_mattermost.py index 3166ddea53..9cb56073a8 100644 --- a/tests/gateway/test_mattermost.py +++ b/tests/gateway/test_mattermost.py @@ -594,3 +594,118 @@ async def test_mattermost_top_level_channel_post_is_thread_root(): assert msg_event.message_id == "top_post_123" +# --------------------------------------------------------------------------- +# Multiplex secondary-profile scope +# --------------------------------------------------------------------------- +# +# __init__'s url/reply_mode, validate_mattermost_config's url, +# _standalone_send's url, and _handle_ws_event's require_mention/ +# free_response_channels/allowed_channels, all previously read raw +# os.getenv unconditionally (only MATTERMOST_TOKEN was already scoped). +# _apply_yaml_config also wrote MATTERMOST_REQUIRE_MENTION/ +# MATTERMOST_FREE_RESPONSE_CHANNELS/MATTERMOST_ALLOWED_CHANNELS into the +# process-global os.environ unconditionally. Under multiplex, os.environ +# holds the DEFAULT profile's YAML-to-env bridge output -- a secondary +# profile with its own (different or absent) Mattermost config would +# silently connect to the default profile's server, or have its +# mention-gating/channel-allowlist decisions driven by the default +# profile's settings. Mirrors the LINE/DingTalk/IRC fix for #98738. + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + tokens = [] + + def install(scope=None): + from agent.secret_scope import set_multiplex_active, set_secret_scope + + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + return tokens[-1] + + yield install + + from agent.secret_scope import reset_secret_scope, set_multiplex_active + + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("MATTERMOST_URL", "https://default.example.com") + monkeypatch.setenv("MATTERMOST_REPLY_MODE", "thread") + monkeypatch.setenv("MATTERMOST_REQUIRE_MENTION", "false") + monkeypatch.setenv("MATTERMOST_FREE_RESPONSE_CHANNELS", "chan_default") + monkeypatch.setenv("MATTERMOST_ALLOWED_CHANNELS", "chan_default") + + +class TestMultiplexProfileScope: + + @pytest.mark.asyncio + async def test_ws_event_gating_uses_scoped_settings_not_default( + self, monkeypatch + ): + """A secondary profile's own require_mention/free_response_channels/ + allowed_channels (installed via the scope) must gate its messages -- + not the default profile's bridged settings.""" + from agent.secret_scope import ( + reset_secret_scope, + set_multiplex_active, + set_secret_scope, + ) + from plugins.platforms.mattermost.adapter import MattermostAdapter + + monkeypatch.setenv("MATTERMOST_REQUIRE_MENTION", "true") + monkeypatch.delenv("MATTERMOST_FREE_RESPONSE_CHANNELS", raising=False) + + adapter = _make_adapter() + adapter._bot_user_id = "bot_user_id" + adapter._bot_username = "hermes-bot" + adapter.handle_message = AsyncMock() + + post_data = { + "id": "post_scoped", + "user_id": "user_123", + "channel_id": "chan_456", + "message": "hello with no mention", + } + event = { + "event": "posted", + "data": { + "post": json.dumps(post_data), + "channel_type": "O", + "sender_name": "@alice", + }, + } + + set_multiplex_active(True) + token = set_secret_scope({"MATTERMOST_REQUIRE_MENTION": "false"}) + try: + await adapter._handle_ws_event(event) + finally: + reset_secret_scope(token) + set_multiplex_active(False) + + # The profile's own scope disables require_mention -- the message + # must be dispatched even without an @mention, despite the default + # profile's env bridge saying require_mention=true. + assert adapter.handle_message.called + + def test_apply_yaml_config_scoped_skips_env_write_and_seeds_extra( + self, multiplex_scope + ): + from plugins.platforms.mattermost.adapter import _apply_yaml_config + + multiplex_scope() + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("MATTERMOST_REQUIRE_MENTION", None) + seeded = _apply_yaml_config({}, {"require_mention": False, "allowed_channels": ["c1"]}) + assert seeded == {"require_mention": False, "allowed_channels": ["c1"]} + # Under a secondary profile's scope the env bridge must be + # skipped -- writing here would leak into every other profile's + # os.environ. + assert "MATTERMOST_REQUIRE_MENTION" not in os.environ + diff --git a/tests/gateway/test_multiplex_adapter_registry.py b/tests/gateway/test_multiplex_adapter_registry.py index 3d0c196cbf..972701ddaf 100644 --- a/tests/gateway/test_multiplex_adapter_registry.py +++ b/tests/gateway/test_multiplex_adapter_registry.py @@ -1,6 +1,9 @@ """Phase 3: secondary-profile adapter registry + same-token conflict detection.""" import logging import asyncio +import threading +import time +import types from contextlib import contextmanager from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -39,6 +42,50 @@ class TestCredentialFingerprint: assert fp1 is not None assert "shared-project-secret" not in fp1 + def test_reads_feishu_app_id(self): + """Feishu/Lark authenticates via app_id/app_secret, not a token. + + Without _app_id in the fingerprint attribute list, every Feishu + adapter in a multiplexed gateway returns None here and the + same-credential conflict check is silently skipped — N profiles + spawn WebSocket clients against the same app, which evict each + other in a 1000 bye loop until all go offline. + """ + class _FeishuAdapter: + def __init__(self): + self._app_id = "cli_a1b2c3" + self._app_secret = "top-secret" + + fp1 = GatewayRunner._adapter_credential_fingerprint(_FeishuAdapter()) + fp2 = GatewayRunner._adapter_credential_fingerprint(_FeishuAdapter()) + + assert fp1 is not None + assert fp1 == fp2 # same app -> same fingerprint -> conflict detected + assert "cli_a1b2c3" not in fp1 # log-safe, never the raw credential + + def test_distinct_feishu_app_ids_distinct_fp(self): + class _FeishuAdapter: + def __init__(self, app_id): + self._app_id = app_id + self._app_secret = "s" + + fp_a = GatewayRunner._adapter_credential_fingerprint(_FeishuAdapter("app-A")) + fp_b = GatewayRunner._adapter_credential_fingerprint(_FeishuAdapter("app-B")) + + assert fp_a is not None and fp_b is not None + assert fp_a != fp_b + + @pytest.mark.parametrize("attr", ["_client_id", "_bot_id"]) + def test_reads_app_style_ids_teams_wecom(self, attr): + """Teams (_client_id) and WeCom (_bot_id) are the same class as Feishu: + id/secret pairs, no token — cloned profiles must collide.""" + a = types.SimpleNamespace(**{attr: "app-1"}) + b = types.SimpleNamespace(**{attr: "app-1"}) + c = types.SimpleNamespace(**{attr: "app-2"}) + fp = GatewayRunner._adapter_credential_fingerprint + assert fp(a) is not None and fp(a) == fp(b) + assert fp(a) != fp(c) + assert "app-1" not in fp(a) def test_reads_config_token(self): """Adapters like Discord store token on `config`, not on self. @@ -182,11 +229,15 @@ def _secondary_recovery_runner(*, running=True): return runner -def _install_secondary_reconnect_context(monkeypatch, runner, adapter, scoped_homes=None): +def _install_secondary_reconnect_context( + monkeypatch, runner, adapter, scoped_homes=None, hydration_flags=None +): @contextmanager - def fake_scope(profile_home): + def fake_scope(profile_home, *, hydrate_secrets=True): if scoped_homes is not None: scoped_homes.append(Path(profile_home)) + if hydration_flags is not None: + hydration_flags.append(hydrate_secrets) yield monkeypatch.setattr(gateway_run, "_profile_runtime_scope", fake_scope) @@ -208,6 +259,93 @@ def _install_secondary_reconnect_context(monkeypatch, runner, adapter, scoped_ho class TestSecondaryProfileFatalRecovery: + @pytest.mark.asyncio + @pytest.mark.parametrize("entry", ["startup", "reconnect"]) + async def test_secondary_hydrates_secrets_off_the_event_loop(self, monkeypatch, entry): + """#99519 class: both secondary entry points (initial start + reconnect) + hydrate external secret sources in a worker thread, exactly once, and + enter the runtime scope with hydration disabled.""" + runner = _secondary_recovery_runner() + replacement = _SecondaryRecoveryAdapter() + hydration_flags = [] + _install_secondary_reconnect_context( + monkeypatch, runner, replacement, hydration_flags=hydration_flags + ) + loop_thread_id = threading.get_ident() + hydration_started = threading.Event() + hydration_finished = threading.Event() + hydration_thread_ids = [] + stop_ticker = asyncio.Event() + ticks_during_hydration = 0 + + def slow_hydrate(profile_home): + hydration_thread_ids.append(threading.get_ident()) + hydration_started.set() + time.sleep(0.05) + hydration_finished.set() + + async def ticker(): + nonlocal ticks_during_hydration + while not stop_ticker.is_set(): + if hydration_started.is_set() and not hydration_finished.is_set(): + ticks_during_hydration += 1 + await asyncio.sleep(0) + + async def connect(adapter, platform, **_kwargs): + assert adapter is replacement + assert platform is Platform.DISCORD + return True + + monkeypatch.setattr( + "hermes_cli.env_loader.hydrate_profile_secret_sources", slow_hydrate + ) + monkeypatch.setattr(runner, "_connect_adapter_with_timeout", connect) + monkeypatch.setattr(runner, "_connect_initial_adapter_with_timeout", connect) + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: {}) + monkeypatch.setattr(runner, "_snapshot_profile_busy_modes", lambda *a, **k: None) + monkeypatch.setattr("hermes_cli.plugins.discover_plugins", lambda: None) + if entry == "startup": + coro = runner._start_one_profile_adapters( + "reviewer", Path("/profiles/reviewer"), {} + ) + else: + coro = runner._run_secondary_profile_reconnect("reviewer", Platform.DISCORD) + ticker_task = asyncio.create_task(ticker()) + work = asyncio.create_task(coro) + try: + assert await asyncio.to_thread(hydration_started.wait, 1.0) + await work + finally: + stop_ticker.set() + await ticker_task + + assert len(hydration_thread_ids) == 1 + assert hydration_thread_ids[0] != loop_thread_id + assert ticks_during_hydration > 0 + assert hydration_flags and set(hydration_flags) == {False} + assert runner._profile_adapters["reviewer"][Platform.DISCORD] is replacement + + @pytest.mark.asyncio + async def test_secondary_initial_connect_syncs_voice_mode_state(self, monkeypatch): + """#84872: a secondary bot gets its persisted /voice state at INITIAL + connect, not only on reconnect.""" + runner = _secondary_recovery_runner() + adapter = _SecondaryRecoveryAdapter() + _install_secondary_reconnect_context(monkeypatch, runner, adapter) + synced = [] + runner._sync_voice_mode_state_to_adapter = synced.append + monkeypatch.setattr("hermes_cli.env_loader.hydrate_profile_secret_sources", lambda h: {}) + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: {}) + monkeypatch.setattr(runner, "_snapshot_profile_busy_modes", lambda *a, **k: None) + monkeypatch.setattr("hermes_cli.plugins.discover_plugins", lambda: None) + + async def connect(a, platform): + return True + + monkeypatch.setattr(runner, "_connect_initial_adapter_with_timeout", connect) + assert await runner._start_one_profile_adapters("reviewer", Path("/profiles/reviewer"), {}) == 1 + assert synced == [adapter] + @pytest.mark.asyncio async def test_retryable_secondary_fatal_reconnects_with_its_profile_scope( self, monkeypatch @@ -341,13 +479,15 @@ class TestSecondaryStartupFailureRecovery: # gateway is already running) to the regular reconnect task, which # publishes the replacement and clears its own slot. await asyncio.wait_for(bridge[0], timeout=0.5) - for _ in range(20): - if ( - runner._profile_adapters.get("reviewer", {}).get(Platform.DISCORD) - is replacement - ): - break - await asyncio.sleep(0) + # The reconnect runner hops to a worker thread for secret hydration, + # so wait on a deadline rather than a fixed number of loop turns. + deadline = time.monotonic() + 1.0 + while ( + runner._profile_adapters.get("reviewer", {}).get(Platform.DISCORD) + is not replacement + and time.monotonic() < deadline + ): + await asyncio.sleep(0.005) assert ( runner._profile_adapters["reviewer"][Platform.DISCORD] is replacement ) @@ -394,13 +534,15 @@ class TestSecondaryStartupFailureRecovery: bridge = list(runner._background_tasks) assert len(bridge) == 1 await asyncio.wait_for(bridge[0], timeout=0.5) - for _ in range(20): - if ( - runner._profile_adapters.get("reviewer", {}).get(Platform.DISCORD) - is replacement - ): - break - await asyncio.sleep(0) + # The reconnect runner hops to a worker thread for secret hydration, + # so wait on a deadline rather than a fixed number of loop turns. + deadline = time.monotonic() + 1.0 + while ( + runner._profile_adapters.get("reviewer", {}).get(Platform.DISCORD) + is not replacement + and time.monotonic() < deadline + ): + await asyncio.sleep(0.005) assert ( runner._profile_adapters["reviewer"][Platform.DISCORD] is replacement ) @@ -435,6 +577,55 @@ class TestSecondaryStartupFailureRecovery: assert runner._background_tasks == set() assert runner._profile_failed_platforms == {} + @pytest.mark.asyncio + async def test_token_lock_initial_failure_parks_fatal_not_retried( + self, monkeypatch + ): + """Salvage of #83183 claim 2: a secondary whose token is held by a live + foreign gateway (``{scope}_lock``, emitted retryable by + ``_acquire_platform_lock``) is an ownership conflict — park it fatal + like ``duplicate_credential`` instead of retry-storming the token.""" + runner = _secondary_recovery_runner() + failed = _SecondaryRecoveryAdapter() + failed.fatal_error_code = "discord-bot-token_lock" + failed.fatal_error_message = "Discord bot token already in use (PID 4242)." + _install_secondary_reconnect_context( + monkeypatch, runner, _SecondaryRecoveryAdapter() + ) + monkeypatch.setattr(runner, "_create_adapter", lambda platform, config: failed) + statuses = [] + monkeypatch.setattr( + runner, + "_update_platform_runtime_status", + lambda key, **kw: statuses.append((key, kw)), + ) + + async def fail_initial_connect(adapter, platform): + return False + + monkeypatch.setattr( + runner, "_connect_initial_adapter_with_timeout", fail_initial_connect + ) + + connected = await runner._start_one_profile_adapters( + "reviewer", "/tmp/reviewer", {} + ) + + assert connected == 0 + assert failed.disconnected is True + assert runner._background_tasks == set() + assert runner._profile_failed_platforms == {} + assert statuses == [ + ( + "reviewer:discord", + { + "platform_state": "fatal", + "error_code": "discord-bot-token_lock", + "error_message": failed.fatal_error_message, + }, + ) + ] + @pytest.mark.asyncio async def test_handoff_failure_is_logged_not_raised(self, monkeypatch, caplog): """If the scheduler raises at bridge handoff, the parked task must not @@ -820,6 +1011,113 @@ class TestSecondaryProfileConfigHandling: assert second == 1 assert runner._profile_adapters["later"][photon] is later + @pytest.mark.asyncio + async def test_secondary_teams_uses_degradable_error(self, monkeypatch): + from gateway.config import GatewayConfig, Platform, PlatformConfig + from gateway.run import SecondaryPortBindingConfigError + + runner = GatewayRunner.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + runner._profile_adapters = {} + + reviewer_cfg = GatewayConfig(multiplex_profiles=True) + reviewer_cfg.platforms = { + Platform("teams"): PlatformConfig(enabled=True, extra={"port": 3978}), + } + monkeypatch.setattr( + "gateway.config.load_gateway_config", lambda: reviewer_cfg + ) + + with pytest.raises(SecondaryPortBindingConfigError) as exc_info: + await runner._start_one_profile_adapters("reviewer", "/tmp/x", {}) + assert "teams" in str(exc_info.value) + assert "reviewer" in str(exc_info.value) + assert "reviewer" not in runner._profile_adapters + + @pytest.mark.asyncio + async def test_secondary_profile_adapter_start_skips_whatsapp(self, monkeypatch): + """WhatsApp is shared process-level ingress like Relay: the bridge is + one authenticated session tied to a single phone number, so a + credential-less secondary profile must be skipped (not stall startup + in a connect/retry loop) while its other platforms start normally.""" + runner = _secondary_recovery_runner() + direct = _SecondaryRecoveryAdapter() + _install_secondary_reconnect_context(monkeypatch, runner, direct) + monkeypatch.setattr( + "gateway.config.load_gateway_config", + lambda: GatewayConfig( + multiplex_profiles=True, + platforms={ + Platform.WHATSAPP: PlatformConfig(enabled=True), + Platform.DISCORD: PlatformConfig(enabled=True, token="profile-token"), + }, + ), + ) + factory_calls = [] + + def _create_adapter(platform, config): + factory_calls.append(platform) + return direct + + async def _connect(adapter, platform): + return True + + monkeypatch.setattr(runner, "_create_adapter", _create_adapter) + monkeypatch.setattr(runner, "_connect_initial_adapter_with_timeout", _connect) + + connected = await runner._start_one_profile_adapters("clientbot", "/tmp/x", {}) + + assert connected == 1 + assert factory_calls == [Platform.DISCORD] + assert runner._profile_adapters["clientbot"] == {Platform.DISCORD: direct} + + +class TestSecondaryProfileHookRegistration: + """A secondary profile's own `hooks:` block must register on ITS + plugin manager, not just the root/default profile's (#92672). + + Startup only calls agent.shell_hooks/outbound_webhooks + register_from_config() once, against the root config, before any + profile scope exists. Without a matching call inside + _start_one_profile_adapters, a secondary profile's config.yaml + `hooks:` block (shell hooks and outbound webhooks) never registers. + """ + + @pytest.mark.asyncio + async def test_registers_shell_hooks_and_webhooks_for_secondary_profile( + self, monkeypatch + ): + runner = _secondary_recovery_runner() + config = GatewayConfig(multiplex_profiles=True, platforms={}) + monkeypatch.setattr("gateway.config.load_gateway_config", lambda: config) + + profile_cfg = { + "hooks": { + "pre_tool_call": [ + {"matcher": "write_file", "command": "~/.hermes/deny.sh"} + ], + "outbound": [ + {"url": "http://127.0.0.1:9000/hook", "events": ["on_session_end"]} + ], + } + } + monkeypatch.setattr("hermes_cli.config.load_config", lambda: profile_cfg) + + seen = [] + monkeypatch.setattr( + "agent.shell_hooks.register_from_config", + lambda cfg, **kwargs: seen.append(("shell", cfg)) or [], + ) + monkeypatch.setattr( + "agent.outbound_webhooks.register_from_config", + lambda cfg: seen.append(("webhook", cfg)) or [], + ) + + await runner._start_one_profile_adapters("second", "/tmp/second", {}) + + assert ("shell", profile_cfg) in seen + assert ("webhook", profile_cfg) in seen + class TestFeishuPortBindingConditional: """Feishu websocket mode does NOT bind a port; only webhook mode does (#52563).""" @@ -848,3 +1146,78 @@ class TestFeishuPortBindingConditional: assert connected == 0 # no error, just nothing connected +class TestSecondarySkipsCredentiallessPlatforms: + """#84079 — multiplex must not build adapters for platforms a profile + has no credential for. + + The shared config.yaml enables a platform once; under multiplex every + secondary profile reloads it inside its own secret scope, so a profile + whose scope lacks the platform credential resolves ``enabled=True`` with + an empty token. Constructing an adapter anyway treats every profile as + configured for the platform — one inbound message fans out across all of + them. These tests lock the credential gate on the secondary startup path + (the primary path got the same gate in #64674; the reconnect path shares + the helper). Also reported independently in #72313. + """ + + def _make_runner(self, monkeypatch, profile_cfg): + runner = GatewayRunner.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + runner._profile_adapters = {} + runner.adapters = {} + created = [] + + def fake_create(platform, platform_config): + created.append((platform, platform_config)) + return _FakeAdapter(token=platform_config.token or None) + + monkeypatch.setattr("gateway.config.load_gateway_config", lambda: profile_cfg) + monkeypatch.setattr(runner, "_create_adapter", fake_create) + monkeypatch.setattr(runner, "_configure_profile_adapter", lambda *a, **k: None) + monkeypatch.setattr( + runner, + "_connect_initial_adapter_with_timeout", + AsyncMock(return_value=True), + ) + return runner, created + + @pytest.mark.asyncio + async def test_credentialless_platform_builds_no_adapter(self, monkeypatch, tmp_path): + """Enabled-in-YAML but no credential in the profile scope -> no adapter.""" + from gateway.config import GatewayConfig, Platform, PlatformConfig + + profile_cfg = GatewayConfig(multiplex_profiles=True) + profile_cfg.platforms = { + # Shared config.yaml enables Slack; profile-b's .env has no + # SLACK_BOT_TOKEN, so its scoped load resolves token="" but + # keeps enabled=True (#84079). + Platform.SLACK: PlatformConfig(enabled=True, token=""), + Platform.TELEGRAM: PlatformConfig(enabled=True, token="telegram-token-b"), + } + runner, created = self._make_runner(monkeypatch, profile_cfg) + + connected = await runner._start_one_profile_adapters("profile-b", tmp_path, {}) + + # Only Telegram (which profile-b has its own credential for) gets an + # adapter; Slack is skipped instead of fanning out a turn per profile. + assert [p for p, _ in created] == [Platform.TELEGRAM] + assert connected == 1 + assert Platform.TELEGRAM in runner._profile_adapters["profile-b"] + assert Platform.SLACK not in runner._profile_adapters["profile-b"] + + @pytest.mark.asyncio + async def test_profile_with_own_credential_still_connects(self, monkeypatch, tmp_path): + """A profile that defines its own credential keeps its adapter.""" + from gateway.config import GatewayConfig, Platform, PlatformConfig + + profile_cfg = GatewayConfig(multiplex_profiles=True) + profile_cfg.platforms = { + Platform.SLACK: PlatformConfig(enabled=True, token="slack-token-b"), + } + runner, created = self._make_runner(monkeypatch, profile_cfg) + + connected = await runner._start_one_profile_adapters("profile-b", tmp_path, {}) + + assert connected == 1 + assert created == [(Platform.SLACK, profile_cfg.platforms[Platform.SLACK])] + assert Platform.SLACK in runner._profile_adapters["profile-b"] diff --git a/tests/gateway/test_multiplex_credential_isolation.py b/tests/gateway/test_multiplex_credential_isolation.py index f5d2d5d8f6..a43ee8f440 100644 --- a/tests/gateway/test_multiplex_credential_isolation.py +++ b/tests/gateway/test_multiplex_credential_isolation.py @@ -88,6 +88,54 @@ class TestProfilePathResolutionUnderMultiplexScope: assert b_seen == prof_b / "skills" +def test_turn_scoped_dotenv_reload_does_not_pollute_process_env(tmp_path, monkeypatch): + """A routed profile reload must stay inside its context-local scope. + + ``load_hermes_dotenv`` has several lazy-import and cron call sites beyond + the gateway's guarded reload helper. Any one of them can run during a + multiplexed turn, so the loader itself must not copy the active profile's + ``.env`` into the shared process environment. + """ + import os + + from agent.secret_scope import get_secret + from gateway.run import _profile_runtime_scope + from hermes_cli.env_loader import load_hermes_dotenv + from hermes_constants import get_hermes_home + + profile_a = tmp_path / "profiles" / "a" + profile_b = tmp_path / "profiles" / "b" + profile_a.mkdir(parents=True) + profile_b.mkdir(parents=True) + (profile_a / ".env").write_text( + "PROFILE_SCOPED_API_KEY=secret-a\n" + "DISCORD_ALLOWED_CHANNELS=profile-a-only\n", + encoding="utf-8", + ) + (profile_b / ".env").write_text( + "PROFILE_SCOPED_API_KEY=secret-b\n" + "DISCORD_ALLOWED_CHANNELS=profile-b-only\n", + encoding="utf-8", + ) + monkeypatch.delenv("PROFILE_SCOPED_API_KEY", raising=False) + monkeypatch.setenv("DISCORD_ALLOWED_CHANNELS", "all-channels") + + ss.set_multiplex_active(True) + with _profile_runtime_scope(profile_a): + assert get_secret("PROFILE_SCOPED_API_KEY") == "secret-a" + assert get_secret("DISCORD_ALLOWED_CHANNELS") == "profile-a-only" + assert load_hermes_dotenv(hermes_home=get_hermes_home()) == [] + assert "PROFILE_SCOPED_API_KEY" not in os.environ + assert os.environ["DISCORD_ALLOWED_CHANNELS"] == "all-channels" + + with _profile_runtime_scope(profile_b): + assert get_secret("PROFILE_SCOPED_API_KEY") == "secret-b" + assert get_secret("DISCORD_ALLOWED_CHANNELS") == "profile-b-only" + assert load_hermes_dotenv(hermes_home=get_hermes_home()) == [] + assert "PROFILE_SCOPED_API_KEY" not in os.environ + assert os.environ["DISCORD_ALLOWED_CHANNELS"] == "all-channels" + + def test_cold_profile_hydrates_external_source_without_global_env( tmp_path, monkeypatch ): @@ -164,5 +212,3 @@ def test_cold_profile_hydrates_external_source_without_global_env( assert calls["count"] == 1 assert "TEST_PROVIDER_API_KEY" not in os.environ assert "EXPLICIT_API_KEY" not in os.environ - - diff --git a/tests/gateway/test_multiplex_interactive_auth.py b/tests/gateway/test_multiplex_interactive_auth.py new file mode 100644 index 0000000000..5ed97aa17c --- /dev/null +++ b/tests/gateway/test_multiplex_interactive_auth.py @@ -0,0 +1,170 @@ +"""Multiplex interactive-auth regressions (#86296, #92840, #72657, #87240 egress). + +Real ``GatewayRunner`` methods on an ``object.__new__`` runner, real +``PairingStore`` files under a temp HERMES_HOME, multiplex active. +""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.pairing import PairingStore +from gateway.profile_routing import ProfileRoute + + +@pytest.fixture +def mux_home(tmp_path, monkeypatch): + from agent import secret_scope + + home = tmp_path / "hh" + (home / "profiles" / "secondary").mkdir(parents=True) + (home / ".env").write_text("") + (home / "profiles" / "secondary" / ".env").write_text("") + monkeypatch.setenv("HERMES_HOME", str(home)) + for key in ( + "TELEGRAM_ALLOWED_USERS", + "TELEGRAM_ALLOW_BOTS", + "GATEWAY_ALLOW_ALL_USERS", + "GATEWAY_ALLOWED_USERS", + "SLACK_ALLOW_ALL_USERS", + "SLACK_ALLOWED_USERS", + ): + monkeypatch.delenv(key, raising=False) + prev = secret_scope.is_multiplex_active() + secret_scope.set_multiplex_active(True) + yield home + secret_scope.set_multiplex_active(prev) + + +def _runner(home): + from gateway.run import GatewayRunner + + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + runner.config.profile_routes = [ + ProfileRoute(name="r", platform="telegram", chat_id="-100555", profile="secondary") + ] + runner.config.platforms = {Platform.TELEGRAM: PlatformConfig(enabled=True, extra={})} + runner.pairing_store = PairingStore(profile="default") + runner.pairing_stores = { + "default": runner.pairing_store, + "secondary": PairingStore(profile="secondary"), + } + runner._primary_profile_name = "default" + runner._profile_adapters = {"secondary": {}} + return runner + + +def _telegram(runner): + from plugins.platforms.telegram.adapter import TelegramAdapter + + tg = object.__new__(TelegramAdapter) + tg.config = PlatformConfig(enabled=True, extra={}) + tg._authorization_check = None + tg._message_handler = runner._primary_message_handler() # closure, no __self__ + runner.adapters = {Platform.TELEGRAM: tg} + tg.set_authorization_check(runner._make_adapter_auth_check(Platform.TELEGRAM)) + return tg + + +def test_routed_primary_callback_uses_routed_pairing_store_and_transport_allowlist(mux_home): + """#86296: shared primary bot + profile_routes → the inline-button caller + is authorized by the ROUTED profile's pairing store, while env allowlists + resolve under the transport (launch) home, exactly like inbound messages.""" + runner = _runner(mux_home) + store = runner.pairing_stores["secondary"] + store._save_json(store._approved_path("telegram"), {"777": {}}) + (mux_home / ".env").write_text("TELEGRAM_ALLOWED_USERS=999\n") + tg = _telegram(runner) + + # Paired only in the routed profile → allowed in the routed chat only. + assert tg._is_callback_user_authorized("777", chat_id="-100555", chat_type="supergroup") is True + assert tg._is_callback_user_authorized("777", chat_id="-100999", chat_type="supergroup") is False + # Transport-home allowlist honored in the routed chat (not the routed profile's empty scope). + assert tg._is_callback_user_authorized("999", chat_id="-100555", chat_type="supergroup") is True + assert tg._is_callback_user_authorized("888", chat_id="-100555", chat_type="supergroup") is False + + +def test_bot_sender_reaches_allow_bots_policy_through_callback(mux_home): + """#92840: the early prefilter must carry ``is_bot`` so TELEGRAM_ALLOW_BOTS + admits bot-authored messages under the multiplex closure handler.""" + from gateway.run import _profile_runtime_scope + + runner = _runner(mux_home) + (mux_home / ".env").write_text("TELEGRAM_ALLOWED_USERS=999\nTELEGRAM_ALLOW_BOTS=all\n") + tg = _telegram(runner) + + def msg(uid, is_bot): + return SimpleNamespace( + from_user=SimpleNamespace(id=uid, is_bot=is_bot, username="x", full_name="X"), + chat=SimpleNamespace(id=-100777, type="supergroup", is_forum=False), + sender_chat=None, + message_thread_id=None, + is_topic_message=False, + ) + + with _profile_runtime_scope(mux_home): + assert tg._is_user_authorized_from_message(msg(4242, True)) is True + assert tg._is_user_authorized_from_message(msg(4343, False)) is False + + +def test_slack_interactive_auth_prefers_wired_profile_check(mux_home, monkeypatch): + """#72657: a multiplexed Slack adapter's button gate resolves through the + wired ``_make_adapter_auth_check`` for its own profile; the DEFAULT + profile's process-env allow-all never leaks in — not through the + injected path, and not through the env-only fallback either.""" + from gateway.run import _profile_runtime_scope + from plugins.platforms.slack.adapter import SlackAdapter + + runner = _runner(mux_home) + runner.adapters = {} + sec_home = mux_home / "profiles" / "secondary" + (sec_home / ".env").write_text("SLACK_ALLOWED_USERS=U_SEC\n") + monkeypatch.setenv("SLACK_ALLOW_ALL_USERS", "true") + + def slack(with_check): + sl = object.__new__(SlackAdapter) + sl.config = PlatformConfig(enabled=True, extra={}) + sl._authorization_check = None + sl._message_handler = runner._make_profile_message_handler("secondary") + if with_check: + runner._profile_adapters = {"secondary": {Platform.SLACK: sl}} + sl.set_authorization_check( + runner._make_adapter_auth_check(Platform.SLACK, profile_name="secondary") + ) + return sl + + with _profile_runtime_scope(sec_home): + wired = slack(True) + assert wired._is_interactive_user_authorized("U_SEC", channel_id="C1") is True + assert wired._is_interactive_user_authorized("U_X", channel_id="C1") is False + assert slack(False)._is_interactive_user_authorized("U_X", channel_id="C1") is False + + +def test_authorization_adapter_ignores_per_turn_active_profile(mux_home): + """#87240 egress half: inside a secondary profile's runtime scope the + default bot must not be handed to that profile (fail-closed None); the + launch profile still resolves ``self.adapters``.""" + from gateway.run import _profile_runtime_scope + + runner = _runner(mux_home) + default_bot = object() + runner.adapters = {Platform.TELEGRAM: default_bot} + + with _profile_runtime_scope(mux_home / "profiles" / "secondary"): + assert runner._authorization_adapter(Platform.TELEGRAM, profile="secondary") is None + assert runner._authorization_adapter(Platform.TELEGRAM, profile="default") is default_bot + + +def test_channel_directory_path_follows_current_home(mux_home): + """#87240: the directory file resolves against the CURRENT profile home, + not the home that happened to import the module.""" + import gateway.channel_directory as cd + from gateway.run import _profile_runtime_scope + + assert cd.DIRECTORY_PATH is None + with _profile_runtime_scope(mux_home / "profiles" / "secondary"): + assert cd._directory_path() == Path(mux_home / "profiles" / "secondary" / "channel_directory.json") + assert cd._directory_path() == Path(mux_home / "channel_directory.json") diff --git a/tests/gateway/test_multiplex_log_routing.py b/tests/gateway/test_multiplex_log_routing.py new file mode 100644 index 0000000000..529018b51a --- /dev/null +++ b/tests/gateway/test_multiplex_log_routing.py @@ -0,0 +1,67 @@ +"""Multiplex gateway log routing (#82936, salvage of #84954). + +``setup_logging(mode="gateway")`` binds agent.log/errors.log/gateway.log to +the launch home. Under ``multiplex_profiles`` every secondary profile's +records — emitted inside ``_profile_runtime_scope`` — used to fan out into +the DEFAULT profile's files. The gateway now enables the #99440 profile +routers at startup so each record lands in its owner's ``logs/``. +""" + +import logging +import types +from pathlib import Path + +import pytest + +import hermes_logging +from gateway import run + + +@pytest.fixture +def clean_logging(): + hermes_logging._reset_queued_handlers() + hermes_logging._logging_initialized = False + yield + hermes_logging._reset_queued_handlers() + hermes_logging._logging_initialized = False + + +def _emit_under(home: Path, name: str, level: int, msg: str) -> None: + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + token = set_hermes_home_override(home) + try: + logging.getLogger(name).log(level, msg) + finally: + reset_hermes_home_override(token) + + +def _contains(home: Path, filename: str, needle: str) -> bool: + path = home / "logs" / filename + return path.exists() and needle in path.read_text() + + +def test_multiplex_gateway_routes_profile_records_to_their_own_logs( + tmp_path, monkeypatch, clean_logging +): + default_home = tmp_path / "default" + beta_home = tmp_path / "default" / "profiles" / "beta" + beta_home.mkdir(parents=True) + homes = [("default", default_home), ("beta", beta_home)] + monkeypatch.setattr(run, "_multiplex_profile_homes", lambda _cfg: homes) + + hermes_logging.setup_logging(hermes_home=default_home, mode="gateway") + + # Single-profile gateway: wiring is inert and handlers stay static. + assert run._enable_multiplex_log_routing(types.SimpleNamespace(multiplex_profiles=False)) is False + assert run._enable_multiplex_log_routing(types.SimpleNamespace(multiplex_profiles=True)) is True + + _emit_under(beta_home, "gateway.run", logging.WARNING, "BETA-GATEWAY-WARN") + _emit_under(default_home, "gateway.run", logging.INFO, "DEFAULT-GATEWAY-INFO") + hermes_logging.flush_log_queue() + + for filename in ("agent.log", "errors.log", "gateway.log"): + assert _contains(beta_home, filename, "BETA-GATEWAY-WARN"), filename + assert not _contains(default_home, filename, "BETA-GATEWAY-WARN"), filename + assert _contains(default_home, "gateway.log", "DEFAULT-GATEWAY-INFO") + assert not _contains(beta_home, "gateway.log", "DEFAULT-GATEWAY-INFO") diff --git a/tests/gateway/test_multiplex_mcp_discovery.py b/tests/gateway/test_multiplex_mcp_discovery.py new file mode 100644 index 0000000000..633b39b755 --- /dev/null +++ b/tests/gateway/test_multiplex_mcp_discovery.py @@ -0,0 +1,121 @@ +"""Multiplexed gateways discover and reload MCP servers per profile (#95518).""" + +from __future__ import annotations + +import threading +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from gateway.config import GatewayConfig, Platform +from gateway.platforms.base import MessageEvent +from gateway.session import SessionSource +from hermes_constants import get_hermes_home, hermes_home_key + + +@pytest.mark.asyncio +async def test_gateway_boot_discovers_mcp_under_every_profile_home( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import gateway.run as gateway_run + from tools import mcp_tool + + homes = [("default", tmp_path / "default"), ("worker", tmp_path / "worker")] + for _name, home in homes: + home.mkdir() + seen: list[tuple[Path, str]] = [] + + def fake_discover() -> list[str]: + seen.append((get_hermes_home(), threading.current_thread().name)) + return [] + + monkeypatch.setattr( + "hermes_cli.profiles.profiles_to_serve", + lambda multiplex, profile_allowlist=None: homes, + ) + monkeypatch.setattr(mcp_tool, "discover_mcp_tools", fake_discover) + + await gateway_run._discover_gateway_mcp_tools(GatewayConfig(multiplex_profiles=True)) + + # Ran once per profile, under that profile's home, off the loop thread. + assert [home for home, _ in seen] == [home for _, home in homes] + assert all(thread != threading.current_thread().name for _, thread in seen) + + +@pytest.mark.asyncio +async def test_reload_mcp_only_touches_requesting_profile( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from gateway.run import GatewayRunner + from tools import mcp_tool + + worker_home = tmp_path / "profiles" / "worker" + worker_home.mkdir(parents=True) + worker_scope = hermes_home_key(worker_home) + + runner = GatewayRunner.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + runner._resolve_profile_home_for_source = MagicMock(return_value=worker_home) + runner._agent_cache = {} + runner._agent_cache_lock = None + runner._async_session_store = SimpleNamespace( + get_or_create_session=MagicMock(side_effect=RuntimeError("skip transcript")), + ) + + monkeypatch.setattr(mcp_tool, "_servers", {"default-srv": object(), "worker-srv": object()}) + monkeypatch.setattr( + mcp_tool, "_server_scope_keys", + {"default-srv": hermes_home_key(tmp_path), "worker-srv": worker_scope}, + ) + seen: list[tuple] = [] + + def fake_shutdown(*, scope=None) -> None: + seen.append(("shutdown", scope, get_hermes_home())) + + def fake_discover() -> list[str]: + seen.append(("discover", get_hermes_home())) + return [] + + monkeypatch.setattr(mcp_tool, "shutdown_mcp_servers", fake_shutdown) + monkeypatch.setattr(mcp_tool, "discover_mcp_tools", fake_discover) + + event = MessageEvent( + text="/reload-mcp", message_id="m1", + source=SessionSource( + platform=Platform.TELEGRAM, user_id="u1", chat_id="c1", + chat_type="dm", profile="worker", + ), + ) + result = await runner._execute_mcp_reload(event) + + # Entered worker's scope itself, shut down only worker's servers, and + # reported only worker's servers (default's untouched connection is not + # "removed"). + assert seen == [ + ("shutdown", worker_scope, worker_home), + ("discover", worker_home), + ] + assert "default-srv" not in result + + +def test_deregister_scope_kwarg_targets_overlay_and_keeps_plugin_confinement() -> None: + from tools.registry import ToolRegistry + + reg = ToolRegistry() + reg.register("mcp__s__t", "mcp-s", {"name": "mcp__s__t", "description": "d"}, + lambda **kw: None, scope="/home/p1") + assert reg.snapshot_registration("mcp__s__t", scope="/home/p1") is not None + + reg.deregister("mcp__s__t") # unscoped: global slot only, overlay untouched + assert reg.snapshot_registration("mcp__s__t", scope="/home/p1") is not None + + reg.deregister("mcp__s__t", scope="/home/p1") + assert reg.snapshot_registration("mcp__s__t", scope="/home/p1") is None + + # A plugin module may not name another profile's overlay. + reg._plugin_module_scopes["hermes_plugins.p"] = {"/home/p1"} + reg._caller_module = staticmethod(lambda: "hermes_plugins.p") + with pytest.raises(PermissionError): + reg.deregister("anything", scope="/home/p2") diff --git a/tests/gateway/test_multiplex_pairing_stores.py b/tests/gateway/test_multiplex_pairing_stores.py index 63c4a9ea9a..78dc2f3e1d 100644 --- a/tests/gateway/test_multiplex_pairing_stores.py +++ b/tests/gateway/test_multiplex_pairing_stores.py @@ -85,3 +85,40 @@ def test_pairing_store_scoped_to_profile_dir(tmp_path, monkeypatch): assert "profiles/ops/platforms/pairing" in str(store._dir).replace("\\", "/"), ( f"store not profile-scoped: {store._dir}" ) + + +def test_routed_pairing_grant_mirror_stays_in_profile_scope(tmp_path, monkeypatch): + """A /pair grant mirrored under a routed profile scope must update THAT + profile's .env and installed scope, never the shared os.environ (#88441, + #77490). Outside multiplex the legacy os.environ publish is unchanged.""" + import os + + from agent import secret_scope as ss + from gateway.pairing import _sync_allowlist_add + from gateway.run import _profile_runtime_scope + from hermes_cli.config import save_env_value + + root = tmp_path / ".hermes" + prof = root / "profiles" / "b" + prof.mkdir(parents=True) + (root / ".env").write_text("DISCORD_ALLOWED_USERS=default-admin\n") + (prof / ".env").write_text("DISCORD_ALLOWED_USERS=b-admin\n") + monkeypatch.setenv("HERMES_HOME", str(root)) + monkeypatch.setenv("DISCORD_ALLOWED_USERS", "default-admin") + + was_active = ss.is_multiplex_active() + ss.set_multiplex_active(True) + try: + with _profile_runtime_scope(prof): + _sync_allowlist_add("discord", "111") + assert ss.get_secret("DISCORD_ALLOWED_USERS") == "b-admin,111" + finally: + ss.set_multiplex_active(was_active) + + assert (prof / ".env").read_text().strip() == "DISCORD_ALLOWED_USERS=b-admin,111" + assert (root / ".env").read_text().strip() == "DISCORD_ALLOWED_USERS=default-admin" + assert os.environ["DISCORD_ALLOWED_USERS"] == "default-admin" + + # Single-profile: no multiplex -> save still publishes to the process env. + save_env_value("DISCORD_ALLOWED_USERS", "default-admin,222") + assert os.environ["DISCORD_ALLOWED_USERS"] == "default-admin,222" diff --git a/tests/gateway/test_multiplex_phase0.py b/tests/gateway/test_multiplex_phase0.py index 836a0d6355..6558d67f54 100644 --- a/tests/gateway/test_multiplex_phase0.py +++ b/tests/gateway/test_multiplex_phase0.py @@ -172,3 +172,27 @@ class TestSessionStoreUnmultiplexedRecovery: assert recovered.session_id == "sess-coder" assert recovered.session_key == "agent:main:telegram:dm:99" assert store._db.reopened == ["sess-coder"] + + @pytest.mark.parametrize( + ("recovered_key", "adopted"), + [ + ("agent:coder:telegram:dm:99", False), # sibling namespace → fail closed + ("agent:main:telegram:dm:99:v1", True), # same namespace → adoptable + ], + ids=["sibling-profile", "same-profile"], + ) + def test_flag_on_fences_recovery_by_requested_namespace( + self, tmp_path, recovered_key, adopted + ): + """#74285: under multiplexing the guard compares the recovered row's + ``agent::`` against the REQUESTED key, never the active profile.""" + row = {"id": "sess", "started_at": 1700000000, "session_key": recovered_key} + store = self._store_with_row(tmp_path, row, multiplex_profiles=True) + store._db_pinned = store._db + with patch("hermes_cli.profiles.get_active_profile_name", return_value="coder"): + recovered = store._recover_session_from_db( + session_key="agent:main:telegram:dm:99", + source=_src(chat_id="99", chat_type="dm"), + now=datetime.fromtimestamp(1700000001), + ) + assert (recovered is not None) is adopted diff --git a/tests/gateway/test_multiplex_profile_authz.py b/tests/gateway/test_multiplex_profile_authz.py index e20176efcb..cd1bbf5a59 100644 --- a/tests/gateway/test_multiplex_profile_authz.py +++ b/tests/gateway/test_multiplex_profile_authz.py @@ -74,6 +74,52 @@ def test_active_profile_stamp_resolves_primary_adapter(monkeypatch): assert runner._authorization_adapter(Platform.WECOM, profile="dev") is default_adapter +def test_scoped_secondary_profile_still_uses_profile_adapters(monkeypatch): + """Runtime scope must not redirect secondary authz to primary adapters. + + ``_make_profile_message_handler`` wraps ``_handle_message`` in + ``_profile_runtime_scope``, which overrides HERMES_HOME so + ``get_active_profile_name()`` equals the secondary profile for that turn. + Authorization must still read ``_profile_adapters[profile]``, not the + empty primary ``self.adapters`` map — otherwise upstream-auth platforms + such as A2A default-deny an already-authenticated peer (#80884). A + secondary profile with NO registry entry still fails closed. + """ + from gateway.run import GatewayRunner + + _clear_auth_env(monkeypatch) + + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + runner.adapters = {} + runner.pairing_store = MagicMock() + runner.pairing_store.is_approved.return_value = False + + secondary = SimpleNamespace( + authorization_is_upstream=True, + enforces_own_access_policy=False, + ) + runner._profile_adapters = {"beta": {Platform("a2a"): secondary}} + # Simulate the scoped turn: active profile name collapses to the secondary. + runner._active_profile_name = lambda: "beta" + + assert runner._authorization_adapter(Platform("a2a"), profile="beta") is secondary + + source = SessionSource( + platform=Platform("a2a"), + chat_id="a2a-context", + user_id="alpha", + user_name="alpha", + chat_type="dm", + profile="beta", + ) + assert runner._is_user_authorized(source) is True + + # Fail-closed guard is untouched: no registry entry -> no default fallback. + runner._profile_adapters = {"beta": {}} + assert runner._authorization_adapter(Platform("a2a"), profile="beta") is None + + def test_secondary_allowlist_dm_behavior_ignores_unauthorized(monkeypatch): """Unauthorized-DM behavior must read the secondary adapter's dm_policy.""" runner, _default_adapter, secondary_adapter = _make_multiplex_runner(monkeypatch) diff --git a/tests/gateway/test_ntfy_plugin.py b/tests/gateway/test_ntfy_plugin.py index 9e992eeb3e..8cb86b7f8f 100644 --- a/tests/gateway/test_ntfy_plugin.py +++ b/tests/gateway/test_ntfy_plugin.py @@ -491,3 +491,82 @@ class TestTruncateHelper: assert _ntfy._truncate_body("hi", context="test") == b"hi" +# --------------------------------------------------------------------------- +# 13. Multiplex secondary-profile scope +# --------------------------------------------------------------------------- +# +# __init__'s server/topic/publish_topic, _env_enablement's topic/server/ +# publish_topic/markdown/home_channel, and check_requirements/validate_config/ +# is_connected's topic reads, all previously read raw os.getenv +# unconditionally (only NTFY_TOKEN was already scoped). Under multiplex, +# os.environ holds the DEFAULT profile's YAML-to-env bridge output -- a +# secondary profile with its own (different or absent) ntfy config would +# silently subscribe to / publish on the default profile's topic, or get +# auto-enabled using the default profile's topic entirely. Mirrors the +# LINE/Buzz/SimpleX fix for #98738. + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + tokens = [] + + def install(scope=None): + from agent.secret_scope import set_multiplex_active, set_secret_scope + + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + return tokens[-1] + + yield install + + from agent.secret_scope import reset_secret_scope, set_multiplex_active + + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("NTFY_TOPIC", "default-topic") + monkeypatch.setenv("NTFY_SERVER_URL", "https://default.example.com") + monkeypatch.setenv("NTFY_PUBLISH_TOPIC", "default-out") + + +class TestMultiplexProfileScope: + + def test_secondary_extra_wins_over_default_profile_env( + self, multiplex_scope, default_profile_env + ): + """The secondary profile's own config.yaml extra is authoritative, + not the default profile's bridged topic/server/publish_topic.""" + multiplex_scope() + cfg = PlatformConfig( + enabled=True, + extra={ + "topic": "profile-topic", + "server": "https://profile.example.com", + "publish_topic": "profile-out", + }, + ) + adapter = NtfyAdapter(cfg) + assert adapter._topic == "profile-topic" + assert adapter._server == "https://profile.example.com" + assert adapter._publish_topic == "profile-out" + + def test_secondary_missing_keys_fail_closed( + self, multiplex_scope, default_profile_env + ): + """Keys absent from the profile's own scope must NOT borrow the + default profile's bridged env values -- that would silently + subscribe/publish on the wrong topic.""" + multiplex_scope() + adapter = NtfyAdapter(PlatformConfig(enabled=True, extra={})) + assert adapter._topic == "" + assert adapter._server == DEFAULT_SERVER + assert adapter._publish_topic == "" + # Nor may the registry auto-enable ntfy for this profile off the default's topic. + assert _env_enablement() is None + assert is_connected(PlatformConfig(enabled=True, extra={})) is False + diff --git a/tests/gateway/test_personality_routed_profile.py b/tests/gateway/test_personality_routed_profile.py new file mode 100644 index 0000000000..9ad41b3628 --- /dev/null +++ b/tests/gateway/test_personality_routed_profile.py @@ -0,0 +1,45 @@ +"""#89161: a routed multiplex profile's personality must reach its turns. + +``GatewayRunner`` used to snapshot ``_ephemeral_system_prompt`` once at boot +from the launch profile's config and hand that string to every routed turn, +so a secondary profile's ``display.personality`` / ``agent.system_prompt`` never +injected. ``_get_system_prompt_for_channel`` now resolves from the config of +the profile currently in scope (``run_sync`` runs inside +``_profile_runtime_scope``). +""" + +from __future__ import annotations + +import gateway.run as gateway_run +from gateway.config import Platform +from gateway.run import GatewayRunner, _profile_runtime_scope + + +def test_routed_profile_prompt_resolves_from_its_own_config(tmp_path, monkeypatch): + default_home = tmp_path / "default" + routed_home = tmp_path / "profiles" / "beta" + default_home.mkdir() + routed_home.mkdir(parents=True) + (default_home / "config.yaml").write_text("agent:\n system_prompt: DEFAULT-PERSONA\n") + (routed_home / "config.yaml").write_text( + "agent:\n system_prompt: BETA-PERSONA\n personalities:\n pirate: ARR\n" + ) + monkeypatch.setattr(gateway_run, "_hermes_home", default_home) + monkeypatch.setenv("HERMES_HOME", str(default_home)) + monkeypatch.delenv("HERMES_EPHEMERAL_SYSTEM_PROMPT", raising=False) + + runner = object.__new__(GatewayRunner) + runner.config = None + + with _profile_runtime_scope(routed_home): + assert runner._get_system_prompt_for_channel(Platform.TELEGRAM, "c") == "BETA-PERSONA" + assert runner._get_system_prompt_for_channel(Platform.TELEGRAM, "c") == "DEFAULT-PERSONA" + + # /personality from the routed chat writes the routed profile and only it. + from hermes_cli.personality import persist_personality + + with _profile_runtime_scope(routed_home): + assert persist_personality("pirate") + assert runner._get_system_prompt_for_channel(Platform.TELEGRAM, "c") == "ARR" + assert "pirate" not in (default_home / "config.yaml").read_text() + assert runner._get_system_prompt_for_channel(Platform.TELEGRAM, "c") == "DEFAULT-PERSONA" diff --git a/tests/gateway/test_platform_reconnect.py b/tests/gateway/test_platform_reconnect.py index 75029072c8..e923fac809 100644 --- a/tests/gateway/test_platform_reconnect.py +++ b/tests/gateway/test_platform_reconnect.py @@ -199,6 +199,55 @@ class TestPlatformReconnectWatcher: ) assert Platform.TELEGRAM in runner.adapters + @pytest.mark.asyncio + @pytest.mark.parametrize("degraded", [False, True]) + async def test_reconnect_stamp_honours_adapter_send_path_degraded(self, degraded): + """connect() returning True is not proof the receive path is live: + Telegram's degraded reconnect returns True while its own ladder + retries. The watcher's status stamp must publish what the adapter + reports, not an unconditional "connected" (#101391).""" + runner = _make_runner() + runner._sync_voice_mode_state_to_adapter = MagicMock() + runner._update_platform_runtime_status = MagicMock() + runner._failed_platforms[Platform.TELEGRAM] = { + "config": PlatformConfig(enabled=True, token="test"), + "attempts": 1, + "next_retry": time.monotonic() - 1, + } + + class _DegradableAdapter(StubAdapter): + @property + def send_path_degraded(self) -> bool: + return degraded + + adapter = _DegradableAdapter(succeed=True) + real_sleep = asyncio.sleep + + with patch.object(runner, "_create_adapter", return_value=adapter): + with patch("gateway.run.build_channel_directory", create=True): + runner._running = True + call_count = 0 + + async def fake_sleep(n): + nonlocal call_count + call_count += 1 + if call_count > 1: + runner._running = False + await real_sleep(0) + + with patch("asyncio.sleep", side_effect=fake_sleep): + await runner._platform_reconnect_watcher() + + stamps = [ + c.kwargs for c in runner._update_platform_runtime_status.call_args_list + if c.args and c.args[0] == Platform.TELEGRAM.value + ] + assert stamps, "watcher never stamped telegram" + final = stamps[-1] + assert final["platform_state"] == ("retrying" if degraded else "connected") + assert final["error_message"] == (adapter.DEGRADED_STATUS_MESSAGE if degraded else None) + assert final["retrying_since"] is None + @pytest.mark.asyncio async def test_cold_connect_defaults_to_is_reconnect_false(self): """The cold-start connect path (_connect_adapter_with_timeout with no diff --git a/tests/gateway/test_profile_resolution.py b/tests/gateway/test_profile_resolution.py index 695b9c7b89..e79e0415ca 100644 --- a/tests/gateway/test_profile_resolution.py +++ b/tests/gateway/test_profile_resolution.py @@ -248,6 +248,69 @@ class TestGatewayRunnerInjection: assert hasattr(BasePlatformAdapter, "gateway_runner") assert BasePlatformAdapter.gateway_runner is None + def test_factory_binds_every_adapter_to_runner(self, monkeypatch): + """``_create_adapter`` binds the runner regardless of which branch + built the adapter (plugin registry OR built-in if/elif) — every + lifecycle path (startup, reconnect, secondary profiles) goes through + it, so this is the single seam that makes profile_routes reachable + for built-ins like Signal (#68332 / #70831).""" + from gateway.config import PlatformConfig + + runner = object.__new__(GatewayRunner) + adapter = MagicMock(spec=BasePlatformAdapter) + monkeypatch.setattr(runner, "_instantiate_adapter", lambda platform, config: adapter) + assert runner._create_adapter(Platform.SIGNAL, PlatformConfig(enabled=True)) is adapter + assert adapter.gateway_runner is runner + monkeypatch.setattr(runner, "_instantiate_adapter", lambda platform, config: None) + assert runner._create_adapter(Platform.SIGNAL, PlatformConfig(enabled=True)) is None + + @pytest.mark.asyncio + async def test_real_signal_factory_routes_inbound_group_event(self, monkeypatch): + """A factory-built (built-in) Signal adapter resolves profile_routes + for a real inbound envelope — fails on main where the Signal branch + returned a bare ``SignalAdapter(config)`` with no runner.""" + from gateway.config import PlatformConfig + + group_id = "test-signal-route" + monkeypatch.setenv("SIGNAL_GROUP_ALLOWED_USERS", group_id) + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig( + multiplex_profiles=True, + profile_routes=[ + ProfileRoute(name="signal", platform="signal", profile="ops", chat_id=f"group:{group_id}"), + ], + ) + adapter = runner._create_adapter( + Platform.SIGNAL, + PlatformConfig(enabled=True, extra={"http_url": "http://127.0.0.1:18080", "account": "+15555550123"}), + ) + assert adapter is not None and adapter.gateway_runner is runner + + captured = {} + + async def capture_event(event): + captured["event"] = event + + adapter.handle_message = capture_event + with patch( + "hermes_cli.profiles.profiles_to_serve", + return_value=[("default", Path("/profiles/default")), ("ops", Path("/profiles/ops"))], + ): + await adapter._handle_envelope({ + "envelope": { + "sourceNumber": "+15555550124", + "sourceName": "Test Operator", + "timestamp": 1700000000000, + "dataMessage": { + "message": "diagnose the cluster", + "groupInfo": {"groupId": group_id, "groupName": "US East 7"}, + }, + }, + }) + source = captured["event"].source + assert source.profile == "ops" + assert build_session_key(source, profile=source.profile).startswith("agent:ops:") + # A concrete adapter we can instantiate without the full platform stack. # ``build_source`` only reads ``self.platform`` and ``self.gateway_runner``, so a diff --git a/tests/gateway/test_profile_routing.py b/tests/gateway/test_profile_routing.py index 73934ac52d..37ebb8f69a 100644 --- a/tests/gateway/test_profile_routing.py +++ b/tests/gateway/test_profile_routing.py @@ -1,5 +1,7 @@ """Tests for gateway/profile_routing.py — profile-based routing.""" +import json + import pytest from gateway.profile_routing import ( ProfileRoute, @@ -50,6 +52,40 @@ class TestParseProfileRoutes: assert parse_profile_routes(None) == [] assert parse_profile_routes([]) == [] + def test_coerces_yaml_native_int_ids_to_str(self): + # PyYAML loads unquoted snowflakes / negative Telegram ids as int; + # inbound SessionSource ids are str, so un-coerced routes never match. + routes = parse_profile_routes([ + {"name": "server", "platform": "discord", "profile": "p", + "guild_id": 111, "chat_id": 222, "thread_id": 333}, + {"name": "tg", "platform": "telegram", "profile": "p", + "chat_id": -1001234567890}, + {"name": "platform-only", "platform": "discord", "profile": "p"}, + ]) + by_name = {r.name: r for r in routes} + assert (by_name["server"].guild_id, by_name["server"].chat_id, + by_name["server"].thread_id) == ("111", "222", "333") + assert match_profile_route( + routes, "discord", guild_id="111", chat_id="222", thread_id="333", + ).name == "server" + assert match_profile_route( + routes, "telegram", chat_id="-1001234567890", + ).name == "tg" + assert (by_name["platform-only"].guild_id, by_name["platform-only"].chat_id, + by_name["platform-only"].thread_id) == (None, None, None) + + def test_non_int_numeric_ids_warn_instead_of_silently_coercing(self, caplog): + # #86470 nuance: float/bool stringify to values that can never match + # an inbound id, so surface the misconfiguration at load time. + with caplog.at_level("WARNING", logger="gateway.profile_routing"): + routes = parse_profile_routes([ + {"name": "f", "platform": "discord", "profile": "p", "chat_id": 123.0}, + {"name": "b", "platform": "discord", "profile": "p", "guild_id": True}, + ]) + assert {r.name for r in routes} == {"f", "b"} + assert match_profile_route(routes, "discord", chat_id="123") is None + assert sum("can never match" in rec.message for rec in caplog.records) == 2 + class TestMatchProfileRoute: @@ -106,3 +142,48 @@ class TestForumPostMatching: parent_chat_id="forum_channel_123") assert m is not None assert m.profile == "forum_profile" + + +class TestWhatsAppChatIdIdentityMatching: + """WhatsApp ``chat_id`` routes match across number / JID / LID forms (the + same alias canonicalization allowlists and session keys already use); + every other platform, and WhatsApp groups, stay exact-compare.""" + + PHONE = "15551234567" + LID = "999999999999999" + + def _write_lid_mapping(self, tmp_path, monkeypatch): + mapping_dir = tmp_path / "platforms" / "whatsapp" / "session" + mapping_dir.mkdir(parents=True) + (mapping_dir / f"lid-mapping-{self.PHONE}.json").write_text(json.dumps(f"{self.LID}@lid")) + (mapping_dir / f"lid-mapping-{self.LID}_reverse.json").write_text( + json.dumps(f"{self.PHONE}@s.whatsapp.net") + ) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + def test_number_route_matches_jid_and_mapped_lid_forms(self, tmp_path, monkeypatch): + self._write_lid_mapping(tmp_path, monkeypatch) + for platform in ("whatsapp", "whatsapp_cloud"): + r = ProfileRoute(name="owner", platform=platform, profile="owner", chat_id=self.PHONE) + assert r.matches(platform, chat_id=f"{self.PHONE}@s.whatsapp.net") + assert r.matches(platform, chat_id=f"{self.PHONE}:47@s.whatsapp.net") + assert r.matches(platform, chat_id=f"{self.LID}@lid") + # Alias fallback also applies to the thread-parent slot. + assert r.matches(platform, chat_id="thread-1", parent_chat_id=f"{self.LID}@lid") + assert not r.matches(platform, chat_id="15550001111@s.whatsapp.net") + + def test_groups_and_other_platforms_stay_exact(self, tmp_path, monkeypatch): + self._write_lid_mapping(tmp_path, monkeypatch) + group = "120363012345678901@g.us" + owner = ProfileRoute(name="owner", platform="whatsapp", profile="owner", chat_id=self.PHONE) + assert not owner.matches("whatsapp", chat_id=group) + grp = ProfileRoute(name="grp", platform="whatsapp", profile="grp", chat_id=group) + assert grp.matches("whatsapp", chat_id=group) + assert not grp.matches("whatsapp", chat_id=f"{self.PHONE}@s.whatsapp.net") + # Stripping @g.us must never turn a group into a phone-identity match. + assert not ProfileRoute( + name="oops", platform="whatsapp", profile="owner", chat_id=group.split("@", 1)[0] + ).matches("whatsapp", chat_id=group) + tg = ProfileRoute(name="tg", platform="telegram", profile="owner", chat_id="640466638") + assert tg.matches("telegram", chat_id="640466638") + assert not tg.matches("telegram", chat_id="640466638@s.whatsapp.net") diff --git a/tests/gateway/test_raft_adapter.py b/tests/gateway/test_raft_adapter.py index 34a739f6e2..552c0da291 100644 --- a/tests/gateway/test_raft_adapter.py +++ b/tests/gateway/test_raft_adapter.py @@ -3,6 +3,7 @@ import asyncio import json import os +from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest @@ -218,3 +219,104 @@ class TestRaftConfig: assert os.environ["RAFT_PROFILE"] == "existing" assert "Keeping RAFT_PROFILE=existing" in capsys.readouterr().out + +# --------------------------------------------------------------------------- +# Multiplex secondary-profile scope (RAFT_PROFILE resolution) +# --------------------------------------------------------------------------- +# +# _spawn_bridge, _env_enablement, and register()'s platform_hint all +# previously read RAFT_PROFILE via raw os.environ.get unconditionally. Under +# a multiplexed secondary profile, os.environ holds the DEFAULT profile's +# YAML-to-env bridge output — a secondary profile with its own RAFT_PROFILE +# (set only in its own .env, resolved via the installed secret scope) would +# silently connect the bridge subprocess / CLI hint to the default profile's +# external Raft workspace/agent identity instead of its own. Mirrors the +# Buzz/SimpleX fix for #98738. + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + tokens = [] + + def install(scope=None): + from agent.secret_scope import set_multiplex_active, set_secret_scope + + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + return tokens[-1] + + yield install + + from agent.secret_scope import reset_secret_scope, set_multiplex_active + + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("RAFT_PROFILE", "default-profile-slug") + + +class _FakeCtx: + """Minimal ``ctx`` capturing ``register_platform``'s kwargs.""" + + def __init__(self): + self.platform_kwargs = None + + def register_platform(self, **kwargs): + self.platform_kwargs = kwargs + + def register_hook(self, *args, **kwargs): + pass + + +class TestMultiplexProfileScope: + + def test_secondary_profile_uses_its_own_slug_and_never_borrows_default( + self, multiplex_scope, default_profile_env, monkeypatch + ): + """Bridge spawn, env-enablement and the register() hint all resolve + the secondary profile's own RAFT_PROFILE; with none of its own the + profile fails closed (bridge not spawned, not auto-enabled).""" + import plugins.platforms.raft.adapter as raft_mod + + monkeypatch.setattr(raft_mod.shutil, "which", lambda name: "/usr/bin/raft") + spawned = [] + monkeypatch.setattr( + raft_mod.subprocess, "Popen", + lambda cmd, **kwargs: spawned.append(cmd) or SimpleNamespace(pid=1), + ) + + multiplex_scope({"RAFT_PROFILE": "secondary-profile-slug"}) + _make_adapter()._spawn_bridge(9999) + assert spawned[-1][:3] == ["/usr/bin/raft", "--profile", "secondary-profile-slug"] + assert _env_enablement() == {"enabled": True} + ctx = _FakeCtx() + register(ctx) + assert "--profile secondary-profile-slug" in ctx.platform_kwargs["platform_hint"] + assert "default-profile-slug" not in ctx.platform_kwargs["platform_hint"] + + spawned.clear() + multiplex_scope({}) + _make_adapter()._spawn_bridge(9999) + assert spawned == [] + assert _env_enablement() is None + + def test_default_profile_unscoped_keeps_env_precedence( + self, monkeypatch, default_profile_env + ): + """Multiplex ON but no scope (the DEFAULT profile constructs + unscoped): env is its own bridge output and still wins.""" + from agent.secret_scope import set_multiplex_active + + set_multiplex_active(True) + try: + assert _env_enablement() == {"enabled": True} + ctx = _FakeCtx() + register(ctx) + assert "--profile default-profile-slug" in ctx.platform_kwargs["platform_hint"] + finally: + set_multiplex_active(False) diff --git a/tests/gateway/test_runner_startup_failures.py b/tests/gateway/test_runner_startup_failures.py index 68fbfdf162..c3f906f171 100644 --- a/tests/gateway/test_runner_startup_failures.py +++ b/tests/gateway/test_runner_startup_failures.py @@ -3,11 +3,31 @@ from unittest.mock import AsyncMock from gateway.config import GatewayConfig, Platform, PlatformConfig from gateway.platforms.base import BasePlatformAdapter -from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE +from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE, is_global_startup_conflict from gateway.run import GatewayRunner from gateway.status import read_runtime_status +@pytest.mark.parametrize( + "code, expected", + [ + ("telegram-bot-token_lock", True), # BasePlatformAdapter._acquire_platform_lock + ("discord-bot-token_lock", True), + ("whatsapp-session_lock", True), + ("feishu_app_lock", True), + ("lock_conflict", True), # buzz / irc / line identity conflicts + ("telegram_connect_error", False), + ("telegram_auth_error", False), + ("relay_membership_required", False), + ("duplicate_credential", False), + ("", False), + (None, False), + ], +) +def test_is_global_startup_conflict_matches_lock_code_families(code, expected): + assert is_global_startup_conflict(code) is expected + + class _RetryableFailureAdapter(BasePlatformAdapter): def __init__(self): super().__init__(PlatformConfig(enabled=True, token="***"), Platform.TELEGRAM) @@ -443,3 +463,115 @@ async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_ await start_gateway(config=GatewayConfig(), replace=False, verbosity=0) assert exc_info.value.code == GATEWAY_FATAL_CONFIG_EXIT_CODE + + +class _ForeignTokenLockAdapter(BasePlatformAdapter): + """Connects exactly like telegram/discord do: production + ``_acquire_platform_lock`` first, which emits ``{scope}_lock`` with + ``retryable=True`` (so a mid-run reconnect can recover, #54167).""" + + def __init__(self): + super().__init__(PlatformConfig(enabled=True, token="***"), Platform.TELEGRAM) + + async def connect(self, *, is_reconnect: bool = False) -> bool: + return self._acquire_platform_lock( + "telegram-bot-token", self.config.token, "Telegram bot token" + ) + + async def disconnect(self) -> None: + self._release_platform_lock() + self._mark_disconnected() + + async def send(self, chat_id, content, reply_to=None, metadata=None): + raise NotImplementedError + + async def get_chat_info(self, chat_id): + return {"id": chat_id} + + +@pytest.mark.asyncio +async def test_live_foreign_token_lock_at_startup_exits_ex_config(monkeypatch, tmp_path): + """Salvage of #83183 claim 1: a LIVE foreign holder of the bot token at + zero-connected startup is a single-writer conflict, not a transient blip. + + ``_acquire_platform_lock`` deliberately emits the conflict retryable so a + *mid-run* reconnect can recover once the holder exits. The startup router + used to key solely off that flag, so the gateway stayed alive, deaf, and + retry-queued forever instead of exiting 78 (EX_CONFIG).""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("HERMES_GATEWAY_LOCK_DIR", str(tmp_path / "locks")) + # A live foreign holder: acquire_scoped_lock reports (False, record). + monkeypatch.setattr( + "gateway.status.acquire_scoped_lock", + lambda scope, identity, metadata=None: ( + False, + {"pid": 424242, "start_time": 1, "hermes_home": "/other/home", "profile": "other"}, + ), + ) + config = GatewayConfig( + platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="***")}, + sessions_dir=tmp_path / "sessions", + ) + runner = GatewayRunner(config) + monkeypatch.setattr( + runner, "_create_adapter", lambda platform, platform_config: _ForeignTokenLockAdapter() + ) + + ok = await runner.start() + + assert ok is True + assert runner.should_exit_cleanly is True + assert runner.exit_code == GATEWAY_FATAL_CONFIG_EXIT_CODE + assert runner._failed_platforms == {} + state = read_runtime_status() + assert state["gateway_state"] == "startup_failed" + assert state["platforms"]["telegram"]["state"] == "fatal" + assert state["platforms"]["telegram"]["error_code"] == "telegram-bot-token_lock" + + +@pytest.mark.asyncio +async def test_token_lock_plus_retryable_peer_stays_alive(monkeypatch, tmp_path): + """A lock conflict alongside a genuinely transient peer failure is the + NS-609 mixed mode: the lock is parked fatal, the peer keeps its retry, and + the gateway stays alive (no exit 78).""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("HERMES_GATEWAY_LOCK_DIR", str(tmp_path / "locks")) + monkeypatch.setattr( + "gateway.status.acquire_scoped_lock", + lambda scope, identity, metadata=None: (False, {"pid": 424242, "start_time": 1}), + ) + config = GatewayConfig( + platforms={ + Platform.TELEGRAM: PlatformConfig(enabled=True, token="***"), + Platform.DISCORD: PlatformConfig(enabled=True, token="***"), + }, + sessions_dir=tmp_path / "sessions", + ) + runner = GatewayRunner(config) + + class _DiscordBlip(_RetryableFailureAdapter): + def __init__(self): + BasePlatformAdapter.__init__( + self, PlatformConfig(enabled=True, token="***"), Platform.DISCORD + ) + + monkeypatch.setattr( + runner, + "_create_adapter", + lambda platform, cfg: ( + _ForeignTokenLockAdapter() if platform is Platform.TELEGRAM else _DiscordBlip() + ), + ) + + ok = await runner.start() + try: + assert ok is True + assert runner.should_exit_cleanly is False + assert runner.exit_code is None + assert set(runner._failed_platforms) == {Platform.DISCORD} + state = read_runtime_status() + assert state["gateway_state"] == "running" + assert state["platforms"]["telegram"]["state"] == "fatal" + assert state["platforms"]["discord"]["state"] == "retrying" + finally: + await runner.stop() diff --git a/tests/gateway/test_scale_to_zero_dashboard_client.py b/tests/gateway/test_scale_to_zero_dashboard_client.py new file mode 100644 index 0000000000..93dbd73f1c --- /dev/null +++ b/tests/gateway/test_scale_to_zero_dashboard_client.py @@ -0,0 +1,284 @@ +"""Scale-to-zero: an attached dashboard/desktop/TUI WS client counts as activity. + +Background (2026-09-02 fleet audit): 13 of 72 active opted-in prod instances +flapped suspend -> proxy-wake every ~60s. The gateway only stamped +``_last_inbound_at`` for messaging inbound, so it suspended under an open +dashboard client; the client's reconnect loop re-poked the Fly-proxied hostname +and autostart resumed the box. The dashboard runs in a separate process on +hosted instances, so the signal crosses over as a marker-file mtime. + +These tests exercise the REAL seams — the pure helpers with a real temp +HERMES_HOME, GatewayRunner._scale_to_zero_is_idle's composition, and +tui_gateway.ws.handle_ws — rather than stubbing the collection under test +(the F25 / #84327 lesson: bugs live at the call site, not in the pure predicate). +""" +from __future__ import annotations + +import asyncio +import os +import time + +import pytest + +from gateway import scale_to_zero as s2z +from gateway.run import GatewayRunner + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + return tmp_path + + +def _stat_denying(target): + """os.stat replacement that denies ONLY the marker path (stdlib callers unaffected).""" + real = os.stat + + def _stat(path, *a, **k): + if os.fspath(path) == os.fspath(target): + raise PermissionError("nope") + return real(path, *a, **k) + + return _stat + + +# --- pure helpers ----------------------------------------------------------- + + +def test_heartbeat_path_lives_under_hermes_home_state(hermes_home): + p = s2z.dashboard_client_heartbeat_path() + assert p == hermes_home / "state" / "dashboard_clients.heartbeat" + + +def test_last_seen_missing_marker_is_none_not_fail_awake(hermes_home): + # Steady state for a box nobody has the dashboard open on: must read as + # "no client", otherwise no instance would ever suspend. + assert s2z.dashboard_client_last_seen() is None + + +def test_touch_creates_state_dir_and_marker(hermes_home): + assert s2z.touch_dashboard_client_heartbeat() is True + p = s2z.dashboard_client_heartbeat_path() + assert p.exists() + seen = s2z.dashboard_client_last_seen() + assert seen is not None and abs(seen - time.time()) < 5 + + +def test_last_seen_returns_raw_mtime_without_staleness_cutoff(hermes_home): + # No liveness cutoff here on purpose: is_idle decides recency. A 1h-old + # marker still reports its mtime; the gateway then finds it outside + # idle_timeout, same as an old _last_inbound_at. + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + mtime = os.stat(p).st_mtime + assert s2z.dashboard_client_last_seen(now=mtime + 10) == mtime + assert s2z.dashboard_client_last_seen(now=mtime + 3600) == mtime + + +def test_last_seen_future_mtime_is_clamped_to_now(hermes_home): + # A wall-clock step-back can leave the marker in the future; it must not + # extend the idle window past "now". + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + future = time.time() + 600 + os.utime(p, (future, future)) + now = time.time() + assert s2z.dashboard_client_last_seen(now=now) == now + + +def test_last_seen_unreadable_marker_fails_awake(hermes_home, monkeypatch): + s2z.touch_dashboard_client_heartbeat() + + monkeypatch.setattr(s2z.os, "stat", _stat_denying(s2z.dashboard_client_heartbeat_path())) + now = 1_000_000.0 + # Unreadable (not missing) => counts as activity right now. + assert s2z.dashboard_client_last_seen(now=now) == now + + +def test_touch_never_raises(hermes_home, monkeypatch): + monkeypatch.setattr(s2z.os, "utime", lambda *a, **k: (_ for _ in ()).throw(OSError("ro"))) + assert s2z.touch_dashboard_client_heartbeat() is False + + +# --- gateway side: the idle predicate composition --------------------------- + + +def _runner(monkeypatch, *, last_inbound_at): + r = GatewayRunner.__new__(GatewayRunner) + r._running = True + r._last_inbound_at = last_inbound_at + r._running_agents = {} + r._background_tasks = set() + r.adapters = {} + monkeypatch.setattr(r, "_scale_to_zero_idle_timeout_seconds", lambda: 120.0, raising=False) + monkeypatch.setattr(r, "_scale_to_zero_has_live_background_work", lambda: False, raising=False) + monkeypatch.setattr("cron.scheduler.get_running_job_ids", lambda: []) + return r + + +def test_idle_without_dashboard_client_unchanged(hermes_home, monkeypatch): + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + assert r._scale_to_zero_is_idle() is True + + +def test_attached_dashboard_client_blocks_idle(hermes_home, monkeypatch): + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + s2z.touch_dashboard_client_heartbeat() + assert r._scale_to_zero_is_idle() is False + + +def test_client_gets_the_same_idle_grace_as_a_message(hermes_home, monkeypatch): + """Last WS frame 100s ago with a 120s idle_timeout => still inside the + window => NOT idle. This is the 2-minute-after-the-app-closes contract; an + earlier draft cut the marker off at 45s and suspended ~50s after + disconnect (observed live on staging).""" + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + old = time.time() - 100 + os.utime(p, (old, old)) + assert r._scale_to_zero_is_idle() is False + + +def test_client_gone_longer_than_idle_timeout_is_idle(hermes_home, monkeypatch): + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + old = time.time() - 121 + os.utime(p, (old, old)) + assert r._scale_to_zero_is_idle() is True + + +def test_marker_predating_gateway_inbound_does_not_matter(hermes_home, monkeypatch): + # Ancient marker from a client that left hours ago, gateway idle 600s. + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + old = time.time() - 7200 + os.utime(p, (old, old)) + assert r._scale_to_zero_is_idle() is True + + +def test_dashboard_client_seen_recently_extends_inbound_clock(hermes_home, monkeypatch): + # Marker 30s old: inbound clock moves to 30s ago, which is + # inside the 120s window => not idle, even though the gateway's own + # _last_inbound_at is ancient. + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + t = time.time() - 30 + os.utime(p, (t, t)) + assert r._scale_to_zero_is_idle() is False + + +def test_newer_gateway_inbound_wins_over_older_marker(hermes_home, monkeypatch): + r = _runner(monkeypatch, last_inbound_at=time.time() - 5) + monkeypatch.setattr(r, "_scale_to_zero_idle_timeout_seconds", lambda: 10.0, raising=False) + s2z.touch_dashboard_client_heartbeat() + p = s2z.dashboard_client_heartbeat_path() + t = time.time() - 40 + os.utime(p, (t, t)) + # Picking the marker (40s > 10s) would read idle; the chat message (5s) wins. + assert r._scale_to_zero_is_idle() is False + # _last_inbound_at itself is not mutated by the read. + assert time.time() - r._last_inbound_at < 10 + + +def test_unreadable_marker_keeps_gateway_awake(hermes_home, monkeypatch): + r = _runner(monkeypatch, last_inbound_at=time.time() - 600) + s2z.touch_dashboard_client_heartbeat() + monkeypatch.setattr(s2z.os, "stat", _stat_denying(s2z.dashboard_client_heartbeat_path())) + assert r._scale_to_zero_is_idle() is False + + +# --- dashboard side: the real handle_ws path touches the marker ------------- + + +def test_handle_ws_connect_touches_marker(hermes_home, monkeypatch): + from tui_gateway import server, ws as ws_mod + + monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", lambda: None) + monkeypatch.setattr(server, "_schedule_startup_orphan_sweep", lambda: None, raising=False) + monkeypatch.setattr(server, "resolve_skin", lambda: "default") + monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None) + monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None) + monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0) + monkeypatch.setattr(ws_mod, "_dashboard_client_touched_at", 0.0) + + class FakeWS: + async def accept(self): + pass + + async def send_text(self, line): + pass + + async def receive_text(self): + raise ws_mod._WebSocketDisconnect() + + async def close(self): + pass + + assert s2z.dashboard_client_last_seen() is None + asyncio.run(ws_mod.handle_ws(FakeWS())) + seen = s2z.dashboard_client_last_seen() + assert seen is not None and abs(seen - time.time()) < 5 + + +def test_handle_ws_inbound_frames_refresh_marker(hermes_home, monkeypatch): + from tui_gateway import server, ws as ws_mod + + monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", lambda: None) + monkeypatch.setattr(server, "_schedule_startup_orphan_sweep", lambda: None, raising=False) + monkeypatch.setattr(server, "resolve_skin", lambda: "default") + monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None) + monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None) + monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0) + monkeypatch.setattr(ws_mod, "_dashboard_client_touched_at", 0.0) + # Disable the throttle so each frame is observable. + monkeypatch.setattr(ws_mod, "_DASHBOARD_CLIENT_TOUCH_MIN_INTERVAL_S", 0.0) + + frames = ['{"jsonrpc":"2.0","method":"gateway.ping","id":1}'] * 2 + touches: list[float] = [] + real_touch = s2z.touch_dashboard_client_heartbeat + + def _spy(path=None): + touches.append(time.time()) + return real_touch(path) + + monkeypatch.setattr(s2z, "touch_dashboard_client_heartbeat", _spy) + + class FakeWS: + async def accept(self): + pass + + async def send_text(self, line): + pass + + async def receive_text(self): + if frames: + return frames.pop() + raise ws_mod._WebSocketDisconnect() + + async def close(self): + pass + + asyncio.run(ws_mod.handle_ws(FakeWS())) + # 1 on connect + 1 per inbound frame. + assert len(touches) == 3 + assert s2z.dashboard_client_last_seen() is not None + + +def test_note_activity_is_throttled(hermes_home, monkeypatch): + from tui_gateway import ws as ws_mod + + calls = {"n": 0} + monkeypatch.setattr( + s2z, "touch_dashboard_client_heartbeat", lambda path=None: calls.__setitem__("n", calls["n"] + 1) or True + ) + monkeypatch.setattr(ws_mod, "_dashboard_client_touched_at", 0.0) + ws_mod._note_dashboard_client_activity(force=True) + ws_mod._note_dashboard_client_activity() + ws_mod._note_dashboard_client_activity() + assert calls["n"] == 1 + ws_mod._note_dashboard_client_activity(force=True) + assert calls["n"] == 2 diff --git a/tests/gateway/test_session_continuity_82616.py b/tests/gateway/test_session_continuity_82616.py index d99bccfa77..7f9a498dc6 100644 --- a/tests/gateway/test_session_continuity_82616.py +++ b/tests/gateway/test_session_continuity_82616.py @@ -201,6 +201,28 @@ class TestPeerResolutionRecency: class TestLoadTranscriptReroutes: + def test_load_transcript_raises_when_message_read_fails(self, tmp_path, monkeypatch): + from gateway.session import SessionStore, TranscriptReadError + + from gateway.config import GatewayConfig + + store = SessionStore(sessions_dir=tmp_path / "gw-failed-read", config=GatewayConfig()) + db = store._db + assert db is not None + monkeypatch.setattr(db, "get_compression_tip", lambda _session_id: None) + + def _malformed(_session_id, *, repair_alternation): + assert repair_alternation is True + raise RuntimeError("database disk image is malformed") + + monkeypatch.setattr(db, "get_messages_as_conversation", _malformed) + + with pytest.raises(TranscriptReadError) as exc_info: + store.load_transcript("existing-session") + + assert exc_info.value.session_id == "existing-session" + assert isinstance(exc_info.value.__cause__, RuntimeError) + def test_load_transcript_follows_reroute_chain(self, tmp_path): from gateway.session import SessionStore diff --git a/tests/gateway/test_session_db_corrupt_fallback.py b/tests/gateway/test_session_db_corrupt_fallback.py new file mode 100644 index 0000000000..9a6be9a114 --- /dev/null +++ b/tests/gateway/test_session_db_corrupt_fallback.py @@ -0,0 +1,70 @@ +"""Gateway SessionStore must divert, not retry forever, after structural corruption. + +Mirrors ``test_session_db_replaced_fallback.py``: once the SessionDB handle +is quarantined (``StateDbCorruptError``) the pending transcript goes to the +JSONL/spool fallback and no FTS surgery runs on the damaged file. +""" + +import json +import sqlite3 + +from gateway.config import GatewayConfig +from gateway.session import SessionStore + + +class _MalformedConn: + def __init__(self, real_conn): + self._real = real_conn + + def execute(self, *args, **kwargs): + raise sqlite3.DatabaseError("database disk image is malformed") + + def __getattr__(self, name): + return getattr(self._real, name) + + +def _assert_diverted(tmp_path, sid, needle): + pending = list((tmp_path / "pending_messages").glob("pending-*.json")) + assert pending, "expected pending_messages/pending-*.json spool" + spooled = False + for path in pending: + payload = json.loads(path.read_text(encoding="utf-8")) + message = (payload.get("data") or {}).get("message") or {} + if needle in str(message.get("content", "")): + spooled = True + break + assert spooled, f"{needle!r} missing from pending spool" + jsonl = tmp_path / "sessions" / f"{sid}.jsonl" + assert jsonl.is_file() + assert needle in jsonl.read_text(encoding="utf-8") + + +def test_corrupt_state_db_diverts_pending_without_fts_rebuild(tmp_path, monkeypatch): + import hermes_state + + live = tmp_path / "state.db" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", live) + + store = SessionStore(sessions_dir=tmp_path, config=GatewayConfig()) + sid = "gw-corrupt" + store._db.create_session(session_id=sid, source="cli") + store.append_to_transcript( + sid, {"role": "user", "content": "before", "timestamp": 1.0} + ) + real_conn = store._db._conn + store._db._conn = _MalformedConn(real_conn) + try: + store.append_to_transcript( + sid, {"role": "user", "content": "after-corrupt", "timestamp": 2.0} + ) + assert store._db._db_corrupt is True + # No FTS surgery ran on either layer. + assert store._db._fts_enabled is True + assert store._db._fts_stale is False + assert store._fts_rebuild_attempted is False + assert sid not in store._dirty_transcripts + _assert_diverted(tmp_path, sid, "after-corrupt") + finally: + store._db._conn = real_conn + store.close_all_db_handles() diff --git a/tests/gateway/test_session_hygiene_turnhold_adoption.py b/tests/gateway/test_session_hygiene_turnhold_adoption.py new file mode 100644 index 0000000000..1b6c209cbd --- /dev/null +++ b/tests/gateway/test_session_hygiene_turnhold_adoption.py @@ -0,0 +1,431 @@ +"""Regression tests for #97963 — hygiene turn-hold must not burn a +watermark-fenced compression attempt. + +The 10s ``hygiene_max_turn_hold_seconds`` budget (#92318) releases the +arriving user turn while a thinking summary model is still streaming its +reasoning prefix. Before the fix, that release ALWAYS cancelled the commit +fence, so 100% of the summary attempt (including the full thinking prefix) +was discarded on every turn — auto-compression permanently failed for any +deployment whose summary model thinks longer than the hold. + +The fix decouples the turn from the compression: when the worker's commit is +watermark-fenced (rows appended after compression start survive its commit +verbatim as concurrent tail), the detached worker KEEPS its commit admission +and the summary is adopted at its own watermark-fenced commit boundary. The +turn is still released at the same budget — the invariant pinned by +``test_session_hygiene_turn_hold_budget_abandons_streaming_wait`` (#90845) +is untouched (that test's worker is NOT watermark-fenced and still takes the +cancel path). +""" + +import asyncio +import importlib +import sys +import threading +import time +import types +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.platforms.base import BasePlatformAdapter, MessageEvent, SendResult +from gateway.session import SessionEntry, SessionSource + + +def _make_history(n_messages: int, content_size: int = 100) -> list: + history = [] + content = "x" * content_size + for i in range(n_messages): + role = "user" if i % 2 == 0 else "assistant" + history.append({"role": role, "content": content, "timestamp": f"t{i}"}) + return history + + +class _CaptureAdapter(BasePlatformAdapter): + def __init__(self): + super().__init__( + PlatformConfig(enabled=True, token="fake-token"), Platform.TELEGRAM + ) + self.sent = [] + + async def connect(self, *, is_reconnect: bool = False) -> bool: + return True + + async def disconnect(self) -> None: + return None + + async def send(self, chat_id, content, reply_to=None, metadata=None): + self.sent.append({"chat_id": chat_id, "content": content}) + return SendResult(success=True, message_id="x") + + async def get_chat_info(self, chat_id: str): + return {"id": chat_id} + + +def _write_turnhold_config(tmp_path): + cfg_path = tmp_path / "config.yaml" + cfg_path.write_text( + "compression:\n" + " enabled: true\n" + " hygiene_timeout_seconds: 60\n" + " hygiene_total_ceiling_seconds: 600\n" + " hygiene_max_turn_hold_seconds: 0.3\n" + " hygiene_failure_cooldown_seconds: 120\n" + ) + + +def _build_runner(gateway_run, adapter, fake_db): + runner = object.__new__(gateway_run.GatewayRunner) + runner.config = GatewayConfig( + platforms={ + Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake-token") + } + ) + runner.adapters = {Platform.TELEGRAM: adapter} + runner._voice_mode = {} + runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False) + runner.session_store = MagicMock() + runner.session_store.get_or_create_session.return_value = SessionEntry( + session_key="agent:main:telegram:dm:12345", + session_id="sess-97963", + created_at=datetime.now(), + updated_at=datetime.now(), + platform=Platform.TELEGRAM, + chat_type="dm", + ) + runner.session_store.load_transcript.return_value = _make_history( + 6, content_size=400 + ) + runner.session_store.has_any_sessions.return_value = True + runner.session_store.rewrite_transcript = MagicMock() + runner.session_store.append_to_transcript = MagicMock() + runner._running_agents = {} + runner._pending_messages = {} + runner._pending_approvals = {} + runner._session_db = SimpleNamespace(_db=fake_db) + runner._is_user_authorized = lambda _source: True + runner._set_session_env = lambda _context: None + runner._run_agent = AsyncMock( + return_value={ + "final_response": "ok", + "messages": [], + "tools": [], + "history_offset": 0, + "last_prompt_tokens": 0, + } + ) + return runner + + +def _make_event(): + return MessageEvent( + text="hello", + source=SessionSource( + platform=Platform.TELEGRAM, + chat_id="12345", + chat_type="dm", + user_id="12345", + ), + message_id="1", + ) + + +def _install_fakes(monkeypatch, gateway_run, tmp_path, agent_cls): + fake_dotenv = types.ModuleType("dotenv") + fake_dotenv.load_dotenv = lambda *args, **kwargs: None + monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv) + fake_run_agent = types.ModuleType("run_agent") + fake_run_agent.AIAgent = agent_cls + monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent) + monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path) + monkeypatch.setattr( + gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"} + ) + monkeypatch.setattr( + "agent.model_metadata.get_model_context_length", + lambda *_args, **_kwargs: 100, + ) + + +async def _drain_deferred(runner, timeout=10.0): + tasks = getattr(runner, "_deferred_agent_cleanup_tasks", None) or set() + if tasks: + await asyncio.wait_for( + asyncio.gather(*list(tasks), return_exceptions=True), timeout + ) + + +@pytest.mark.asyncio +async def test_turn_hold_keeps_admission_and_adopts_watermark_fenced_summary( + monkeypatch, tmp_path +): + """A watermark-fenced worker keeps its commit admission at turn-hold + expiry; its late summary is ADOPTED (committed), not discarded — while + the turn itself is still released at the budget (#90845 invariant). + """ + worker_started = threading.Event() + release_worker = threading.Event() + committed = threading.Event() + cleanup_done = threading.Event() + fake_db = MagicMock() + fake_db.get_compression_failure_cooldown.return_value = None + + class FencedStreamingAgent: + last_instance = None + + def __init__(self, **kwargs): + self.session_id = kwargs.get("session_id", "sess-97963") + self._session_db = kwargs.get("session_db") + self._last_compaction_in_place = False + self.context_compressor = SimpleNamespace( + bind_session_state=MagicMock(), + _last_compress_aborted=False, + _last_aux_model_failure_model=None, + ) + self.shutdown_memory_provider = MagicMock() + self.close = MagicMock(side_effect=cleanup_done.set) + type(self).last_instance = self + + def _compress_context( + self, messages, *_args, commit_fence=None, **_kwargs + ): + # Real compress_context marks the fence right after capturing + # the active-row watermark under the durable compression lock. + if commit_fence is not None: + commit_fence.mark_commit_watermark_fenced() + worker_started.set() + # Thinking-model shape: continuous progress, no commit yet — + # only the turn-hold budget can release the waiting turn. + # Bounded spin: a failing assertion before release_worker.set() + # must not leave this executor thread alive forever (pytest + # would hang at interpreter exit joining executor threads). + _spin_started = time.monotonic() + while not release_worker.is_set(): + if time.monotonic() - _spin_started > 20: + return (messages, None) + if commit_fence is not None: + commit_fence.touch_progress() + time.sleep(0.01) + if commit_fence is not None and not commit_fence.begin_commit(): + return (messages, None) + try: + self._session_db.archive_and_compact( + self.session_id, + [{"role": "assistant", "content": "summary"}], + watermark=6, + ) + self._last_compaction_in_place = True + committed.set() + return ([{"role": "assistant", "content": "summary"}], None) + finally: + if commit_fence is not None: + commit_fence.finish_commit() + + gateway_run = importlib.import_module("gateway.run") + _write_turnhold_config(tmp_path) + _install_fakes(monkeypatch, gateway_run, tmp_path, FencedStreamingAgent) + + adapter = _CaptureAdapter() + runner = _build_runner(gateway_run, adapter, fake_db) + + started = time.monotonic() + result = await asyncio.wait_for(runner._handle_message(_make_event()), timeout=15) + elapsed = time.monotonic() - started + + # #90845/#92318 invariant intact: the turn is released at the budget. + assert result == "ok" + assert elapsed < 5.0, f"turn held for {elapsed:.1f}s despite the turn-hold budget" + assert worker_started.is_set() + assert runner._run_agent.await_count == 1 + + # (b) NO retry-after was armed while the attempt is still running — + # arming it would block the agent-side preflight from adopting the + # finished summary ("same-session cooldown active", #97963). + assert not fake_db.record_compression_failure_cooldown.called, ( + "keep-admission path must not arm the retry-after while the " + "detached attempt is still running" + ) + + # The detached worker finishes late; its commit is ADMITTED (adoption), + # not refused — the summary attempt is no longer burned. + release_worker.set() + await asyncio.wait_for(asyncio.to_thread(committed.wait, 5), timeout=6) + assert committed.is_set(), ( + "watermark-fenced worker must keep its commit admission after " + "turn-hold expiry (fence was cancelled — attempt burned)" + ) + fake_db.archive_and_compact.assert_called_once() + # The commit went through the watermark-fenced path (concurrent tail + # rows above the watermark survive the compaction). + assert fake_db.archive_and_compact.call_args.kwargs.get("watermark") == 6 + + await _drain_deferred(runner) + await asyncio.wait_for(asyncio.to_thread(cleanup_done.wait, 5), timeout=6) + FencedStreamingAgent.last_instance.close.assert_called_once() + + # Successful adoption resets the hygiene failure streak and still never + # advances it (the deferral is not a failure). + assert not fake_db.increment_hygiene_failure_streak.called + assert fake_db.reset_hygiene_failure_streak.called + # Deferral notice still reaches the user. + sent = [m["content"] for m in adapter.sent] + assert any( + "deferred" in c.lower() or "still streaming" in c.lower() for c in sent + ), f"turn-hold must send deferral notice, got: {sent}" + + +@pytest.mark.asyncio +async def test_turn_hold_kept_admission_arms_flat_retry_only_when_nothing_commits( + monkeypatch, tmp_path +): + """If the kept-admission worker ends WITHOUT committing (summary failed + / attempt superseded), the flat non-escalating retry-after is restored so + sustained traffic does not spawn-and-abandon a compressor every turn — + but only AFTER the attempt truly ended, and without touching the streak. + """ + worker_started = threading.Event() + release_worker = threading.Event() + fake_db = MagicMock() + fake_db.get_compression_failure_cooldown.return_value = None + + class FencedNoCommitAgent: + def __init__(self, **kwargs): + self.session_id = kwargs.get("session_id", "sess-97963") + self._session_db = kwargs.get("session_db") + self._last_compaction_in_place = False + self.context_compressor = SimpleNamespace( + bind_session_state=MagicMock(), + _last_compress_aborted=False, + _last_aux_model_failure_model=None, + ) + self.shutdown_memory_provider = MagicMock() + self.close = MagicMock() + + def _compress_context( + self, messages, *_args, commit_fence=None, **_kwargs + ): + if commit_fence is not None: + commit_fence.mark_commit_watermark_fenced() + worker_started.set() + _spin_started = time.monotonic() + while not release_worker.is_set(): + if time.monotonic() - _spin_started > 20: + return (messages, None) + if commit_fence is not None: + commit_fence.touch_progress() + time.sleep(0.01) + # Summary failed — return unchanged, no commit. + return (messages, None) + + gateway_run = importlib.import_module("gateway.run") + _write_turnhold_config(tmp_path) + _install_fakes(monkeypatch, gateway_run, tmp_path, FencedNoCommitAgent) + + adapter = _CaptureAdapter() + runner = _build_runner(gateway_run, adapter, fake_db) + + result = await asyncio.wait_for(runner._handle_message(_make_event()), timeout=15) + assert result == "ok" + assert worker_started.is_set() + # While the attempt still runs: no cooldown, so preflight adoption + # stays possible. + assert not fake_db.record_compression_failure_cooldown.called + + release_worker.set() + await _drain_deferred(runner) + # Let the done-callback fire. + for _ in range(100): + if fake_db.record_compression_failure_cooldown.called: + break + await asyncio.sleep(0.05) + + # Nothing committed → flat retry-after restored (spacing), streak intact. + assert fake_db.record_compression_failure_cooldown.called, ( + "a kept-admission attempt that ends without committing must restore " + "the flat turn-hold retry-after spacing" + ) + args = fake_db.record_compression_failure_cooldown.call_args[0] + retry = args[1] - time.time() + assert retry <= 120, ( + f"retry-after must stay flat (~60s), got {retry:.0f}s" + ) + assert "turn-hold" in (args[2] or "") + assert not fake_db.increment_hygiene_failure_streak.called, ( + "turn-hold deferral must never advance the failure streak" + ) + + +@pytest.mark.asyncio +async def test_turn_hold_without_watermark_fence_still_cancels( + monkeypatch, tmp_path +): + """A worker whose commit is NOT watermark-fenced (no session_db / + watermark capture failed) must still be cancelled at turn-hold expiry — + a late unfenced commit could clobber newer turns. Never worse than the + status quo. (Complements the pinned #90845 test, which exercises the + same path through the public surface.) + """ + worker_started = threading.Event() + release_worker = threading.Event() + fake_db = MagicMock() + fake_db.get_compression_failure_cooldown.return_value = None + + class UnfencedStreamingAgent: + def __init__(self, **kwargs): + self.session_id = kwargs.get("session_id", "sess-97963") + self._session_db = kwargs.get("session_db") + self._last_compaction_in_place = False + self.context_compressor = SimpleNamespace( + bind_session_state=MagicMock(), + _last_compress_aborted=False, + _last_aux_model_failure_model=None, + ) + self.shutdown_memory_provider = MagicMock() + self.close = MagicMock() + + def _compress_context( + self, messages, *_args, commit_fence=None, **_kwargs + ): + # Deliberately NO mark_commit_watermark_fenced(). + worker_started.set() + _spin_started = time.monotonic() + while not release_worker.is_set(): + if time.monotonic() - _spin_started > 20: + return (messages, None) + if commit_fence is not None: + commit_fence.touch_progress() + time.sleep(0.01) + if commit_fence is not None and not commit_fence.begin_commit(): + return (messages, None) + try: + self._session_db.archive_and_compact( + self.session_id, + [{"role": "assistant", "content": "too late"}], + ) + return ([{"role": "assistant", "content": "too late"}], None) + finally: + if commit_fence is not None: + commit_fence.finish_commit() + + gateway_run = importlib.import_module("gateway.run") + _write_turnhold_config(tmp_path) + _install_fakes(monkeypatch, gateway_run, tmp_path, UnfencedStreamingAgent) + + adapter = _CaptureAdapter() + runner = _build_runner(gateway_run, adapter, fake_db) + + result = await asyncio.wait_for(runner._handle_message(_make_event()), timeout=15) + assert result == "ok" + assert worker_started.is_set() + + release_worker.set() + await _drain_deferred(runner) + await asyncio.sleep(0.2) + # The unfenced late commit was refused — discard as before the fix. + fake_db.archive_and_compact.assert_not_called() + # Legacy path still records the flat retry-after immediately. + assert fake_db.record_compression_failure_cooldown.called + assert not fake_db.increment_hygiene_failure_streak.called diff --git a/tests/gateway/test_shutdown_executor_quiesce.py b/tests/gateway/test_shutdown_executor_quiesce.py new file mode 100644 index 0000000000..4f0233f0ec --- /dev/null +++ b/tests/gateway/test_shutdown_executor_quiesce.py @@ -0,0 +1,266 @@ +"""Gateway shutdown quiesces its thread pool before closing state.db (#101093). + +``_shutdown_executor()`` used to run *after* the SessionDB close block in +``_stop_impl``, and it never waited: ``cancel_futures`` only drops work that has +not started, and cancelling the awaiting task does not stop the worker thread +behind a ``run_in_executor`` future. So blocking DB work could still be running +when ``SessionDB.close()`` checkpointed the WAL and let SQLite unlink the +sidecar. The late write then reopens the handle (#94736) and mints a fresh WAL +generation behind that checkpoint, leaving teardown to checkpoint the same file +a second time from a connection the shutdown log never accounts for -- the +close-time page-write damage in #101093 and the split WAL generation in #101064. + +The order is now: quiesce (bounded) -> close. +""" + +import asyncio +import concurrent.futures +import threading +import time +from collections import OrderedDict + +import pytest + +import gateway.run as gw_mod + + +class _FakeSessionDB: + """Records when the gateway closed it, on a shared event log.""" + + def __init__(self, events, name): + self._events = events + self._name = name + + def close(self): + self._events.append(f"close:{self._name}") + + +class _FakeGateway: + """Minimal stand-in with just enough state for ``stop()`` to run.""" + + def __init__(self, events): + self._events = events + self._running = True + self._draining = False + self._restart_requested = False + self._restart_detached = False + self._restart_via_service = False + self._stop_task = None + self._exit_cleanly = False + self._exit_with_failure = False + self._exit_reason = None + self._exit_code = None + self._restart_drain_timeout = 0.01 + self._running_agents = {} + self._running_agents_ts = {} + self._agent_cache = OrderedDict() + self._agent_cache_lock = threading.Lock() + self.adapters = {} + self._background_tasks = set() + self._failed_platforms = [] + self._shutdown_event = asyncio.Event() + self._pending_messages = {} + self._pending_approvals = {} + self._busy_ack_ts = {} + self._executor_lock = threading.Lock() + self._executor_closing = False + self._executor = concurrent.futures.ThreadPoolExecutor( + max_workers=2, thread_name_prefix="quiesce-test" + ) + self._session_db = _FakeSessionDB(events, "session_db") + self.session_store = None + + # -- shutdown collaborators the real stop() reaches into --------------- + + def _running_agent_count(self): + return len(self._running_agents) + + def _active_cron_job_count(self): + return 0 + + def _active_api_run_count(self): + return 0 + + def _update_runtime_status(self, *_a, **_kw): + pass + + def _clear_plugin_message_injector(self): + pass + + async def _run_in_executor_with_context(self, func, *args): + return func(*args) + + async def _cleanup_agent_resources_off_loop(self, agent, *, context=""): + self._cleanup_agent_resources(agent) + + async def _notify_active_sessions_of_shutdown(self): + pass + + async def _cancel_secondary_profile_reconnect_tasks(self): + pass + + async def _drain_active_agents(self, timeout, cron_timeout=None): + return {}, False + + async def _finalize_shutdown_agents(self, agents): + pass + + def _cleanup_agent_resources(self, agent): + pass + + def _evict_cached_agent(self, key): + pass + + def _release_running_agent_state(self, session_key, **_kwargs): + self._running_agents.pop(session_key, None) + self._running_agents_ts.pop(session_key, None) + return False + + def close_all_session_db_handles(self): + pass + + +@pytest.mark.asyncio +async def test_running_executor_work_finishes_before_session_db_close(): + """A future already running when stop() begins writes before the close.""" + events = [] + gw = _FakeGateway(events) + started = threading.Event() + + def _blocking_db_write(): + started.set() + # Longer than the rest of the shutdown tail (~0.4s), shorter than the + # 2s quiesce ceiling: without the wait the close lands first. + time.sleep(1.0) + events.append("worker_write") + + future = gw._executor.submit(_blocking_db_write) + assert started.wait(2.0), "worker never started" + + await gw_mod.GatewayRunner.stop(gw) + future.result(timeout=5) + + assert "worker_write" in events, "worker never ran" + assert "close:session_db" in events, "SessionDB was never closed" + assert events.index("worker_write") < events.index("close:session_db"), ( + f"state.db was closed while a worker was still writing: {events}" + ) + + +@pytest.mark.asyncio +async def test_executor_refuses_new_work_before_session_db_close(): + """``_executor_closing`` is set before the close, so no fresh pool is minted.""" + events = [] + gw = _FakeGateway(events) + + real_close = gw._session_db.close + + def _close_and_probe(): + # The flag must already be set by the time the DB is closed, or a + # coroutine reaching _get_executor() here would spin up a new pool and + # run more blocking DB work against the handle being torn down. + events.append(f"closing_flag:{gw._executor_closing}") + real_close() + + gw._session_db.close = _close_and_probe + + await gw_mod.GatewayRunner.stop(gw) + + assert "closing_flag:True" in events, events + with pytest.raises(RuntimeError): + gw_mod.GatewayRunner._get_executor(gw) + + +@pytest.mark.asyncio +async def test_stuck_worker_skips_the_session_db_close(): + """A worker that outlives the quiesce budget must not be raced by close(). + + Reporting the live worker with a "may reopen state.db" warning is not + enough: the close()/checkpoint itself is the operation that raced the + late write and produced the wrong-page-number corruption in #101093, + so the close path has to be skipped whenever a worker survives the + budget, not merely logged around. + """ + events = [] + gw = _FakeGateway(events) + release = threading.Event() + started = threading.Event() + + def _stuck(): + started.set() + release.wait(5.0) + events.append("worker_write") + + future = gw._executor.submit(_stuck) + assert started.wait(2.0), "worker never started" + + # Force the quiesce budget to 0 so the worker is deterministically still + # alive when `_shutdown_executor` returns, without sleeping through the + # real 2s ceiling. + original_timeout = gw_mod._EXECUTOR_QUIESCE_TIMEOUT + gw_mod._EXECUTOR_QUIESCE_TIMEOUT = 0.0 + try: + await gw_mod.GatewayRunner.stop(gw) + finally: + gw_mod._EXECUTOR_QUIESCE_TIMEOUT = original_timeout + + assert "close:session_db" not in events, ( + f"SessionDB was closed/checkpointed while a worker was still alive: {events}" + ) + + release.set() + future.result(timeout=5) + assert "worker_write" in events, "worker never finished" + + +def test_shutdown_executor_defaults_to_no_wait(): + """The no-argument call keeps the historical fire-and-forget contract.""" + gw = _FakeGateway([]) + release = threading.Event() + started = threading.Event() + + def _slow(): + started.set() + release.wait(5.0) + + future = gw._executor.submit(_slow) + assert started.wait(2.0) + + began = time.monotonic() + still_live = gw_mod.GatewayRunner._shutdown_executor(gw) + elapsed = time.monotonic() - began + + assert elapsed < 0.5, f"default call waited {elapsed:.2f}s" + assert still_live == 1 + release.set() + future.result(timeout=5) + + +def test_shutdown_executor_reports_a_stuck_worker(): + """A worker that outlives the budget is reported, not waited on forever.""" + gw = _FakeGateway([]) + release = threading.Event() + started = threading.Event() + + def _stuck(): + started.set() + release.wait(5.0) + + future = gw._executor.submit(_stuck) + assert started.wait(2.0) + + began = time.monotonic() + still_live = gw_mod.GatewayRunner._shutdown_executor(gw, drain_timeout=0.2) + elapsed = time.monotonic() - began + + assert still_live == 1 + assert 0.15 <= elapsed < 2.0, f"budget not honoured: {elapsed:.2f}s" + release.set() + future.result(timeout=5) + + +def test_shutdown_executor_without_executor_returns_zero(): + gw = _FakeGateway([]) + gw._executor.shutdown(wait=True) + gw._executor = None + assert gw_mod.GatewayRunner._shutdown_executor(gw, drain_timeout=1.0) == 0 diff --git a/tests/gateway/test_shutdown_flush.py b/tests/gateway/test_shutdown_flush.py index efe6f59572..f966ea896d 100644 --- a/tests/gateway/test_shutdown_flush.py +++ b/tests/gateway/test_shutdown_flush.py @@ -11,6 +11,7 @@ import pytest from gateway.shutdown_flush import ( _serialise_value, + flush_overflow_to_file, flush_pending_to_file, recover_pending_to_db, ) @@ -168,3 +169,72 @@ def test_get_flush_dir_uses_get_hermes_home(tmp_path, monkeypatch): assert result == tmp_path / "pending_messages" + + +# ── FIFO overflow tail durability (#99882) ───────────────────────────── + + +def _overflow_event(text: str, session_id: str = "20260901_120000_fifo"): + event = MagicMock() + event.text = text + event.session_id = session_id + event.platform = "telegram" + event.sender_id = "1572286605" + event.sender_name = "tester" + event.reply_to = None + event.media = None + event.raw_event = None + return event + + +def test_flush_overflow_writes_one_payload_per_event_in_arrival_order(tmp_path, monkeypatch): + """The FIFO tail (queued_events) must survive shutdown like the slot does. + + Each overflow entry is its own recover_pending_to_db-compatible payload, + with ``seq`` recording arrival order inside the session. + """ + flush_dir = _make_flush_dir(tmp_path) + monkeypatch.setattr("gateway.shutdown_flush._get_flush_dir", lambda: flush_dir) + + count = flush_overflow_to_file( + { + "agent:main:telegram:dm:1": [ + _overflow_event("follow-up B"), + _overflow_event("follow-up C"), + ], + "agent:main:telegram:dm:2": [], + "": [_overflow_event("keyless — skipped")], + }, + reason="shutdown", + ) + assert count == 2 + payloads = sorted( + (json.loads(f.read_text(encoding="utf-8")) for f in flush_dir.glob("*.json")), + key=lambda p: p["seq"], + ) + assert [p["data"]["text"] for p in payloads] == ["follow-up B", "follow-up C"] + assert {p["session_key"] for p in payloads} == {"agent:main:telegram:dm:1"} + assert all(p["reason"] == "shutdown" for p in payloads) + + +def test_flushed_overflow_is_replayed_by_recover_pending_to_db(tmp_path, monkeypatch): + """Round-trip: overflow payloads use the slot-flush shape, so the existing + startup recovery inserts them as user rows without any new reader.""" + flush_dir = _make_flush_dir(tmp_path) + monkeypatch.setattr("gateway.shutdown_flush._get_flush_dir", lambda: flush_dir) + flush_overflow_to_file({"agent:main:telegram:dm:1": [_overflow_event("orphan-1")]}) + + db = MagicMock() + recovered = recover_pending_to_db(session_db=db) + assert recovered == 1 + db.append_message.assert_called_once() + kwargs = db.append_message.call_args.kwargs + assert kwargs["session_id"] == "20260901_120000_fifo" + assert kwargs["role"] == "user" + assert kwargs["content"] == "orphan-1" + assert list(flush_dir.glob("*.json")) == [] + + +def test_flush_overflow_noop_on_empty(): + assert flush_overflow_to_file({}) == 0 + assert flush_overflow_to_file({"k": []}) == 0 diff --git a/tests/gateway/test_shutdown_watchdog.py b/tests/gateway/test_shutdown_watchdog.py index b46437383b..ec0e93e5c3 100644 --- a/tests/gateway/test_shutdown_watchdog.py +++ b/tests/gateway/test_shutdown_watchdog.py @@ -8,11 +8,18 @@ structurally unable to fire. These tests pin the out-of-loop backstop from __future__ import annotations import asyncio +import contextlib import json +import logging +import os +import shutil +import tempfile import threading import time +from pathlib import Path from unittest.mock import patch +import gateway.shutdown_watchdog as shutdown_watchdog_module import pytest from gateway.shutdown_watchdog import ( @@ -66,3 +73,105 @@ def test_arm_shutdown_watchdog_fires_with_dump_and_exit(tmp_path): assert get_shutdown_watchdog_dump_path(tmp_path).name == "gateway-shutdown-watchdog.log" + + +async def _run_heartbeat_until_payload(tmp_path, timeout_s=10.0): + """Run loop_heartbeat_forever as a task until a heartbeat payload exists. + + Returns (task, payload). Cancels the task and awaits it (suppressing + CancelledError) before returning so the tick server is closed cleanly. + """ + task = asyncio.ensure_future( + loop_heartbeat_forever(interval_s=1.0, home=tmp_path) + ) + heartbeat_path = get_loop_heartbeat_path(tmp_path) + deadline = time.monotonic() + timeout_s + payload = None + while time.monotonic() < deadline: + if heartbeat_path.is_file(): + with contextlib.suppress(OSError, json.JSONDecodeError): + payload = json.loads(heartbeat_path.read_text(encoding="utf-8")) + if payload: + break + payload = None + await asyncio.sleep(0.05) + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + if payload is None: + pytest.fail( + f"heartbeat payload did not appear at {heartbeat_path} within " + f"{timeout_s}s" + ) + return payload + + +@pytest.fixture() +def short_home(): + """Short HERMES_HOME for tests that bind a real AF_UNIX socket. + + pytest's tmp_path nests deep enough on CI runners / macOS that + ``state/gateway.loop-tick..sock`` exceeds the sockaddr_un limit and + bind() raises ``OSError: AF_UNIX path too long`` — which the producer + swallows into ``loop_tick_socket=False``, falsely failing the POSIX arm + test. Same pattern as tests/hermes_cli/test_update_wedged_gateway.py. + """ + path = Path(tempfile.mkdtemp(prefix="hsw-")) + try: + yield path + finally: + shutil.rmtree(path, ignore_errors=True) + + +@pytest.mark.asyncio +async def test_loop_tick_witness_arms_over_tcp_on_windows( + short_home, caplog, monkeypatch +): + """Non-POSIX never touches AF_UNIX; the witness arms over TCP loopback.""" + tmp_path = short_home + # Pretend the platform is Windows as seen from the module under test. + # A plain monkeypatch of the global os.name would flip pathlib.Path + # dispatch (Path.__new__ reads os.name at runtime) and crash pytest's + # own tmp-dir machinery, so swap the module's `os` binding for a proxy + # whose `.name` is "nt" and which delegates everything else to real os. + class _WindowsOsProxy: + name = "nt" + + def __getattr__(self, item): + return getattr(os, item) + + monkeypatch.setattr(shutdown_watchdog_module, "os", _WindowsOsProxy()) + + start_unix_server_calls = [] + + def _forbid_start_unix_server(*args, **kwargs): + start_unix_server_calls.append((args, kwargs)) + raise AssertionError("start_unix_server must not be called on non-POSIX") + + with patch.object( + shutdown_watchdog_module.asyncio, + "start_unix_server", + side_effect=_forbid_start_unix_server, + ), caplog.at_level(logging.DEBUG, logger="gateway.shutdown_watchdog"): + payload = await _run_heartbeat_until_payload(tmp_path) + + # (a) the AF_UNIX server was never attempted + assert start_unix_server_calls == [] + # (b) no warning about an unavailable tick socket + assert not [ + r + for r in caplog.records + if r.levelname == "WARNING" + and "Loop tick socket unavailable" in r.getMessage() + ] + # (c) the witness is armed over TCP and the port is published + assert payload["loop_tick_socket"] is True + assert 0 < int(payload["loop_tick_tcp_port"]) <= 65535 + # (d) the POSIX socket node was never created + assert not list(tmp_path.glob("**/gateway.loop-tick.*.sock")) + + +@pytest.mark.asyncio +async def test_loop_tick_witness_arms_on_posix(short_home): + payload = await _run_heartbeat_until_payload(short_home) + assert payload["loop_tick_socket"] is True diff --git a/tests/gateway/test_signal.py b/tests/gateway/test_signal.py index 078d787d43..fb668dc34d 100644 --- a/tests/gateway/test_signal.py +++ b/tests/gateway/test_signal.py @@ -270,7 +270,10 @@ class TestSignalAttachmentFetch: adapter._rpc, captured = _stub_rpc({"data": b64_data}) - with patch("gateway.platforms.signal.cache_image_from_bytes", return_value="/tmp/test.png"): + with patch( + "gateway.platforms.signal.cache_image_from_bytes_async", + new=AsyncMock(return_value="/tmp/test.png"), + ): await adapter._fetch_attachment("attachment-123") call = captured[0] @@ -1327,7 +1330,10 @@ class TestSignalContentlessEnvelope: b64_data = base64.b64encode(png_data).decode() adapter._rpc, _ = _stub_rpc({"data": b64_data}) - with patch("gateway.platforms.signal.cache_image_from_bytes", return_value="/tmp/img.png"): + with patch( + "gateway.platforms.signal.cache_image_from_bytes_async", + new=AsyncMock(return_value="/tmp/img.png"), + ): await adapter._handle_envelope({ "envelope": { "sourceNumber": "+155****9999", diff --git a/tests/gateway/test_simplex_plugin.py b/tests/gateway/test_simplex_plugin.py index 1a88d56513..90d3aa8ed1 100644 --- a/tests/gateway/test_simplex_plugin.py +++ b/tests/gateway/test_simplex_plugin.py @@ -388,3 +388,73 @@ def _make_file_chat_item(file_path: str, file_name: str) -> dict: } + + +# --------------------------------------------------------------------------- +# Multiplex secondary-profile scope +# --------------------------------------------------------------------------- +# +# Every SIMPLEX_* read (auto_accept / group_allowed in __init__, ws_url in the +# registry gates, everything in _env_enablement) went through raw os.getenv, +# which under multiplexing holds the DEFAULT profile's YAML-to-env bridge +# output -- a secondary profile silently borrowed the default's daemon URL, +# group allowlist and auto-accept setting. Reads now go through the module's +# ``_get_scoped_secret`` (profile .env AND extra both honored; scoped miss +# fails closed; unscoped default profile keeps env precedence). + + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + from agent.secret_scope import ( + reset_secret_scope, + set_multiplex_active, + set_secret_scope, + ) + + tokens = [] + + def install(scope=None): + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + + yield install + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("SIMPLEX_WS_URL", "ws://default:5225") + monkeypatch.setenv("SIMPLEX_GROUP_ALLOWED", "*") + monkeypatch.setenv("SIMPLEX_AUTO_ACCEPT", "true") + + +def test_multiplex_scoped_miss_does_not_borrow_default_profile_env( + multiplex_scope, default_profile_env +): + """A secondary profile with no SimpleX config of its own must not be + auto-enabled off the default's daemon URL, nor inherit its wide-open + group allowlist.""" + from gateway.config import PlatformConfig + + multiplex_scope({"SOMETHING_ELSE": "x"}) + assert _env_enablement() is None + assert check_requirements() is False + assert is_connected(PlatformConfig(enabled=True, extra={})) is False + adapter = SimplexAdapter(PlatformConfig(enabled=True, extra={"auto_accept": False})) + assert adapter.group_allow_from == set() + assert adapter.auto_accept is False + + +def test_multiplex_scope_reads_profile_own_env_not_default( + multiplex_scope, default_profile_env +): + """A secondary profile's own .env (installed as the scope) is honored -- + the extra-only shape would have ignored it.""" + multiplex_scope({"SIMPLEX_WS_URL": "ws://profile:5225", "SIMPLEX_GROUP_ALLOWED": "g1"}) + seeded = _env_enablement() + assert seeded == {"ws_url": "ws://profile:5225", "group_allowed": "g1"} + assert check_requirements() is True diff --git a/tests/gateway/test_slack_api_human_senders.py b/tests/gateway/test_slack_api_human_senders.py new file mode 100644 index 0000000000..5bde8bd103 --- /dev/null +++ b/tests/gateway/test_slack_api_human_senders.py @@ -0,0 +1,94 @@ +"""Tests for the Slack ``api_human_users`` allowlist. + +A message posted through the Web API with a *user* token (``xoxp-``) is +authored by a real person, but it arrives with the posting ``app_id`` and no +``client_msg_id`` — the #35777 app/bot signature — so +``_event_declares_bot_sender`` drops it. ``platforms.slack.extra.api_human_users`` +allowlists those *users* (never apps: an app's own ``xoxb`` bot posts carry +the same user+app_id shape). +""" + +import sys +from unittest.mock import MagicMock + +import pytest + + +# Mock slack-bolt / slack-sdk the same way test_slack_mention.py does. +def _ensure_slack_mock(): + if "slack_bolt" in sys.modules and hasattr(sys.modules["slack_bolt"], "__file__"): + return + slack_bolt = MagicMock() + slack_bolt.async_app.AsyncApp = MagicMock + slack_bolt.adapter.socket_mode.async_handler.AsyncSocketModeHandler = MagicMock + slack_sdk = MagicMock() + slack_sdk.web.async_client.AsyncWebClient = MagicMock + for name, mod in [ + ("slack_bolt", slack_bolt), + ("slack_bolt.async_app", slack_bolt.async_app), + ("slack_bolt.adapter", slack_bolt.adapter), + ("slack_bolt.adapter.socket_mode", slack_bolt.adapter.socket_mode), + ( + "slack_bolt.adapter.socket_mode.async_handler", + slack_bolt.adapter.socket_mode.async_handler, + ), + ("slack_sdk", slack_sdk), + ("slack_sdk.web", slack_sdk.web), + ("slack_sdk.web.async_client", slack_sdk.web.async_client), + ]: + sys.modules.setdefault(name, mod) + sys.modules.setdefault("aiohttp", MagicMock()) + + +_ensure_slack_mock() + +import plugins.platforms.slack.adapter as _slack_mod # noqa: E402 + +_slack_mod.SLACK_AVAILABLE = True + +from plugins.platforms.slack.adapter import SlackAdapter # noqa: E402 + +from gateway.config import Platform, PlatformConfig # noqa: E402 + + +HUMAN_ID = "U_human" + + +def _make_adapter(extra=None): + adapter = object.__new__(SlackAdapter) + adapter.platform = Platform.SLACK + adapter.config = PlatformConfig(enabled=True, extra=dict(extra or {})) + return adapter + + +def _api_post(**overrides): + """A user-token chat.postMessage as delivered over Socket Mode: + real ``user``, app_id stamp, no ``client_msg_id``.""" + event = {"type": "message", "user": HUMAN_ID, "app_id": "A_frontend", "text": "hi"} + event.update(overrides) + return event + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + monkeypatch.delenv("SLACK_API_HUMAN_USERS", raising=False) + + +def test_api_post_is_bot_by_default(): + assert _make_adapter()._event_declares_bot_sender(_api_post()) is True + + +def test_allowlisted_user_api_post_is_human(): + adapter = _make_adapter({"api_human_users": ["U_other", HUMAN_ID]}) + assert adapter._event_declares_bot_sender(_api_post()) is False + # Same predicate everywhere: no other user, and no user-less app post, rides it. + assert adapter._event_declares_bot_sender(_api_post(user="U_stranger")) is True + assert adapter._event_declares_bot_sender({"app_id": "A_frontend", "text": "hi"}) is True + + +def test_bot_markers_win_over_allowlist(): + """Allowlisting a user never admits genuine bot posts, so the app's own + ``xoxb`` traffic (bot_id / subtype=bot_message) cannot loop back in.""" + adapter = _make_adapter({"api_human_users": HUMAN_ID}) + assert adapter._event_declares_bot_sender(_api_post(subtype="bot_message")) is True + assert adapter._event_declares_bot_sender(_api_post(bot_id="B_stamp")) is True diff --git a/tests/gateway/test_slash_command_profile_scope.py b/tests/gateway/test_slash_command_profile_scope.py new file mode 100644 index 0000000000..5a3dba7397 --- /dev/null +++ b/tests/gateway/test_slash_command_profile_scope.py @@ -0,0 +1,94 @@ +"""Gateway slash commands must do their blocking work inside the routed profile. + +The multiplexed inbound handler wraps the whole message in +``_profile_runtime_scope``, which installs the routed profile's ``HERMES_HOME`` +override and its secret scope as **contextvars**. A bare +``loop.run_in_executor(None, ...)`` starts the worker with an EMPTY context, so +``SessionDB()`` / ``get_hermes_home()`` inside the worker resolve the LAUNCH +home — /insights reported the default profile's conversations from another +profile's chat. ``/compress`` already routes through +``_run_in_executor_with_context``; every other hop in the mixin must too. + +Drives the real mixin methods and the real ``_profile_runtime_scope``: the +contextvar loss is a property of the hop, so mocking the hop away would test +nothing. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def profile_home(tmp_path, monkeypatch): + root = tmp_path / ".hermes" + home = root / "profiles" / "coder" + home.mkdir(parents=True) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.setenv("HERMES_HOME", str(root)) + return home + + +@pytest.fixture +def runner(): + """Minimal host exposing the mixin plus the runner's executor helpers.""" + from gateway.run import GatewayRunner + from gateway.slash_commands import GatewaySlashCommandsMixin + + class _Runner(GatewaySlashCommandsMixin): + _run_in_executor_with_context = GatewayRunner._run_in_executor_with_context + _get_executor = GatewayRunner._get_executor + + r = _Runner() + r.adapters = {} + r._pending_skills_reload_notes = {} + return r + + +class _Event: + def __init__(self, args: str = ""): + self._args = args + self.source = None + + def get_command_args(self) -> str: + return self._args + + +@pytest.mark.asyncio +async def test_insights_opens_session_db_under_the_routed_home( + runner, profile_home, monkeypatch +): + import agent.insights as insights_mod + import hermes_state + from gateway.run import _profile_runtime_scope + from hermes_constants import get_hermes_home + + seen: dict = {} + + class _RecordingDB: + def __init__(self, *a, **kw): + seen["home"] = str(get_hermes_home()) + + def close(self): + pass + + class _Engine: + def __init__(self, db): + pass + + def generate(self, **kw): + return {} + + def format_gateway(self, report): + return "ok" + + monkeypatch.setattr(hermes_state, "SessionDB", _RecordingDB) + monkeypatch.setattr(insights_mod, "InsightsEngine", _Engine) + + with _profile_runtime_scope(profile_home): + result = await runner._handle_insights_command(_Event("")) + + assert result == "ok" + assert seen["home"] == str(profile_home) diff --git a/tests/gateway/test_slash_config_writes_routed_profile.py b/tests/gateway/test_slash_config_writes_routed_profile.py new file mode 100644 index 0000000000..c1893156f5 --- /dev/null +++ b/tests/gateway/test_slash_config_writes_routed_profile.py @@ -0,0 +1,70 @@ +"""Slash-command config writes must land in the routed profile's config.yaml. + +Regression for #87939 / #75684: the multiplexed inbound handler already runs +every slash handler inside ``_profile_runtime_scope`` (routed HERMES_HOME +override), but several handlers built their write path from the module +constant ``gateway.run._hermes_home`` — the LAUNCH home — so ``/reasoning +--global``, ``/fast``, ``/memory approval``, ``/skills approval``, ``/verbose`` +and ``/footer`` persisted into the default profile's config.yaml. They now go +through ``_gateway_config_home()`` like the reads do. +""" + +from __future__ import annotations + +import pytest +import yaml + +import gateway.run as gateway_run +from gateway.run import GatewayRunner, _profile_runtime_scope +from gateway.slash_commands import GatewaySlashCommandsMixin + + +class _Runner(GatewaySlashCommandsMixin): + _run_in_executor_with_context = GatewayRunner._run_in_executor_with_context + _get_executor = GatewayRunner._get_executor + + def _session_key_for_source(self, _source): + return "k" + + def _evict_cached_agent(self, _session_key): + pass + + +class _Event: + def __init__(self, args: str = ""): + self._args = args + self.source = None + + def get_command_args(self) -> str: + return self._args + + +@pytest.fixture +def homes(tmp_path, monkeypatch): + default_home = tmp_path / "default" + routed_home = tmp_path / "profiles" / "beta" + default_home.mkdir() + routed_home.mkdir(parents=True) + (default_home / "config.yaml").write_text("agent:\n reasoning_effort: medium\n") + (routed_home / "config.yaml").write_text("agent:\n reasoning_effort: none\n") + monkeypatch.setattr(gateway_run, "_hermes_home", default_home) + monkeypatch.setenv("HERMES_HOME", str(default_home)) + return default_home, routed_home + + +@pytest.mark.asyncio +async def test_slash_config_writes_hit_routed_profile_and_leave_default_untouched(homes): + default_home, routed_home = homes + default_before = (default_home / "config.yaml").read_bytes() + runner = _Runner() + + with _profile_runtime_scope(routed_home): + assert runner._save_gateway_config_key("agent.reasoning_effort", "high") + await runner._handle_memory_command(_Event("approval on")) + await runner._handle_skills_command(_Event("approval on")) + + routed = yaml.safe_load((routed_home / "config.yaml").read_text()) + assert routed["agent"]["reasoning_effort"] == "high" + assert routed["memory"]["write_approval"] is True + assert routed["skills"]["write_approval"] is True + assert (default_home / "config.yaml").read_bytes() == default_before diff --git a/tests/gateway/test_teams.py b/tests/gateway/test_teams.py index 7f7e734bfc..203eb66872 100644 --- a/tests/gateway/test_teams.py +++ b/tests/gateway/test_teams.py @@ -702,12 +702,12 @@ class TestTeamsBotFrameworkAttachments: adapter._fetch_attachment_bytes = AsyncMock(return_value=b"\x89PNG fake") adapter._get_botframework_token = AsyncMock(return_value="tok") - def fake_cache_media_bytes(data, **kwargs): + async def fake_cache_media_bytes(data, **kwargs): return SimpleNamespace( path="/tmp/img.png", media_type="image/png", kind="image" ) - with patch.object(_teams_mod, "cache_media_bytes", fake_cache_media_bytes): + with patch.object(_teams_mod, "cache_media_bytes_async", fake_cache_media_bytes): activity = self._make_activity([self._bf_image_attachment()]) await adapter._on_message(self._make_ctx(activity)) @@ -943,7 +943,10 @@ class TestTeamsBotFrameworkAttachments: adapter = self._make_adapter() adapter._fetch_attachment_bytes = AsyncMock(return_value=b"error page") - with patch.object(_teams_mod, "cache_media_bytes", lambda *a, **kw: None): + async def _no_media(*a, **kw): + return None + + with patch.object(_teams_mod, "cache_media_bytes_async", _no_media): with patch.object(_teams_mod.logger, "warning") as warn: activity = self._make_activity([self._bf_image_attachment()]) await adapter._on_message(self._make_ctx(activity)) diff --git a/tests/gateway/test_telegram_callback_auth_fail_closed.py b/tests/gateway/test_telegram_callback_auth_fail_closed.py index ee92721e4e..db7d60d2a8 100644 --- a/tests/gateway/test_telegram_callback_auth_fail_closed.py +++ b/tests/gateway/test_telegram_callback_auth_fail_closed.py @@ -87,3 +87,46 @@ class TestCallbackAuthFailClosed: assert adapter._is_callback_user_authorized("12345") is True +class TestCallbackAuthPrefersInjectedCheck: + """_is_callback_user_authorized must use the auth callback GatewayRunner + injects via set_authorization_check before the _message_handler.__self__ + introspection. + + A secondary multiplexed adapter's _message_handler is a profile closure + (no __self__), so the introspection path resolves to nothing and the old + code fell through to the env-only fallback — which knows nothing about + profile config allowlists or the pairing store. The injected callback is + registered for every gateway-connected adapter, including multiplexed + secondaries, and delegates to the full _is_user_authorized chain. + """ + + def test_injected_check_used_when_handler_is_a_closure(self, monkeypatch): + """Multiplexed shape: closure handler (no __self__) + injected check + registered → the injected check decides, not the env fallback.""" + monkeypatch.delenv("TELEGRAM_ALLOWED_USERS", raising=False) + monkeypatch.delenv("GATEWAY_ALLOW_ALL_USERS", raising=False) + adapter = _make_adapter() + adapter._message_handler = lambda *a, **kw: None # no __self__ + seen = {} + + def _check(user_id, chat_type=None, chat_id=None): + seen.update(user_id=user_id, chat_type=chat_type, chat_id=chat_id) + return user_id == "999" + + adapter._authorization_check = _check + + # Env fallback would deny (empty allowlist); the injected check allows. + assert adapter._is_callback_user_authorized( + "999", chat_id="777", chat_type="supergroup" + ) is True + assert seen == {"user_id": "999", "chat_type": "group", "chat_id": "777"} + + def test_injected_check_deny_wins_over_env_allowlist(self, monkeypatch): + """The injected check is authoritative when registered — an env + allowlist entry must not override its deny.""" + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "12345") + adapter = _make_adapter() + adapter._message_handler = lambda *a, **kw: None + adapter._authorization_check = lambda user_id, chat_type=None, chat_id=None: False + + assert adapter._is_callback_user_authorized("12345") is False diff --git a/tests/gateway/test_telegram_documents.py b/tests/gateway/test_telegram_documents.py index a82003ce9d..fd3b406a08 100644 --- a/tests/gateway/test_telegram_documents.py +++ b/tests/gateway/test_telegram_documents.py @@ -308,7 +308,10 @@ class TestMediaGroups: msg1 = _make_message(caption="two images", photo=[first_photo]) msg2 = _make_message(photo=[second_photo]) - with patch("plugins.platforms.telegram.adapter.cache_image_from_bytes", side_effect=["/tmp/burst-one.jpg", "/tmp/burst-two.jpg"]): + with patch( + "plugins.platforms.telegram.adapter.cache_image_from_bytes_async", + new=AsyncMock(side_effect=["/tmp/burst-one.jpg", "/tmp/burst-two.jpg"]), + ): await adapter._handle_media_message(_make_update(msg1), MagicMock()) await adapter._handle_media_message(_make_update(msg2), MagicMock()) assert adapter.handle_message.await_count == 0 diff --git a/tests/gateway/test_telegram_send_path_health.py b/tests/gateway/test_telegram_send_path_health.py index a16faa4ecd..91e0a01123 100644 --- a/tests/gateway/test_telegram_send_path_health.py +++ b/tests/gateway/test_telegram_send_path_health.py @@ -76,3 +76,110 @@ async def test_send_short_flood_still_retries_inline(monkeypatch): sleep.assert_awaited_once_with(2.0) +def test_mark_connected_publishes_connected_when_healthy(): + """A normal connect (never degraded) still publishes platform_state=connected.""" + adapter = _make_adapter() + adapter._send_path_degraded = False + + with patch.object(adapter, "_write_runtime_status_safe") as write_status: + adapter._mark_connected() + + write_status.assert_called_once() + _, kwargs = write_status.call_args + assert kwargs["platform_state"] == "connected" + + +def test_mark_connected_publishes_retrying_when_send_path_degraded(): + """connect() can return True while polling never proved a first getUpdates + round-trip (the degraded branch, or a reconnect where require_progress is + skipped). _mark_connected() must not publish "connected" for that case -- + it is indistinguishable from a healthy adapter to anything reading + gateway_state.json (#101391).""" + adapter = _make_adapter() + adapter._send_path_degraded = True + + with patch.object(adapter, "_write_runtime_status_safe") as write_status: + adapter._mark_connected() + + write_status.assert_called_once() + _, kwargs = write_status.call_args + assert kwargs["platform_state"] == "retrying" + + +def test_record_polling_progress_republishes_connected_after_degraded_connect(): + """Once getUpdates actually proves a round-trip after a degraded connect, + the previously-published "retrying" status must be corrected back to + "connected" -- otherwise it stays wedged until the next disconnect.""" + adapter = _make_adapter() + generation, _event = adapter._begin_polling_generation() + # Simulate connect() having already run and published the degraded state. + adapter._running = True + + with patch.object(adapter, "_write_runtime_status_safe") as write_status: + adapter._record_polling_progress(generation) + + write_status.assert_called_once() + _, kwargs = write_status.call_args + assert kwargs["platform_state"] == "connected" + assert adapter._send_path_degraded is False + + +def test_mid_session_polling_death_publishes_retrying_while_running(): + """#101391's measured incident: a HEALTHY connect published "connected", + then getUpdates silently died mid-session and nothing republished for 11h. + The recovery ladder's entry point must flip the file to "retrying".""" + adapter = _make_adapter() + adapter._running = True + adapter._send_path_degraded = False + adapter._polling_error_task = None + + class _Loop: + def create_task(self, coro): + coro.close() + return MagicMock(done=lambda: False) + + with patch.object(adapter, "_write_runtime_status_safe") as write_status, \ + patch("asyncio.get_running_loop", return_value=_Loop()): + adapter._schedule_polling_recovery(RuntimeError("boom"), reason="heartbeat probe") + + assert adapter._send_path_degraded is True + write_status.assert_called_once() + _, kwargs = write_status.call_args + assert kwargs["platform_state"] == "retrying" + assert kwargs["error_message"] == TelegramAdapter.DEGRADED_STATUS_MESSAGE + + +def test_polling_death_before_connect_does_not_publish(): + """Not yet running (cold connect still in progress): connect()'s own + _mark_connected publishes; the recovery path must not write early.""" + adapter = _make_adapter() + adapter._running = False + adapter._polling_error_task = None + + class _Loop: + def create_task(self, coro): + coro.close() + return MagicMock(done=lambda: False) + + with patch.object(adapter, "_write_runtime_status_safe") as write_status, \ + patch("asyncio.get_running_loop", return_value=_Loop()): + adapter._schedule_polling_recovery(RuntimeError("boom"), reason="polling bootstrap") + + write_status.assert_not_called() + + +@pytest.mark.parametrize("running, fatal", [(False, False), (True, True)]) +def test_record_polling_progress_does_not_flip_when_not_running_or_fatal(running, fatal): + """Cold connect: progress arrives while _running is still False -- the + connect path publishes, not the flip. Fatal: never overwrite "fatal".""" + adapter = _make_adapter() + generation, _event = adapter._begin_polling_generation() + adapter._running = running + if fatal: + adapter._fatal_error_message = "dead" + + with patch.object(adapter, "_write_runtime_status_safe") as write_status: + adapter._record_polling_progress(generation) + + write_status.assert_not_called() + assert adapter._send_path_degraded is False diff --git a/tests/gateway/test_telegram_topic_profile_isolation_76423.py b/tests/gateway/test_telegram_topic_profile_isolation_76423.py new file mode 100644 index 0000000000..32d8323335 --- /dev/null +++ b/tests/gateway/test_telegram_topic_profile_isolation_76423.py @@ -0,0 +1,118 @@ +"""Issue #76423 — SessionDB: telegram topic tables namespace by profile.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from hermes_state import SessionDB + + +CHAT = "208214988" + + +def _session(db, sid, profile_name=None): + db.create_session(session_id=sid, source="telegram", user_id=CHAT, profile_name=profile_name) + + +def test_legacy_rows_migrate_only_to_default(tmp_path: Path): + """v1 shape (no CASCADE, old user index) → v3: rows land in 'default' only.""" + db_path = tmp_path / "legacy.db" + conn = sqlite3.connect(str(db_path)) + conn.executescript( + f""" + CREATE TABLE state_meta (key TEXT PRIMARY KEY, value TEXT); + INSERT INTO state_meta(key, value) VALUES ('telegram_dm_topic_schema_version', '1'); + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT, user_id TEXT, model TEXT, + model_config TEXT, system_prompt TEXT, parent_session_id TEXT, + started_at REAL, ended_at REAL, end_reason TEXT, + message_count INTEGER DEFAULT 0, tool_call_count INTEGER DEFAULT 0, + input_tokens INTEGER DEFAULT 0, output_tokens INTEGER DEFAULT 0 + ); + INSERT INTO sessions(id, source, user_id, started_at) + VALUES ('legacy-sess', 'telegram', '{CHAT}', 1.0); + CREATE TABLE telegram_dm_topic_mode ( + chat_id TEXT PRIMARY KEY, user_id TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + activated_at REAL NOT NULL, updated_at REAL NOT NULL, + has_topics_enabled INTEGER, allows_users_to_create_topics INTEGER, + capability_checked_at REAL, intro_message_id TEXT, pinned_message_id TEXT + ); + INSERT INTO telegram_dm_topic_mode(chat_id, user_id, enabled, activated_at, updated_at) + VALUES ('{CHAT}', '{CHAT}', 1, 1.0, 1.0); + CREATE TABLE telegram_dm_topic_bindings ( + chat_id TEXT NOT NULL, thread_id TEXT NOT NULL, user_id TEXT NOT NULL, + session_key TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES sessions(id), + managed_mode TEXT NOT NULL DEFAULT 'auto', + linked_at REAL NOT NULL, updated_at REAL NOT NULL, + PRIMARY KEY (chat_id, thread_id) + ); + CREATE INDEX idx_telegram_dm_topic_bindings_user + ON telegram_dm_topic_bindings(user_id, chat_id); + INSERT INTO telegram_dm_topic_bindings + VALUES ('{CHAT}', '99', '{CHAT}', 'k', 'legacy-sess', 'auto', 1.0, 1.0); + """ + ) + conn.close() + + db = SessionDB(db_path=db_path) + db.apply_telegram_topic_migration() + assert db.get_meta("telegram_dm_topic_schema_version") == "3" + assert db.is_telegram_topic_mode_enabled( + chat_id=CHAT, user_id=CHAT, profile_name="default", + ) + assert not db.is_telegram_topic_mode_enabled( + chat_id=CHAT, user_id=CHAT, profile_name="coder", + ) + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="99", profile_name="default", + )["session_id"] == "legacy-sess" + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="99", profile_name="coder", + ) is None + fk = db._conn.execute("PRAGMA foreign_key_list('telegram_dm_topic_bindings')").fetchall() + assert any(row[2] == "sessions" and row[6] == "CASCADE" for row in fk) + db.close() + + +def test_mode_and_bindings_isolated_across_profiles(tmp_path: Path): + db = SessionDB(db_path=tmp_path / "state.db") + _session(db, "sess-a", "alpha") + _session(db, "sess-b", "beta") + + db.enable_telegram_topic_mode(chat_id=CHAT, user_id=CHAT, profile_name="alpha") + db.enable_telegram_topic_mode(chat_id=CHAT, user_id=CHAT, profile_name="beta") + db.disable_telegram_topic_mode(chat_id=CHAT, profile_name="alpha") + assert not db.is_telegram_topic_mode_enabled(chat_id=CHAT, user_id=CHAT, profile_name="alpha") + assert db.is_telegram_topic_mode_enabled(chat_id=CHAT, user_id=CHAT, profile_name="beta") + + db.bind_telegram_topic( + chat_id=CHAT, thread_id="77", user_id=CHAT, + session_key="ka", session_id="sess-a", profile_name="alpha", + ) + db.bind_telegram_topic( + chat_id=CHAT, thread_id="77", user_id=CHAT, + session_key="kb", session_id="sess-b", profile_name="beta", + ) + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="77", profile_name="alpha", + )["session_id"] == "sess-a" + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="77", profile_name="beta", + )["session_id"] == "sess-b" + + assert db.delete_telegram_topic_binding( + chat_id=CHAT, thread_id="77", profile_name="alpha", + ) == 1 + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="77", profile_name="alpha", + ) is None + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="77", profile_name="beta", + ) is not None + # Omitted kwarg == the single-profile "default" namespace, not a wildcard. + assert not db.is_telegram_topic_mode_enabled(chat_id=CHAT, user_id=CHAT) + assert db.get_telegram_topic_binding(chat_id=CHAT, thread_id="77") is None + db.close() diff --git a/tests/gateway/test_telegram_topic_profile_routing_76423.py b/tests/gateway/test_telegram_topic_profile_routing_76423.py new file mode 100644 index 0000000000..9f9f16e1ac --- /dev/null +++ b/tests/gateway/test_telegram_topic_profile_routing_76423.py @@ -0,0 +1,93 @@ +"""Issue #76423 — Gateway routes source.profile into telegram topic state.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from hermes_state import SessionDB +from gateway.config import Platform +from gateway.session import SessionSource + + +CHAT = "208214988" + + +def _source(profile=None, thread_id="42"): + return SessionSource( + platform=Platform.TELEGRAM, + user_id=CHAT, + chat_id=CHAT, + user_name="tester", + chat_type="dm", + thread_id=thread_id, + profile=profile, + ) + + +def test_gateway_uses_source_profile_not_global(tmp_path: Path): + from gateway.run import GatewayRunner + + assert GatewayRunner._telegram_topic_profile_name(_source("coder")) == "coder" + assert GatewayRunner._telegram_topic_profile_name(_source(None)) == "default" + + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-coder", source="telegram", user_id=CHAT, profile_name="coder") + db.enable_telegram_topic_mode(chat_id=CHAT, user_id=CHAT, profile_name="coder") + + runner = object.__new__(GatewayRunner) + runner._session_db = db + assert runner._telegram_topic_mode_enabled(_source("coder")) is True + assert runner._telegram_topic_mode_enabled(_source("other")) is False + assert runner._telegram_topic_mode_enabled(_source(None)) is False + + runner._record_telegram_topic_binding( + _source("coder", "42"), + SimpleNamespace(session_key="k", session_id="sess-coder"), + ) + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="42", profile_name="coder", + ) is not None + assert db.get_telegram_topic_binding( + chat_id=CHAT, thread_id="42", profile_name="default", + ) is None + db.close() + + +def test_routed_profile_flows_into_prune_via_send_metadata(tmp_path: Path): + """profile_routes: the transport adapter may be the primary (default) bot + while the turn is routed to another profile — the outbound metadata built + by the gateway carries the routed profile, and prune uses it over the + adapter's own stamp (#76423).""" + from gateway.run import GatewayRunner + from plugins.platforms.telegram.adapter import TelegramAdapter + + runner = object.__new__(GatewayRunner) + runner._thread_metadata_for_target = lambda *a, **k: {"thread_id": "99"} + meta = runner._thread_metadata_for_source(_source("coder", "99")) + assert meta["hermes_profile"] == "coder" + assert "hermes_profile" not in runner._thread_metadata_for_source(_source(None, "99")) + + # Cooldowns are keyed (profile, chat): alpha's reminder must not gag beta. + assert runner._should_send_telegram_lobby_reminder(_source("alpha")) is True + assert runner._should_send_telegram_lobby_reminder(_source("beta")) is True + assert runner._should_send_telegram_lobby_reminder(_source("alpha")) is False + + db = SessionDB(db_path=tmp_path / "state.db") + db.create_session(session_id="sess-default", source="telegram", user_id=CHAT) + db.create_session(session_id="sess-coder", source="telegram", user_id=CHAT, profile_name="coder") + for prof, sid in (("default", "sess-default"), ("coder", "sess-coder")): + db.bind_telegram_topic( + chat_id=CHAT, thread_id="99", user_id=CHAT, + session_key=f"k-{prof}", session_id=sid, profile_name=prof, + ) + + adapter = object.__new__(TelegramAdapter) + adapter.platform = Platform.TELEGRAM + adapter._session_store = SimpleNamespace(_db=db) + adapter._hermes_profile_name = "default" # transport = primary bot + adapter._prune_stale_dm_topic_binding(CHAT, "99", metadata=meta) + + assert db.get_telegram_topic_binding(chat_id=CHAT, thread_id="99", profile_name="coder") is None + assert db.get_telegram_topic_binding(chat_id=CHAT, thread_id="99", profile_name="default") is not None + db.close() diff --git a/tests/gateway/test_transcript_read_failure_100788.py b/tests/gateway/test_transcript_read_failure_100788.py new file mode 100644 index 0000000000..b4dd6c3708 --- /dev/null +++ b/tests/gateway/test_transcript_read_failure_100788.py @@ -0,0 +1,103 @@ +"""A failed transcript read must not masquerade as an empty history (#100788). + +The gateway restore path (``_handle_message``) already fails closed on +current main (#100910). This file covers the surviving half of PR #100887: +the slash-command handlers, which used to let ``TranscriptReadError`` +propagate into the dispatch wrapper and reply with nothing at all. + +Incident shape: a malformed ``state.db`` made every +``SessionStore.load_transcript`` raise; the except-block swallowed it and +returned ``[]``. Restore then rebuilt the turn from "no history", so a +long-running chat silently restarted as a brand-new conversation and the +model happily answered as if nothing had ever been discussed. + +Two guarantees under test: + A. ``load_transcript`` raises ``TranscriptReadError`` on a read failure, + while a genuinely empty session still returns ``[]``. + B. Slash-command handlers that read the transcript reply with + ``HISTORY_UNREADABLE`` instead of raising into the dispatch wrapper + (which logs and sends nothing). + +Offline: SQLite on tmp_path only, no network. +""" + +import sqlite3 + +import pytest + +from gateway.config import GatewayConfig +from gateway.session import SessionStore, TranscriptReadError + + +@pytest.fixture +def store(tmp_path): + return SessionStore(sessions_dir=tmp_path / "gw", config=GatewayConfig()) + + +# -------------------------------------------------------------------------- +# A. read failure != empty transcript (landed on main via #100910; kept as +# the contract the slash-command handlers below rely on) +# -------------------------------------------------------------------------- + + +class TestLoadTranscriptReadFailure: + def test_read_failure_raises_instead_of_returning_empty(self, store, monkeypatch): + db = store._db + assert db is not None + db.create_session("s1", "telegram", session_key="telegram:1") + db.append_message("s1", "user", "the conversation we must not forget") + + boom = sqlite3.DatabaseError("database disk image is malformed") + + def _raise(*_args, **_kwargs): + raise boom + + monkeypatch.setattr(db, "get_messages_as_conversation", _raise) + + with pytest.raises(TranscriptReadError) as excinfo: + store.load_transcript("s1") + + assert excinfo.value.session_id == "s1" + assert excinfo.value.__cause__ is boom + + def test_genuinely_empty_session_still_returns_empty_list(self, store): + db = store._db + assert db is not None + db.create_session("s2", "telegram", session_key="telegram:2") + + assert store.load_transcript("s2") == [] + + def test_no_db_still_returns_empty_list(self, store): + # "No DB for this session" really is an empty transcript, not a + # failure — that path must keep its [] contract. + store._db = None + assert store.load_transcript("nope") == [] + + +# -------------------------------------------------------------------------- +# B. slash-command handlers surface the failure instead of dying silently. +# Before: the handler raised, base.py's dispatch wrapper logged +# "Command '/x' dispatch failed" and the user got NO reply at all. +# -------------------------------------------------------------------------- + + +class TestSlashCommandsOnUnreadableTranscript: + def test_history_unreadable_text_is_explicit(self): + from gateway.slash_commands import HISTORY_UNREADABLE + + assert "unreadable" in HISTORY_UNREADABLE + assert "not a new conversation" in HISTORY_UNREADABLE + + def test_every_transcript_reading_handler_catches_the_error(self): + """No `await ...load_transcript(` in the mixin may be left uncaught.""" + import inspect + import re + + from gateway import slash_commands as sc + + src = inspect.getsource(sc) + # Each awaited load_transcript must sit inside a try: whose handlers + # include TranscriptReadError within the following ~6 lines. + for m in re.finditer(r"await self\.async_session_store\.load_transcript\(", src): + window = src[m.end() : m.end() + 400] + assert "except TranscriptReadError" in window, src[m.start() - 200 : m.end() + 100] diff --git a/tests/gateway/test_turn_request_overrides.py b/tests/gateway/test_turn_request_overrides.py index c985125176..bd5da602d8 100644 --- a/tests/gateway/test_turn_request_overrides.py +++ b/tests/gateway/test_turn_request_overrides.py @@ -53,7 +53,7 @@ def test_provider_request_overrides_merged_under_fast_mode(monkeypatch): """/fast active: provider extra_body AND the service-tier marker both survive.""" monkeypatch.setattr( "hermes_cli.models.resolve_fast_mode_overrides", - lambda model_id: {"service_tier": "priority"}, + lambda model_id, **_route: {"service_tier": "priority"}, ) runner = _runner(service_tier="priority") rk = _runtime_kwargs(request_overrides=PROVIDER_OVERRIDES) diff --git a/tests/gateway/test_voice_mode_platform_isolation.py b/tests/gateway/test_voice_mode_platform_isolation.py index 68485ee14c..799029911f 100644 --- a/tests/gateway/test_voice_mode_platform_isolation.py +++ b/tests/gateway/test_voice_mode_platform_isolation.py @@ -9,7 +9,9 @@ same key. The fix prefixes keys with platform value: 'telegram:123' vs import json import tempfile from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest from gateway.config import Platform @@ -115,6 +117,88 @@ class TestSyncVoiceModeStateToAdapter: assert mock_adapter._auto_tts_disabled_chats == {"123"} +class TestVoiceModeProfileIsolation: + """Two multiplexed bots in one Discord channel keep independent /voice + state and voice transcripts dispatch through the bot that heard them + (#75198 voice half).""" + + @staticmethod + def _discord_adapter(owner=None): + from unittest.mock import AsyncMock + + a = MagicMock() + a.platform = Platform.DISCORD + a._owner_profile = owner + a._voice_text_channels = {111: 123} + a._voice_sources = {} + a._voice_input_callback = None + a._on_voice_disconnect = None + a._voice_mode_getter = None + a._auto_tts_enabled_chats = set() + a._auto_tts_disabled_chats = set() + a._client = MagicMock() + a._client.get_channel = MagicMock(return_value=None) + a.handle_message = AsyncMock() + return a + + @pytest.mark.asyncio + async def test_voice_state_and_transcripts_stay_with_the_owning_bot(self, tmp_path): + from types import SimpleNamespace + + from gateway.platforms.base import MessageEvent, MessageType, SessionSource + + runner = _make_runner() + runner._VOICE_MODE_PATH = tmp_path / "voice.json" + runner._is_user_authorized = lambda source: True + default_ad = self._discord_adapter() + bot2_ad = self._discord_adapter(owner="bot2") + runner.adapters = {Platform.DISCORD: default_ad} + runner._profile_adapters = {"bot2": {Platform.DISCORD: bot2_ad}} + # Inbound event from bot2's transport in channel 123 (same id the + # default bot also sees). + src = SessionSource(platform=Platform.DISCORD, chat_id="123", user_id="u1", + chat_type="channel", profile="bot2") + src._transport_adapter_ref = lambda: bot2_ad + + await runner._handle_voice_command( + MessageEvent(text="/voice tts", message_type=MessageType.TEXT, source=src) + ) + assert runner._voice_mode == {"bot2:discord:123": "all"} + assert "123" in bot2_ad._auto_tts_enabled_chats + assert "123" not in default_ad._auto_tts_enabled_chats + + # A transcript captured by bot2's adapter runs through bot2, not default. + runner._bind_voice_input_callback(bot2_ad) + await bot2_ad._voice_input_callback(guild_id=111, user_id=42, transcript="hi") + bot2_ad.handle_message.assert_awaited_once() + default_ad.handle_message.assert_not_awaited() + assert bot2_ad.handle_message.call_args[0][0].source.profile == "bot2" + + # Timeout cleanup from bot2's channel disables bot2's auto-TTS only. + join = MessageEvent(text="/voice channel", message_type=MessageType.TEXT, source=src) + join.raw_message = SimpleNamespace(guild_id=111, guild=None) + bot2_ad.join_voice_channel = AsyncMock(return_value=True) + ch = MagicMock(); ch.name = "General" + bot2_ad.get_user_voice_channel = AsyncMock(return_value=ch) + await runner._handle_voice_channel_join(join) + bot2_ad._on_voice_disconnect("123") + assert runner._voice_mode["bot2:discord:123"] == "off" + assert "123" in bot2_ad._auto_tts_disabled_chats + assert "123" not in default_ad._auto_tts_disabled_chats + + def test_sync_restores_only_the_owning_profiles_chats(self): + runner = _make_runner() + runner._voice_mode = {"discord:1": "all", "bot2:discord:2": "all"} + default_ad = MagicMock(); default_ad.platform = Platform.DISCORD + default_ad._owner_profile = None; default_ad._auto_tts_enabled_chats = set() + bot2_ad = MagicMock(); bot2_ad.platform = Platform.DISCORD + bot2_ad._owner_profile = "bot2"; bot2_ad._auto_tts_enabled_chats = set() + runner._sync_voice_mode_state_to_adapter(default_ad) + runner._sync_voice_mode_state_to_adapter(bot2_ad) + assert default_ad._auto_tts_enabled_chats == {"1"} + assert bot2_ad._auto_tts_enabled_chats == {"2"} + + # --------------------------------------------------------------------------- # Helper # --------------------------------------------------------------------------- diff --git a/tests/gateway/test_webhook_adapter.py b/tests/gateway/test_webhook_adapter.py index 4f5cdb1390..7ea8e8acde 100644 --- a/tests/gateway/test_webhook_adapter.py +++ b/tests/gateway/test_webhook_adapter.py @@ -1011,6 +1011,63 @@ class TestMultiplexProfileWebhookAuthentication: ) assert default_profile.status == 404 + @pytest.mark.asyncio + async def test_routed_profile_skills_resolve_under_that_profile( + self, tmp_path, monkeypatch + ): + """A /p// route's ``skills:`` must load from that profile's + skills/ dir (#67277). Before the fix the lookup ran with no profile + scope, so it scanned the launch profile and logged "Skill not found". + """ + import agent.skill_commands as sc_mod + + worker = tmp_path / "profiles" / "worker" + skill_dir = worker / "skills" / "worker-only" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: worker-only\ndescription: w\n---\n\nBody of worker-only.\n" + ) + (worker / "config.yaml").write_text("{}\n") + (worker / ".env").write_text("") + monkeypatch.setattr( + "hermes_cli.profiles.get_profile_dir", lambda name: tmp_path / "profiles" / name + ) + route_secret = "worker-route-secret-abc123" + adapter = _make_adapter( + routes={ + "gh": { + "profile": "worker", + "secret": route_secret, + "prompt": "PR: {action}", + "skills": ["worker-only"], + } + }, + host="127.0.0.1", + ) + self._configure_profiles(adapter, tmp_path, monkeypatch) + seen = [] + + async def _capture(event): + seen.append(event) + + adapter.handle_message = _capture + body = b'{"action":"opened"}' + headers = { + "Content-Type": "application/json", + "X-Hub-Signature-256": _github_signature(body, route_secret), + } + with ( + patch.object(sc_mod, "_skill_commands", {}), + patch.object(sc_mod, "_skill_commands_home", None), + ): + async with TestClient(TestServer(self._app(adapter))) as cli: + resp = await cli.post("/p/worker/webhooks/gh", data=body, headers=headers) + assert resp.status == 202 + await asyncio.sleep(0.05) + assert len(seen) == 1 + assert seen[0].source.profile == "worker" + assert "Body of worker-only." in seen[0].text + def test_route_profile_validation_fails_closed(): assert WebhookAdapter._route_allows_profile({}, None) is True diff --git a/tests/gateway/test_wecom.py b/tests/gateway/test_wecom.py index a46a1caded..96886df995 100644 --- a/tests/gateway/test_wecom.py +++ b/tests/gateway/test_wecom.py @@ -76,6 +76,40 @@ class TestWeComAdapterAuthzScope: assert adapter._dm_policy == "pairing" assert adapter._allow_from == [] + def test_scoped_construction_reads_bot_id_from_scope_not_environ(self, multiplex_on, monkeypatch): + """bot_id must honor the same scope as its neighboring _secret read + (both are read on adjacent lines in __init__) -- a secondary profile's + own bot_id must never fall back to the default profile's os.environ + value.""" + from agent import secret_scope + from plugins.platforms.wecom.adapter import WeComAdapter + + monkeypatch.setenv("WECOM_BOT_ID", "default-profile-bot-id") + monkeypatch.setenv("WECOM_SECRET", "default-profile-secret") + token = secret_scope.set_secret_scope( + {"WECOM_BOT_ID": "scoped-bot-id", "WECOM_SECRET": "scoped-secret"} + ) + try: + adapter = WeComAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._bot_id == "scoped-bot-id" + assert adapter._secret == "scoped-secret" + + def test_scoped_miss_does_not_leak_default_profiles_bot_id(self, multiplex_on, monkeypatch): + from agent import secret_scope + from plugins.platforms.wecom.adapter import DEFAULT_WS_URL, WeComAdapter + + monkeypatch.setenv("WECOM_BOT_ID", "default-profile-bot-id") + monkeypatch.setenv("WECOM_WEBSOCKET_URL", "wss://default-profile.example/ws") + token = secret_scope.set_secret_scope({"SOMETHING_ELSE": "x"}) + try: + adapter = WeComAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._bot_id == "" + assert adapter._ws_url == DEFAULT_WS_URL + class TestWeComConnect: diff --git a/tests/gateway/test_weixin.py b/tests/gateway/test_weixin.py index ed20aeb8f3..936fef0f3e 100644 --- a/tests/gateway/test_weixin.py +++ b/tests/gateway/test_weixin.py @@ -167,6 +167,57 @@ class TestWeixinStatePersistence: assert json.loads(account_path.read_text(encoding="utf-8")) == original + @pytest.mark.asyncio + async def test_context_token_persist_runs_off_event_loop_thread(self, tmp_path): + """atomic_json_write() calls os.fsync(), which blocks until the write + reaches stable storage. ContextTokenStore.set() runs on the event + loop for every inbound message carrying a context_token + (_process_message), so the persist step must be offloaded to a + thread — mirrors test_directory_write_runs_off_event_loop_thread in + test_channel_directory.py for the same #83906 bug class.""" + import threading + + store = ContextTokenStore(str(tmp_path)) + loop_thread = threading.get_ident() + write_threads = [] + + def fake_write(path, data, *args, **kwargs): + write_threads.append(threading.get_ident()) + + with patch("gateway.platforms.weixin.atomic_json_write", side_effect=fake_write): + await store.set("acct-1", "user-1", "ctx-token-abc") + + assert store.get("acct-1", "user-1") == "ctx-token-abc" + assert write_threads + assert all(tid != loop_thread for tid in write_threads) + + @pytest.mark.asyncio + async def test_concurrent_context_token_persists_land_in_order(self, tmp_path): + """Two in-flight set() calls (two concurrent inbound messages) must not + let an older snapshot overwrite a newer one on disk. Without + serialization the first (slow) flush lands last and drops user-2.""" + import asyncio as _asyncio + import time + + store = ContextTokenStore(str(tmp_path)) + writes = [] + calls = [0] + + def slow_first_write(path, data, *args, **kwargs): + idx = calls[0] + calls[0] += 1 + if idx == 0: + time.sleep(0.05) + writes.append(dict(data)) + + with patch("gateway.platforms.weixin.atomic_json_write", side_effect=slow_first_write): + first = _asyncio.create_task(store.set("acct-1", "user-1", "t1")) + await _asyncio.sleep(0.005) + second = _asyncio.create_task(store.set("acct-1", "user-2", "t2")) + await _asyncio.gather(first, second) + + assert writes[-1] == {"user-1": "t1", "user-2": "t2"} + class TestWeixinQrLogin: @pytest.mark.asyncio @@ -685,8 +736,8 @@ class TestWeixinVoiceAlwaysDownloaded: adapter._poll_session = Mock() fake_audio_bytes = b"\\x00\\x01\\x02FAKE_SILK" - monkeypatch.setattr(weixin, "cache_audio_from_bytes", - lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + monkeypatch.setattr(weixin, "cache_audio_from_bytes_async", + AsyncMock(side_effect=lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}"))) async def _fake_download(session, *, cdn_base_url, encrypted_query_param, aes_key_b64, full_url, timeout_seconds): @@ -739,8 +790,8 @@ class TestWeixinVoiceAlwaysDownloaded: adapter._cdn_base_url = "https://example.invalid" adapter._poll_session = Mock() - monkeypatch.setattr(weixin, "cache_audio_from_bytes", - lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + monkeypatch.setattr(weixin, "cache_audio_from_bytes_async", + AsyncMock(side_effect=lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}"))) async def _fake_download(session, *, cdn_base_url, encrypted_query_param, aes_key_b64, full_url, timeout_seconds): @@ -803,8 +854,8 @@ class TestWeixinVoiceGatewayHandoff: adapter._token = None adapter._cdn_base_url = "https://example.invalid" - monkeypatch.setattr(weixin, "cache_audio_from_bytes", - lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + monkeypatch.setattr(weixin, "cache_audio_from_bytes_async", + AsyncMock(side_effect=lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}"))) async def _fake_download(*a, **k): return b"\x00\x01FAKE_SILK" monkeypatch.setattr(weixin, "_download_and_decrypt_media", _fake_download) diff --git a/tests/hermes_cli/test_active_sessions.py b/tests/hermes_cli/test_active_sessions.py index 5741aed178..0fe223101f 100644 --- a/tests/hermes_cli/test_active_sessions.py +++ b/tests/hermes_cli/test_active_sessions.py @@ -12,6 +12,17 @@ import pytest from hermes_cli import active_sessions + +def _backdate_leases(*homes, age_seconds=600.0): + """Age every lease in the given registries past the self-orphan grace.""" + for home in homes: + state_path = active_sessions._state_path(home) + entries = active_sessions._read_entries(state_path) + for entry in entries: + entry["started_at"] = time.time() - age_seconds + active_sessions._write_entries(state_path, entries) + + def test_resolve_max_concurrent_sessions_values(caplog): assert active_sessions.resolve_max_concurrent_sessions({}) is None assert active_sessions.resolve_max_concurrent_sessions({"max_concurrent_sessions": None}) is None @@ -164,6 +175,7 @@ def test_release_orphaned_leases_reclaims_only_unowned_own_pid_entries(tmp_path, + [{"lease_id": "elsewhere", "session_id": "other", "surface": "cli", "pid": os.getpid() }], ) + _backdate_leases(tmp_path / ".hermes") assert active_sessions.release_orphaned_leases({kept.lease_id, "elsewhere"}) == 1 assert sorted( entry["session_id"] @@ -172,6 +184,47 @@ def test_release_orphaned_leases_reclaims_only_unowned_own_pid_entries(tmp_path, assert orphan is not None +def test_release_orphaned_leases_sweeps_profile_runtime_registries( + tmp_path, monkeypatch +): + root = tmp_path / "hermes" + profile = root / "profiles" / "worker" + profile.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(root)) + + root_lease, root_error = active_sessions.try_acquire_active_session( + session_id="root-orphan", surface="desktop", config={}, registry_home=root + ) + profile_lease, profile_error = active_sessions.try_acquire_active_session( + session_id="profile-orphan", + surface="desktop", + config={}, + registry_home=profile, + ) + assert root_lease is not None and root_error is None + assert profile_lease is not None and profile_error is None + + # A lease written seconds ago is never an orphan: a sibling finalize that + # snapshotted its live ids before this acquire must not reap it (#101415). + assert active_sessions.release_orphaned_leases(set()) == 0 + _backdate_leases(root, profile) + assert active_sessions.release_orphaned_leases(set()) == 2 + assert active_sessions.active_session_registry_snapshot(root) == [] + assert active_sessions.active_session_registry_snapshot(profile) == [] + + +def test_drop_self_orphans_spares_foreign_and_vouched_leases(): + own = os.getpid() + entries = [ + {"lease_id": "orphan", "pid": own}, + {"lease_id": "live", "pid": own}, + {"lease_id": "foreign", "pid": own + 1}, + ] + + assert active_sessions._drop_self_orphans(entries, None) == entries + assert active_sessions._drop_self_orphans(entries, {"live"}) == entries[1:] + + def test_release_under_profile_home_override_targets_acquisition_registry( tmp_path, monkeypatch ): @@ -566,3 +619,30 @@ def test_release_wins_against_transfer_waiting_on_same_lease_lock( assert lease.released is True assert active_sessions.active_session_registry_snapshot() == [] + + +def test_liveness_guard_keeps_a_just_acquired_own_lease_it_cannot_vouch_for( + tmp_path, monkeypatch +): + """Race in #101415's fix: the finalizing session snapshots its live lease + ids, then a sibling session acquires a lease before the registry lock is + taken. That lease is absent from the snapshot but is not an orphan.""" + home = tmp_path / ".hermes" + monkeypatch.setenv("HERMES_HOME", str(home)) + fresh, error = active_sessions.try_acquire_active_session( + session_id="fresh", surface="desktop", config={}, registry_home=home + ) + assert fresh is not None and error is None + + with active_sessions.active_session_liveness_guard( + "fresh", registry_home=home, own_live_lease_ids=set() + ) as active: + assert active is True + assert [e["lease_id"] for e in active_sessions.active_session_registry_snapshot(home)] == [fresh.lease_id] + + _backdate_leases(home) + with active_sessions.active_session_liveness_guard( + "fresh", registry_home=home, own_live_lease_ids=set() + ) as active: + assert active is False + assert active_sessions.active_session_registry_snapshot(home) == [] diff --git a/tests/hermes_cli/test_alibaba_coding_plan_cn_provider_listing.py b/tests/hermes_cli/test_alibaba_coding_plan_cn_provider_listing.py new file mode 100644 index 0000000000..6dc59e3338 --- /dev/null +++ b/tests/hermes_cli/test_alibaba_coding_plan_cn_provider_listing.py @@ -0,0 +1,30 @@ +"""alibaba-coding-plan and alibaba-coding-plan-cn must not both appear in the +/model picker off a single shared key (#101122). + +The CN profile now has its own ALIBABA_CODING_PLAN_CN_API_KEY (checked first), +keeping the shared ALIBABA_CODING_PLAN_API_KEY / DASHSCOPE_API_KEY as ordered +fallbacks so existing CN users are not broken. The picker hides a ``-cn`` row +whose only lit vars are shared with a lit non-CN sibling row. +""" + +import os +from unittest.mock import patch + +from hermes_cli.model_switch import list_authenticated_providers + +_CLEAR = {k: "" for k in ("ALIBABA_CODING_PLAN_API_KEY", "ALIBABA_CODING_PLAN_CN_API_KEY", "DASHSCOPE_API_KEY")} + + +def _alibaba_slugs(current_provider=""): + return [p["slug"] for p in list_authenticated_providers(current_provider=current_provider) if "coding-plan" in p["slug"]] + + +@patch.dict(os.environ, {**_CLEAR, "ALIBABA_CODING_PLAN_CN_API_KEY": "sk-cn-fake"}, clear=False) +def test_alibaba_cn_appears_when_only_cn_key_set(): + assert _alibaba_slugs() == ["alibaba-coding-plan-cn"] + + +@patch.dict(os.environ, {**_CLEAR, "ALIBABA_CODING_PLAN_API_KEY": "sk-intl-fake"}, clear=False) +def test_alibaba_cn_does_not_appear_when_only_intl_key_set(): + """#101122: the shared intl key alone must light only the intl row.""" + assert _alibaba_slugs() == ["alibaba-coding-plan"] diff --git a/tests/hermes_cli/test_backup.py b/tests/hermes_cli/test_backup.py index fe2d1dd9c4..b7209dffc4 100644 --- a/tests/hermes_cli/test_backup.py +++ b/tests/hermes_cli/test_backup.py @@ -1337,6 +1337,32 @@ class TestQuickSnapshot: assert "state.db" not in data.get("files", {}) assert "state.db" in data.get("failed_dbs", []) + def test_restore_refused_db_is_not_counted(self, hermes_home, monkeypatch): + """A refused live-safe restore (holder detected, backup leg failed) must + not be counted as a restored file — `hermes import` reports it, and + /snapshot restore must not claim success for that file either.""" + import hermes_cli.backup as backup_mod + from hermes_cli.backup import create_quick_snapshot, restore_quick_snapshot + + snap_id = create_quick_snapshot(hermes_home=hermes_home) + monkeypatch.setattr(backup_mod, "_safe_restore_db", lambda src, dst: False) + restored_log: list[str] = [] + real_info = backup_mod.logger.info + monkeypatch.setattr( + backup_mod.logger, "info", + lambda msg, *a, **kw: restored_log.append(msg % a if a else msg) or real_info(msg, *a, **kw), + ) + + restore_quick_snapshot(snap_id, hermes_home=hermes_home) + + manifest = json.loads( + (backup_mod._quick_snapshot_root(hermes_home) / snap_id / "manifest.json").read_text() + ) + non_db = [rel for rel in manifest.get("files", {}) if not rel.endswith(".db")] + summary = [line for line in restored_log if line.startswith("Restored ")] + assert summary, restored_log + assert summary[-1].startswith(f"Restored {len(non_db)} files"), summary[-1] + def test_restore_state_db_live_connection(self, hermes_home): """Restoring state.db must update data visible through a live connection. @@ -2227,3 +2253,175 @@ class TestImportHonorsHermesHomeOverride: backup_mod.run_import(args) assert calls and calls[0].get("context") == "import" + + +# --------------------------------------------------------------------------- +# Live session database import (issue #100960) +# --------------------------------------------------------------------------- + +def _write_session_db(path: Path, sessions: int, messages_per_session: int) -> None: + """Create a minimal Hermes-shaped session database at *path*.""" + conn = sqlite3.connect(str(path)) + try: + conn.execute( + "CREATE TABLE IF NOT EXISTS sessions " + "(session_id TEXT PRIMARY KEY, message_count INTEGER)" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS messages " + "(id INTEGER PRIMARY KEY, session_id TEXT, content TEXT)" + ) + for s in range(sessions): + sid = f"sess-{s}" + conn.execute( + "INSERT INTO sessions VALUES (?, ?)", (sid, messages_per_session) + ) + for m in range(messages_per_session): + conn.execute( + "INSERT INTO messages (session_id, content) VALUES (?, ?)", + (sid, f"{sid}-msg-{m}"), + ) + conn.commit() + finally: + conn.close() + + +class TestImportLiveSessionDatabase: + """`hermes import` must not swap the inode of a database Hermes holds open. + + Publishing state.db with a rename leaves any live gateway/dashboard/WebUI + connection reading and writing the unlinked inode, so its sessions vanish + from the database everyone else opens and nothing is logged (#100960). + """ + + def _zip_with_db(self, zip_path: Path, db_path: Path) -> None: + with zipfile.ZipFile(zip_path, "w") as zf: + zf.write(db_path, "state.db") + + def _prepare(self, tmp_path, monkeypatch, live=(3, 4), backup=(2, 2)): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + + live_db = home / "state.db" + _write_session_db(live_db, *live) + + staged = tmp_path / "backup-state.db" + _write_session_db(staged, *backup) + zip_path = tmp_path / "backup.zip" + self._zip_with_db(zip_path, staged) + return home, live_db, zip_path + + def test_live_holder_sees_imported_rows(self, tmp_path, monkeypatch): + """A connection open across the import converges on the imported data.""" + from hermes_cli.backup import run_import + + home, live_db, zip_path = self._prepare(tmp_path, monkeypatch) + + holder = sqlite3.connect(str(live_db)) + # Read first so the connection has cached pages of the pre-import file. + assert holder.execute("SELECT COUNT(*) FROM messages").fetchone()[0] == 12 + inode_before = os.stat(live_db).st_ino + + try: + run_import(Namespace(zipfile=str(zip_path), force=True)) + assert holder.execute("SELECT COUNT(*) FROM messages").fetchone()[0] == 4 + finally: + holder.close() + + assert os.stat(live_db).st_ino == inode_before + assert _count_rows(live_db) == (2, 4) + + def test_older_backup_reports_replaced_sessions(self, tmp_path, monkeypatch, capsys): + """Importing a backup that predates recorded work says what it dropped.""" + from hermes_cli.backup import run_import + + home, live_db, zip_path = self._prepare(tmp_path, monkeypatch) + run_import(Namespace(zipfile=str(zip_path), force=True)) + + out = capsys.readouterr().out + assert "Session data replaced by older backup contents" in out + assert "3 session(s) / 12 message(s) -> 2 / 4" in out + + def test_newer_backup_reports_nothing(self, tmp_path, monkeypatch, capsys): + """No warning when the import does not shrink the database.""" + from hermes_cli.backup import run_import + + home, live_db, zip_path = self._prepare( + tmp_path, monkeypatch, live=(1, 1), backup=(3, 4) + ) + run_import(Namespace(zipfile=str(zip_path), force=True)) + + out = capsys.readouterr().out + assert "Session data replaced by older backup contents" not in out + + def test_refused_restore_is_reported_and_leaves_db_intact( + self, tmp_path, monkeypatch, capsys + ): + """A refused live-safe restore is a warning, not a counted success.""" + import hermes_cli.backup as backup_mod + + home, live_db, zip_path = self._prepare(tmp_path, monkeypatch) + monkeypatch.setattr(backup_mod, "_safe_restore_db", lambda src, dst: False) + + backup_mod.run_import(Namespace(zipfile=str(zip_path), force=True)) + + out = capsys.readouterr().out + assert "files skipped" in out + assert "state.db" in out + # The pre-import database is still the one on disk. + assert _count_rows(live_db) == (3, 12) + + def test_sidecar_members_are_not_installed_beside_a_restored_db( + self, tmp_path, monkeypatch + ): + """A `state.db-wal` member from an old/hand-built archive must not be + os.replace'd next to the page-restored database: it describes a + different image and SQLite would replay it on the next open.""" + from hermes_cli.backup import run_import + + home, live_db, zip_path = self._prepare(tmp_path, monkeypatch) + with zipfile.ZipFile(zip_path, "a") as zf: + zf.writestr("state.db-wal", b"foreign-wal-from-archive") + zf.writestr("state.db-shm", b"foreign-shm") + zf.writestr("state.db-journal", b"foreign-journal") + + run_import(Namespace(zipfile=str(zip_path), force=True)) + + for suffix, payload in ( + ("-wal", b"foreign-wal-from-archive"), + ("-shm", b"foreign-shm"), + ("-journal", b"foreign-journal"), + ): + sidecar = live_db.with_name("state.db" + suffix) + assert not sidecar.exists() or sidecar.read_bytes() != payload, suffix + assert _count_rows(live_db) == (2, 4) + + def test_missing_target_takes_the_plain_publish(self, tmp_path, monkeypatch): + """A fresh install has no inode to preserve; the member still lands.""" + from hermes_cli.backup import run_import + + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + + staged = tmp_path / "backup-state.db" + _write_session_db(staged, 2, 3) + zip_path = tmp_path / "backup.zip" + self._zip_with_db(zip_path, staged) + + run_import(Namespace(zipfile=str(zip_path), force=True)) + assert _count_rows(home / "state.db") == (2, 6) + + +def _count_rows(db_path: Path) -> tuple[int, int]: + conn = sqlite3.connect(str(db_path)) + try: + return ( + conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0], + conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0], + ) + finally: + conn.close() diff --git a/tests/hermes_cli/test_config.py b/tests/hermes_cli/test_config.py index 6c247ae9ee..4f7e05eee7 100644 --- a/tests/hermes_cli/test_config.py +++ b/tests/hermes_cli/test_config.py @@ -10,6 +10,7 @@ import yaml from hermes_cli.config import ( DEFAULT_CONFIG, + InvalidUserConfigError, check_config_version, get_hermes_home, ensure_hermes_home, @@ -728,7 +729,9 @@ class TestConfigMigrationSecretPrompts: saved = {} monkeypatch.setattr(cfg_mod, "sanitize_env_file", lambda: 0) - monkeypatch.setattr(cfg_mod, "check_config_version", lambda: (999, 999)) + monkeypatch.setattr( + cfg_mod, "check_config_version", lambda **_kwargs: (999, 999) + ) monkeypatch.setattr(cfg_mod, "get_missing_config_fields", lambda: []) monkeypatch.setattr(cfg_mod, "get_missing_skill_config_vars", lambda: []) monkeypatch.setattr( @@ -773,6 +776,48 @@ class TestConfigVersionDetection: assert load_config()["_config_version"] == DEFAULT_CONFIG["_config_version"] assert check_config_version() == (0, DEFAULT_CONFIG["_config_version"]) + _LATEST = DEFAULT_CONFIG["_config_version"] + # (bytes, strict match, tolerant return): tolerant malformed YAML keeps + # the historical latest/latest fallback; a parseable non-mapping root is + # reported as legacy (0). + _INVALID_CONFIG_CASES = [ + pytest.param( + b"model: [unterminated\n", "not valid YAML", (_LATEST, _LATEST), id="malformed-yaml" + ), + pytest.param(b"- just_a_list\n", "must be a mapping", (0, _LATEST), id="list-root"), + pytest.param(b"[]\n", "must be a mapping", (0, _LATEST), id="empty-list-root"), + ] + + @pytest.mark.parametrize("config_bytes, match, tolerant", _INVALID_CONFIG_CASES) + def test_strict_check_rejects_invalid_config( + self, tmp_path, config_bytes, match, tolerant + ): + config_path = tmp_path / "config.yaml" + config_path.write_bytes(config_bytes) + + with patch.dict(os.environ, {"HERMES_HOME": str(tmp_path)}): + with pytest.raises(InvalidUserConfigError, match=match): + check_config_version(raise_on_parse_error=True) + # Tolerant callers keep the historical non-raising behavior. + assert check_config_version() == tolerant + + @pytest.mark.parametrize("config_bytes, match, _tolerant", _INVALID_CONFIG_CASES) + def test_migration_rejects_invalid_config_before_sanitizing_env( + self, tmp_path, config_bytes, match, _tolerant + ): + config_path = tmp_path / "config.yaml" + config_path.write_bytes(config_bytes) + env_path = tmp_path / ".env" + env_bytes = b"OPENAI_API_KEY=test-without-final-newline" + env_path.write_bytes(env_bytes) + + with patch.dict(os.environ, {"HERMES_HOME": str(tmp_path)}): + with pytest.raises(InvalidUserConfigError, match=match): + migrate_config(interactive=False, quiet=True) + + assert config_path.read_bytes() == config_bytes + assert env_path.read_bytes() == env_bytes + class TestConfigSupportFloor: """Auto-migration support floor (v12). @@ -886,7 +931,9 @@ class TestConfigSupportFloor: }, "memory": {"write_approval": True}, "model": {"default": "openai/gpt-5.4", "provider": "openrouter"}, - "model_catalog": {"ttl_hours": 1}, + # v25 lowered the old 24h default to 1h; v40 drops that 1h default so + # the shipped ttl_minutes (20) applies. + "model_catalog": {}, "plugins": {"enabled": []}, "stt": {"provider": "local"}, } @@ -905,7 +952,7 @@ class TestConfigSupportFloor: # default (opt-in) so the write invariant strips it from disk. "agent": {}, "model": {"default": "anthropic/claude-fable-5", "provider": "nous"}, - "model_catalog": {"ttl_hours": 1}, + "model_catalog": {}, "plugins": {"disabled": ["foo"], "enabled": []}, } diff --git a/tests/hermes_cli/test_config_env_expansion.py b/tests/hermes_cli/test_config_env_expansion.py index 207ae5625f..6571015245 100644 --- a/tests/hermes_cli/test_config_env_expansion.py +++ b/tests/hermes_cli/test_config_env_expansion.py @@ -122,3 +122,31 @@ class TestLoadCliConfigExpansion: config = load_cli_config() assert config["auxiliary"]["vision"]["api_key"] == "${UNSET_CLI_VAR_ABC}" + + +class TestExpansionUnderProfileScope: + """``${VAR}`` refs must resolve against the active profile's secret scope, + not the shared process environment (#84079): under multiplex every + secondary profile otherwise "had" the default profile's token and fanned + out. Outside multiplex the scope is an overlay and environ still applies.""" + + def test_scoped_ref_never_reads_another_profiles_environ(self, monkeypatch): + from agent import secret_scope as ss + + monkeypatch.setenv("MATRIX_ACCESS_TOKEN", "default-token") + was_active = ss.is_multiplex_active() + ss.set_multiplex_active(True) + token = ss.set_secret_scope({"OTHER_KEY": "x"}) # profile-b: no matrix token + try: + assert _expand_env_vars("${MATRIX_ACCESS_TOKEN}") == "${MATRIX_ACCESS_TOKEN}" + assert _expand_env_vars("${env:MATRIX_ACCESS_TOKEN}") == "${env:MATRIX_ACCESS_TOKEN}" + finally: + ss.reset_secret_scope(token) + token = ss.set_secret_scope({"MATRIX_ACCESS_TOKEN": "c-token"}) + try: + assert _expand_env_vars("${MATRIX_ACCESS_TOKEN}") == "c-token" + finally: + ss.reset_secret_scope(token) + ss.set_multiplex_active(was_active) + # Unscoped (default profile / single-profile CLI): legacy environ read. + assert _expand_env_vars("${MATRIX_ACCESS_TOKEN}") == "default-token" diff --git a/tests/hermes_cli/test_config_set_platforms_redirect.py b/tests/hermes_cli/test_config_set_platforms_redirect.py new file mode 100644 index 0000000000..803a70e58f --- /dev/null +++ b/tests/hermes_cli/test_config_set_platforms_redirect.py @@ -0,0 +1,157 @@ +"""Regression tests for #71047 (Problem A): per-platform display settings. + +`hermes config set platforms.. ` must write to +`display.platforms..` — the path the gateway actually +reads (gateway/display_config.py::resolve_display_setting). Writing to the +top-level `platforms.` block is silently ignored by the runtime, so the +edit appeared to succeed while having no effect. +""" + +from pathlib import Path + +import pytest +import yaml + + +def _write_config(hermes_home: Path, data: dict) -> Path: + hermes_home.mkdir(parents=True, exist_ok=True) + config_path = hermes_home / "config.yaml" + config_path.write_text(yaml.dump(data)) + return config_path + + +def _set(monkeypatch, hermes_home, key, value, force=False): + """Isolated call to set_config_value against a temp HERMES_HOME.""" + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + # set_config_value resolves the home live via get_config_path()/get_hermes_home() + from hermes_cli.config import set_config_value + set_config_value(key, value, force=force) + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + home = tmp_path / ".hermes" + # A config that already has a top-level platforms block (connection keys) + # AND a display.platforms block, mirroring the real-world report. + cfg = { + "model": {"default": "test-model", "provider": "openrouter"}, + "platforms": { + "telegram": {"token": "secret-bot-token"}, + }, + "display": { + "skin": "default", + "platforms": { + "telegram": {"show_reasoning": True}, + }, + }, + } + _write_config(home, cfg) + return home + + +class TestPerPlatformDisplayRedirect: + def test_streaming_redirects_to_display_platforms(self, hermes_home, monkeypatch): + """platforms.telegram.streaming must land under display.platforms.""" + _set(monkeypatch, hermes_home, "platforms.telegram.streaming", "false") + + result = yaml.safe_load((hermes_home / "config.yaml").read_text()) + # Redirected target exists and is correct + assert result["display"]["platforms"]["telegram"]["streaming"] is False + # Top-level platforms.telegram must NOT gain a streaming key + assert "streaming" not in result["platforms"]["telegram"] + # Connection key untouched + assert result["platforms"]["telegram"]["token"] == "secret-bot-token" + + def test_show_reasoning_redirects(self, hermes_home, monkeypatch): + _set(monkeypatch, hermes_home, "platforms.telegram.show_reasoning", "false") + result = yaml.safe_load((hermes_home / "config.yaml").read_text()) + assert result["display"]["platforms"]["telegram"]["show_reasoning"] is False + + def test_tool_progress_redirects(self, hermes_home, monkeypatch): + # ``off`` is coerced to False by the bool-aware coercion in + # set_config_value; gateway/display_config._normalise turns False back + # into the canonical "off" string at read time, so the persisted value + # is the bool. + _set(monkeypatch, hermes_home, "platforms.discord.tool_progress", "off") + result = yaml.safe_load((hermes_home / "config.yaml").read_text()) + assert result["display"]["platforms"]["discord"]["tool_progress"] is False + + def test_connection_key_not_redirected(self, hermes_home, monkeypatch): + """A real connection key (token) stays in top-level platforms..""" + _set(monkeypatch, hermes_home, "platforms.telegram.token", "new-token") + result = yaml.safe_load((hermes_home / "config.yaml").read_text()) + assert result["platforms"]["telegram"]["token"] == "new-token" + # Nothing leaked into display.platforms.telegram.token + assert "token" not in result["display"]["platforms"]["telegram"] + + def test_no_top_level_platforms_created_when_missing(self, tmp_path, monkeypatch): + """When there is no pre-existing top-level platforms block, a display + setting write must not invent one.""" + home = tmp_path / ".hermes" + _write_config(home, {"model": {"default": "m"}}) + _set(monkeypatch, home, "platforms.telegram.streaming", "true") + result = yaml.safe_load((home / "config.yaml").read_text()) + assert result["display"]["platforms"]["telegram"]["streaming"] is True + assert "platforms" not in result # no stray top-level platforms block + + +class TestRedirectSiblingSurfaces: + """The canonicalization must hold for every CLI surface that takes a dotted + key — set, get, unset — and the written value must be what the gateway's + resolver actually reads (the #71047 symptom was CLI and runtime disagreeing). + """ + + def test_get_mirrors_gateway_resolution_after_set(self, hermes_home, monkeypatch, capsys): + from gateway.display_config import resolve_display_setting + from hermes_cli.config import get_config_value + + _set(monkeypatch, hermes_home, "platforms.telegram.streaming", "false") + capsys.readouterr() + get_config_value("platforms.telegram.streaming") + assert capsys.readouterr().out.strip() == "false" + + raw = yaml.safe_load((hermes_home / "config.yaml").read_text()) + assert resolve_display_setting(raw, "telegram", "streaming") is False + + def test_unset_removes_the_redirected_leaf(self, hermes_home, monkeypatch): + from hermes_cli.config import unset_config_value + + _set(monkeypatch, hermes_home, "platforms.telegram.streaming", "false") + unset_config_value("platforms.telegram.streaming") + result = yaml.safe_load((hermes_home / "config.yaml").read_text()) + assert "streaming" not in result["display"]["platforms"]["telegram"] + # Sibling display override and connection block untouched. + assert result["display"]["platforms"]["telegram"]["show_reasoning"] is True + assert result["platforms"]["telegram"] == {"token": "secret-bot-token"} + + def test_unset_missing_redirected_leaf_exits_nonzero(self, hermes_home, monkeypatch): + from hermes_cli.config import unset_config_value + + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + with pytest.raises(SystemExit) as exc: + unset_config_value("platforms.telegram.streaming") + assert exc.value.code == 1 + + def test_set_prints_redirect_note(self, hermes_home, monkeypatch, capsys): + _set(monkeypatch, hermes_home, "platforms.telegram.streaming", "false") + out = capsys.readouterr().out + assert "saved as display.platforms.telegram.streaming" in out + assert "Set display.platforms.telegram.streaming = False" in out + + def test_redirect_helper_only_touches_known_display_keys(self): + from gateway.display_config import OVERRIDEABLE_KEYS + from hermes_cli.config import _redirect_platform_display_key + + for setting in OVERRIDEABLE_KEYS: + canonical, note = _redirect_platform_display_key(f"platforms.discord.{setting}") + assert canonical == f"display.platforms.discord.{setting}" + assert note + for key in ( + "platforms.telegram.token", + "platforms.telegram.reply_to_mode", + "platforms.telegram.extra.foo", # 4 segments — not a display leaf + "platforms.telegram", + "display.platforms.telegram.streaming", # already canonical + "streaming.enabled", + ): + assert _redirect_platform_display_key(key) == (key, None) diff --git a/tests/hermes_cli/test_container_boot.py b/tests/hermes_cli/test_container_boot.py index 8134af5a9c..6f7346187a 100644 --- a/tests/hermes_cli/test_container_boot.py +++ b/tests/hermes_cli/test_container_boot.py @@ -130,6 +130,49 @@ def test_running_profile_is_registered_and_autostarted(tmp_path: Path) -> None: assert not (svc / "down").exists() +@pytest.mark.parametrize( + "config_value,env_value,expected", + [ + pytest.param("true", None, "registered", id="config-only-multiplex"), + pytest.param("true", "false", "started", id="env-false-overrides-config"), + ], +) +def test_boot_honors_config_multiplex_profiles( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + config_value: str, + env_value: str | None, + expected: str, +) -> None: + """Container boot must resolve multiplex_profiles like the gateway does: + config.yaml opt-in honored (#85413), env override keeps precedence.""" + scandir = tmp_path / "run-service" + scandir.mkdir() + _make_profile(tmp_path, "coder", state="running") + (tmp_path / "config.yaml").write_text( + f"multiplex_profiles: {config_value}\n", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + if env_value is None: + monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False) + else: + monkeypatch.setenv("GATEWAY_MULTIPLEX_PROFILES", env_value) + + actions = reconcile_profile_gateways( + hermes_home=tmp_path, + scandir=scandir, + dry_run=False, + ) + + assert _named_actions(actions) == [ReconcileAction( + profile="coder", + prior_state="running", + action=expected, + )] + assert (scandir / "gateway-coder" / "down").exists() is (expected == "registered") + + def test_registered_profile_has_finish_script(tmp_path: Path) -> None: """The finish script must be written so s6 stops restarting on fatal config errors (exit 78 → exit 125). See #51228.""" @@ -301,5 +344,3 @@ def _write_lifecycle_sentinel(profile_dir: Path, payload: dict) -> None: (state_dir / "gateway.lifecycle.json").write_text(json.dumps(payload)) - - diff --git a/tests/hermes_cli/test_copilot_in_model_list.py b/tests/hermes_cli/test_copilot_in_model_list.py index 83832b0c33..2889627964 100644 --- a/tests/hermes_cli/test_copilot_in_model_list.py +++ b/tests/hermes_cli/test_copilot_in_model_list.py @@ -3,6 +3,8 @@ import os from unittest.mock import patch +import pytest + from hermes_cli.model_switch import list_authenticated_providers @@ -20,3 +22,63 @@ def test_copilot_picker_uses_live_catalog_when_available(): assert copilot is not None assert copilot["models"] == live_models assert copilot["total_models"] == len(live_models) + + +# --- copilot-acp: external_process availability (#63662) ------------------- +# +# copilot-acp holds no API key, OAuth token, or credential-pool entry by +# design — the spawned `copilot --acp --stdio` subprocess brings its own auth. +# The picker loop used to filter it out unconditionally (has_creds never had +# an external_process branch), so the provider was invisible in every picker +# even with a perfectly resolvable executable. + + +@pytest.fixture() +def _no_other_copilot_creds(monkeypatch): + """Make sure copilot-acp visibility comes ONLY from executable resolution: + no env tokens, no configured ACP endpoint, no auth-store entry, no seeded + credential pool.""" + # COPILOT_ACP_BASE_URL is not a credential, but an `acp+tcp://` value marks + # the provider configured with no executable at all (hermes_cli/auth.py), so + # a host that sets it would decide the outcome instead of the test. + for var in ("GH_TOKEN", "GITHUB_TOKEN", "HERMES_COPILOT_ACP_COMMAND", + "COPILOT_CLI_PATH", "COPILOT_ACP_BASE_URL"): + monkeypatch.delenv(var, raising=False) + import hermes_cli.auth as auth + import hermes_cli.model_switch as model_switch + + monkeypatch.setattr(auth, "_load_auth_store", lambda: {}) + monkeypatch.setattr(model_switch, "_credential_pool_is_usable", lambda *a, **k: False) + + +def test_copilot_acp_listed_when_executable_resolves(tmp_path, monkeypatch, _no_other_copilot_creds): + fake = tmp_path / ("copilot.exe" if os.name == "nt" else "copilot") + fake.write_text("", encoding="utf-8") + fake.chmod(0o755) + monkeypatch.setenv("HERMES_COPILOT_ACP_COMMAND", str(fake)) + + with patch("agent.models_dev.fetch_models_dev", return_value={}), \ + patch("hermes_cli.models._resolve_copilot_catalog_api_key", return_value=None), \ + patch("hermes_cli.models._fetch_github_models", return_value=[]): + providers = list_authenticated_providers(current_provider="openrouter", max_models=50) + + acp = next((p for p in providers if p["slug"] == "copilot-acp"), None) + + assert acp is not None, "copilot-acp must be listed when its executable resolves" + assert acp["models"], "copilot-acp row must offer at least the curated fallback models" + + +def test_copilot_acp_hidden_when_executable_missing(monkeypatch, _no_other_copilot_creds): + # `copilot` may genuinely be installed on a dev machine — force the + # resolution miss so the test pins behaviour, not the host's PATH. + import hermes_cli.auth as auth + + monkeypatch.setattr(auth.shutil, "which", lambda *_a, **_k: None) + + with patch("agent.models_dev.fetch_models_dev", return_value={}), \ + patch("hermes_cli.models._resolve_copilot_catalog_api_key", return_value=None), \ + patch("hermes_cli.models._fetch_github_models", return_value=[]): + providers = list_authenticated_providers(current_provider="openrouter", max_models=50) + + assert all(p["slug"] != "copilot-acp" for p in providers), \ + "copilot-acp must stay hidden when no executable resolves" diff --git a/tests/hermes_cli/test_credential_pool_off_loop.py b/tests/hermes_cli/test_credential_pool_off_loop.py new file mode 100644 index 0000000000..6ffe6a6137 --- /dev/null +++ b/tests/hermes_cli/test_credential_pool_off_loop.py @@ -0,0 +1,227 @@ +"""Regression tests for the #91912 salvage — credential-pool handlers off-loop +and the bounded Copilot token exchange. + +The 2026-08-22 incident: ``GET /api/credentials/pool`` ran ``load_pool()`` on +the uvicorn event loop; for the copilot provider that reaches +``urllib.request.urlopen`` whose ``timeout`` does not bound ``getaddrinfo``, +so a networkless host froze the whole dashboard backend for 17 minutes. +""" + +from __future__ import annotations + +import asyncio +import threading +import time +from unittest.mock import patch + +import pytest + +from hermes_cli import copilot_auth + + +# --------------------------------------------------------------------------- +# _urlopen_bounded +# --------------------------------------------------------------------------- + + +class TestUrlopenBounded: + def test_returns_response_when_worker_completes(self): + sentinel = object() + with patch("urllib.request.urlopen", return_value=sentinel): + assert copilot_auth._urlopen_bounded("req", 1.0) is sentinel + + def test_reraises_worker_exception(self): + with patch("urllib.request.urlopen", side_effect=OSError("boom")): + with pytest.raises(OSError, match="boom"): + copilot_auth._urlopen_bounded("req", 1.0) + + def test_hard_cap_fires_on_hung_resolver_and_closes_late_response(self, monkeypatch): + """A urlopen that hangs past timeout + grace must raise TimeoutError + promptly, and when the abandoned worker later *succeeds* it must close + the response instead of leaking the socket.""" + monkeypatch.setattr(copilot_auth, "_DNS_GRACE_SECONDS", 0.05) + release = threading.Event() + closed = threading.Event() + + class _LateResponse: + def close(self): + closed.set() + + def hung_urlopen(req, timeout): + release.wait(timeout=5) + return _LateResponse() + + with patch("urllib.request.urlopen", side_effect=hung_urlopen): + started = time.monotonic() + with pytest.raises(TimeoutError, match="hard cap"): + copilot_auth._urlopen_bounded("req", 0.05) + elapsed = time.monotonic() - started + assert elapsed < 2.0 + release.set() + assert closed.wait(timeout=2), "late response was not closed" + + +# --------------------------------------------------------------------------- +# single-flight exchange +# --------------------------------------------------------------------------- + + +class TestExchangeSingleFlight: + @pytest.fixture(autouse=True) + def _clean_caches(self, monkeypatch, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + copilot_auth._jwt_cache.clear() + copilot_auth._exchange_failure_cache.clear() + copilot_auth._exchange_locks.clear() + yield + copilot_auth._jwt_cache.clear() + copilot_auth._exchange_failure_cache.clear() + copilot_auth._exchange_locks.clear() + + def test_concurrent_callers_share_one_exchange(self, monkeypatch): + """N concurrent callers for the same token must perform ONE network + exchange; the rest wait on the lock and hit the populated cache.""" + monkeypatch.setattr(copilot_auth, "_load_jwt_from_disk", lambda fp: None) + monkeypatch.setattr(copilot_auth, "_save_jwt_to_disk", lambda *a, **k: None) + calls = [] + gate = threading.Event() + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return b'{"token": "tid=1;exp=9", "expires_at": 4102444800}' + + def fake_bounded(req, timeout): + calls.append(threading.get_ident()) + gate.wait(timeout=5) # hold the first exchange open while others queue + return _Resp() + + monkeypatch.setattr(copilot_auth, "_urlopen_bounded", fake_bounded) + + results = [] + threads = [ + threading.Thread(target=lambda: results.append(copilot_auth.exchange_copilot_token("ghu_" + "x" * 30))) + for _ in range(8) + ] + for t in threads: + t.start() + time.sleep(0.2) # let every caller reach the lock + assert len(calls) == 1 + gate.set() + for t in threads: + t.join(timeout=5) + + assert len(calls) == 1 + assert len(results) == 8 + assert {r[0] for r in results} == {"tid=1;exp=9"} + + def test_waiters_observe_negative_cache_after_failed_exchange(self, monkeypatch): + """When the single in-flight exchange fails, queued callers must not + each start their own exchange — they see the negative cache.""" + monkeypatch.setattr(copilot_auth, "_load_jwt_from_disk", lambda fp: None) + monkeypatch.setattr(copilot_auth, "_EXCHANGE_MAX_ATTEMPTS", 1) + calls = [] + gate = threading.Event() + + def fake_bounded(req, timeout): + calls.append(1) + gate.wait(timeout=5) + raise TimeoutError("hard cap") + + monkeypatch.setattr(copilot_auth, "_urlopen_bounded", fake_bounded) + errors = [] + + def run(): + try: + copilot_auth.exchange_copilot_token("ghu_" + "y" * 30) + except ValueError as exc: + errors.append(str(exc)) + + threads = [threading.Thread(target=run) for _ in range(5)] + for t in threads: + t.start() + time.sleep(0.2) + gate.set() + for t in threads: + t.join(timeout=5) + + assert len(calls) == 1 + assert len(errors) == 5 + assert any("recently failed" in e for e in errors) + + def test_negative_cache_short_circuits_before_taking_the_lock(self, monkeypatch): + """While one exchange holds the lock, a caller whose fingerprint is + already in the failure cache must raise immediately rather than park + an executor thread behind the holder.""" + fp = copilot_auth._token_fingerprint("ghu_" + "z" * 30) + copilot_auth._exchange_failure_cache[fp] = time.time() + 60 + lock = copilot_auth._exchange_lock_for(fp) + lock.acquire() # simulate an in-flight holder + try: + started = time.monotonic() + with pytest.raises(ValueError, match="recently failed"): + copilot_auth.exchange_copilot_token("ghu_" + "z" * 30) + assert time.monotonic() - started < 0.5 + finally: + lock.release() + + +# --------------------------------------------------------------------------- +# web_server credential-pool handlers off the loop +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_credential_pool_runs_off_event_loop(monkeypatch): + import hermes_cli.auth as auth_mod + from hermes_cli import web_server + + loop_thread = threading.get_ident() + seen = {} + + def fake_read_pool(*args, **kwargs): + seen["thread"] = threading.get_ident() + return {} + + monkeypatch.setattr(auth_mod, "read_credential_pool", fake_read_pool) + result = await web_server.list_credential_pool() + + assert result == {"providers": []} + assert seen["thread"] != loop_thread + + +@pytest.mark.asyncio +async def test_list_credential_pool_keeps_loop_responsive(monkeypatch): + """A 200 ms blocking pool read must not freeze a concurrent ticker.""" + import hermes_cli.auth as auth_mod + from hermes_cli import web_server + + def slow_read(*args, **kwargs): + time.sleep(0.5) + return {} + + monkeypatch.setattr(auth_mod, "read_credential_pool", slow_read) + + gaps = [] + stop = asyncio.Event() + + async def ticker(): + last = time.perf_counter() + while not stop.is_set(): + await asyncio.sleep(0.005) + now = time.perf_counter() + gaps.append(now - last) + last = now + + t = asyncio.create_task(ticker()) + await web_server.list_credential_pool() + stop.set() + await t + # 0.25 s threshold vs a 0.5 s blocking read: a regression (read on the + # loop) trips it by 2x, while runner-noise descheduling would need >200 ms. + assert max(gaps) < 0.25, f"event loop stalled for {max(gaps) * 1000:.0f} ms" diff --git a/tests/hermes_cli/test_cron.py b/tests/hermes_cli/test_cron.py index f48ce6e0e6..1712e0ddab 100644 --- a/tests/hermes_cli/test_cron.py +++ b/tests/hermes_cli/test_cron.py @@ -129,6 +129,44 @@ class TestCronCommandLifecycle: assert jobs[0]["name"] == "Skill combo" +class TestUnverifiedDeliveryVisibility: + """An evidence-free live-adapter ack (Slack/Matrix/Mattermost bare + ``SendResult(success=True)``) is accepted as delivered, but the UNVERIFIED + state must be visible in ``hermes cron list`` and ``hermes cron doctor``, + not only in a WARNING log line.""" + + def _seed(self): + job = create_job(prompt="Nightly brief", schedule="every 1h", deliver="slack:C0123456") + jobs = load_jobs() + jobs[0]["last_status"] = "ok" + jobs[0]["last_delivery_unverified"] = ["slack:C0123456"] + save_jobs(jobs) + return job + + def test_list_shows_unverified_delivery(self, tmp_cron_dir, capsys): + job = self._seed() + cron_command(Namespace(cron_command="list", all=True, json=False)) + out = capsys.readouterr().out + assert job["id"] in out + assert "Delivery UNVERIFIED" in out + assert "slack:C0123456" in out + assert "without message_id/raw_response" in out + + def test_list_is_quiet_when_delivery_was_verified(self, tmp_cron_dir, capsys): + create_job(prompt="Nightly brief", schedule="every 1h", deliver="slack:C0123456") + cron_command(Namespace(cron_command="list", all=True, json=False)) + assert "UNVERIFIED" not in capsys.readouterr().out + + def test_doctor_reports_unverified_delivery(self, tmp_cron_dir, capsys): + job = self._seed() + rc = cron_command(Namespace(cron_command="doctor")) + out = capsys.readouterr().out + assert rc == 1 + assert job["id"] in out + assert "last delivery unverified" in out + assert "slack:C0123456" in out + + class TestCronDoctor: def test_doctor_reports_cron_health_issues(self, tmp_cron_dir, capsys): job = create_job(prompt="Daily digest", schedule="every 1h", script="missing.py") @@ -160,6 +198,28 @@ class TestCronDoctor: assert rc == 0 assert "✓ Cron doctor found no issues" in out + def test_doctor_reports_delivery_failure_once(self, tmp_cron_dir, capsys): + """A delivery_failed run is a delivery issue, not a failed agent run. + + The agent succeeded (last_error is None), so the generic last-run-failed + line would only ever say "unknown error" — double-reporting the same + incident (#83993). + """ + create_job(prompt="Daily digest", schedule="every 1h") + jobs = load_jobs() + jobs[0]["last_status"] = "delivery_failed" + jobs[0]["last_error"] = None + jobs[0]["last_delivery_error"] = "telegram timeout" + save_jobs(jobs) + + rc = cron_command(Namespace(cron_command="doctor")) + + out = capsys.readouterr().out + assert rc == 1 + assert "last delivery failed: telegram timeout" in out + assert "last run failed" not in out + assert "unknown error" not in out + def test_doctor_flags_overdue_next_run(self, tmp_cron_dir, capsys): from datetime import datetime, timedelta, timezone @@ -193,6 +253,48 @@ class TestCronDoctor: assert "✓ Cron doctor found no issues" in out +class TestCronListStatusRendering: + """`cron list` must never paint an undelivered run as a success (#83993).""" + + def test_delivery_failed_is_not_green_ok(self, tmp_cron_dir, capsys, monkeypatch): + monkeypatch.setattr("hermes_cli.gateway.find_gateway_pids", lambda: [1]) + # capsys is not a tty, so force colors on to check the paint itself. + monkeypatch.setattr("hermes_cli.colors.should_use_color", lambda: True) + create_job(prompt="Daily digest", schedule="every 1h") + jobs = load_jobs() + jobs[0]["last_run_at"] = "2026-09-01T09:00:00+00:00" + jobs[0]["last_status"] = "delivery_failed" + jobs[0]["last_error"] = None + jobs[0]["last_delivery_error"] = "telegram timeout" + save_jobs(jobs) + + cron_command(Namespace(cron_command="list", all=True)) + + out = capsys.readouterr().out + last_run_line = next(l for l in out.splitlines() if "Last run:" in l) + assert "delivery_failed" in last_run_line + assert "telegram timeout" in last_run_line, ( + "the delivery detail lives in last_delivery_error, not last_error" + ) + assert cron_cli.Colors.GREEN not in last_run_line + + def test_ok_run_still_green(self, tmp_cron_dir, capsys, monkeypatch): + monkeypatch.setattr("hermes_cli.gateway.find_gateway_pids", lambda: [1]) + monkeypatch.setattr("hermes_cli.colors.should_use_color", lambda: True) + create_job(prompt="Daily digest", schedule="every 1h") + jobs = load_jobs() + jobs[0]["last_run_at"] = "2026-09-01T09:00:00+00:00" + jobs[0]["last_status"] = "ok" + save_jobs(jobs) + + cron_command(Namespace(cron_command="list", all=True)) + + out = capsys.readouterr().out + last_run_line = next(l for l in out.splitlines() if "Last run:" in l) + assert f"{cron_cli.Colors.GREEN}ok" in last_run_line + assert "delivery_failed" not in last_run_line + + class TestGatewayNotRunningWarning: """`cron create` / `cron list` must warn when the gateway (and thus the cron ticker) isn't running, since jobs only fire inside the gateway. @@ -446,3 +548,41 @@ class TestCronRunBackgroundDispatch: assert rc == 0 assert "Running in background (delegation del-xyz)." in out assert "failed" not in out.lower() + + +class TestSlashCronListLastStatus: + """The in-chat ``/cron list`` (cli_commands_mixin) renders every + ``last_status`` literal explicitly — ``delivery_failed`` names the delivery + reason (last_error is None for those runs) instead of printing the bare + literal next to a run that looks otherwise fine.""" + + def _run_list(self, tmp_cron_dir, capsys): + from hermes_cli.cli_commands_mixin import CLICommandsMixin + + class _Host(CLICommandsMixin): + pass + + _Host()._handle_cron_command("/cron list --all") + return capsys.readouterr().out + + def test_delivery_failed_names_the_delivery_error(self, tmp_cron_dir, capsys): + create_job(prompt="Nightly brief", schedule="every 1h", deliver="telegram:1") + jobs = load_jobs() + jobs[0]["last_run_at"] = "2026-09-01T07:00:00+00:00" + jobs[0]["last_status"] = "delivery_failed" + jobs[0]["last_error"] = None + jobs[0]["last_delivery_error"] = "telegram: 502 Bad Gateway" + save_jobs(jobs) + + out = self._run_list(tmp_cron_dir, capsys) + assert "Last run: 2026-09-01T07:00:00+00:00 (delivery_failed: telegram: 502 Bad Gateway)" in out + + def test_ok_stays_plain(self, tmp_cron_dir, capsys): + create_job(prompt="Nightly brief", schedule="every 1h") + jobs = load_jobs() + jobs[0]["last_run_at"] = "2026-09-01T07:00:00+00:00" + jobs[0]["last_status"] = "ok" + save_jobs(jobs) + + out = self._run_list(tmp_cron_dir, capsys) + assert "(ok)" in out diff --git a/tests/hermes_cli/test_cron_fire_dashboard.py b/tests/hermes_cli/test_cron_fire_dashboard.py index aa898bf78e..d6d406398d 100644 --- a/tests/hermes_cli/test_cron_fire_dashboard.py +++ b/tests/hermes_cli/test_cron_fire_dashboard.py @@ -250,6 +250,41 @@ def test_fire_endpoint_multiplex_profile_prefix(tmp_path, monkeypatch): assert url == "http://127.0.0.1:8642/p/worker_alpha/api/cron/fire" +def test_fire_endpoint_multiplex_reads_port_from_default_listener(tmp_path, monkeypatch): + """Multiplex mode: only the DEFAULT profile's api_server is bound, so a + secondary's fire URL must use the default home's port — not the + secondary's own config.yaml/.env port, which nothing listens on + (PR #84755). Real config files, real load_config().""" + default_home = tmp_path / "root" + worker_home = default_home / "profiles" / "worker_alpha" + default_home.mkdir() + worker_home.mkdir(parents=True) + (default_home / "config.yaml").write_text( + "gateway:\n multiplex_profiles: true\n" + "platforms:\n api_server:\n extra:\n port: 8650\n", + encoding="utf-8", + ) + (worker_home / "config.yaml").write_text( + "platforms:\n api_server:\n enabled: false\n extra:\n port: 8702\n", + encoding="utf-8", + ) + (worker_home / ".env").write_text("API_SERVER_PORT=8701\n", encoding="utf-8") + monkeypatch.setenv("HERMES_HOME", str(default_home)) + monkeypatch.delenv("API_SERVER_PORT", raising=False) + monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False) + monkeypatch.setattr(web_server, "_cron_default_profile", lambda: "default") + + url = web_server._gateway_fire_endpoint("worker_alpha", worker_home) + + assert url == "http://127.0.0.1:8650/p/worker_alpha/api/cron/fire" + # The GATEWAY_MULTIPLEX_PROFILES env override is still honored (parity + # with gateway/config.py): forcing it off restores per-profile routing. + monkeypatch.setenv("GATEWAY_MULTIPLEX_PROFILES", "0") + assert web_server._gateway_fire_endpoint("worker_alpha", worker_home) == ( + "http://127.0.0.1:8702/api/cron/fire" + ) + + # ── OOF-266: intentional-stop drop + Retry-After on transient 503 ───────── diff --git a/tests/hermes_cli/test_cross_profile_kill_refusal.py b/tests/hermes_cli/test_cross_profile_kill_refusal.py new file mode 100644 index 0000000000..7b05e98d65 --- /dev/null +++ b/tests/hermes_cli/test_cross_profile_kill_refusal.py @@ -0,0 +1,217 @@ +"""Cross-profile kill refusal regression tests (#89315). + +A poisoned/contaminated ``gateway.pid`` inside one profile's HERMES_HOME can +truthfully name ANOTHER profile's live gateway (its ``hermes_home`` stamp +records the real owner). ``gateway stop`` / the restart force-kill escalation +/ ``profile delete`` must refuse to signal such a PID instead of starting the +mutual cross-profile SIGTERM restart loop from the issue report. + +These tests exercise the REAL code paths against real PID files, a real +flock-held gateway lock, and a real dummy child process — no mocks of the +code under test. +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from gateway.status import recorded_gateway_home_conflicts + + +def _spawn_gateway_lookalike(bin_dir: Path, lock_path: Path) -> subprocess.Popen: + """Real child process whose argv matches the gateway runtime matcher.""" + bin_dir.mkdir(parents=True, exist_ok=True) + lock_path.parent.mkdir(parents=True, exist_ok=True) + script = bin_dir / "hermes" + if sys.platform == "win32": + body = "import time\ntime.sleep(120)\n" + else: + body = ( + "import fcntl, time\n" + f"fh = open({str(lock_path)!r}, 'a+')\n" + "fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)\n" + "time.sleep(120)\n" + ) + script.write_text(f"#!{sys.executable}\n{body}", encoding="utf-8") + if sys.platform != "win32": + script.chmod(0o755) + cmd = [str(script), "gateway", "run"] + else: + cmd = [sys.executable, str(script), "gateway", "run"] + proc = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline and not lock_path.exists(): + if proc.poll() is not None: + raise RuntimeError("gateway lookalike died at startup") + time.sleep(0.05) + return proc + + +def _pid_record(proc: subprocess.Popen, script: Path, owner_home: Path) -> dict: + from gateway.status import get_process_start_time + + return { + "pid": proc.pid, + "kind": "hermes-gateway", + "argv": [str(script), "gateway", "run"], + "start_time": get_process_start_time(proc.pid), + "hermes_home": str(owner_home), + } + + +class TestRecordedGatewayHomeConflicts: + def test_conflicting_home_detected(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profiles" / "tim")) + record = {"pid": 1, "hermes_home": str(tmp_path)} + assert recorded_gateway_home_conflicts(record) is True + + def test_same_home_accepted(self, tmp_path, monkeypatch): + home = tmp_path / "profiles" / "tim" + monkeypatch.setenv("HERMES_HOME", str(home)) + record = {"pid": 1, "hermes_home": str(home)} + assert recorded_gateway_home_conflicts(record) is False + + def test_legacy_record_without_home_proves_nothing(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + assert recorded_gateway_home_conflicts({"pid": 1}) is False + assert recorded_gateway_home_conflicts(None) is False + assert recorded_gateway_home_conflicts({"pid": 1, "hermes_home": " "}) is False + + def test_expected_home_override(self, tmp_path): + target = tmp_path / "profiles" / "tim" + record = {"pid": 1, "hermes_home": str(tmp_path)} + assert ( + recorded_gateway_home_conflicts(record, expected_home=target) is True + ) + assert ( + recorded_gateway_home_conflicts(record, expected_home=tmp_path) is False + ) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock harness") +class TestCrossProfileStopRefusal: + def test_stop_profile_gateway_refuses_other_profiles_pid( + self, tmp_path, monkeypatch + ): + """Profile B's ``gateway stop`` must not SIGTERM profile A's gateway. + + On main this path is already safe upstream of any guard: + ``get_running_pid()`` filters a pid record owned by another profile + (and unlinks the poisoned pid file) before ``stop_profile_gateway`` + ever sees a pid — so the contract here is "returns False, other + profile's process untouched, poisoned pid file gone", not a printed + refusal. + """ + root_home = tmp_path / "root-home" + tim_home = tmp_path / "root-home" / "profiles" / "tim" + tim_home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(tim_home)) + + proc = _spawn_gateway_lookalike( + tmp_path / "bin", tim_home / "gateway.lock" + ) + try: + record = _pid_record(proc, tmp_path / "bin" / "hermes", root_home) + (tim_home / "gateway.pid").write_text(json.dumps(record)) + + from hermes_cli import gateway as gateway_cli + + assert gateway_cli.stop_profile_gateway() is False + assert not (tim_home / "gateway.pid").exists(), ( + "poisoned cross-profile pid file should have been unlinked" + ) + time.sleep(0.5) + assert proc.poll() is None, ( + "cross-profile SIGTERM fired: profile A's gateway was killed" + ) + finally: + proc.kill() + proc.wait(timeout=10) + + def test_stop_profile_gateway_still_stops_own_gateway( + self, tmp_path, monkeypatch + ): + """Same-home records keep stopping normally (no false refusal).""" + tim_home = tmp_path / "profiles" / "tim" + tim_home.mkdir(parents=True) + monkeypatch.setenv("HERMES_HOME", str(tim_home)) + + proc = _spawn_gateway_lookalike( + tmp_path / "bin", tim_home / "gateway.lock" + ) + try: + record = _pid_record(proc, tmp_path / "bin" / "hermes", tim_home) + (tim_home / "gateway.pid").write_text(json.dumps(record)) + + from hermes_cli import gateway as gateway_cli + + assert gateway_cli.stop_profile_gateway() is True + deadline = time.monotonic() + 15.0 + while time.monotonic() < deadline and proc.poll() is None: + time.sleep(0.1) + assert proc.poll() is not None, "own gateway was not stopped" + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock harness") +class TestProfileDeleteStopRefusal: + def test_stop_gateway_process_refuses_other_profiles_pid( + self, tmp_path, capsys + ): + """``profile delete`` must not kill a gateway owned by another home.""" + root_home = tmp_path / "root-home" + tim_home = root_home / "profiles" / "tim" + tim_home.mkdir(parents=True) + + proc = _spawn_gateway_lookalike( + tmp_path / "bin", tim_home / "gateway.lock" + ) + try: + record = _pid_record(proc, tmp_path / "bin" / "hermes", root_home) + (tim_home / "gateway.pid").write_text(json.dumps(record)) + + from hermes_cli.profiles import _stop_gateway_process + + _stop_gateway_process(tim_home) + out = capsys.readouterr().out + assert "Refusing to stop" in out + time.sleep(0.5) + assert proc.poll() is None, ( + "profile delete killed another profile's gateway" + ) + finally: + proc.kill() + proc.wait(timeout=10) + + def test_stop_gateway_process_still_stops_own_gateway(self, tmp_path): + tim_home = tmp_path / "profiles" / "tim" + tim_home.mkdir(parents=True) + + proc = _spawn_gateway_lookalike( + tmp_path / "bin", tim_home / "gateway.lock" + ) + try: + record = _pid_record(proc, tmp_path / "bin" / "hermes", tim_home) + (tim_home / "gateway.pid").write_text(json.dumps(record)) + + from hermes_cli.profiles import _stop_gateway_process + + _stop_gateway_process(tim_home) + deadline = time.monotonic() + 15.0 + while time.monotonic() < deadline and proc.poll() is None: + time.sleep(0.1) + assert proc.poll() is not None, "own gateway was not stopped" + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) diff --git a/tests/hermes_cli/test_desktop_exe_integrity.py b/tests/hermes_cli/test_desktop_exe_integrity.py index 3644e49a68..1defe0f716 100644 --- a/tests/hermes_cli/test_desktop_exe_integrity.py +++ b/tests/hermes_cli/test_desktop_exe_integrity.py @@ -277,8 +277,12 @@ def _ns(**kw): @pytest.mark.platforms("windows") def test_build_only_fails_when_pack_produces_corrupt_exe(tmp_path, monkeypatch, capsys): """The updater chain's contract: a rebuild whose Hermes.exe cannot launch - must exit nonzero (so hermes-setup's retry-once kicks in) and must restore - the previous working build instead of leaving the corrupt one. + must exit nonzero (so hermes-setup's retry-once kicks in) and must leave + the previous working build in place instead of installing the corrupt one. + + Stage-and-swap (#86443): the pack lands in a staging dir; the integrity + gate runs on the STAGED exe and a failure discards staging without ever + touching the live ``win-unpacked`` tree. ``platforms("windows")``: the whole chain is Windows-gated — ``win-unpacked`` candidate discovery in ``_desktop_packaged_executable`` and the integrity @@ -290,12 +294,20 @@ def test_build_only_fails_when_pack_produces_corrupt_exe(tmp_path, monkeypatch, (desktop_dir / "package.json").write_text("{}", encoding="utf-8") monkeypatch.setattr(cli_main, "PROJECT_ROOT", root) - exe = desktop_dir / "release" / "win-unpacked" / "Hermes.exe" - make_pe(exe, PE_AMD64, truncate_to=0x300) # what the failed pack produced - make_pe(desktop_dir / "release" / "win-unpacked.bak" / "Hermes.exe", PE_AMD64) + live_exe = desktop_dir / "release" / "win-unpacked" / "Hermes.exe" + make_pe(live_exe, PE_AMD64) # the previous, working app + live_bytes = live_exe.read_bytes() install_ok = subprocess.CompletedProcess(["npm", "ci"], 0) - pack_ok = subprocess.CompletedProcess(["npm", "run", "pack"], 0) + + def pack_into_staging(cmd, *args, **kwargs): + # electron-builder honours -c.directories.output=; emulate a + # pack that "succeeds" but writes a truncated exe there. + out_flag = next((a for a in cmd if str(a).startswith("-c.directories.output=")), None) + assert out_flag is not None, "pack must be redirected into a staging dir" + staging = Path(str(out_flag).split("=", 1)[1]) + make_pe(staging / "win-unpacked" / "Hermes.exe", PE_AMD64, truncate_to=0x300) + return subprocess.CompletedProcess(list(cmd), 0) with patch("hermes_cli.main.shutil.which", return_value="/usr/bin/npm"), \ patch("hermes_cli.main._resolve_node_runtime_npm", return_value="npm.cmd"), \ @@ -306,16 +318,17 @@ def test_build_only_fails_when_pack_produces_corrupt_exe(tmp_path, monkeypatch, patch("hermes_cli.main._desktop_stamp_path", return_value=tmp_path / "stamp.json"), \ patch("hermes_cli.main._write_desktop_build_stamp") as mock_stamp, \ patch("hermes_cli.main._windows_native_machine", return_value="AMD64"), \ - patch("hermes_cli.main.subprocess.run", return_value=pack_ok), \ + patch("hermes_cli.main.subprocess.run", side_effect=pack_into_staging), \ pytest.raises(SystemExit) as exc: cli_main.cmd_gui(_ns()) assert exc.value.code == 1 - # The previous working exe was restored... - assert cli_main._parse_pe_machine(exe) == PE_AMD64 + # The previous working exe was never touched... + assert live_exe.read_bytes() == live_bytes + assert cli_main._parse_pe_machine(live_exe) == PE_AMD64 + # ...the staged corrupt tree was discarded... + assert not list((desktop_dir / "release").glob(".staging-*")) # ...and the poisoned build was never stamped as good. mock_stamp.assert_not_called() out = capsys.readouterr().out assert "integrity check" in out - - diff --git a/tests/hermes_cli/test_doctor.py b/tests/hermes_cli/test_doctor.py index b85e4ca53d..bd21a462fb 100644 --- a/tests/hermes_cli/test_doctor.py +++ b/tests/hermes_cli/test_doctor.py @@ -281,18 +281,34 @@ def test_doctor_reports_vercel_backend_diagnostics(monkeypatch, tmp_path): class TestDoctorMemoryProviderSection: """The ◆ Memory Provider section should respect memory.provider config.""" - def _make_hermes_home(self, tmp_path, provider=""): + def _make_hermes_home(self, tmp_path, provider="", memory_config=None): """Create a minimal HERMES_HOME with config.yaml.""" home = tmp_path / ".hermes" home.mkdir(parents=True, exist_ok=True) import yaml - config = {"memory": {"provider": provider}} if provider else {"memory": {}} + config = dict(memory_config or {}) + if provider: + config["provider"] = provider + config = {"memory": config} (home / "config.yaml").write_text(yaml.dump(config)) return home - def _run_doctor_and_capture(self, monkeypatch, tmp_path, provider=""): + def _run_doctor_and_capture( + self, + monkeypatch, + tmp_path, + provider="", + *, + memory_config=None, + stale_builtin_files=False, + ): """Run doctor and capture stdout.""" - home = self._make_hermes_home(tmp_path, provider) + home = self._make_hermes_home(tmp_path, provider, memory_config) + if stale_builtin_files: + memories = home / "memories" + memories.mkdir() + (memories / "MEMORY.md").write_text("stale memory", encoding="utf-8") + (memories / "USER.md").write_text("stale user", encoding="utf-8") monkeypatch.setattr(doctor_mod, "HERMES_HOME", home) monkeypatch.setattr(doctor_mod, "PROJECT_ROOT", tmp_path / "project") monkeypatch.setattr(doctor_mod, "_DHH", str(home)) @@ -350,6 +366,26 @@ class TestDoctorMemoryProviderSection: assert "Memory Provider" in out assert "Built-in memory active" not in out + @pytest.mark.parametrize("memory_enabled", [False, True]) + def test_stale_builtin_files_reported_only_when_store_enabled( + self, monkeypatch, tmp_path, memory_enabled + ): + # #100668: disabled built-in stores must not surface stale files as active. + out = self._run_doctor_and_capture( + monkeypatch, + tmp_path, + provider="mnemosyne", + memory_config={ + "memory_enabled": memory_enabled, + "user_profile_enabled": False, + }, + stale_builtin_files=True, + ) + + assert ("MEMORY.md exists" in out) is memory_enabled + assert "USER.md exists" not in out + assert ("Built-in memory files disabled by config" in out) is not memory_enabled + def test_run_doctor_accepts_named_provider_from_providers_section(monkeypatch, tmp_path): home = tmp_path / ".hermes" diff --git a/tests/hermes_cli/test_external_process_auth_status.py b/tests/hermes_cli/test_external_process_auth_status.py new file mode 100644 index 0000000000..221459b10d --- /dev/null +++ b/tests/hermes_cli/test_external_process_auth_status.py @@ -0,0 +1,292 @@ +"""Tests for external-process provider auth status and Accounts-tab wiring. + +Covers the copilot-acp fix class: + * ``get_auth_status()`` dispatches on ``auth_type == "external_process"`` + (not a hardcoded slug), so future ACP-style providers inherit the + behaviour automatically. + * ``auth_verified``/``auth_source`` carry positive credential evidence + (env token or on-disk GitHub Copilot credential store) while remaining + honest — no evidence means unknown, never "signed out". + * The Accounts-tab sign-in ``cli_command`` reflects the executable the + user actually configured (``HERMES_COPILOT_ACP_COMMAND`` / + ``COPILOT_CLI_PATH``), and its default is a valid Copilot CLI + invocation (``copilot login`` — ``copilot /login`` is not a command). +""" + +import os + +import pytest + +from hermes_cli.auth import ( + get_auth_status, + get_external_process_provider_status, +) + + +@pytest.fixture() +def _clean_copilot_env(monkeypatch): + """Neutralize host state so tests pin behaviour, not this machine.""" + for var in ( + "COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN", + "HERMES_COPILOT_ACP_COMMAND", "COPILOT_CLI_PATH", + "HERMES_COPILOT_ACP_ARGS", "COPILOT_ACP_BASE_URL", + ): + monkeypatch.delenv(var, raising=False) + + +# --- get_auth_status dispatches on auth_type, not slug ---------------------- + + +def test_get_auth_status_dispatches_external_process_by_auth_type( + tmp_path, monkeypatch, _clean_copilot_env +): + fake = tmp_path / ("copilot.exe" if os.name == "nt" else "copilot") + fake.write_text("", encoding="utf-8") + fake.chmod(0o755) + monkeypatch.setenv("HERMES_COPILOT_ACP_COMMAND", str(fake)) + # Point HOME somewhere empty so on-disk credential stores don't leak in. + monkeypatch.setenv("HOME", str(tmp_path)) + + status = get_auth_status("copilot-acp") + + # The external_process status shape, not the {"logged_in": False} + # fallthrough — proves the dispatcher reached the right branch. + assert status.get("provider") == "copilot-acp" + assert status.get("configured") is True + assert status.get("resolved_command") == str(fake) + assert "auth_verified" in status + + +def test_external_process_status_rejects_wrong_auth_type(): + # A provider that exists but is not external_process must be refused — + # the generic dispatcher relies on this guard. + assert get_external_process_provider_status("openrouter") == {"configured": False} + assert get_external_process_provider_status("no-such-provider") == {"configured": False} + + +# --- auth_verified: positive evidence only ---------------------------------- + + +def test_auth_verified_false_without_evidence(tmp_path, monkeypatch, _clean_copilot_env): + monkeypatch.setenv("HOME", str(tmp_path)) # no ~/.config/github-copilot + status = get_external_process_provider_status("copilot-acp") + assert status["auth_verified"] is False + assert status["auth_source"] is None + + +def test_auth_verified_from_supported_env_token(tmp_path, monkeypatch, _clean_copilot_env): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("GH_TOKEN", "gho_" + "x" * 36) # supported OAuth prefix + + status = get_external_process_provider_status("copilot-acp") + + assert status["auth_verified"] is True + assert status["auth_source"] == "env: GH_TOKEN" + + +def test_classic_pat_is_not_login_evidence(tmp_path, monkeypatch, _clean_copilot_env): + # ghp_* classic PATs are rejected by the Copilot API — presence of one + # must not be presented as a working login. + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("GH_TOKEN", "ghp_" + "x" * 36) + + status = get_external_process_provider_status("copilot-acp") + + assert status["auth_verified"] is False + + +def test_auth_verified_from_on_disk_credential_store(tmp_path, monkeypatch, _clean_copilot_env): + monkeypatch.setenv("HOME", str(tmp_path)) + store = tmp_path / ".config" / "github-copilot" + store.mkdir(parents=True) + (store / "hosts.json").write_text( + '{"github.com": {"oauth_token": "gho_test"}}', encoding="utf-8" + ) + + status = get_external_process_provider_status("copilot-acp") + + assert status["auth_verified"] is True + assert status["auth_source"] == "~/.config/github-copilot/hosts.json" + + +def test_empty_credential_store_is_not_evidence(tmp_path, monkeypatch, _clean_copilot_env): + monkeypatch.setenv("HOME", str(tmp_path)) + store = tmp_path / ".config" / "github-copilot" + store.mkdir(parents=True) + (store / "hosts.json").write_text("{}", encoding="utf-8") # logged out + + status = get_external_process_provider_status("copilot-acp") + + assert status["auth_verified"] is False + + +def test_auth_verified_from_copilot_cli_plaintext_store(tmp_path, monkeypatch, _clean_copilot_env): + # `copilot login` without an OS keychain writes the token into + # ~/.copilot/config.json (JSONC, with //-comment header lines). + monkeypatch.setenv("HOME", str(tmp_path)) + cfg_dir = tmp_path / ".copilot" + cfg_dir.mkdir() + (cfg_dir / "config.json").write_text( + "// User settings belong in settings.json.\n" + "// This file is managed automatically.\n" + "{\n" + ' "copilotTokens": {"https://github.com:someuser": "gho_test"},\n' + ' "lastLoggedInUser": {"host": "https://github.com", "login": "someuser"}\n' + "}\n", + encoding="utf-8", + ) + + status = get_external_process_provider_status("copilot-acp") + + assert status["auth_verified"] is True + assert status["auth_source"] == "~/.copilot/config.json" + + +def test_copilot_cli_store_without_tokens_is_not_evidence(tmp_path, monkeypatch, _clean_copilot_env): + # A config.json exists after first launch even before any login — + # its presence alone must not read as signed-in. + monkeypatch.setenv("HOME", str(tmp_path)) + cfg_dir = tmp_path / ".copilot" + cfg_dir.mkdir() + (cfg_dir / "config.json").write_text( + '// managed\n{"firstLaunchAt": "2026-01-01T00:00:00Z", "copilotTokens": {}}\n', + encoding="utf-8", + ) + + status = get_external_process_provider_status("copilot-acp") + + assert status["auth_verified"] is False + + +# --- desktop picker explicit-only filter ------------------------------------ + + +def test_explicit_filter_keeps_signed_in_external_process_row(tmp_path, monkeypatch, _clean_copilot_env): + # A verified CLI login leaves no trace in active_provider/config/env — + # the explicit-only desktop filter must treat it like the Anthropic OAuth + # carve-out and keep the row. + from hermes_cli.inventory import _filter_explicit_provider_rows + + monkeypatch.setenv("HOME", str(tmp_path)) + cfg_dir = tmp_path / ".copilot" + cfg_dir.mkdir() + (cfg_dir / "config.json").write_text( + '{"copilotTokens": {"https://github.com:u": "gho_test"}}', encoding="utf-8" + ) + + class _Ctx: + current_provider = "nous" + + rows = [{"slug": "copilot-acp", "models": ["gpt-5.4"]}] + kept = _filter_explicit_provider_rows(rows, _Ctx()) + + assert any(r["slug"] == "copilot-acp" for r in kept), \ + "signed-in copilot-acp must survive the explicit-only picker filter" + + +def test_explicit_filter_drops_unverified_external_process_row(tmp_path, monkeypatch, _clean_copilot_env): + # Merely having the executable on PATH is ambient discovery, not an + # explicit configuration — the desktop filter keeps its narrower contract. + from hermes_cli.inventory import _filter_explicit_provider_rows + + monkeypatch.setenv("HOME", str(tmp_path)) # no credential stores + + class _Ctx: + current_provider = "nous" + + rows = [{"slug": "copilot-acp", "models": ["gpt-5.4"]}] + kept = _filter_explicit_provider_rows(rows, _Ctx()) + + assert all(r["slug"] != "copilot-acp" for r in kept) + + +# --- Accounts-tab cli_command ------------------------------------------------ + + +def test_catalog_sign_in_command_is_a_valid_copilot_invocation(): + from hermes_cli.web_server import _OAUTH_PROVIDER_CATALOG + + entry = next(e for e in _OAUTH_PROVIDER_CATALOG if e["id"] == "copilot-acp") + # `copilot /login` is not a valid invocation — slash-commands only exist + # inside an interactive session. The catalog must hand users a command + # that actually starts a login flow. + assert entry["cli_command"] == "copilot login" + + +def test_cli_command_reflects_configured_executable(tmp_path, monkeypatch, _clean_copilot_env): + from hermes_cli.web_server import _external_process_cli_command + + fake = tmp_path / ("copilot.exe" if os.name == "nt" else "copilot") + fake.write_text("", encoding="utf-8") + fake.chmod(0o755) + monkeypatch.setenv("HERMES_COPILOT_ACP_COMMAND", str(fake)) + + rendered = _external_process_cli_command("copilot-acp", "copilot login") + + assert rendered == f"{fake} login" + + +def test_cli_command_untouched_for_non_external_providers(_clean_copilot_env): + from hermes_cli.web_server import _external_process_cli_command + + assert _external_process_cli_command("nous", "hermes auth add nous") == "hermes auth add nous" + + +def test_cli_command_default_when_no_override(monkeypatch, _clean_copilot_env): + from hermes_cli.web_server import _external_process_cli_command + + assert _external_process_cli_command("copilot-acp", "copilot login") == "copilot login" + + +# --- live catalog key from the Copilot CLI store ----------------------------- + + +def test_catalog_key_resolves_from_copilot_cli_store(tmp_path, monkeypatch, _clean_copilot_env): + # A user whose ONLY credential is `copilot login` must still get the live + # model catalog — otherwise the picker silently falls back to the stale + # curated list (visibly wrong vs. what their subscription serves). + from unittest.mock import patch as mock_patch + + from hermes_cli import models as models_mod + + monkeypatch.setenv("HOME", str(tmp_path)) + cfg_dir = tmp_path / ".copilot" + cfg_dir.mkdir() + (cfg_dir / "config.json").write_text( + "// managed\n" + '{"copilotTokens": {"https://github.com:u": "gho_' + "x" * 36 + '"}}\n', + encoding="utf-8", + ) + + with mock_patch.object( + models_mod, "_resolve_copilot_catalog_api_key", wraps=models_mod._resolve_copilot_catalog_api_key + ), mock_patch( + "hermes_cli.copilot_auth.exchange_copilot_token", + return_value=("exchanged-api-token", 0.0, None), + ), mock_patch( + "hermes_cli.auth.resolve_api_key_provider_credentials", + side_effect=Exception("no env creds"), + ), mock_patch( + "hermes_cli.auth.read_credential_pool", return_value=[] + ): + key = models_mod._resolve_copilot_catalog_api_key() + + assert key == "exchanged-api-token" + + +def test_catalog_key_empty_when_cli_store_absent(tmp_path, monkeypatch, _clean_copilot_env): + from unittest.mock import patch as mock_patch + + from hermes_cli import models as models_mod + + monkeypatch.setenv("HOME", str(tmp_path)) # no ~/.copilot at all + + with mock_patch( + "hermes_cli.auth.resolve_api_key_provider_credentials", + side_effect=Exception("no env creds"), + ), mock_patch( + "hermes_cli.auth.read_credential_pool", return_value=[] + ): + key = models_mod._resolve_copilot_catalog_api_key() + + assert key == "" diff --git a/tests/hermes_cli/test_external_process_provider_seam.py b/tests/hermes_cli/test_external_process_provider_seam.py new file mode 100644 index 0000000000..47f3f47668 --- /dev/null +++ b/tests/hermes_cli/test_external_process_provider_seam.py @@ -0,0 +1,86 @@ +"""An external-process (ACP) provider can ship from outside this tree. + +The known-provider gate, the binary, the argv and the env var names used to be +spelled out for one vendor, so a profile registered from a plugin died with +"Unknown provider" before any client was built. This registers a provider the +way a standalone package does — before importing ``hermes_cli`` — and walks the +real resolution path, asserting ``copilot-acp`` is unchanged alongside it. +""" + +from __future__ import annotations + +import os +import stat + +import pytest + +from providers import register_provider +from providers.base import ProviderProfile + + +class _AcmeACPProfile(ProviderProfile): + def create_client(self, **kwargs): + return ("acme-client", kwargs) + + def fetch_models(self, **kwargs): + return None + + +register_provider( + _AcmeACPProfile( + name="acme-acp", + aliases=("acme",), + display_name="Acme ACP", + base_url="acp://acme", + auth_type="external_process", + process_command="acme-cli", + process_args=("--acp",), + process_command_env_vars=("ACME_CLI_PATH",), + process_args_env_var="ACME_ACP_ARGS", + ) +) + + +@pytest.fixture +def fake_cli(tmp_path, monkeypatch): + bindir = tmp_path / "bin" + bindir.mkdir() + for name in ("acme-cli", "copilot", "custom-acme"): + exe = bindir / name + exe.write_text("#!/bin/sh\nexit 0\n") + exe.chmod(exe.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("PATH", f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}") + return bindir + + +def test_an_out_of_tree_external_process_provider_resolves_end_to_end(fake_cli, monkeypatch): + from hermes_cli.auth import PROVIDER_REGISTRY, resolve_external_process_provider_credentials, resolve_provider + from hermes_cli.runtime_provider import resolve_runtime_provider + + assert PROVIDER_REGISTRY["acme"] is PROVIDER_REGISTRY["acme-acp"] + assert PROVIDER_REGISTRY["acme-acp"].auth_type == "external_process" + assert resolve_provider("acme") == "acme-acp" + + creds = resolve_external_process_provider_credentials("acme-acp") + assert (creds["command"], creds["args"], creds["api_key"]) == (str(fake_cli / "acme-cli"), ["--acp"], "acme-acp") + + monkeypatch.setenv("ACME_CLI_PATH", str(fake_cli / "custom-acme")) + monkeypatch.setenv("ACME_ACP_ARGS", "--acp=true --verbose") + creds = resolve_external_process_provider_credentials("acme-acp") + assert (creds["command"], creds["args"]) == (str(fake_cli / "custom-acme"), ["--acp=true", "--verbose"]) + + runtime = resolve_runtime_provider(requested="acme", target_model="acme") + assert (runtime["provider"], runtime["base_url"], runtime["source"]) == ("acme-acp", "acp://acme", "process") + + +def test_copilot_acp_launch_details_are_unchanged(fake_cli, monkeypatch): + from hermes_cli.auth import resolve_external_process_provider_credentials + from hermes_cli.runtime_provider import resolve_runtime_provider + + creds = resolve_external_process_provider_credentials("copilot-acp") + assert creds["command"] == str(fake_cli / "copilot") + assert (creds["args"], creds["api_key"], creds["base_url"]) == (["--acp", "--stdio"], "copilot-acp", "acp://copilot") + + monkeypatch.setenv("COPILOT_CLI_PATH", str(fake_cli / "custom-acme")) + assert resolve_external_process_provider_credentials("copilot-acp")["command"] == str(fake_cli / "custom-acme") + assert resolve_runtime_provider(requested="copilot-acp", target_model="x")["base_url"] == "acp://copilot" diff --git a/tests/hermes_cli/test_fast_serve_launch.py b/tests/hermes_cli/test_fast_serve_launch.py new file mode 100644 index 0000000000..a0f3961596 --- /dev/null +++ b/tests/hermes_cli/test_fast_serve_launch.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import argparse +import sys + +import hermes_cli.config as config_mod +import hermes_cli.main as main_mod +from hermes_cli.subcommands.dashboard import build_dashboard_parser, build_serve_parser + + +def _capture(_args) -> None: + return None + + +def test_lean_serve_parser_matches_full_subcommand_parser() -> None: + root = argparse.ArgumentParser() + subparsers = root.add_subparsers(dest="command") + build_dashboard_parser(subparsers, cmd_dashboard=_capture, cmd_dashboard_register=_capture) + lean = build_serve_parser(cmd_dashboard=_capture) + + argv = [ + "--host", "127.0.0.1", "--port", "0", "--no-open", + "--ssh-session-token-file", "token.txt", "--ssh-owner-nonce", "0123456789abcdef", + ] + + assert vars(lean.parse_args(argv)) == vars(root.parse_args(["serve", *argv])) + + +def test_fast_serve_launch_dispatches_only_unambiguous_serve(monkeypatch) -> None: + captured = [] + monkeypatch.setattr(config_mod, "get_container_exec_info", lambda: None) + monkeypatch.setattr(main_mod, "cmd_dashboard", captured.append) + + monkeypatch.setattr(sys, "argv", ["hermes", "serve", "--host", "127.0.0.1", "--port", "0"]) + assert main_mod._try_fast_serve_launch() is True + assert (captured[0].command, captured[0].headless_backend, captured[0].no_open, captured[0].port) == ( + "serve", True, True, 0, + ) + + # Every ambiguous shape falls back to the full parser: unknown flags, + # help, the opt-out, and container routing. + for argv in (["serve", "--future-flag"], ["serve", "--help"], ["chat"]): + monkeypatch.setattr(sys, "argv", ["hermes", *argv]) + assert main_mod._try_fast_serve_launch() is False + monkeypatch.setenv("HERMES_DISABLE_FAST_SERVE_LAUNCH", "1") + monkeypatch.setattr(sys, "argv", ["hermes", "serve"]) + assert main_mod._try_fast_serve_launch() is False + monkeypatch.delenv("HERMES_DISABLE_FAST_SERVE_LAUNCH") + monkeypatch.setattr(config_mod, "get_container_exec_info", lambda: {"name": "managed"}) + assert main_mod._try_fast_serve_launch() is False + assert len(captured) == 1 diff --git a/tests/hermes_cli/test_fts_optimize_notice.py b/tests/hermes_cli/test_fts_optimize_notice.py new file mode 100644 index 0000000000..8ba415c0a3 --- /dev/null +++ b/tests/hermes_cli/test_fts_optimize_notice.py @@ -0,0 +1,52 @@ +"""Regression coverage for FTS storage upgrade discoverability.""" + +import sqlite3 +from types import SimpleNamespace + + +def test_update_notice_offers_v1_trigram_tool_calls_rebuild(tmp_path, monkeypatch, capsys): + """A deployed v1 trigram projection still receives the opt-in notice.""" + from hermes_cli import update_cmd + import hermes_constants + import hermes_state + + db_path = tmp_path / "state.db" + db_path.touch() + conn = sqlite3.connect(db_path) + conn.executescript( + """ + CREATE TABLE state_meta (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE messages_fts (content TEXT, tool_name TEXT, tool_calls TEXT); + CREATE TABLE messages_fts_trigram (content TEXT, tool_name TEXT, tool_calls TEXT); + """ + ) + + class FakeSessionDB: + def __init__(self, **_kwargs): + self._conn = conn + + def close(self): + pass + + _db_needs_fts_storage_upgrade = staticmethod( + hermes_state.SessionDB._db_needs_fts_storage_upgrade + ) + + monkeypatch.setattr(hermes_constants, "get_hermes_home", lambda: tmp_path) + monkeypatch.setattr(hermes_state, "SessionDB", FakeSessionDB) + # Report a large state.db without patching Path.stat globally: a + # 1-arg lambda on the class breaks pathlib.exists(follow_symlinks=...) + # for every caller in the process (pytest's own teardown included). + real_stat = update_cmd.Path.stat + + def _stat(path, *args, **kwargs): + if path.name == "state.db": + return SimpleNamespace(st_size=512 * 1024 ** 2) + return real_stat(path, *args, **kwargs) + + monkeypatch.setattr(update_cmd.Path, "stat", _stat) + + update_cmd._print_fts_optimize_available_notice() + + assert "hermes sessions optimize-storage" in capsys.readouterr().out + conn.close() diff --git a/tests/hermes_cli/test_gateway_multiplex_status.py b/tests/hermes_cli/test_gateway_multiplex_status.py new file mode 100644 index 0000000000..0c516db590 --- /dev/null +++ b/tests/hermes_cli/test_gateway_multiplex_status.py @@ -0,0 +1,61 @@ +"""PR #69118: a named profile served by the default multiplexer reports as running. + +``hermes gateway status`` / ``gateway list`` / ``profile list`` keyed liveness +off the profile's own gateway.pid, so a satellite profile served by the default +multiplexer showed "not running" even though the multiplexer was its live +inbound process. All three now consult the same +``named_profile_served_by_running_multiplexer()`` lookup the start guard and +cron liveness use. +""" + +from __future__ import annotations + +import io +import os +from contextlib import redirect_stdout +from types import SimpleNamespace + + +def _fake_multiplexer(monkeypatch, tmp_path, *, multiplex: bool): + import hermes_constants + import gateway.status as status + + (tmp_path / "profiles" / "beta").mkdir(parents=True) + (tmp_path / "config.yaml").write_text( + f"gateway:\n multiplex_profiles: {'true' if multiplex else 'false'}\n" + ) + (tmp_path / "gateway.pid").write_text(str(os.getpid())) + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "profiles" / "beta")) + monkeypatch.setattr(hermes_constants, "_default_hermes_root_memo", None) + monkeypatch.setattr(status, "_pid_exists", lambda pid: True) + + +def _run_status(): + from hermes_cli import gateway as gw + + buf = io.StringIO() + with redirect_stdout(buf): + gw._gateway_command_inner( + SimpleNamespace(gateway_command="status", deep=False, full=False, system=False) + ) + return buf.getvalue().splitlines()[0] + + +def test_served_named_profile_reports_running(monkeypatch, tmp_path): + from hermes_cli.profiles import list_profiles + + _fake_multiplexer(monkeypatch, tmp_path, multiplex=True) + + beta = next(p for p in list_profiles() if p.name == "beta") + assert beta.gateway_running is True + assert _run_status().startswith("✓ Gateway is running via the default-profile multiplexer") + + +def test_unserved_named_profile_still_reports_stopped(monkeypatch, tmp_path): + from hermes_cli.profiles import list_profiles + + _fake_multiplexer(monkeypatch, tmp_path, multiplex=False) + + beta = next(p for p in list_profiles() if p.name == "beta") + assert beta.gateway_running is False + assert _run_status().startswith("✗ Gateway is not running") diff --git a/tests/hermes_cli/test_gateway_restart_loop.py b/tests/hermes_cli/test_gateway_restart_loop.py index 65fc7994b8..a74d00a5b9 100644 --- a/tests/hermes_cli/test_gateway_restart_loop.py +++ b/tests/hermes_cli/test_gateway_restart_loop.py @@ -588,6 +588,33 @@ class TestTerminalToolGatewayLifecycleGuard: assert result["exit_code"] == 1 assert "KeepAlive" in result["error"] + def test_oversized_root_skips_launchctl_prescan_and_fails_closed( + self, monkeypatch + ): + """#78398: an over-budget root must never reach shlex — not even via + the launchctl pre-scan that runs before the full guard.""" + import cron.lifecycle_guard as lifecycle_guard + import tools.terminal_tool as tt + + self._patch_env(monkeypatch, self._make_fake_env(), inside_gateway=True) + monkeypatch.setattr( + lifecycle_guard, "_MAX_LIFECYCLE_SCAN_BYTES", 8, raising=False + ) + monkeypatch.setattr( + lifecycle_guard, "_MAX_LIFECYCLE_SCAN_LINE_BYTES", 8, raising=False + ) + + def explode_if_tokenized(*args, **kwargs): + raise AssertionError("over-budget root reached shlex") + + monkeypatch.setattr(lifecycle_guard.shlex, "shlex", explode_if_tokenized) + + result = json.loads(tt.terminal_tool(command="x" * 9)) + + assert result["exit_code"] == 1 + assert "command or referenced script" in result["error"] + assert "KeepAlive" not in result["error"] + @pytest.mark.parametrize("command", [ # Neutral, non-hermes label: label-independent detection is the point # (#62891 second reproduction used `ai.hermes.svc-reload-tmp`). diff --git a/tests/hermes_cli/test_goals.py b/tests/hermes_cli/test_goals.py index 625ccbe111..413a9330ed 100644 --- a/tests/hermes_cli/test_goals.py +++ b/tests/hermes_cli/test_goals.py @@ -798,3 +798,38 @@ class TestContractAndBackgroundCompose: assert verdict == "wait" assert wait_directive and wait_directive.get("pid") == 4242 + +class TestBlockedVerdict: + """#100954: a genuinely unachievable goal must be refused, not completed.""" + + def test_parse_judge_response_accepts_blocked(self): + from hermes_cli.goals import _parse_judge_response + + verdict, reason, parse_failed, _wd = _parse_judge_response( + '{"verdict": "blocked", "reason": "the repo was deleted"}' + ) + assert verdict == "blocked" + assert reason == "the repo was deleted" + assert parse_failed is False + + def test_blocked_verdict_pauses_goal_instead_of_done(self, hermes_home): + from unittest.mock import patch + from hermes_cli.goals import GoalManager + + mgr = GoalManager(session_id="blocked-sid") + mgr.set("delete a repository that does not exist") + with patch( + "hermes_cli.goals.judge_goal", + return_value=("blocked", "the repo does not exist", False, None, False), + ): + decision = mgr.evaluate_after_turn( + "The repo cannot be deleted: it does not exist." + ) + + assert decision["verdict"] == "blocked" + assert decision["status"] == "paused" + assert decision["should_continue"] is False + assert "unachievable" in decision["message"].lower() + assert mgr.state is not None + assert mgr.state.status == "paused" + assert "unachievable" in (mgr.state.paused_reason or "").lower() diff --git a/tests/hermes_cli/test_gui_command.py b/tests/hermes_cli/test_gui_command.py index 80fc10dd03..37f6678ffb 100644 --- a/tests/hermes_cli/test_gui_command.py +++ b/tests/hermes_cli/test_gui_command.py @@ -99,7 +99,41 @@ def _make_packaged_executable(root: Path, monkeypatch) -> Path: return exe -@pytest.mark.platforms("linux") +def _staging_dir_from(cmd) -> Path: + """Extract the ``-c.directories.output=

`` electron-builder override + ``cmd_gui`` appends to ``npm run pack`` (stage-and-swap, #86443).""" + for arg in cmd: + if isinstance(arg, str) and arg.startswith("-c.directories.output="): + return Path(arg.split("=", 1)[1]) + raise AssertionError(f"no staging output override in {cmd!r}") + + +def _packaged_exe_rel() -> Path: + """Packaged-exe path relative to electron-builder's output dir on THIS host.""" + if sys.platform == "darwin": + return Path("mac-arm64") / "Hermes.app" / "Contents" / "MacOS" / "Hermes" + if sys.platform == "win32": + return Path("win-unpacked") / "Hermes.exe" + return Path("linux-unpacked") / "hermes" + + +def _pack_into_staging(root: Path, content: str = "", returncode: int = 0): + """``subprocess.run`` side effect mimicking a real ``npm run pack``: lays + the packaged app down inside the STAGING dir named on the command line + (never in release/), then returns *returncode*. Non-pack commands (the + launch) return success.""" + def _run(cmd, **kwargs): + if len(cmd) >= 3 and cmd[1:3] == ["run", "pack"]: + exe = _staging_dir_from(cmd) / _packaged_exe_rel() + exe.parent.mkdir(parents=True, exist_ok=True) + exe.write_text(content, encoding="utf-8") + if sys.platform not in ("darwin", "win32"): + (exe.parent / "chrome-sandbox").write_text("", encoding="utf-8") + return subprocess.CompletedProcess(cmd, returncode) + return subprocess.CompletedProcess(cmd, 0) + return _run + + def test_gui_installs_packages_and_launches_desktop_app(tmp_path, monkeypatch): # Exercises the npm-pack → packaged-exe launch path; Windows desktop is # MSIX-only (Electron autoUpdater) and takes a different launch route. @@ -119,7 +153,7 @@ def test_gui_installs_packages_and_launches_desktop_app(tmp_path, monkeypatch): patch("hermes_cli.main._desktop_macos_relaunchable_fixup"), \ patch("hermes_cli.main._desktop_linux_sandbox_fixup", return_value=True), \ patch("hermes_cli.main._register_linux_desktop_entry"), \ - patch("hermes_cli.main.subprocess.run", side_effect=[pack_ok, launch_ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit) as exc: cli_main.cmd_gui(_ns()) @@ -131,7 +165,13 @@ def test_gui_installs_packages_and_launches_desktop_app(tmp_path, monkeypatch): assert mock_install.call_args.kwargs["capture_output"] is False install_env = mock_install.call_args.kwargs["env"] assert install_env is not None and "PATH" in install_env - assert mock_run.call_args_list[0].args[0] == ["/usr/bin/npm", "run", "pack"] + pack_cmd = mock_run.call_args_list[0].args[0] + assert pack_cmd[:4] == ["/usr/bin/npm", "run", "pack", "--"] + # Stage-and-swap (#86443): the pack targets a staging dir beside release/, + # never release/ itself. + staging = _staging_dir_from(pack_cmd) + assert staging.parent == desktop_dir and staging.name.startswith(".staging-") + assert not staging.exists() # swapped into release/ and cleaned up assert mock_run.call_args_list[0].kwargs["cwd"] == desktop_dir launched = mock_run.call_args_list[1].args[0] if sys.platform.startswith("linux"): @@ -289,23 +329,29 @@ def test_gui_does_not_retry_after_packaged_executable_exists(tmp_path, monkeypat """ root = _make_desktop_tree(tmp_path) monkeypatch.setattr(cli_main, "PROJECT_ROOT", root) - # Executable EXISTS at failure time → late failure, not a corrupt download. - _make_packaged_executable(root, monkeypatch) + live_exe = _make_packaged_executable(root, monkeypatch) + live_exe.write_text("good build", encoding="utf-8") monkeypatch.delenv("ELECTRON_MIRROR", raising=False) install_ok = subprocess.CompletedProcess(["npm", "ci"], 0) - pack_fail = subprocess.CompletedProcess(["npm", "run", "pack"], 1) + # Executable EXISTS in the STAGING output at failure time → late failure + # (e.g. signing), not a corrupt download. With stage-and-swap (#86443) the + # discriminator reads the staging dir, so the fake pack lays it down there. + pack_fail = _pack_into_staging(root, content="half-signed", returncode=1) with patch("hermes_cli.main.shutil.which", return_value="/usr/bin/npm"), \ patch("hermes_cli.main._run_npm_install_deterministic", return_value=install_ok), \ patch("hermes_cli.main._desktop_macos_relaunchable_fixup"), \ patch("hermes_cli.main._purge_electron_build_cache", return_value=[Path("/c/electron.zip")]) as mock_purge, \ patch("hermes_cli.main._redownload_electron_dist", return_value=True) as mock_dl, \ - patch("hermes_cli.main.subprocess.run", return_value=pack_fail) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=pack_fail) as mock_run, \ pytest.raises(SystemExit) as exc: cli_main.cmd_gui(_ns()) assert exc.value.code == 1 + # The live app was never touched by the failed pack (#86443). + assert live_exe.read_text(encoding="utf-8") == "good build" + assert not list((root / "apps" / "desktop").glob(".staging-*")) # Neither destructive recovery runs, and there is exactly ONE pack attempt. mock_purge.assert_not_called() mock_dl.assert_not_called() @@ -1094,7 +1140,7 @@ def test_gui_bridges_ozone_hint_to_launch_env(tmp_path, monkeypatch): patch("hermes_cli.main._desktop_linux_sandbox_fixup", return_value=True), \ patch("hermes_cli.config.load_config", return_value=cfg), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns()) @@ -1110,7 +1156,7 @@ def test_gui_bridges_ozone_hint_to_launch_env(tmp_path, monkeypatch): patch("hermes_cli.main._desktop_linux_sandbox_fixup", return_value=True), \ patch("hermes_cli.config.load_config", return_value=cfg), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run2, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run2, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns()) @@ -1192,7 +1238,7 @@ def test_gui_linux_packaged_launch_bridges_detected_password_store(tmp_path, mon patch("hermes_cli.config.load_config", return_value={}), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ patch("hermes_cli.main._detect_linux_password_store", return_value="gnome-libsecret"), \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns()) @@ -1215,7 +1261,7 @@ def test_gui_linux_source_launch_bridges_detected_password_store(tmp_path, monke patch("hermes_cli.config.load_config", return_value={}), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ patch("hermes_cli.main._detect_linux_password_store", return_value="kwallet6"), \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns(source=True)) @@ -1243,7 +1289,7 @@ def test_gui_config_password_store_skips_detection(tmp_path, monkeypatch): patch("hermes_cli.config.load_config", return_value=cfg), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ patch("hermes_cli.main._detect_linux_password_store") as mock_detect, \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns()) @@ -1272,7 +1318,7 @@ def test_gui_explicit_password_store_env_wins_over_config_and_detection(tmp_path patch("hermes_cli.config.load_config", return_value=cfg), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ patch("hermes_cli.main._detect_linux_password_store") as mock_detect, \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns()) @@ -1298,10 +1344,178 @@ def test_gui_password_store_bridge_is_linux_only(tmp_path, monkeypatch): patch("hermes_cli.config.load_config", return_value={}), \ patch("hermes_cli.linux_desktop_entry.install_desktop_entry", return_value=None), \ patch("hermes_cli.main._detect_linux_password_store") as mock_detect, \ - patch("hermes_cli.main.subprocess.run", side_effect=[ok, ok]) as mock_run, \ + patch("hermes_cli.main.subprocess.run", side_effect=_pack_into_staging(root)) as mock_run, \ pytest.raises(SystemExit): cli_main.cmd_gui(_ns()) mock_detect.assert_not_called() launch_env = mock_run.call_args_list[1].kwargs["env"] assert "HERMES_DESKTOP_PASSWORD_STORE" not in launch_env + + +# --------------------------------------------------------------------------- +# #86443: stage-and-swap — a failed Desktop rebuild must never remove the +# working app. electron-builder packs IN PLACE (before-pack.mjs wipes +# release/ first), so cmd_gui now packs into a staging dir and only +# renames it over release/ after the staged result verifies. +# --------------------------------------------------------------------------- + + +def _gui_build_patches(root: Path, run_side_effect): + return [ + patch("hermes_cli.main.shutil.which", return_value="/usr/bin/npm"), + patch("hermes_cli.main._run_npm_install_deterministic", + return_value=subprocess.CompletedProcess(["npm", "ci"], 0)), + patch("hermes_cli.main._desktop_build_needed", return_value=True), + patch("hermes_cli.main._write_desktop_build_stamp"), + patch("hermes_cli.main._desktop_macos_relaunchable_fixup"), + patch("hermes_cli.main._register_linux_desktop_entry"), + patch("hermes_cli.main._stop_desktop_processes_locking_build", return_value=[]), + patch("hermes_cli.main._purge_electron_build_cache", return_value=[]), + patch("hermes_cli.main._redownload_electron_dist", return_value=False), + patch("hermes_cli.main.subprocess.run", side_effect=run_side_effect), + ] + + +def test_swap_staged_desktop_app_promotes_staged_tree_and_drops_previous(tmp_path): + root = _make_desktop_tree(tmp_path) + desktop_dir = root / "apps" / "desktop" + live_exe = desktop_dir / "release" / _packaged_exe_rel() + live_exe.parent.mkdir(parents=True) + live_exe.write_text("old", encoding="utf-8") + staging = cli_main._desktop_staging_dir(desktop_dir) + staged_exe = staging / _packaged_exe_rel() + staged_exe.parent.mkdir(parents=True) + staged_exe.write_text("new", encoding="utf-8") + + promoted = cli_main._swap_staged_desktop_app(desktop_dir, staging) + + assert promoted == live_exe + assert live_exe.read_text(encoding="utf-8") == "new" + assert not staging.exists() + assert sorted(p.name for p in (desktop_dir / "release").iterdir()) == [_packaged_exe_rel().parts[0]] + + +def test_swap_staged_desktop_app_without_staged_exe_keeps_live_app(tmp_path): + """Zero-exit pack that produced nothing: live app untouched, staging gone.""" + root = _make_desktop_tree(tmp_path) + desktop_dir = root / "apps" / "desktop" + live_exe = desktop_dir / "release" / _packaged_exe_rel() + live_exe.parent.mkdir(parents=True) + live_exe.write_text("old", encoding="utf-8") + staging = cli_main._desktop_staging_dir(desktop_dir) + (staging / "linux-unpacked" / "resources").mkdir(parents=True) # partial tree, no exe + + assert cli_main._swap_staged_desktop_app(desktop_dir, staging) is None + assert live_exe.read_text(encoding="utf-8") == "old" + assert not staging.exists() + + +def test_swap_staged_desktop_app_rolls_back_when_second_rename_fails(tmp_path, monkeypatch): + root = _make_desktop_tree(tmp_path) + desktop_dir = root / "apps" / "desktop" + live_exe = desktop_dir / "release" / _packaged_exe_rel() + live_exe.parent.mkdir(parents=True) + live_exe.write_text("old", encoding="utf-8") + staging = cli_main._desktop_staging_dir(desktop_dir) + staged_exe = staging / _packaged_exe_rel() + staged_exe.parent.mkdir(parents=True) + staged_exe.write_text("new", encoding="utf-8") + + real_rename = cli_main.os.rename + calls = {"n": 0} + + def flaky_rename(src, dst): + calls["n"] += 1 + if calls["n"] == 2: # staged → live + raise OSError("EXDEV simulated") + return real_rename(src, dst) + + monkeypatch.setattr(cli_main.os, "rename", flaky_rename) + assert cli_main._swap_staged_desktop_app(desktop_dir, staging) is None + assert live_exe.read_text(encoding="utf-8") == "old" + assert not (live_exe.parent.parent / (live_exe.parent.name + ".previous")).exists() + + +def test_gui_failed_pack_leaves_previous_app_untouched(tmp_path, monkeypatch, capsys): + """Every pack attempt fails → the pre-existing app is exactly as it was, + no staging dir remains, exit is non-zero.""" + root = _make_desktop_tree(tmp_path) + desktop_dir = root / "apps" / "desktop" + monkeypatch.setattr(cli_main, "PROJECT_ROOT", root) + live_exe = _make_packaged_executable(root, monkeypatch) + live_exe.write_text("good build", encoding="utf-8") + monkeypatch.setenv("ELECTRON_MIRROR", "https://example.test/electron/") + + def failing_pack(cmd, **kwargs): + # Mimic before-pack.mjs wiping appOutDir inside the OUTPUT dir it was + # given, then dying (corrupt Electron zip → ENOENT on rename). + out = _staging_dir_from(cmd) / _packaged_exe_rel().parts[0] + out.mkdir(parents=True, exist_ok=True) + (out / "resources").mkdir(exist_ok=True) + return subprocess.CompletedProcess(cmd, 1) + + patches = _gui_build_patches(root, failing_pack) + for p in patches: + p.start() + try: + with pytest.raises(SystemExit) as exc: + cli_main.cmd_gui(_ns(build_only=True)) + finally: + for p in patches: + p.stop() + + assert exc.value.code == 1 + assert live_exe.read_text(encoding="utf-8") == "good build" + assert not list(desktop_dir.glob(".staging-*")) + assert not list((desktop_dir / "release").glob("*.previous")) + out = capsys.readouterr().out + assert "previous desktop app was left untouched" in out + + +def test_gui_successful_pack_swaps_new_app_into_release(tmp_path, monkeypatch): + root = _make_desktop_tree(tmp_path) + desktop_dir = root / "apps" / "desktop" + monkeypatch.setattr(cli_main, "PROJECT_ROOT", root) + live_exe = _make_packaged_executable(root, monkeypatch) + live_exe.write_text("old build", encoding="utf-8") + + patches = _gui_build_patches(root, _pack_into_staging(root, content="new build")) + for p in patches: + p.start() + try: + cli_main.cmd_gui(_ns(build_only=True)) + finally: + for p in patches: + p.stop() + + assert live_exe.read_text(encoding="utf-8") == "new build" + assert not list(desktop_dir.glob(".staging-*")) + assert not list((desktop_dir / "release").glob("*.previous")) + + +def test_gui_zero_exit_pack_without_artifact_keeps_previous_app(tmp_path, monkeypatch, capsys): + root = _make_desktop_tree(tmp_path) + desktop_dir = root / "apps" / "desktop" + monkeypatch.setattr(cli_main, "PROJECT_ROOT", root) + live_exe = _make_packaged_executable(root, monkeypatch) + live_exe.write_text("good build", encoding="utf-8") + + def empty_pack(cmd, **kwargs): + _staging_dir_from(cmd).mkdir(parents=True, exist_ok=True) + return subprocess.CompletedProcess(cmd, 0) + + patches = _gui_build_patches(root, empty_pack) + for p in patches: + p.start() + try: + with pytest.raises(SystemExit) as exc: + cli_main.cmd_gui(_ns(build_only=True)) + finally: + for p in patches: + p.stop() + + assert exc.value.code == 1 + assert live_exe.read_text(encoding="utf-8") == "good build" + assert not list(desktop_dir.glob(".staging-*")) + assert "produced no launchable app" in capsys.readouterr().out diff --git a/tests/hermes_cli/test_inventory_pricing.py b/tests/hermes_cli/test_inventory_pricing.py index 5fb7c39490..d626593f34 100644 --- a/tests/hermes_cli/test_inventory_pricing.py +++ b/tests/hermes_cli/test_inventory_pricing.py @@ -5,6 +5,9 @@ columns + Free/Pro badges and gate paid models on free Nous accounts, the same way the `hermes model` CLI picker does. """ +from threading import Event +from time import monotonic + import hermes_cli.inventory as inv import hermes_cli.models as models_mod @@ -101,3 +104,389 @@ def test_apply_pricing_omits_sale_when_original_not_cheaper(monkeypatch): assert "discount_percent" not in rows[0]["pricing"]["a/eq"] +def test_model_options_cold_pricing_fetch_runs_off_the_request_path(monkeypatch): + """A cold pricing endpoint must not delay the first picker payload.""" + fetch_started = Event() + release_fetch = Event() + + def fake_pricing(_slug, *, force_refresh=False, cached_only=False): + if cached_only: + return {} + fetch_started.set() + release_fetch.wait(timeout=5) + return {} + + row = { + "slug": "openrouter", + "name": "OpenRouter", + "models": ["vendor/model"], + "total_models": 1, + "is_current": True, + "is_user_defined": False, + "source": "built-in", + } + monkeypatch.setattr(models_mod, "get_pricing_for_provider", fake_pricing) + monkeypatch.setattr( + "hermes_cli.model_switch.list_authenticated_providers", + lambda **_kwargs: [row], + ) + monkeypatch.setattr(inv, "_moa_provider_row", lambda *_args, **_kwargs: None) + monkeypatch.setattr(inv, "_apply_capabilities", lambda _rows: None) + monkeypatch.setattr(inv, "_apply_featured", lambda _rows: None) + monkeypatch.setattr(inv, "_pricing_prewarm_threads", {}) + + try: + started_at = monotonic() + payload = inv.build_model_options_payload( + inv.ConfigContext( + current_provider="openrouter", + current_model="vendor/model", + current_base_url="", + user_providers={}, + custom_providers=[], + ) + ) + elapsed = monotonic() - started_at + assert payload["providers"][0]["slug"] == "openrouter" + assert "pricing" not in payload["providers"][0] + assert elapsed < 2.0, f"cold picker blocked for {elapsed:.2f}s" + assert fetch_started.wait(timeout=1), "pricing should prewarm in the background" + finally: + threads = list(inv._pricing_prewarm_threads.values()) + release_fetch.set() + for thread in threads: + thread.join(timeout=2) + + +def test_cold_nous_entitlement_keeps_models_unselectable(monkeypatch): + """A cold nonblocking response must not expose paid models fail-open.""" + monkeypatch.setattr( + models_mod, "get_pricing_for_provider", lambda *_args, **_kwargs: {} + ) + monkeypatch.setattr(models_mod, "get_cached_nous_free_tier", lambda: None) + rows = [{"slug": "nous", "models": ["free/model", "paid/model"]}] + + inv._apply_pricing(rows, cached_only=True) + + assert rows[0]["free_tier_pending"] is True + assert rows[0]["unavailable_models"] == ["free/model", "paid/model"] + # The whole list renders locked — the picker's per-provider warning + # surface must say why, without clobbering an existing auth warning. + assert "entitlement" in rows[0]["warning"] + + rows = [{"slug": "nous", "models": ["m"], "warning": "paste NOUS_API_KEY to activate"}] + inv._apply_pricing(rows, cached_only=True) + assert rows[0]["warning"] == "paste NOUS_API_KEY to activate" + + +def test_prewarm_preserves_context_and_runs_once_per_profile(tmp_path, monkeypatch): + """Concurrent multiplex profiles retain their own home and secret scope.""" + from agent.secret_scope import ( + current_secret_scope, + reset_secret_scope, + set_secret_scope, + ) + from hermes_constants import ( + hermes_home_key, + reset_hermes_home_override, + set_hermes_home_override, + ) + + monkeypatch.setattr(inv, "_pricing_prewarm_threads", {}) + release = Event() + started = {"a": Event(), "b": Event()} + observed = {} + + def capture_context(_rows): + scope = current_secret_scope() + label = scope["PROFILE_MARKER"] + observed[label] = (hermes_home_key(), dict(scope)) + started[label].set() + release.wait(timeout=5) + + monkeypatch.setattr(inv, "_apply_pricing", capture_context) + + threads = [] + try: + for label in ("a", "b"): + home = tmp_path / label + home_token = set_hermes_home_override(str(home)) + secret_token = set_secret_scope({"PROFILE_MARKER": label}) + try: + threads.append(inv._prewarm_pricing_async([{"models": []}])) + finally: + reset_secret_scope(secret_token) + reset_hermes_home_override(home_token) + + assert threads[0] is not threads[1] + assert started["a"].wait(timeout=1) + assert started["b"].wait(timeout=1) + assert observed["a"] == ( + hermes_home_key(tmp_path / "a"), + {"PROFILE_MARKER": "a"}, + ) + assert observed["b"] == ( + hermes_home_key(tmp_path / "b"), + {"PROFILE_MARKER": "b"}, + ) + finally: + release.set() + for thread in threads: + if thread is not None: + thread.join(timeout=2) + + +def test_prewarm_deduplicates_inflight_scope_and_cleans_up(monkeypatch): + """Rapid opens share one worker, then a completed scope can run again.""" + monkeypatch.setattr(inv, "_pricing_prewarm_threads", {}) + started = Event() + release = Event() + calls = [] + + def blocked_prewarm(_rows): + calls.append(None) + started.set() + release.wait(timeout=5) + + monkeypatch.setattr(inv, "_apply_pricing", blocked_prewarm) + rows = [{"slug": "openrouter", "models": ["vendor/model"]}] + + first = inv._prewarm_pricing_async(rows) + try: + assert started.wait(timeout=1) + second = inv._prewarm_pricing_async(rows) + assert second is first + assert len(calls) == 1 + finally: + release.set() + first.join(timeout=2) + + assert not first.is_alive() + assert inv._pricing_prewarm_threads == {} + + retry = inv._prewarm_pricing_async(rows) + retry.join(timeout=2) + assert retry is not first + assert len(calls) == 2 + assert inv._pricing_prewarm_threads == {} + + +def test_prewarm_endpoint_rotation_starts_a_new_worker(tmp_path, monkeypatch): + """A live endpoint-A worker must not suppress endpoint B for its profile.""" + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + endpoint_a = "https://endpoint-a.example" + endpoint_b = "https://endpoint-b.example" + active_endpoint = {"value": endpoint_a} + started = {endpoint_a: Event(), endpoint_b: Event()} + release_a = Event() + expected = { + endpoint_a: {"a/model": {"prompt": "1", "completion": "2"}}, + endpoint_b: {"b/model": {"prompt": "3", "completion": "4"}}, + } + monkeypatch.setattr(inv, "_pricing_prewarm_threads", {}) + monkeypatch.setattr(models_mod, "_pricing_cache", {}) + monkeypatch.setattr(models_mod, "_pricing_cache_retry_after", {}) + monkeypatch.setattr(models_mod, "_pricing_provider_cache_keys", {}) + monkeypatch.setattr( + models_mod, + "_resolve_nous_pricing_credentials", + lambda: ("", active_endpoint["value"]), + ) + + def fetch_pricing(*, base_url, **_kwargs): + started[base_url].set() + if base_url == endpoint_a: + release_a.wait(timeout=5) + return models_mod._cache_catalog(base_url, expected[base_url]) + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", fetch_pricing) + monkeypatch.setattr( + inv, + "_apply_pricing", + lambda _rows: models_mod.get_pricing_for_provider("nous"), + ) + + token = set_hermes_home_override(str(tmp_path / "profile")) + threads = [] + try: + threads.append( + inv._prewarm_pricing_async( + [{"slug": "nous", "models": ["a/model"]}], + current_provider="nous", + current_base_url=endpoint_a, + ) + ) + assert started[endpoint_a].wait(timeout=1) + + active_endpoint["value"] = endpoint_b + threads.append( + inv._prewarm_pricing_async( + [{"slug": "nous", "models": ["b/model"]}], + current_provider="nous", + current_base_url=endpoint_b, + ) + ) + + assert threads[0] is not threads[1] + assert started[endpoint_b].wait(timeout=1) + threads[1].join(timeout=2) + assert not threads[1].is_alive() + assert models_mod.get_pricing_for_provider( + "nous", cached_only=True + ) == expected[endpoint_b] + finally: + release_a.set() + for thread in threads: + if thread is not None: + thread.join(timeout=2) + reset_hermes_home_override(token) + + +def test_prewarm_nous_rotation_when_another_provider_is_current(tmp_path, monkeypatch): + """Nous endpoint identity must not depend on Nous being selected.""" + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + endpoint_a = "https://endpoint-a.example" + endpoint_b = "https://endpoint-b.example" + active_endpoint = {"value": endpoint_a} + started = {endpoint_a: Event(), endpoint_b: Event()} + release_a = Event() + expected = { + endpoint_a: {"a/model": {"prompt": "1", "completion": "2"}}, + endpoint_b: {"b/model": {"prompt": "3", "completion": "4"}}, + } + monkeypatch.setattr(inv, "_pricing_prewarm_threads", {}) + monkeypatch.setattr(models_mod, "_pricing_cache", {}) + monkeypatch.setattr(models_mod, "_pricing_cache_retry_after", {}) + monkeypatch.setattr(models_mod, "_pricing_provider_cache_keys", {}) + monkeypatch.setattr( + models_mod, + "get_cached_nous_inference_base_url", + lambda: active_endpoint["value"], + ) + monkeypatch.setattr( + models_mod, + "_resolve_nous_pricing_credentials", + lambda: ("", active_endpoint["value"]), + ) + + def fetch_pricing(*, base_url, **_kwargs): + started[base_url].set() + if base_url == endpoint_a: + release_a.wait(timeout=5) + return models_mod._cache_catalog(base_url, expected[base_url]) + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", fetch_pricing) + monkeypatch.setattr( + inv, + "_apply_pricing", + lambda _rows: models_mod.get_pricing_for_provider("nous"), + ) + + token = set_hermes_home_override(str(tmp_path / "profile")) + threads = [] + try: + threads.append( + inv._prewarm_pricing_async( + [{"slug": "nous", "models": ["a/model"]}], + current_provider="openrouter", + current_base_url="https://openrouter.ai/api/v1", + ) + ) + assert started[endpoint_a].wait(timeout=1) + + active_endpoint["value"] = endpoint_b + threads.append( + inv._prewarm_pricing_async( + [{"slug": "nous", "models": ["b/model"]}], + current_provider="openrouter", + current_base_url="https://openrouter.ai/api/v1", + ) + ) + + assert threads[0] is not threads[1] + assert started[endpoint_b].wait(timeout=1) + threads[1].join(timeout=2) + assert not threads[1].is_alive() + assert models_mod.get_pricing_for_provider( + "nous", cached_only=True + ) == expected[endpoint_b] + finally: + release_a.set() + for thread in threads: + if thread is not None: + thread.join(timeout=2) + reset_hermes_home_override(token) + + +def test_cached_only_pricing_returns_a_warm_value_without_fetching(monkeypatch): + """Cache-only picker reads preserve pricing once the prewarm completes.""" + cache_key = "https://openrouter.ai/api" + expected = {"vendor/model": {"prompt": "0.000001", "completion": "0.000002"}} + monkeypatch.setattr(models_mod, "_pricing_cache", {cache_key: expected}) + monkeypatch.setattr(models_mod, "_pricing_cache_retry_after", {}) + monkeypatch.setattr(models_mod, "_pricing_provider_cache_keys", {}) + monkeypatch.setattr( + models_mod, + "fetch_models_with_pricing", + lambda **_kwargs: (_ for _ in ()).throw(AssertionError("network fetch started")), + ) + + assert models_mod.get_pricing_for_provider( + "openrouter", cached_only=True + ) == expected + + +def test_cached_only_dynamic_pricing_is_profile_scoped(tmp_path, monkeypatch): + """Alternating profiles read the endpoint each profile warmed.""" + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + endpoint_a = "https://profile-a.example" + endpoint_b = "https://profile-b.example" + expected_a = {"a/model": {"prompt": "1", "completion": "2"}} + expected_b = {"b/model": {"prompt": "3", "completion": "4"}} + monkeypatch.setattr( + models_mod, + "_pricing_cache", + {endpoint_a: expected_a, endpoint_b: expected_b}, + ) + monkeypatch.setattr(models_mod, "_pricing_cache_retry_after", {}) + monkeypatch.setattr(models_mod, "_pricing_provider_cache_keys", {}) + active_endpoint = {"value": endpoint_a} + monkeypatch.setattr( + models_mod, + "_resolve_nous_pricing_credentials", + lambda: ("", active_endpoint["value"]), + ) + monkeypatch.setattr( + models_mod, + "fetch_models_with_pricing", + lambda **kwargs: models_mod._pricing_cache[kwargs["base_url"]], + ) + + def in_profile(home, endpoint, *, cached_only): + token = set_hermes_home_override(str(home)) + active_endpoint["value"] = endpoint + try: + return models_mod.get_pricing_for_provider( + "nous", cached_only=cached_only + ) + finally: + reset_hermes_home_override(token) + + assert in_profile(tmp_path / "a", endpoint_a, cached_only=False) == expected_a + assert in_profile(tmp_path / "b", endpoint_b, cached_only=False) == expected_b + assert in_profile(tmp_path / "a", endpoint_b, cached_only=True) == expected_a + assert in_profile(tmp_path / "b", endpoint_a, cached_only=True) == expected_b + + diff --git a/tests/hermes_cli/test_kanban_gateway_restart_handoff.py b/tests/hermes_cli/test_kanban_gateway_restart_handoff.py new file mode 100644 index 0000000000..d0efb6ed89 --- /dev/null +++ b/tests/hermes_cli/test_kanban_gateway_restart_handoff.py @@ -0,0 +1,178 @@ +"""Managed-gateway isolation for dispatcher-owned Kanban workers.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from hermes_cli import kanban_db as kb + + +@pytest.fixture +def worker_setup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, kb.Task]: + root = tmp_path / ".hermes" + profile = root / "profiles" / "coder" + profile.mkdir(parents=True) + root.joinpath("config.yaml").write_text("{}\n", encoding="utf-8") + profile.joinpath("config.yaml").write_text("{}\n", encoding="utf-8") + monkeypatch.setenv("HERMES_HOME", str(root)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.setattr(kb, "_resolve_hermes_argv", lambda: ["hermes"]) + + workspace = tmp_path / "candidate-worktree" + workspace.mkdir() + task = kb.Task( + id="t_candidate_restart", + title="activate candidate", + body=None, + assignee="coder", + status="running", + priority=0, + created_by="test", + created_at=1, + started_at=1, + completed_at=None, + workspace_kind="worktree", + workspace_path=str(workspace), + claim_lock="host:dispatcher", + claim_expires=999, + tenant=None, + branch_name="wt/t_candidate_restart", + current_run_id=23, + ) + return workspace, task + + +@pytest.mark.linux_only +def test_managed_gateway_worker_is_spawned_in_restart_safe_scope( + worker_setup: tuple[Path, kb.Task], monkeypatch: pytest.MonkeyPatch +) -> None: + workspace, task = worker_setup + captured_cmd: list[str] = [] + captured_env: dict[str, str] = {} + captured_cwd: str | None = None + + class FakeProc: + pid = 4242 + + def fake_popen(cmd, **kwargs): + nonlocal captured_cwd + captured_cmd.extend(cmd) + captured_env.update(kwargs.get("env") or {}) + captured_cwd = kwargs.get("cwd") + return FakeProc() + + monkeypatch.setenv("INVOCATION_ID", "managed-gateway-test") + monkeypatch.setenv("ANTHROPIC_API_KEY", "must-not-cross-profile") + monkeypatch.setattr("agent.secret_scope.is_multiplex_active", lambda: True) + monkeypatch.setattr(subprocess, "Popen", fake_popen) + monkeypatch.setattr("tools.process_registry._is_supervised_gateway_process", lambda: True) + monkeypatch.setattr("tools.process_registry._systemd_run_user_scope_available", lambda: True) + monkeypatch.setattr("tools.process_registry._worker_memory_max_bytes", lambda: 536_870_912) + monkeypatch.setattr("shutil.which", lambda name: "/usr/bin/systemd-run") + + assert kb._default_spawn(task, str(workspace)) == 4242 + assert captured_cmd[:4] == ["/usr/bin/systemd-run", "--user", "--scope", "--quiet"] + unit_index = captured_cmd.index("--unit") + assert captured_cmd[unit_index + 1] == "hermes-worker-kanban-t_candidate_restart-run-23" + assert "MemoryMax=536870912" in captured_cmd + separator = captured_cmd.index("--") + assert captured_cmd[separator + 1 : separator + 4] == ["hermes", "-p", "coder"] + assert captured_cwd == str(workspace) + assert captured_env["HERMES_KANBAN_TASK"] == task.id + assert captured_env["HERMES_KANBAN_RUN_ID"] == "23" + assert "ANTHROPIC_API_KEY" not in captured_env + + +@pytest.mark.linux_only +def test_managed_gateway_worker_spawn_fails_closed_without_scope( + worker_setup: tuple[Path, kb.Task], monkeypatch: pytest.MonkeyPatch +) -> None: + workspace, task = worker_setup + popen_calls: list[list[str]] = [] + monkeypatch.setenv("INVOCATION_ID", "managed-gateway-test") + monkeypatch.setattr(subprocess, "Popen", lambda cmd, **kwargs: popen_calls.append(list(cmd))) + monkeypatch.setattr("tools.process_registry._is_supervised_gateway_process", lambda: True) + monkeypatch.setattr("tools.process_registry._systemd_run_user_scope_available", lambda: False) + + with pytest.raises(RuntimeError, match="restart-safe systemd scope"): + kb._default_spawn(task, str(workspace)) + assert popen_calls == [] + + +@pytest.mark.linux_only +def test_managed_gateway_scope_builder_fails_closed_if_binary_disappears( + worker_setup: tuple[Path, kb.Task], monkeypatch: pytest.MonkeyPatch +) -> None: + workspace, task = worker_setup + monkeypatch.setenv("INVOCATION_ID", "managed-gateway-test") + monkeypatch.setattr("tools.process_registry._is_supervised_gateway_process", lambda: True) + monkeypatch.setattr("tools.process_registry._systemd_run_user_scope_available", lambda: True) + monkeypatch.setattr("shutil.which", lambda _name: None) + monkeypatch.setattr(subprocess, "Popen", lambda *_args, **_kwargs: pytest.fail("unsafe direct spawn")) + + with pytest.raises(RuntimeError, match="restart-safe systemd scope"): + kb._default_spawn(task, str(workspace)) + + +def test_standalone_dispatcher_keeps_direct_worker_spawn( + worker_setup: tuple[Path, kb.Task], monkeypatch: pytest.MonkeyPatch +) -> None: + workspace, task = worker_setup + captured_cmd: list[str] = [] + + class FakeProc: + pid = 4243 + + monkeypatch.setattr(subprocess, "Popen", lambda cmd, **kwargs: captured_cmd.extend(cmd) or FakeProc()) + monkeypatch.setattr("tools.process_registry._is_supervised_gateway_process", lambda: False) + monkeypatch.setattr( + "tools.process_registry._systemd_run_user_scope_available", + lambda: pytest.fail("scope probe must not run outside managed gateway"), + ) + + assert kb._default_spawn(task, str(workspace)) == 4243 + assert captured_cmd[:3] == ["hermes", "-p", "coder"] + + +@pytest.mark.linux_only +def test_real_user_systemd_scope_preserves_worker_context( + worker_setup: tuple[Path, kb.Task], monkeypatch: pytest.MonkeyPatch +) -> None: + from tools import process_registry + + if not process_registry._systemd_run_user_scope_available(): + pytest.skip("systemd-run --user --scope is unavailable on this host") + + workspace, task = worker_setup + receipt = workspace / "worker-receipt.json" + script = ( + "import json, os, pathlib, sys, time; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({" + "'pid': os.getpid(), 'cwd': os.getcwd(), " + "'task': os.environ.get('HERMES_KANBAN_TASK'), " + "'run': os.environ.get('HERMES_KANBAN_RUN_ID'), " + "'cgroup': pathlib.Path('/proc/self/cgroup').read_text()})); time.sleep(0.5)" + ) + monkeypatch.setattr(kb, "_resolve_hermes_argv", lambda: [sys.executable, "-c", script, str(receipt)]) + monkeypatch.setenv("INVOCATION_ID", "managed-gateway-test") + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + + pid = kb._default_spawn(task, str(workspace)) + deadline = time.monotonic() + 5 + while not receipt.exists() and time.monotonic() < deadline: + time.sleep(0.05) + + assert receipt.exists() + payload = json.loads(receipt.read_text(encoding="utf-8")) + assert payload["pid"] == pid + assert payload["cwd"] == str(workspace) + assert payload["task"] == task.id + assert payload["run"] == "23" + assert ".scope" in payload["cgroup"] + assert "hermes-gateway.service" not in payload["cgroup"] diff --git a/tests/hermes_cli/test_kanban_goal_mode.py b/tests/hermes_cli/test_kanban_goal_mode.py index 61ece645ff..00616bfd7d 100644 --- a/tests/hermes_cli/test_kanban_goal_mode.py +++ b/tests/hermes_cli/test_kanban_goal_mode.py @@ -205,3 +205,21 @@ class TestCLIJudgeGate: rc, complete_calls = self._run(monkeypatch, goal_mode=False) assert rc == 0 assert complete_calls == ["t1"] + + def test_judge_blocked_verdict_rejects_completion(self, monkeypatch, capsys): + """#100954: an unachievable goal must not complete silently. + + The judge's ``blocked`` verdict is a refusal, not a completion — + ``complete_task`` must never run and stderr must steer the user + toward re-scoping / recording the block. + """ + rc, complete_calls = self._run( + monkeypatch, + verdict="blocked", + reason="the target repository does not exist", + ) + err = capsys.readouterr().err + assert rc != 0, "blocked verdict must reject the completion" + assert complete_calls == [], "an unachievable goal must never reach complete_task" + assert "unachievable" in err.lower() + assert "kanban block" in err.lower() diff --git a/tests/hermes_cli/test_kanban_notify.py b/tests/hermes_cli/test_kanban_notify.py index ec01f5a5d3..3eea0a1920 100644 --- a/tests/hermes_cli/test_kanban_notify.py +++ b/tests/hermes_cli/test_kanban_notify.py @@ -1113,6 +1113,34 @@ def test_gc_spares_reopened_task_even_when_old(kanban_home): conn.close() +def _set_task_status(kb, conn, tid, status): + """Force a task into ``status`` with a matching status event.""" + with kb.write_txn(conn): + conn.execute("UPDATE tasks SET status = ? WHERE id = ?", (status, tid)) + kb._append_event(conn, tid, "status", {"status": status}) + + +def test_gc_purges_blocked_task_that_never_done(kanban_home): + import hermes_cli.kanban_db as kb + + conn = kb.connect() + try: + tid = kb.create_task(conn, title="stuck blocked", assignee="worker1") + kb.add_notify_sub( + conn, task_id=tid, platform="telegram", chat_id="c-blocked", + notifier_profile="default", + ) + _set_task_status(kb, conn, tid, "blocked") + _backdate_task(kb, conn, tid, days=45) + + purged = kb.purge_stale_done_notify_subs(conn, max_age_days=30) + + assert purged == 1 + assert kb.list_notify_subs(conn, tid) == [] + finally: + conn.close() + + def test_gc_archived_rows_already_removed_by_unsub(kanban_home): import hermes_cli.kanban_db as kb diff --git a/tests/hermes_cli/test_linux_desktop_entry.py b/tests/hermes_cli/test_linux_desktop_entry.py index 4485f3ab84..cc7b0c4306 100644 --- a/tests/hermes_cli/test_linux_desktop_entry.py +++ b/tests/hermes_cli/test_linux_desktop_entry.py @@ -2,7 +2,10 @@ from __future__ import annotations +import io +import os import stat +import struct import sys from pathlib import Path @@ -28,6 +31,26 @@ def _make_project(tmp_path: Path) -> Path: return root +def _png_ihdr(width: int, height: int) -> bytes: + """Minimal PNG prefix whose IHDR the installer can parse (no pixels).""" + return ( + b"\x89PNG\r\n\x1a\n" + + b"\x00\x00\x00\r" + + b"IHDR" + + struct.pack(">II", width, height) + ) + + +def _stub_install(tmp_path, monkeypatch) -> None: + hermes_bin = tmp_path / "bin" / "hermes" + hermes_bin.parent.mkdir(exist_ok=True) + hermes_bin.write_text("", encoding="utf-8") + monkeypatch.setattr( + "hermes_cli.relaunch.resolve_hermes_bin", lambda: str(hermes_bin) + ) + monkeypatch.setattr(lde, "refresh_desktop_databases", lambda _dir: []) + + def _parse(entry_text: str) -> dict: values = {} for line in entry_text.splitlines(): @@ -100,8 +123,8 @@ def test_install_prefers_themed_icon_from_hicolor(tmp_path, xdg_home, monkeypatc # And the icon really landed in the hicolor tree: the fixture icon is # a fake PNG (no valid IHDR), so the size is unknown and the icon - # lands under scalable/. - dest = xdg_home / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png" + # lands under 256x256/ (indexed; never scalable, which is SVG-only). + dest = xdg_home / "icons" / "hicolor" / "256x256" / "apps" / "hermes.png" assert dest.is_file() assert dest.read_bytes() == lde.icon_path(root).read_bytes() @pytest.mark.platforms("linux") @@ -995,8 +1018,8 @@ def test_probe_accepts_shell_launcher_wrapper(tmp_path, xdg_home, monkeypatch): def test_install_icon_handles_truncated_png_header(tmp_path, xdg_home, monkeypatch): """A truncated PNG (valid signature + IHDR tag, <24 bytes) must not - raise struct.error out of the fail-safe: it lands in scalable/ like - any other unknown-size image.""" + raise struct.error out of the fail-safe: it lands in 256x256/ like + any other unknown-size raster.""" root = _make_project(tmp_path) icon = lde.icon_path(root) icon.write_bytes( @@ -1014,5 +1037,96 @@ def test_install_icon_handles_truncated_png_header(tmp_path, xdg_home, monkeypat values = _parse(entry.read_text(encoding="utf-8")) assert values["Icon"] == "hermes" - dest = xdg_home / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png" + dest = xdg_home / "icons" / "hicolor" / "256x256" / "apps" / "hermes.png" assert dest.is_file() + + +def test_hicolor_subdir_puts_rasters_in_indexed_dirs_never_scalable(): + """Panel lookup uses fixed sizes. scalable/ is SVG-only.""" + assert lde._hicolor_subdir(None) == "256x256" + assert lde._hicolor_subdir((1024, 1024)) == "256x256" + assert lde._hicolor_subdir((512, 512)) == "512x512" + assert lde._hicolor_subdir((256, 256)) == "256x256" + assert lde._hicolor_subdir((48, 48)) == "48x48" + assert lde._hicolor_subdir((24, 24)) == "24x24" + assert lde._hicolor_subdir((64, 32)) == "256x256" + + +def test_install_places_1024_png_in_256x256_not_scalable( + tmp_path, xdg_home, monkeypatch +): + """The shipped desktop asset is 1024×1024. A PNG in scalable/ is what + Cinnamon's panel rasterizes as a mangled low-res icon.""" + root = _make_project(tmp_path) + lde.icon_path(root).write_bytes(_png_ihdr(1024, 1024)) + _stub_install(tmp_path, monkeypatch) + + entry = lde.install_desktop_entry(root) + values = _parse(entry.read_text(encoding="utf-8")) + + dest = xdg_home / "icons" / "hicolor" / "256x256" / "apps" / "hermes.png" + stale = xdg_home / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png" + assert values["Icon"] == "hermes" + assert dest.is_file() + assert dest.read_bytes() == lde.icon_path(root).read_bytes() + assert not stale.exists() + + +def test_install_removes_stale_scalable_png(tmp_path, xdg_home, monkeypatch): + """v2026.8.31 wrote the PNG into scalable/. A later hermes desktop + must delete that leftover so Cinnamon does not keep using it.""" + root = _make_project(tmp_path) + lde.icon_path(root).write_bytes(_png_ihdr(1024, 1024)) + _stub_install(tmp_path, monkeypatch) + + stale = xdg_home / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png" + stale.parent.mkdir(parents=True) + stale.write_bytes(b"old scalable png") + + lde.install_desktop_entry(root) + + dest = xdg_home / "icons" / "hicolor" / "256x256" / "apps" / "hermes.png" + assert dest.is_file() + assert not stale.exists() + + +def test_install_exact_48_png_uses_48x48_dir(tmp_path, xdg_home, monkeypatch): + root = _make_project(tmp_path) + lde.icon_path(root).write_bytes(_png_ihdr(48, 48)) + _stub_install(tmp_path, monkeypatch) + + lde.install_desktop_entry(root) + + dest = xdg_home / "icons" / "hicolor" / "48x48" / "apps" / "hermes.png" + assert dest.is_file() + assert not ( + xdg_home / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png" + ).exists() + + +def test_install_resizes_decodable_png_to_panel_sizes( + tmp_path, xdg_home, monkeypatch +): + """A decodeable PNG is Lanczos-resized so the 24px slot is actually 24px.""" + from PIL import Image + + root = _make_project(tmp_path) + im = Image.new("RGBA", (64, 64), (255, 255, 255, 255)) + for x in range(16, 48): + for y in range(16, 48): + im.putpixel((x, y), (0, 0, 0, 255)) + buf = io.BytesIO() + im.save(buf, format="PNG") + lde.icon_path(root).write_bytes(buf.getvalue()) + _stub_install(tmp_path, monkeypatch) + + lde.install_desktop_entry(root) + + dest_24 = xdg_home / "icons" / "hicolor" / "24x24" / "apps" / "hermes.png" + dest_256 = xdg_home / "icons" / "hicolor" / "256x256" / "apps" / "hermes.png" + stale = xdg_home / "icons" / "hicolor" / "scalable" / "apps" / "hermes.png" + assert dest_24.is_file() + assert dest_256.is_file() + assert not stale.exists() + assert struct.unpack(">II", dest_24.read_bytes()[16:24]) == (24, 24) + assert struct.unpack(">II", dest_256.read_bytes()[16:24]) == (256, 256) diff --git a/tests/hermes_cli/test_loops.py b/tests/hermes_cli/test_loops.py index 3a630c4206..a99f9f6a0b 100644 --- a/tests/hermes_cli/test_loops.py +++ b/tests/hermes_cli/test_loops.py @@ -443,6 +443,20 @@ class TestTickLifecycle: decision = mgr.complete_tick("3 tests still failing") assert decision["stopped"] is False + def test_until_judge_blocked_pauses(self, hermes_home): + """An unachievable stop condition pauses the loop instead of spinning to the tick budget.""" + from hermes_cli.loops import LoopManager + + mgr = LoopManager(session_id="t11b") + state = mgr.set("poll", interval_seconds=300, until="the deleted repo's CI is green") + state.next_due_at = time.time() - 1 + mgr.fire_tick() + with patch("hermes_cli.goals.judge_goal", return_value=("blocked", "repo no longer exists", False, None, False)): + decision = mgr.complete_tick("The repository was deleted; there is no CI to watch.") + assert decision["stopped"] is True + assert decision["status"] == "paused" + assert "unachievable" in decision["message"] + def test_until_judge_error_fails_open(self, hermes_home): from hermes_cli.loops import LoopManager diff --git a/tests/hermes_cli/test_mcp_startup.py b/tests/hermes_cli/test_mcp_startup.py index 9c4b94a182..f9be0410ac 100644 --- a/tests/hermes_cli/test_mcp_startup.py +++ b/tests/hermes_cli/test_mcp_startup.py @@ -102,6 +102,96 @@ def test_prepare_agent_startup_backgrounds_blocking_mcp_for_chat(monkeypatch): stop.set() +def test_prepare_agent_startup_skips_discovery_when_chat_resolves_to_tui( + monkeypatch, +): + """Bare ``hermes`` / ``hermes chat`` on a TTY with ``display.interface: + tui`` resolves to the TUI via ``_resolve_use_tui``, but does NOT pass + ``--tui`` or ``HERMES_TUI``. Discovery must be skipped in the wrapper: + the TUI gateway owns it, and the wrapper would otherwise hold a dead + MCP server for the entire session (3 copies per TUI instance). + """ + calls = {"background": 0, "inline": 0} + + monkeypatch.setattr(main_mod, "_resolve_use_tui", lambda _args: True) + monkeypatch.setattr( + mcp_startup, + "start_background_mcp_discovery", + lambda **_kwargs: calls.__setitem__("background", calls["background"] + 1), + ) + monkeypatch.setitem( + sys.modules, + "hermes_cli.plugins", + types.SimpleNamespace(discover_plugins=lambda: None), + ) + monkeypatch.setitem( + sys.modules, + "hermes_cli.config", + types.SimpleNamespace( + read_raw_config=lambda: {"mcp_servers": {"demo": {"transport": "stdio"}}}, + load_config=lambda: {}, + ), + ) + monkeypatch.setitem( + sys.modules, + "agent.shell_hooks", + types.SimpleNamespace(register_from_config=lambda *_a, **_k: None), + ) + monkeypatch.setitem( + sys.modules, + "tools.mcp_tool", + types.SimpleNamespace( + discover_mcp_tools=lambda: calls.__setitem__("inline", calls["inline"] + 1), + ), + ) + + main_mod._prepare_agent_startup(_agent_args(command=None)) + + assert calls["background"] == 0 + assert calls["inline"] == 0 + assert mcp_startup._mcp_discovery_thread is None + + +def test_prepare_agent_startup_keeps_discovery_for_non_chat_commands( + monkeypatch, +): + """Non-chat commands never launch the TUI, so they must keep their own + MCP discovery even when the ambient display config resolves to TUI — + ``_is_tui_chat_launch`` must not consult ``_resolve_use_tui`` there.""" + calls = {"inline": 0} + + monkeypatch.setattr(main_mod, "_resolve_use_tui", lambda _args: True) + monkeypatch.setitem( + sys.modules, + "hermes_cli.plugins", + types.SimpleNamespace(discover_plugins=lambda: None), + ) + monkeypatch.setitem( + sys.modules, + "hermes_cli.config", + types.SimpleNamespace( + read_raw_config=lambda: {"mcp_servers": {"demo": {"transport": "stdio"}}}, + load_config=lambda: {}, + ), + ) + monkeypatch.setitem( + sys.modules, + "agent.shell_hooks", + types.SimpleNamespace(register_from_config=lambda *_a, **_k: None), + ) + monkeypatch.setitem( + sys.modules, + "tools.mcp_tool", + types.SimpleNamespace( + discover_mcp_tools=lambda: calls.__setitem__("inline", calls["inline"] + 1), + ), + ) + + main_mod._prepare_agent_startup(_agent_args(command="mcp", mcp_action="serve")) + + assert calls["inline"] == 1 + + def test_background_mcp_discovery_suppresses_interactive_oauth(monkeypatch): state = {"active": False, "during_discover": None} @@ -199,3 +289,108 @@ def _install_retry_stubs(monkeypatch, *, connected: bool, calls: dict): ) + + +# --- -t/--toolsets MCP spawn filter (#19000) -------------------------------- + + +@pytest.fixture +def _reset_mcp_server_filter(): + saved = mcp_startup._mcp_server_filter + try: + yield + finally: + mcp_startup._mcp_server_filter = saved + + +@pytest.mark.parametrize( + ("toolsets", "expected"), + [ + (None, None), + ("", None), + ("all", None), + (["*"], None), + ("terminal,web", ["terminal", "web"]), + (["terminal", "code-mcp,web"], ["terminal", "code-mcp", "web"]), + ], +) +def test_set_mcp_server_filter_normalizes(_reset_mcp_server_filter, toolsets, expected): + assert mcp_startup.set_mcp_server_filter(toolsets) == expected + assert mcp_startup.get_mcp_server_filter() == expected + + +def test_discover_mcp_tools_spawns_only_allowed_servers(monkeypatch): + """The filter must narrow the spawn set before any server is connected; + built-in toolset names in the list are ignored.""" + from tools import mcp_tool + + servers = { + "code-mcp": {"command": "true"}, + "docs-mcp": {"command": "true"}, + } + seen: dict[str, dict] = {} + + sdk_probes = {"n": 0} + + def _fake_ensure_sdk(): + sdk_probes["n"] += 1 + return True + + monkeypatch.setattr(mcp_tool, "_load_mcp_config", lambda: dict(servers)) + monkeypatch.setattr(mcp_tool, "_ensure_mcp_sdk", _fake_ensure_sdk) + monkeypatch.setattr(mcp_tool, "_try_acquire_mcp_discovery_lock", lambda: mcp_tool._LOCK_UNAVAILABLE) + monkeypatch.setattr(mcp_tool, "_release_mcp_discovery_lock", lambda *_a, **_k: None, raising=False) + + def _fake_register(cfgs): + seen.update(cfgs) + return [] + + monkeypatch.setattr(mcp_tool, "register_mcp_servers", _fake_register) + monkeypatch.setattr(mcp_tool, "_servers", {}) + monkeypatch.setattr(mcp_tool, "_server_connecting", set()) + + # Everything (no filter) — both would be registered. + mcp_tool.discover_mcp_tools() + assert set(seen) == {"code-mcp", "docs-mcp"} + + # `-t terminal,code-mcp` — only the matching server; "terminal" is a no-op. + seen.clear() + mcp_tool.discover_mcp_tools(allowed_mcp_names=["terminal", "code-mcp"]) + assert set(seen) == {"code-mcp"} + + # `-t terminal` — no MCP server in the filter: skip the whole MCP load, + # including the ~260ms `mcp` SDK import. + seen.clear() + sdk_probes["n"] = 0 + assert mcp_tool.discover_mcp_tools(allowed_mcp_names=["terminal"]) == [] + assert seen == {} + assert sdk_probes["n"] == 0 + + +def test_background_discovery_honors_server_filter(monkeypatch, _reset_mcp_server_filter): + calls: list = [] + monkeypatch.setitem( + sys.modules, + "tools.mcp_tool", + types.SimpleNamespace(discover_mcp_tools=lambda allowed_mcp_names=None: calls.append(allowed_mcp_names)), + ) + monkeypatch.setitem( + sys.modules, + "tools.mcp_oauth", + types.SimpleNamespace(suppress_interactive_oauth=nullcontext), + ) + mcp_startup.set_mcp_server_filter("terminal,code-mcp") + mcp_startup._discover_mcp_tools_without_interactive_oauth() + assert calls == [["terminal", "code-mcp"]] + + +def test_prepare_agent_startup_installs_server_filter(monkeypatch, _reset_mcp_server_filter): + monkeypatch.setitem( + sys.modules, + "hermes_cli.plugins", + types.SimpleNamespace(discover_plugins=lambda: None), + ) + monkeypatch.setattr(main_mod, "_should_background_mcp_startup", lambda args: False) + monkeypatch.setattr(main_mod, "_command_has_dedicated_mcp_startup", lambda args: True) + main_mod._prepare_agent_startup(_agent_args(toolsets="terminal,code-mcp")) + assert mcp_startup.get_mcp_server_filter() == ["terminal", "code-mcp"] diff --git a/tests/hermes_cli/test_model_catalog.py b/tests/hermes_cli/test_model_catalog.py index b4d8e8a40a..3e9c1844ff 100644 --- a/tests/hermes_cli/test_model_catalog.py +++ b/tests/hermes_cli/test_model_catalog.py @@ -307,6 +307,30 @@ class TestProviderOverride: assert result == [("override/model", "custom")] +class TestRefreshCadence: + def test_default_ttl_is_twenty_minutes_and_legacy_hours_honoured(self): + from hermes_cli import model_catalog + + with patch("hermes_cli.config.load_config", return_value={"model_catalog": {"ttl_minutes": 20}}): + assert model_catalog.refresh_interval_seconds() == 20 * 60 + # A user-set legacy ttl_hours still wins while ttl_minutes sits at its default. + with patch("hermes_cli.config.load_config", return_value={"model_catalog": {"ttl_minutes": 20, "ttl_hours": 3}}): + assert model_catalog.refresh_interval_seconds() == 3 * 3600 + + def test_refresh_catalogs_forces_every_source(self): + from hermes_cli import model_catalog + + with patch.object(model_catalog, "_load_catalog_config", return_value={ + "enabled": True, "url": "http://master", "ttl_hours": 1.0, "providers": {}, + }), patch.object(model_catalog, "get_catalog", return_value=_valid_manifest()) as gc, \ + patch("hermes_cli.models.fetch_openrouter_models") as orm, \ + patch("hermes_cli.models.fetch_nous_recommended_models") as nous: + assert model_catalog.refresh_catalogs() is True + gc.assert_called_once_with(force_refresh=True) + orm.assert_called_once_with(force_refresh=True) + nous.assert_called_once_with(force_refresh=True) + + class TestIntegrationWithModelsModule: """Exercise the fallback paths via the real callers in hermes_cli.models.""" diff --git a/tests/hermes_cli/test_model_data_policy_guard.py b/tests/hermes_cli/test_model_data_policy_guard.py index 493ccd4bb8..2546dba15d 100644 --- a/tests/hermes_cli/test_model_data_policy_guard.py +++ b/tests/hermes_cli/test_model_data_policy_guard.py @@ -11,9 +11,10 @@ def test_fires_on_meta_contributor(): assert isinstance(w, DataTrainingWarning) assert w.model == "muse-spark-1.2-contributor" assert "train" in w.message.lower() - assert "muse-spark-1.2" in w.message # points to the no-training alternative - # Aligns with Meta's own pricing doc language + figures. - assert "$0.10" in w.message and "$0.20" in w.message and "$0.002" in w.message + assert "-contributor" in w.message.lower() or "contributor" in w.message.lower() # mentions the tier + # Points to Meta's live pricing page instead of hardcoding prices. + assert "pricing and rate limits" in w.message.lower() + assert "$0.10" not in w.message and "$0.20" not in w.message and "$0.002" not in w.message assert "prompts and completions" in w.message.lower() assert "dev.meta.ai/docs/pricing-rate-limits" in w.message diff --git a/tests/hermes_cli/test_model_switch_persist_default.py b/tests/hermes_cli/test_model_switch_persist_default.py index 11394c4222..b53c8913e1 100644 --- a/tests/hermes_cli/test_model_switch_persist_default.py +++ b/tests/hermes_cli/test_model_switch_persist_default.py @@ -52,6 +52,24 @@ class TestResolvePersistBehavior: with _config({"model": {"persist_switch_by_default": True}}): assert resolve_persist_behavior(False, False, explicit_provider="") is True + def test_first_pick_persists_then_session_only(self): + # #90235 / #86414: the ONE policy every surface (CLI, gateway, Desktop + # picker) defers to. With no default ever configured, the first pick + # persists (even with --provider, which is how the Desktop picker + # always sends it) so resolve_provider never falls through to a stray + # env key on restart. Once a default exists, a plain pick is + # session-only unless --global / persist_switch_by_default. + with _config({"model": {}}): + assert resolve_persist_behavior(False, False, explicit_provider="anthropic") is True + with _config({"model": ""}): + assert resolve_persist_behavior(False, False) is True + with _config({"model": {"default": "gpt-5.6", "provider": "openai-codex"}}): + assert resolve_persist_behavior(False, False, explicit_provider="openai-api") is False + assert resolve_persist_behavior(False, False) is False + assert resolve_persist_behavior(True, False, explicit_provider="openai-api") is True + with _config({"model": "gpt-5.6"}): + assert resolve_persist_behavior(False, False) is False + # --------------------------------------------------------------------------- # helper diff --git a/tests/hermes_cli/test_models.py b/tests/hermes_cli/test_models.py index bba81a9ecc..855bc248a5 100644 --- a/tests/hermes_cli/test_models.py +++ b/tests/hermes_cli/test_models.py @@ -299,10 +299,10 @@ class TestCheckNousFreeTierCache: """Tests for the TTL cache on check_nous_free_tier().""" def setup_method(self): - _models_mod._free_tier_cache = None + _models_mod._free_tier_cache.clear() def teardown_method(self): - _models_mod._free_tier_cache = None + _models_mod._free_tier_cache.clear() @patch("hermes_cli.nous_account.get_nous_portal_account_info") def test_result_is_cached(self, mock_account): @@ -320,6 +320,42 @@ class TestCheckNousFreeTierCache: assert result2 is True assert mock_account.call_count == 1 + @patch("hermes_cli.nous_account.get_nous_portal_account_info") + def test_cache_only_cold_lookup_does_not_call_portal(self, mock_account): + assert check_nous_free_tier(cached_only=True) is False + mock_account.assert_not_called() + + @patch("hermes_cli.nous_account.get_nous_portal_account_info") + def test_entitlement_cache_is_profile_scoped(self, mock_account, tmp_path): + from hermes_constants import ( + hermes_home_key, + reset_hermes_home_override, + set_hermes_home_override, + ) + + def account_for_active_profile(*, force_fresh=False): + is_free = hermes_home_key() == hermes_home_key(tmp_path / "free") + return NousPortalAccountInfo( + logged_in=True, + source="jwt", + fresh=force_fresh, + paid_service_access=not is_free, + ) + + mock_account.side_effect = account_for_active_profile + + def check_in(home): + token = set_hermes_home_override(str(home)) + try: + return check_nous_free_tier() + finally: + reset_hermes_home_override(token) + + assert check_in(tmp_path / "free") is True + assert check_in(tmp_path / "paid") is False + assert check_in(tmp_path / "free") is True + assert mock_account.call_count == 2 + @patch("hermes_cli.nous_account.get_nous_portal_account_info") def test_force_fresh_bypasses_cache(self, mock_account): diff --git a/tests/hermes_cli/test_noninteractive_git.py b/tests/hermes_cli/test_noninteractive_git.py index 2b8e48b814..04843413c1 100644 --- a/tests/hermes_cli/test_noninteractive_git.py +++ b/tests/hermes_cli/test_noninteractive_git.py @@ -56,6 +56,47 @@ class TestNoninteractiveGitEnv: env = noninteractive_git_env({"GIT_TERMINAL_PROMPT": "1"}) assert env["GIT_TERMINAL_PROMPT"] == "0" + def test_strips_ambient_git_config_injection(self): + env = noninteractive_git_env( + { + "GIT_CONFIG_COUNT": "2", + "GIT_CONFIG_KEY_0": "core.pager", + "GIT_CONFIG_VALUE_0": "less", + "GIT_CONFIG_KEY_1": "core.hooksPath", + "GIT_CONFIG_VALUE_1": ".git/hooks", + "GIT_CONFIG_PARAMETERS": "'core.pager=less'", + } + ) + + assert env["GIT_CONFIG_COUNT"] != "2" + assert "GIT_CONFIG_PARAMETERS" not in env + values = { + env[f"GIT_CONFIG_KEY_{idx}"]: env[f"GIT_CONFIG_VALUE_{idx}"] + for idx in range(int(env["GIT_CONFIG_COUNT"])) + } + assert values["core.pager"] == "cat" + assert values["core.hooksPath"] == os.devnull + assert values["credential.helper"] == "" + + def test_disables_pagers_hooks_editors_and_user_config(self): + env = noninteractive_git_env({}) + values = { + env[f"GIT_CONFIG_KEY_{idx}"]: env[f"GIT_CONFIG_VALUE_{idx}"] + for idx in range(int(env["GIT_CONFIG_COUNT"])) + } + + assert env["GIT_CONFIG_GLOBAL"] == os.devnull + assert env["GIT_CONFIG_SYSTEM"] == os.devnull + assert env["GIT_CONFIG_NOSYSTEM"] == "1" + assert env["GIT_PAGER"] == "cat" + assert env["PAGER"] == "cat" + assert env["GIT_EDITOR"] == "true" + assert values["core.fsmonitor"] == "false" + assert values["core.hooksPath"] == os.devnull + assert values["core.editor"] == "true" + assert values["sequence.editor"] == "true" + assert values["diff.external"] == "" + # --------------------------------------------------------------------------- # 2. Real-git E2E: 401 remote fails fast instead of prompting diff --git a/tests/hermes_cli/test_nous_auth_keepalive.py b/tests/hermes_cli/test_nous_auth_keepalive.py index 9e633a1417..8d07f4da3b 100644 --- a/tests/hermes_cli/test_nous_auth_keepalive.py +++ b/tests/hermes_cli/test_nous_auth_keepalive.py @@ -1,5 +1,69 @@ from hermes_cli import nous_auth_keepalive as keepalive +# Both lifetimes have been observed on real installs. +OBSERVED_LIFETIMES_SECONDS = (3594, 899) + + +def test_refresh_always_fires_before_expiry_for_observed_lifetimes(): + """Simulate the tick schedule and assert no credential expires unrefreshed. + + This is the property that actually matters: for every lifetime, some tick + must decide to refresh while the credential is still valid. Ticking faster + alone does not guarantee it -- the refresh horizon has to cover the gap + between ticks too. + """ + for lifetime in OBSERVED_LIFETIMES_SECONDS: + tick = keepalive._tick_seconds( + keepalive.NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS, lifetime + ) + horizon = keepalive._refresh_horizon_seconds( + tick, keepalive.NOUS_INVOKE_JWT_MIN_TTL_SECONDS + ) + + # Walk the ticks and find the first one that refreshes. + refreshed_at = None + elapsed = 0 + while elapsed <= lifetime: + if lifetime - elapsed <= horizon: + refreshed_at = elapsed + break + elapsed += tick + + assert refreshed_at is not None, f"never refreshed for lifetime={lifetime}" + assert refreshed_at < lifetime, ( + f"refresh at {refreshed_at}s came at/after expiry {lifetime}s " + f"(tick={tick}, horizon={horizon})" + ) + + +def test_interval_precedence_and_disable(monkeypatch): + def _config(section): + monkeypatch.setattr(keepalive, "_nous_config", lambda: section) + + # An absent section leaves the module default in place. + _config({}) + assert ( + keepalive._interval_seconds(None) + == keepalive.NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS + ) + + _config({keepalive.NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY: 600}) + assert keepalive._interval_seconds(None) == 600 + # An explicit argument still outranks config.yaml. + assert keepalive._interval_seconds(300) == 300 + + # A malformed value falls back to the default rather than disabling. + _config({keepalive.NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY: "not-a-number"}) + assert ( + keepalive._interval_seconds(None) + == keepalive.NOUS_AUTH_KEEPALIVE_INTERVAL_SECONDS + ) + + # Zero remains the documented way to turn the keepalive off. + _config({keepalive.NOUS_AUTH_KEEPALIVE_INTERVAL_CONFIG_KEY: 0}) + assert keepalive._interval_seconds(None) == 0 + assert keepalive.start_nous_auth_keepalive() is None + def test_keepalive_refreshes_stale_pool_entry(monkeypatch): class _Entry: diff --git a/tests/hermes_cli/test_opencode_free_live_catalog.py b/tests/hermes_cli/test_opencode_free_live_catalog.py index 24fd7fc6a8..67ac93b64a 100644 --- a/tests/hermes_cli/test_opencode_free_live_catalog.py +++ b/tests/hermes_cli/test_opencode_free_live_catalog.py @@ -41,6 +41,7 @@ _LIVE_FREE_MODELS = [ "nemotron-3-ultra-free", "nemotron-3.5-lightning-free", "muse-spark-1.2-contributor-free", + "muse-spark-1.3-contributor-free", ] # The raw live /zen/v1/models dump also lists paid/subscription + KEYED-free IDs diff --git a/tests/hermes_cli/test_openrouter_reasoning_metadata.py b/tests/hermes_cli/test_openrouter_reasoning_metadata.py index 09517e0952..f4e6d5d620 100644 --- a/tests/hermes_cli/test_openrouter_reasoning_metadata.py +++ b/tests/hermes_cli/test_openrouter_reasoning_metadata.py @@ -266,3 +266,29 @@ class TestOpenRouterProfileClamp: ) # Unknown capability → passthrough unchanged (no silent downgrade). assert extra_body["reasoning"]["effort"] == "ultra" + + def test_disable_omitted_for_mandatory_route_kept_otherwise(self, monkeypatch): + """A reasoning-mandatory route 400s on ``{enabled: false}`` — omit it; + a route that can disable still gets the user's disable verbatim.""" + import hermes_cli.models as models_mod + from providers import get_provider_profile + + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", { + "z-ai/glm-5.3-flash": { + "supports_reasoning": True, + "supported_efforts": ["max", "high", "low"], + "mandatory": True, + }, + "z-ai/glm-5.1": {"supports_reasoning": True, "supported_efforts": None, "mandatory": False}, + }) + profile = get_provider_profile("openrouter") + for disable in ({"enabled": False}, {"enabled": True, "effort": "none"}): + extra_body, _top = profile.build_api_kwargs_extras( + reasoning_config=disable, supports_reasoning=True, model="z-ai/glm-5.3-flash", + ) + assert "reasoning" not in extra_body, disable + extra_body, _top = profile.build_api_kwargs_extras( + reasoning_config={"enabled": False}, supports_reasoning=True, model="z-ai/glm-5.1", + ) + assert extra_body["reasoning"] == {"enabled": False} diff --git a/tests/hermes_cli/test_platform_actions.py b/tests/hermes_cli/test_platform_actions.py index c32bdad9cd..8a7d66e05a 100644 --- a/tests/hermes_cli/test_platform_actions.py +++ b/tests/hermes_cli/test_platform_actions.py @@ -39,6 +39,17 @@ def _runner_with(adapters: dict): return patch("gateway.run._gateway_runner_ref", lambda: runner) +def _multiplex_runner_with(*, default: dict, profiles: dict, active_profile: str = "default"): + """A runner using the REAL GatewayAuthorizationMixin resolution ladder.""" + from gateway.authz_mixin import GatewayAuthorizationMixin + + runner = GatewayAuthorizationMixin.__new__(GatewayAuthorizationMixin) + runner.adapters = default + runner._profile_adapters = profiles + runner._active_profile_name = lambda: active_profile + return patch("gateway.run._gateway_runner_ref", lambda: runner) + + def _telegram_adapter(connected=True): a = MagicMock() a.platform = Platform.TELEGRAM @@ -264,6 +275,49 @@ class TestVerbRouting: assert result["error"] == "invalid_argument" +class TestMultiplexProfileRouting: + """A plugin acting during a secondary profile's turn must act through THAT + profile's adapter, never the default profile's — the fail-closed contract + of GatewayAuthorizationMixin._authorization_adapter (#85245).""" + + def test_secondary_profile_routes_to_its_own_adapter_not_default(self): + actions = PlatformActions("p") + default_adapter = _telegram_adapter() + team_b_adapter = _telegram_adapter() + with ( + _grant(True), + _multiplex_runner_with( + default={Platform.TELEGRAM: default_adapter}, + profiles={"team-b": {Platform.TELEGRAM: team_b_adapter}}, + ), + patch("hermes_cli.profiles.get_active_profile_name", return_value="team-b"), + ): + result = asyncio.run(actions.add_reaction("telegram", "1", "2", "x")) + assert result["ok"] is True + team_b_adapter._set_reaction.assert_awaited_once() + default_adapter._set_reaction.assert_not_awaited() + + @pytest.mark.parametrize( + "resolver", + [ + {"return_value": "team-b"}, # stamped profile, no registry entry + {"side_effect": RuntimeError("boom")}, # profile resolution itself fails + ], + ids=["no-registry-entry", "resolution-error"], + ) + def test_unresolvable_profile_fails_closed_never_default_bot(self, resolver): + actions = PlatformActions("p") + default_adapter = _telegram_adapter() + with ( + _grant(True), + _multiplex_runner_with(default={Platform.TELEGRAM: default_adapter}, profiles={}), + patch("hermes_cli.profiles.get_active_profile_name", **resolver), + ): + result = asyncio.run(actions.add_reaction("telegram", "1", "2", "x")) + assert result["error"] == "adapter_not_registered" + default_adapter._set_reaction.assert_not_awaited() + + class TestPluginContextWiring: def test_ctx_platform_actions_bound_to_plugin_id(self): from hermes_cli.plugins import PluginContext, PluginManager, PluginManifest diff --git a/tests/hermes_cli/test_plugins.py b/tests/hermes_cli/test_plugins.py index 5ae4d5aee1..d44abafb4f 100644 --- a/tests/hermes_cli/test_plugins.py +++ b/tests/hermes_cli/test_plugins.py @@ -933,6 +933,13 @@ class TestDeliveryParity: class TestForceReloadSymmetry: """Force rediscovery restores non-plugin state it wiped (#64178).""" + @pytest.fixture(autouse=True) + def _cleanup_shell_hook_registry(self): + yield + import agent.shell_hooks as shell_hooks_mod + + shell_hooks_mod.reset_for_tests() + def test_force_reload_re_registers_shell_hooks(self, monkeypatch): """config.yaml shell hooks are re-wired after force=True (#60036).""" calls = [] @@ -992,6 +999,7 @@ class TestForceReloadSymmetry: def test_re_register_config_hooks_clears_idempotence_set(self, monkeypatch): import agent.shell_hooks as shell_hooks_mod + from hermes_constants import get_hermes_home recorded = {} monkeypatch.setattr( @@ -1002,8 +1010,9 @@ class TestForceReloadSymmetry: monkeypatch.setattr( "hermes_cli.config.load_config", lambda: {"hooks": {}} ) + home_key = str(get_hermes_home().expanduser().resolve()) with shell_hooks_mod._registered_lock: - shell_hooks_mod._registered.add(("post_llm_call", None, "echo hi")) + shell_hooks_mod._registered.add((home_key, "post_llm_call", None, "echo hi")) shell_hooks_mod.re_register_config_hooks() @@ -1219,6 +1228,50 @@ class TestForceReloadSymmetry: assert _PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE in result hold.set() + def test_force_reload_of_one_profile_does_not_orphan_another(self, monkeypatch): + """Real two-manager regression: force-reloading profile A's plugin + manager must leave profile B's shell hook registered exactly once — + not duplicated, not dropped (#92682 review). + """ + import hermes_cli.plugins as plugins_mod + import agent.shell_hooks as shell_hooks_mod + + cfg = {"hooks": {"on_session_start": [{"command": "/bin/true"}]}} + monkeypatch.setenv("HERMES_ACCEPT_HOOKS", "1") + monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg) + monkeypatch.setattr( + PluginManager, "_discover_and_load_inner", lambda self_inner: None, + ) + + monkeypatch.setenv("HERMES_HOME", "/tmp/profile-a") + mgr_a = PluginManager() + plugins_mod._plugin_manager = mgr_a + shell_hooks_mod.register_from_config(cfg, accept_hooks=True) + + monkeypatch.setenv("HERMES_HOME", "/tmp/profile-b") + mgr_b = PluginManager() + plugins_mod._plugin_manager = mgr_b + shell_hooks_mod.register_from_config(cfg, accept_hooks=True) + + assert len(mgr_a._hooks.get("on_session_start", [])) == 1 + assert len(mgr_b._hooks.get("on_session_start", [])) == 1 + + # Force-reload A. Its own manager's hook is wiped and restored; + # B's manager (and idempotence key) must be untouched. + mgr_a.discover_and_load(force=True) + + assert len(mgr_a._hooks.get("on_session_start", [])) == 1 + assert len(mgr_b._hooks.get("on_session_start", [])) == 1 + + # B's later adapter reconnect re-runs register_from_config(); its + # idempotence key must still be intact, so this must be a no-op + # rather than appending a second callback to B's live manager. + monkeypatch.setenv("HERMES_HOME", "/tmp/profile-b") + second = shell_hooks_mod.register_from_config(cfg, accept_hooks=True) + + assert second == [] + assert len(mgr_b._hooks.get("on_session_start", [])) == 1 + class TestPreToolCallBlocking: """Tests for the pre_tool_call block directive helper.""" diff --git a/tests/hermes_cli/test_profiles_sidebar_cache.py b/tests/hermes_cli/test_profiles_sidebar_cache.py index 5bb113a028..ead0c6b08b 100644 --- a/tests/hermes_cli/test_profiles_sidebar_cache.py +++ b/tests/hermes_cli/test_profiles_sidebar_cache.py @@ -128,6 +128,30 @@ class SidebarCacheTests(unittest.TestCase): self.assertEqual(scan(), {"ok": True}) self.assertEqual(calls, 2) + def test_does_not_cache_payloads_that_carry_profile_errors(self): + # A 200 with a non-empty errors[] is how a failed profile scan is + # reported. Caching it for the TTL keeps the empty recents page in + # front of a store that has already recovered. + calls = 0 + + @profiles._sidebar_singleflight_cache + def scan(): + nonlocal calls + calls += 1 + if calls == 1: + return { + "errors": [{"profile": "default", "error": "disk I/O error"}], + "recents": {"sessions": []}, + } + return {"errors": [], "recents": {"sessions": [{"id": "yesterday"}]}} + + first = scan() + second = scan() + + self.assertEqual(first["errors"][0]["error"], "disk I/O error") + self.assertEqual(second["recents"]["sessions"], [{"id": "yesterday"}]) + self.assertEqual(calls, 2) + def test_can_be_disabled(self): calls = 0 diff --git a/tests/hermes_cli/test_proxy_off_loop.py b/tests/hermes_cli/test_proxy_off_loop.py new file mode 100644 index 0000000000..257331055a --- /dev/null +++ b/tests/hermes_cli/test_proxy_off_loop.py @@ -0,0 +1,523 @@ +"""`hermes proxy` must resolve upstream credentials off the event loop. + +``UpstreamAdapter`` is a synchronous contract (``adapters/base.py`` — every +method is a plain ``def``), and both shipped adapters implement it with +blocking I/O: + + * ``NousPortalAdapter.get_credential`` takes ``_auth_store_lock()``, a + *cross-process* advisory lock with ``AUTH_LOCK_TIMEOUT_SECONDS = 15.0`` + (``hermes_cli/auth.py:110``), reads ``auth.json`` from disk, and may issue a + token-refresh POST. Its terminal-error path takes that lock a second time to + persist the quarantined state. + * ``NousPortalAdapter.get_retry_credential`` routes to that same + ``_get_credential`` with ``force_refresh=True``, so the refresh POST it only + *may* perform above is unconditional here. + * ``XAIGrokAdapter`` reads its key pool off disk under a ``threading.Lock``. + Its ``get_retry_credential`` loads the pool and calls + ``try_refresh_current`` / ``mark_exhausted_and_rotate`` under that lock. + +``create_app`` registers two ``async def`` handlers, so calling those methods +directly from a handler freezes the proxy's single event loop — and with it +every other in-flight streaming completion — for the whole duration. + +The primary assertions here are **thread identity**, not latency. A latency +assertion measured with an HTTP client on the blocked loop is vacuous: the +client's own timer cannot advance until the block ends, so it reports a fast +response on code that was provably frozen. Thread identity has no such failure +mode and no timing sensitivity. + +The harness mirrors ``tests/hermes_cli/test_proxy.py``: the proxy and a fake +upstream run as real aiohttp servers on ephemeral ports, driven by +``asyncio.run``. That keeps everything on exactly one event loop, which is what +makes the loop-starvation observations meaningful, and it avoids taking a +pytest-aiohttp dependency for one test file. +""" + +from __future__ import annotations + +import asyncio +import threading +import time +from typing import Any, Dict, List, Optional + +import pytest + +from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential + +aiohttp = pytest.importorskip("aiohttp") +from aiohttp import web # noqa: E402 + +from hermes_cli.proxy.server import create_app # noqa: E402 + + +# How long the fake adapter blocks. Long enough that a starved loop records +# zero heartbeats, short enough to keep the suite fast. The thread-identity +# assertions do not depend on this value at all. +_STALL_SECONDS = 0.5 + +# Heartbeat cadence. A healthy loop fires ~50 ticks across the stall above; a +# blocked one fires exactly 0, so the threshold has three orders of magnitude +# of headroom on a loaded runner. +_HEARTBEAT_INTERVAL = 0.01 +_MIN_TICKS_ACROSS_STALL = 3 + + +class _RecordingAdapter(UpstreamAdapter): + """Adapter that records the thread each blocking call ran on. + + ``get_credential`` and ``is_authenticated`` are plain synchronous methods + that sleep, standing in for the auth-store lock and token refresh under the + real adapters. Each also samples the loop-heartbeat counter on entry and + exit, so ``ticks_across_*`` is the number of loop iterations that got to run + *while the adapter was blocking*. + """ + + def __init__( + self, + base_url: str, + *, + stall: float = 0.0, + ticks: Optional[List[int]] = None, + raise_on_credential: bool = False, + retry_bearer: Optional[str] = None, + raise_on_retry: bool = False, + ) -> None: + self._base_url = base_url + self._stall = stall + self._ticks = ticks if ticks is not None else [0] + self._raise_on_credential = raise_on_credential + self._retry_bearer = retry_bearer + self._raise_on_retry = raise_on_retry + self.credential_thread: Optional[int] = None + self.authenticated_thread: Optional[int] = None + self.retry_thread: Optional[int] = None + self.retry_status_code: Optional[int] = None + self.ticks_across_credential: Optional[int] = None + self.ticks_across_is_authenticated: Optional[int] = None + self.ticks_across_retry: Optional[int] = None + + @property + def name(self) -> str: + return "recording" + + @property + def display_name(self) -> str: + return "Recording Provider" + + @property + def allowed_paths(self): + return frozenset({"/chat/completions"}) + + def is_authenticated(self) -> bool: + self.authenticated_thread = threading.get_ident() + before = self._ticks[0] + if self._stall: + time.sleep(self._stall) + self.ticks_across_is_authenticated = self._ticks[0] - before + return True + + def get_credential(self) -> UpstreamCredential: + self.credential_thread = threading.get_ident() + before = self._ticks[0] + if self._stall: + time.sleep(self._stall) + self.ticks_across_credential = self._ticks[0] - before + if self._raise_on_credential: + raise RuntimeError("simulated auth failure") + return UpstreamCredential( + bearer="test-bearer", + base_url=self._base_url, + expires_at="2099-01-01T00:00:00Z", + ) + + def get_retry_credential(self, *, failed_credential, status_code): + _ = failed_credential + self.retry_thread = threading.get_ident() + self.retry_status_code = status_code + before = self._ticks[0] + if self._stall: + time.sleep(self._stall) + self.ticks_across_retry = self._ticks[0] - before + if self._raise_on_retry: + raise RuntimeError("simulated retry-credential failure") + if self._retry_bearer is None: + return None + return UpstreamCredential( + bearer=self._retry_bearer, + base_url=self._base_url, + expires_at="2099-01-01T00:00:00Z", + ) + + +async def _start_runner(app: "web.Application"): + """Spin up an aiohttp app on an ephemeral localhost port. Returns (runner, base_url).""" + runner = web.AppRunner(app, access_log=None) + await runner.setup() + site = web.TCPSite(runner, host="127.0.0.1", port=0) + await site.start() + sockets = list(site._server.sockets) # type: ignore[union-attr] + port = sockets[0].getsockname()[1] + return runner, f"http://127.0.0.1:{port}" + + +def _build_fake_upstream(captured: Dict[str, Any]) -> "web.Application": + async def echo(request): + body = await request.read() + captured["requests"].append( + {"path": request.path, "auth": request.headers.get("Authorization")} + ) + return web.json_response({"echoed": True, "body": body.decode("utf-8") if body else ""}) + + app = web.Application() + app.router.add_route("*", "/v1/chat/completions", echo) + return app + + +def _build_rejecting_upstream( + captured: Dict[str, Any], *, reject_status: int, accept_bearer: str +) -> "web.Application": + """Upstream that rejects every bearer except ``accept_bearer``. + + Drives ``handle_proxy``'s ``status in {401, 429}`` branch: the first forward + carries the initial credential and comes back rejected, the retry carries the + rotated one and succeeds. + """ + + async def gated(request): + auth = request.headers.get("Authorization") + captured["requests"].append({"path": request.path, "auth": auth}) + if auth != f"Bearer {accept_bearer}": + return web.json_response( + {"error": {"message": "rejected"}}, status=reject_status + ) + return web.json_response({"echoed": True}) + + app = web.Application() + app.router.add_route("*", "/v1/chat/completions", gated) + return app + + +async def _heartbeat(ticks: List[int], running: List[bool]) -> None: + """Tick a counter on the event loop until told to stop.""" + while running[0]: + ticks[0] += 1 + await asyncio.sleep(_HEARTBEAT_INTERVAL) + + +# --------------------------------------------------------------------------- +# handle_proxy -> get_credential +# --------------------------------------------------------------------------- + + +def test_get_credential_runs_off_the_event_loop(): + """The blocking credential resolution must not execute on the loop thread. + + On the unfixed handler ``adapter.get_credential()`` is called inline, so the + recorded thread is the loop's own and this assertion fails. + """ + async def run(): + loop_thread = threading.get_ident() + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured)) + adapter = _RecordingAdapter(f"{upstream_base}/v1") + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", json={} + ) as resp: + assert resp.status == 200 + await resp.read() + + assert adapter.credential_thread is not None, "get_credential was never called" + assert adapter.credential_thread != loop_thread, ( + "get_credential ran on the event-loop thread " + f"({adapter.credential_thread}); it blocks on a cross-process " + "auth-store lock and must be offloaded" + ) + # The forward itself still worked, with our bearer attached. + assert captured["requests"][0]["auth"] == "Bearer test-bearer" + finally: + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + +def test_event_loop_keeps_running_while_credentials_resolve(): + """A stalled credential resolution must not starve the rest of the loop. + + Measured from a heartbeat task *on the loop*, sampled by the adapter itself + on entry and exit — not from an HTTP client, whose clock cannot advance + while the loop is blocked and which would therefore report a false pass. + """ + async def run(): + ticks = [0] + running = [True] + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured)) + adapter = _RecordingAdapter( + f"{upstream_base}/v1", stall=_STALL_SECONDS, ticks=ticks + ) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + beat = asyncio.create_task(_heartbeat(ticks, running)) + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", json={} + ) as resp: + await resp.read() + + assert adapter.ticks_across_credential is not None + assert adapter.ticks_across_credential >= _MIN_TICKS_ACROSS_STALL, ( + f"only {adapter.ticks_across_credential} loop iterations ran during a " + f"{_STALL_SECONDS}s credential resolution — the event loop was frozen" + ) + finally: + running[0] = False + beat.cancel() + await asyncio.gather(beat, return_exceptions=True) + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + +def test_credential_failure_still_maps_to_401(): + """Offloading must not change the error contract. + + ``asyncio.to_thread`` re-raises the worker's exception in the awaiting + frame, so the handler's existing ``except Exception`` still produces the + ``upstream_auth_failed`` 401. This one is deliberately *not* in the + red-before set — it guards the behaviour the fix must leave alone. + """ + async def run(): + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured)) + adapter = _RecordingAdapter(f"{upstream_base}/v1", raise_on_credential=True) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", json={} + ) as resp: + assert resp.status == 401 + payload = await resp.json() + + assert payload["error"]["code"] == "upstream_auth_failed" + assert "simulated auth failure" in payload["error"]["message"] + # The request never reached the upstream. + assert captured["requests"] == [] + finally: + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# handle_proxy -> get_retry_credential (the 401/429 rotation path) +# --------------------------------------------------------------------------- + + +def test_get_retry_credential_runs_off_the_event_loop(): + """The 401/429 rotation must not resolve its credential on the loop thread. + + ``get_retry_credential`` is the third and last blocking method on the + ``UpstreamAdapter`` contract, and it is the most expensive of them: + ``NousPortalAdapter`` routes it into ``_get_credential(force_refresh=True)``, + so the token-refresh POST that ``get_credential`` performs only near expiry + is unconditional here, and it happens under the same 15s cross-process + ``_auth_store_lock()``. ``XAIGrokAdapter`` loads its key pool off disk and + rotates it under ``self._lock``. + + Called inline, that whole rotation runs on the loop thread and this + assertion fails. + """ + async def run(): + loop_thread = threading.get_ident() + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner( + _build_rejecting_upstream( + captured, reject_status=401, accept_bearer="rotated-bearer" + ) + ) + adapter = _RecordingAdapter( + f"{upstream_base}/v1", retry_bearer="rotated-bearer" + ) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", json={} + ) as resp: + assert resp.status == 200 + await resp.read() + + assert adapter.retry_thread is not None, "get_retry_credential was never called" + assert adapter.retry_thread != loop_thread, ( + "get_retry_credential ran on the event-loop thread " + f"({adapter.retry_thread}); it force-refreshes the upstream token " + "under a cross-process auth-store lock and must be offloaded" + ) + # The rotation itself still worked: rejected bearer, then ours. + assert adapter.retry_status_code == 401 + assert [r["auth"] for r in captured["requests"]] == [ + "Bearer test-bearer", + "Bearer rotated-bearer", + ] + finally: + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + +def test_event_loop_keeps_running_while_the_retry_credential_resolves(): + """A stalled 429 rotation must not starve the rest of the loop. + + Same loop-side heartbeat as the credential test, sampled by the adapter on + entry and exit. A 429 rotation is precisely when the proxy is busiest, so + this is the worst moment to freeze every other in-flight completion. + """ + async def run(): + ticks = [0] + running = [True] + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner( + _build_rejecting_upstream( + captured, reject_status=429, accept_bearer="rotated-bearer" + ) + ) + adapter = _RecordingAdapter( + f"{upstream_base}/v1", + stall=_STALL_SECONDS, + ticks=ticks, + retry_bearer="rotated-bearer", + ) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + beat = asyncio.create_task(_heartbeat(ticks, running)) + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", json={} + ) as resp: + await resp.read() + + assert adapter.ticks_across_retry is not None + assert adapter.ticks_across_retry >= _MIN_TICKS_ACROSS_STALL, ( + f"only {adapter.ticks_across_retry} loop iterations ran during a " + f"{_STALL_SECONDS}s 429 credential rotation — the event loop was frozen" + ) + finally: + running[0] = False + beat.cancel() + await asyncio.gather(beat, return_exceptions=True) + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + +def test_retry_credential_failure_still_returns_the_upstream_rejection(): + """Offloading must not change the rotation's error contract. + + ``asyncio.to_thread`` re-raises the worker's exception in the awaiting + frame, so the handler's ``except Exception -> retry_cred = None`` still + swallows it and streams the upstream's own 401 back. Deliberately *not* in + the red-before set — it guards behaviour the fix must leave alone. + """ + async def run(): + captured: Dict[str, Any] = {"requests": []} + upstream_runner, upstream_base = await _start_runner( + _build_rejecting_upstream( + captured, reject_status=401, accept_bearer="never-offered" + ) + ) + adapter = _RecordingAdapter(f"{upstream_base}/v1", raise_on_retry=True) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{proxy_base}/v1/chat/completions", json={} + ) as resp: + assert resp.status == 401 + await resp.read() + + # One forward only — the failed rotation must not be retried. + assert len(captured["requests"]) == 1 + finally: + await proxy_runner.cleanup() + await upstream_runner.cleanup() + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# handle_health -> is_authenticated +# --------------------------------------------------------------------------- + + +def test_is_authenticated_runs_off_the_event_loop(): + """`/health` must not resolve auth state on the loop thread. + + ``adapters/base.py`` documents ``is_authenticated`` as "cheap — no network + calls", but ``NousPortalAdapter`` implements it via ``_read_state()``, which + takes the same 15s cross-process ``_auth_store_lock()``. ``/health`` is what + a supervisor, systemd unit, container healthcheck or load balancer polls, so + it is the endpoint least able to afford a lock wait. + """ + async def run(): + loop_thread = threading.get_ident() + adapter = _RecordingAdapter("http://127.0.0.1:1/v1") + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{proxy_base}/health") as resp: + assert resp.status == 200 + payload = await resp.json() + + assert payload["authenticated"] is True + assert adapter.authenticated_thread is not None, "is_authenticated was never called" + assert adapter.authenticated_thread != loop_thread, ( + "is_authenticated ran on the event-loop thread " + f"({adapter.authenticated_thread}); it takes the cross-process " + "auth-store lock and must be offloaded" + ) + finally: + await proxy_runner.cleanup() + + asyncio.run(run()) + + +def test_event_loop_keeps_running_while_health_resolves_auth_state(): + """A contended auth store must not freeze the loop behind `/health`. + + Same loop-side heartbeat measurement as the credential test — a client on + the blocked loop cannot observe its own starvation. + """ + async def run(): + ticks = [0] + running = [True] + adapter = _RecordingAdapter( + "http://127.0.0.1:1/v1", stall=_STALL_SECONDS, ticks=ticks + ) + proxy_runner, proxy_base = await _start_runner(create_app(adapter)) + beat = asyncio.create_task(_heartbeat(ticks, running)) + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{proxy_base}/health") as resp: + await resp.read() + + assert adapter.ticks_across_is_authenticated is not None + assert adapter.ticks_across_is_authenticated >= _MIN_TICKS_ACROSS_STALL, ( + f"only {adapter.ticks_across_is_authenticated} loop iterations ran during " + f"a {_STALL_SECONDS}s /health auth check — the event loop was frozen" + ) + finally: + running[0] = False + beat.cancel() + await asyncio.gather(beat, return_exceptions=True) + await proxy_runner.cleanup() + + asyncio.run(run()) diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py index 998fae07b2..e7ada02063 100644 --- a/tests/hermes_cli/test_relay_shared_metrics_runtime.py +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -25,6 +25,12 @@ class _Request: self.content = content +class _ToolExecutionResult: + def __init__(self, result: Any, annotation: Any = None) -> None: + self.result = result + self.annotation = annotation + + class _Relay: def __init__(self) -> None: self.events: list[tuple[Any, ...]] = [] @@ -39,6 +45,7 @@ class _Relay: Agent="agent", Function="function", Tool="tool" ) self.LLMRequest = _Request + self.ToolExecutionResult = _ToolExecutionResult self.scope = SimpleNamespace( push=self._scope_push, pop=self._scope_pop, @@ -174,11 +181,13 @@ class _Relay: def _tool_call_end( self, handle: Any, - result: dict[str, Any], + result: _ToolExecutionResult, **kwargs: Any, ) -> None: + assert isinstance(result, _ToolExecutionResult) + payload = result.result start = self._tool_starts.pop(handle) - self.events.append(("tool.call_end", handle, result, kwargs)) + self.events.append(("tool.call_end", handle, payload, kwargs)) event = SimpleNamespace( kind="scope", category="tool", @@ -190,7 +199,7 @@ class _Relay: **kwargs["metadata"], "otel.status_code": "OK", }, - data=result, + data=payload, ) for callback in list(self._callbacks.values()): callback(event) diff --git a/tests/hermes_cli/test_restart_plan_reconciliation.py b/tests/hermes_cli/test_restart_plan_reconciliation.py index eebe427728..b87cc98f7a 100644 --- a/tests/hermes_cli/test_restart_plan_reconciliation.py +++ b/tests/hermes_cli/test_restart_plan_reconciliation.py @@ -159,6 +159,124 @@ def test_external_supervisor_counts_as_restarted(): assert outcomes[0]["outcome"] == "restarted" +def test_unmanaged_serve_runtime_under_default_profile_is_unaccounted(): + """#100479: an sshd-spawned `serve --isolated` has no systemd unit and + shares the default profile with the gateway. A gateway-only restart + must not be read as covering it — it must trip the tripwire instead.""" + serve_runtime = RuntimeRecord( + kind="serve", + profile="default", + pid=900, + supervisor="manual-serve", + restart_via=_restart_mechanism("manual-serve", "default"), + ) + outcomes = match_runtime_outcomes( + _plan(_rt("default", 100, supervisor="systemd"), serve_runtime), + restarted_services=["hermes-gateway"], relaunched_profiles=[], + externally_supervised_profiles=[], killed_pids=set(), failed_units=[], + ) + by_pid = {o["pid"]: o["outcome"] for o in outcomes} + assert by_pid[100] == "restarted" + assert by_pid[900] == "unaccounted" + assert report_unaccounted_runtimes(outcomes) is True + + +def _serve(profile: str, pid: int, kind: str = "serve") -> RuntimeRecord: + return RuntimeRecord( + kind=kind, + profile=profile, + pid=pid, + supervisor="manual-serve", + restart_via=_restart_mechanism("manual-serve", profile), + ) + + +def test_serve_never_borrows_relaunched_or_external_gateway_profile(): + """Sibling site of #100479: the relaunched_profiles / external-supervisor + bookkeeping is gateway vocabulary too. A manual gateway relaunch under + ``default`` (or a named profile) says nothing about a serve that shares + the profile name.""" + outcomes = match_runtime_outcomes( + _plan(_rt("default", 100), _serve("default", 900), + _rt("work", 101), _serve("work", 901, kind="dashboard")), + restarted_services=[], relaunched_profiles=["default"], + externally_supervised_profiles=["work"], killed_pids=set(), failed_units=[], + ) + by_pid = {o["pid"]: o["outcome"] for o in outcomes} + assert by_pid == { + 100: "restarted", 900: "unaccounted", 101: "restarted", 901: "unaccounted" + } + + +def test_named_profile_serve_does_not_match_gateway_profile_unit(): + """``hermes-gateway-work.service`` restarted must not credit the ``work`` + serve — the old substring match (``"work" in unit``) did exactly that.""" + outcomes = match_runtime_outcomes( + _plan(_rt("work", 101, supervisor="systemd"), _serve("work", 901)), + restarted_services=["hermes-gateway-work.service"], relaunched_profiles=[], + externally_supervised_profiles=[], killed_pids=set(), failed_units=[], + ) + by_pid = {o["pid"]: o["outcome"] for o in outcomes} + assert by_pid == {101: "restarted", 901: "unaccounted"} + + +def test_serve_reconciles_against_its_own_unit_vocabulary(): + """A serve IS covered when a ``hermes-serve*`` unit for its profile was + restarted (or failed) — scope-qualified identities included.""" + outcomes = match_runtime_outcomes( + _plan(_serve("default", 900), _serve("work", 901), + _serve("ops", 902, kind="dashboard"), _serve("qa", 903)), + restarted_services=["hermes-gateway", "user/hermes-serve", + "hermes-serve-work.service", "hermes-dashboard-ops"], + relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids=set(), failed_units=["hermes-serve-qa.service"], + ) + by_pid = {o["pid"]: o["outcome"] for o in outcomes} + assert by_pid == {900: "restarted", 901: "restarted", 902: "restarted", 903: "failed"} + # exact names: ``work`` must not claim ``hermes-serve-workbench`` + outcomes = match_runtime_outcomes( + _plan(_serve("work", 901)), + restarted_services=["hermes-serve-workbench.service"], relaunched_profiles=[], + externally_supervised_profiles=[], killed_pids=set(), failed_units=[], + ) + assert outcomes[0]["outcome"] == "unaccounted" + + +def test_serve_outcome_follows_incarnation_probe_when_provided(): + """With the (pid, create_time) survivor probe result, liveness decides: + a pre-update serve that is gone was replaced (restarted); one still + alive is unaccounted — even when a hermes-serve unit was restarted.""" + plan = _plan(_serve("default", 900), _serve("default", 901, kind="dashboard")) + outcomes = match_runtime_outcomes( + plan, restarted_services=["hermes-serve.service"], relaunched_profiles=[], + externally_supervised_profiles=[], killed_pids=set(), failed_units=[], + stale_serve_pids={900}, + ) + by_pid = {o["pid"]: o["outcome"] for o in outcomes} + assert by_pid == {900: "unaccounted", 901: "restarted"} + # killed pid still wins as "stopped"; probe None => fail closed + outcomes = match_runtime_outcomes( + plan, restarted_services=[], relaunched_profiles=[], + externally_supervised_profiles=[], killed_pids={901}, failed_units=[], + stale_serve_pids=None, + ) + by_pid = {o["pid"]: o["outcome"] for o in outcomes} + assert by_pid == {900: "unaccounted", 901: "stopped"} + + +def test_unaccounted_serve_report_names_serve_remedy_not_gateway_restart(capsys): + outcomes = match_runtime_outcomes( + _plan(_serve("default", 900)), + restarted_services=["hermes-gateway"], relaunched_profiles=[], + externally_supervised_profiles=[], killed_pids=set(), failed_units=[], + ) + assert report_unaccounted_runtimes(outcomes) is True + out = capsys.readouterr().out + assert "serve [default] pid 900" in out + assert "hermes-serve.service" in out + assert "hermes gateway restart" not in out + + def test_mixed_fleet_only_the_missed_one_escalates(capsys): outcomes = match_runtime_outcomes( _plan( diff --git a/tests/hermes_cli/test_serve_mcp_discovery_after_bind.py b/tests/hermes_cli/test_serve_mcp_discovery_after_bind.py new file mode 100644 index 0000000000..0ae23f019e --- /dev/null +++ b/tests/hermes_cli/test_serve_mcp_discovery_after_bind.py @@ -0,0 +1,70 @@ +"""Desktop `serve` starts background MCP discovery only after the socket binds. + +The MCP SDK import (~350ms) used to run on a thread started BEFORE +web_server was imported, holding the GIL against the main thread's own +import path and delaying the READY sentinel the Desktop waits on. +""" + +from __future__ import annotations + +import logging +import threading + +import hermes_cli.mcp_startup as mcp_startup +import hermes_cli.web_server as web_server +from tests.hermes_cli.test_dashboard_auth_gate import _stub_uvicorn_run + + +def _reset_discovery_state(monkeypatch): + monkeypatch.setattr(mcp_startup, "_mcp_discovery_started", False) + monkeypatch.setattr(mcp_startup, "_mcp_discovery_thread", None) + monkeypatch.setattr(mcp_startup, "_mcp_discovery_deferred", None) + + +def test_desktop_serve_arms_mcp_discovery_only_after_ready_sentinel(monkeypatch): + _reset_discovery_state(monkeypatch) + order: list[str] = [] + monkeypatch.setattr( + mcp_startup, + "start_background_mcp_discovery", + lambda *, logger, thread_name: order.append("discovery:" + thread_name), + ) + monkeypatch.setattr(web_server, "_write_machine_sentinel_line", lambda line: order.append("sentinel")) + _stub_uvicorn_run(monkeypatch) + + web_server.start_server( + host="127.0.0.1", port=0, open_browser=False, headless=True, + start_mcp_discovery_after_bind=True, + ) + timer = mcp_startup._mcp_discovery_deferred + assert order == ["sentinel"] and isinstance(timer, threading.Timer) + timer.cancel() + # An agent build inside the delay window pulls discovery forward itself. + mcp_startup.wait_for_mcp_discovery(timeout=0) + assert order == ["sentinel", "discovery:dashboard-mcp-discovery"] + assert mcp_startup._mcp_discovery_deferred is None + + # Without the flag (dashboard / non-Desktop serve) start_server does not + # start discovery itself — cmd_dashboard's pre-import path still owns it. + order.clear() + _reset_discovery_state(monkeypatch) + web_server.start_server(host="127.0.0.1", port=0, open_browser=False, headless=True) + assert order == ["sentinel"] and mcp_startup._mcp_discovery_deferred is None + + +def test_deferred_discovery_fires_once_and_is_idempotent(monkeypatch): + _reset_discovery_state(monkeypatch) + calls: list[str] = [] + monkeypatch.setattr( + mcp_startup, + "start_background_mcp_discovery", + lambda *, logger, thread_name: calls.append(thread_name), + ) + log = logging.getLogger("test") + mcp_startup.defer_background_mcp_discovery(logger=log, thread_name="t", delay=60) + mcp_startup.defer_background_mcp_discovery(logger=log, thread_name="t", delay=60) # second arm is a no-op + first = mcp_startup._mcp_discovery_deferred + mcp_startup._start_deferred_mcp_discovery_now() + mcp_startup._start_deferred_mcp_discovery_now() + assert calls == ["t"] + assert first is not None and mcp_startup._mcp_discovery_deferred is None diff --git a/tests/hermes_cli/test_session_list_reader_disposable.py b/tests/hermes_cli/test_session_list_reader_disposable.py new file mode 100644 index 0000000000..1109c1b5d8 --- /dev/null +++ b/tests/hermes_cli/test_session_list_reader_disposable.py @@ -0,0 +1,96 @@ +"""Read-only session-list opens must stay disposable. + +Two properties that a "keep the read-only handle for the process lifetime" +optimisation silently destroys. Both are asserted against the real +``_open_session_db_at_path`` read path the sidebar poll uses, because both +failures are invisible in a unit test that mocks the store. + +1. **The store on disk is the truth.** Recovering a corrupt ``state.db`` + is a file swap (``mv state.db state.db.corrupt-…; cp -a recovered.db + state.db``) performed while the backend is stopped, but a poll can also + race a restore. A reader pinned to the old inode keeps serving + pre-recovery rows forever, so the user "recovers" and still sees the + broken list. + +2. **Forensic backup must stay reachable.** ``offline_file_access`` refuses + raw byte access while ANY tracked connection is registered for the path, + because a raw ``close()`` would cancel this process's POSIX advisory locks + (howtocorrupt §2.2). ``_backup_db_file`` (the copy taken BEFORE a malformed + store is repaired) and ``_db_fingerprint`` (the repair-attempt ledger key) + both go through it. A never-closed list reader makes both fail for the rest + of the process, so a repair runs without its forensic backup and the + ledger degrades to a size-only key. +""" + +from __future__ import annotations + +import shutil + +from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access +from hermes_cli.web_server import _open_session_db_at_path +from hermes_state import SessionDB, _db_fingerprint + + +def _ids(db) -> list: + return [row["id"] for row in db.list_sessions_rich(limit=10, compact_rows=True)] + + +def test_poll_observes_a_replaced_state_db(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + old.create_session("before-recovery", source="cli") + old.close() + + first = _open_session_db_at_path(db_path, read_only=True) + try: + assert _ids(first) == ["before-recovery"] + finally: + first.close() + + # `hermes sessions recover` writes a clean database, which the operator + # then installs over the corrupt one. + recovered = tmp_path / "recovered-state.db" + rebuilt = SessionDB(db_path=recovered) + rebuilt.create_session("after-recovery", source="cli") + rebuilt.close() + + for suffix in ("-wal", "-shm"): + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + sidecar.unlink() + db_path.unlink() + shutil.copy2(recovered, db_path) + + second = _open_session_db_at_path(db_path, read_only=True) + try: + assert _ids(second) == ["after-recovery"] + finally: + second.close() + + +def test_poll_leaves_forensic_backup_reachable(tmp_path): + db_path = tmp_path / "state.db" + writer = SessionDB(db_path=db_path) + writer.create_session("s1", source="cli") + writer.close() + + baseline = _db_fingerprint(db_path) + assert baseline is not None + + poll = _open_session_db_at_path(db_path, read_only=True) + try: + assert _ids(poll) == ["s1"] + finally: + poll.close() + + # The raw-copy path a malformed-store repair takes before it touches + # anything must still be permitted after the poll. + try: + with offline_file_access(db_path, what="forensic-backup"): + pass + except LiveConnectionError as exc: # pragma: no cover - failure detail + raise AssertionError( + f"a session-list poll left a tracked connection open: {exc}" + ) from exc + + assert _db_fingerprint(db_path) == baseline diff --git a/tests/hermes_cli/test_session_recovery.py b/tests/hermes_cli/test_session_recovery.py index 3cabe5a750..42b5db314b 100644 --- a/tests/hermes_cli/test_session_recovery.py +++ b/tests/hermes_cli/test_session_recovery.py @@ -648,4 +648,188 @@ def test_partial_recovery_clears_only_unreadable_system_prompt_refs( conn.close() +def _insert_delivery_obligations(path: Path, rows: list[tuple[object, ...]]) -> None: + from gateway.delivery_ledger import _initialize_schema + conn = sqlite3.connect(str(path), isolation_level=None) + try: + _initialize_schema(conn) + conn.executemany( + """INSERT INTO delivery_obligations ( + obligation_id, session_key, platform, chat_id, thread_id, + content, state, attempts, created_at, updated_at, + owner_pid, owner_started_at, last_error, adapter_profile + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + rows, + ) + finally: + conn.close() + + +def test_recovery_copies_delivery_obligations(tmp_path: Path) -> None: + """Owed replies must survive salvage — #100313 lost 6 obligation rows.""" + + source = tmp_path / "state.db" + output = tmp_path / "recovered.db" + _make_source(source) + now = 1_720_000_000.0 + _insert_delivery_obligations( + source, + [ + ( + "ob-pending", + "telegram:1:chat-1", + "telegram", + "chat-1", + None, + "owed reply", + "pending", + 0, + now, + now, + 4242, + 99, + None, + "default", + ), + ( + "ob-delivered", + "telegram:1:chat-1", + "telegram", + "chat-1", + None, + "already sent", + "delivered", + 1, + now, + now + 1, + None, + None, + None, + "default", + ), + ], + ) + + inspection = inspect_session_database(source, work_dir=tmp_path) + assert inspection["tables"]["delivery_obligations"]["available"] is True + assert inspection["tables"]["delivery_obligations"]["rows"] == 2 + + report = recover_session_database(source, output, work_dir=tmp_path) + copied = report["copy"]["delivery_obligations"] + assert copied["status"] == "complete" + assert copied["copied_rows"] == 2 + assert report["verification"]["table_counts"]["delivery_obligations"] == 2 + assert report["complete"] is True + assert report["verified"] is True + assert report["installed"] is False + + conn = sqlite3.connect(str(output)) + try: + recovered = conn.execute( + """SELECT obligation_id, state, content, owner_pid, adapter_profile + FROM delivery_obligations ORDER BY obligation_id""" + ).fetchall() + finally: + conn.close() + assert recovered == [ + ("ob-delivered", "delivered", "already sent", None, "default"), + ("ob-pending", "pending", "owed reply", 4242, "default"), + ] + + +def test_recovery_without_delivery_ledger_is_not_lossy(tmp_path: Path) -> None: + """CLI-only stores never created the lazy table; that is not data loss.""" + + source = tmp_path / "state.db" + output = tmp_path / "recovered.db" + _make_source(source) + + report = recover_session_database(source, output, work_dir=tmp_path) + assert report["copy"]["delivery_obligations"]["status"] == "missing" + assert "delivery_obligations" not in report["verification"]["table_counts"] + assert report["complete"] is True + assert report["verified"] is True + + + + + +def test_recovery_flags_delivery_obligation_count_mismatch_as_loss( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A source-vs-destination ledger count mismatch must not verify as complete. + + The destination table is created through the registered initializer; a + real SQL trigger that silently drops one row stands in for the "rows went + missing on the way over" failure the verifier has to catch. + """ + + from hermes_cli import session_recovery + + source = tmp_path / "state.db" + output = tmp_path / "recovered.db" + _make_source(source) + now = 1_720_000_000.0 + _insert_delivery_obligations( + source, + [ + ("ob-a", "k", "telegram", "chat-1", None, "a", "pending", 0, now, now, None, None, None, "default"), + ("ob-b", "k", "telegram", "chat-1", None, "b", "pending", 0, now, now, None, None, None, "default"), + ], + ) + + real_init = session_recovery._AUXILIARY_TABLE_SCHEMAS["delivery_obligations"] + + def lossy_init(conn: sqlite3.Connection) -> None: + real_init(conn) + conn.execute( + """CREATE TRIGGER drop_ob_b BEFORE INSERT ON delivery_obligations + WHEN NEW.obligation_id = 'ob-b' BEGIN SELECT RAISE(IGNORE); END""" + ) + + monkeypatch.setitem( + session_recovery._AUXILIARY_TABLE_SCHEMAS, "delivery_obligations", lossy_init + ) + + report = recover_session_database(source, output, work_dir=tmp_path) + assert report["verification"]["table_counts"]["delivery_obligations"] == 1 + assert report["complete"] is False + assert any( + "delivery_obligations count is 1, expected 2" in error + for error in report["verification"]["errors"] + ) + + +def test_lost_and_found_direct_copy_creates_lazy_delivery_ledger(tmp_path: Path) -> None: + """The .recover lane copies the ledger even though SessionDB never made it.""" + + from hermes_cli.session_lost_and_found import _copy_direct_tables + + recovered_source = tmp_path / "lost_and_found.db" + now = 1_720_000_000.0 + _insert_delivery_obligations( + recovered_source, + [ + ("ob-1", "k", "telegram", "chat-1", None, "one", "pending", 0, now, now, None, None, None, "default"), + ("ob-2", "k", "telegram", "chat-1", None, "two", "failed", 3, now, now, None, None, "boom", "default"), + ], + ) + output = tmp_path / "rebuilt.db" + SessionDB(db_path=output).close() + + lf_conn = sqlite3.connect(str(recovered_source), isolation_level=None) + dest = sqlite3.connect(str(output), isolation_level=None) + try: + assert not dest.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='delivery_obligations'" + ).fetchall() + copied = _copy_direct_tables(lf_conn, dest) + assert copied["delivery_obligations"] == 2 + rows = dest.execute( + "SELECT obligation_id, state, last_error FROM delivery_obligations ORDER BY obligation_id" + ).fetchall() + finally: + lf_conn.close() + dest.close() + assert rows == [("ob-1", "pending", None), ("ob-2", "failed", "boom")] diff --git a/tests/hermes_cli/test_session_recovery_lost_and_found.py b/tests/hermes_cli/test_session_recovery_lost_and_found.py index d93bd69b6f..5831bbd602 100644 --- a/tests/hermes_cli/test_session_recovery_lost_and_found.py +++ b/tests/hermes_cli/test_session_recovery_lost_and_found.py @@ -17,6 +17,7 @@ import pytest from hermes_state import SessionDB from hermes_cli import session_recovery from hermes_cli.session_lost_and_found import ( + STUB_TITLE_PREFIX, classify_lost_and_found_row, map_lost_and_found_rows, rebuild_fts_indexes, @@ -569,3 +570,292 @@ def test_fingerprint_error_enumerates_parent_cli_session( assert "CLI session" in message assert "fresh shell" in message assert "snapshot" in message + + +# ── issue #101409: mis-mapped salvage must not be reported verified ───────── + + +def _map_salvage_rows( + tmp_path: Path, + *, + blank_session_started_at: bool, + blank_message_timestamp: bool, +) -> sqlite3.Connection: + """Map synthetic lost_and_found cells into a fresh template DB. + + With either ``blank_*`` flag the cells mimic an upgraded source's + *physical* column order (#101409): whatever lands on the declared + ``started_at``/``timestamp`` position is not an epoch timestamp, so + the NOT NULL substitute turns it into 0.0 on every row. + """ + + schema_ref = tmp_path / "schema-ref.db" + SessionDB(db_path=schema_ref).close() + schema = sqlite3.connect(str(schema_ref)) + try: + sessions_columns = [ + str(row[1]) for row in schema.execute("PRAGMA table_info(sessions)") + ] + messages_columns = [ + str(row[1]) for row in schema.execute("PRAGMA table_info(messages)") + ] + finally: + schema.close() + current_width = len(sessions_columns) + + lf_path = tmp_path / "lost_and_found.db" + lf_conn = sqlite3.connect(str(lf_path), isolation_level=None) + try: + lf_cells = ", ".join(f"c{i}" for i in range(current_width)) + lf_conn.execute( + "CREATE TABLE lost_and_found (rootpgno INTEGER, pgno INTEGER, " + "nfield INTEGER, id INTEGER, " + lf_cells + ")" + ) + + def insert(nfield: int, rowid: int, values: list) -> None: + padded = list(values) + [None] * (current_width - len(values)) + placeholders = ", ".join("?" for _ in range(4 + current_width)) + lf_conn.execute( + "INSERT INTO lost_and_found VALUES (" + placeholders + ")", + [2, 5, nfield, rowid, *padded], + ) + + def session_row(session_id: str) -> list: + # title is UNIQUE (idx_sessions_title_unique) — keep it distinct + # per row so the probe isolates timestamp mis-mapping. + row = { + "id": session_id, + "source": "telegram", + "started_at": None + if blank_session_started_at + else 1_754_000_000.0, + "message_count": 2, + "title": f"mis-mapped probe {session_id}", + } + return [row.get(column) for column in sessions_columns] + + for index in range(3): + insert( + current_width, + index + 1, + session_row(f"20260101_01010{index}_aaa00{index}"), + ) + + for index in range(2): + message = { + "id": None, + "session_id": "20260101_010100_aaa000", + "role": "user", + "content": "payload", + "timestamp": None + if blank_message_timestamp + else 1_754_000_100.0 + index, + } + insert( + 23, + 100 + index, + [message.get(column) for column in messages_columns[:23]], + ) + finally: + lf_conn.close() + + output = tmp_path / "mapped.db" + SessionDB(db_path=output).close() + lf_conn = sqlite3.connect(str(lf_path), isolation_level=None) + dest = sqlite3.connect(str(output), isolation_level=None) + try: + dest.execute("PRAGMA foreign_keys=OFF") + map_lost_and_found_rows(lf_conn, dest) + finally: + lf_conn.close() + dest.close() + return sqlite3.connect(str(output), isolation_level=None) + + +def test_plausibility_gate_flags_positional_mis_mapping( + tmp_path: Path, +) -> None: + """A salvage whose timestamps all landed below the epoch floor was + mapped onto the wrong columns and must be flagged, not verified + (#101409).""" + + conn = _map_salvage_rows( + tmp_path, + blank_session_started_at=True, + blank_message_timestamp=False, + ) + try: + # The mapper happily inserted every row; structural checks pass. + assert conn.execute("SELECT COUNT(*) FROM sessions").fetchone()[0] == 3 + assert conn.execute( + "SELECT COUNT(*) FROM sessions WHERE started_at = 0.0" + ).fetchone()[0] == 3 + + errors = session_recovery._lost_and_found_plausibility_errors(conn) + assert len(errors) == 1 + assert "sessions.started_at" in errors[0] + finally: + conn.close() + + +def test_plausibility_gate_flags_mis_mapped_message_timestamps( + tmp_path: Path, +) -> None: + conn = _map_salvage_rows( + tmp_path, + blank_session_started_at=False, + blank_message_timestamp=True, + ) + try: + errors = session_recovery._lost_and_found_plausibility_errors(conn) + assert len(errors) == 1 + assert "messages.timestamp" in errors[0] + finally: + conn.close() + + +def test_plausibility_gate_passes_correctly_mapped_salvage( + tmp_path: Path, +) -> None: + """Well-mapped rows — and partially damaged ones (a torn cell on some + rows is expected salvage noise) — must not trip the gate: it fires + only on a *systematic* violation.""" + + conn = _map_salvage_rows( + tmp_path, + blank_session_started_at=False, + blank_message_timestamp=False, + ) + try: + # Damage one of three sessions the way a torn cell would. + conn.execute( + "UPDATE sessions SET started_at = 0.0 WHERE id = ?", + ("20260101_010101_aaa001",), + ) + conn.commit() + + assert session_recovery._lost_and_found_plausibility_errors(conn) == [] + finally: + conn.close() + + +def _rebuild_with_started_at_appended(conn: sqlite3.Connection) -> None: + """Give ``sessions`` the physical layout of an upgraded DB: ``started_at`` + lands at the END (as ``ALTER TABLE ADD COLUMN`` would place a column that + the current template declares mid-definition). Data is preserved.""" + info = list(conn.execute("PRAGMA table_info(sessions)")) + declared = [row[1] for row in info] + + def coldef(row): + _, name, ctype, notnull, dflt, pk = row + parts = [f'"{name}" {ctype}'] + if pk: + parts.append("PRIMARY KEY") + if notnull: + parts.append("NOT NULL") + if dflt is not None: + parts.append(f"DEFAULT {dflt}") + return " ".join(parts) + + reordered = [r for r in info if r[1] != "started_at"] + [r for r in info if r[1] == "started_at"] + cols = ", ".join(f'"{c}"' for c in declared) + conn.executescript("PRAGMA foreign_keys=OFF;") + conn.execute("CREATE TABLE sessions_new (" + ", ".join(coldef(r) for r in reordered) + ")") + conn.execute(f"INSERT INTO sessions_new({cols}) SELECT {cols} FROM sessions") + conn.executescript("DROP TABLE sessions; ALTER TABLE sessions_new RENAME TO sessions;") + + +@pytest.mark.skipif( + not HAVE_SQLITE3_CLI, + reason="sqlite3 CLI not on PATH; .recover is a shell-only feature", +) +def test_lost_and_found_lane_refuses_to_verify_a_physically_shifted_source( + tmp_path: Path, +) -> None: + """#101409 end to end: a source whose physical column order differs from + the template's declared order maps every cell onto the wrong column. The + output still passes integrity/FK/FTS, so only the plausibility gate can + stop the report from claiming ``verified``.""" + source = tmp_path / "upgraded.db" + output = tmp_path / "upgraded-recovered.db" + db = SessionDB(db_path=source) + try: + for n in range(3): + sid = f"20260812_1400{n:02d}_def{n:03x}" + db.create_session(sid, "cli", cwd=f"/tmp/shift-{n}") + db.set_session_title(sid, f"shift {n}") + for m in range(4): + db.append_message(sid, "user" if m % 2 == 0 else "assistant", f"payload {n} {m}") + finally: + db.close() + conn = sqlite3.connect(str(source), isolation_level=None) + try: + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + conn.execute("PRAGMA journal_mode=DELETE") + _rebuild_with_started_at_appended(conn) + conn.execute("VACUUM") + physical = [r[1] for r in conn.execute("PRAGMA table_info(sessions)")] + assert physical[-1] == "started_at" + finally: + conn.close() + # The reporter's damage: page 1 (header + sqlite_master) overwritten, so + # ``.recover`` cannot name any table and every row lands in + # lost_and_found, to be mapped positionally onto the template. + with open(source, "r+b") as fh: + fh.write(b"\0" * _page_size(source.read_bytes())) + + report = recover_session_database(source, output, work_dir=tmp_path, allow_partial=True) + + assert report["mode"] == "lost_and_found_salvage" + # Mis-mapped rows that trip a NOT NULL / type constraint are stubbed, not + # mapped (the reporter saw 190 of 1,875) — at least one lands positionally. + assert report["lost_and_found"]["mapped"]["sessions"] >= 1 + assert report["verification"]["healthy"] is False + assert report["verified"] is False + assert any("sessions.started_at is implausible" in e for e in report["verification"]["errors"]) + out = sqlite3.connect(str(output)) + try: + # The mis-mapping the gate caught: every mapped (non-stub) session got + # the NOT NULL substitute where its real start time should be. + mapped = out.execute( + f"SELECT started_at FROM sessions WHERE COALESCE(title, '') NOT LIKE '{STUB_TITLE_PREFIX}%'" + ).fetchall() + assert mapped and all(row[0] == 0.0 for row in mapped) + finally: + out.close() + + +def test_plausibility_gate_ignores_stub_only_sessions(tmp_path: Path) -> None: + """Stub rows from ``stub_missing_parent_sessions`` legitimately carry + ``started_at = 0.0``; a salvage where only stubs survived is depleted, + not mis-mapped, and must not be flagged.""" + output = tmp_path / "stubs.db" + SessionDB(db_path=output).close() + conn = sqlite3.connect(str(output)) + try: + now = 1_750_000_000.0 + conn.execute( + "INSERT INTO sessions (id, source, started_at, title) VALUES (?, ?, ?, ?)", + ("20260812_140000_aaa000", "recovered", 0.0, "[best-effort recovered 1] session metadata was unreadable"), + ) + conn.execute( + "INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)", + ("20260812_140000_aaa000", "user", "hi", now), + ) + conn.commit() + assert session_recovery._lost_and_found_plausibility_errors(conn) == [] + # One genuinely mapped row with a real timestamp keeps it clean too... + conn.execute( + "INSERT INTO sessions (id, source, started_at, title) VALUES (?, ?, ?, ?)", + ("20260812_140001_aaa001", "cli", now, None), + ) + conn.commit() + assert session_recovery._lost_and_found_plausibility_errors(conn) == [] + # ...and a mapped row at 0.0 with a NULL title (the mis-mapped shape: + # blank titles) is still counted as mapped, not as a stub. + conn.execute("UPDATE sessions SET started_at = 0.0 WHERE id = '20260812_140001_aaa001'") + conn.commit() + errors = session_recovery._lost_and_found_plausibility_errors(conn) + assert len(errors) == 1 and "sessions.started_at" in errors[0] + finally: + conn.close() diff --git a/tests/hermes_cli/test_set_config_value.py b/tests/hermes_cli/test_set_config_value.py index e4c5f8ca15..d83a2af3ca 100644 --- a/tests/hermes_cli/test_set_config_value.py +++ b/tests/hermes_cli/test_set_config_value.py @@ -53,6 +53,7 @@ class TestExplicitAllowlist: "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN", "SLACK_APP_TOKEN", + "API_SERVER_KEY", ]) def test_explicit_key_routes_to_env(self, key, _isolated_hermes_home): set_config_value(key, "test-value-123") diff --git a/tests/hermes_cli/test_setup_blank_slate.py b/tests/hermes_cli/test_setup_blank_slate.py index b401a2069e..e08d67c8e3 100644 --- a/tests/hermes_cli/test_setup_blank_slate.py +++ b/tests/hermes_cli/test_setup_blank_slate.py @@ -53,6 +53,14 @@ class TestBlankSlateMinimalToolsets: from tools.registry import registry as _tool_registry _entry = _tool_registry.get_entry("vision_analyze") monkeypatch.setattr(_entry, "check_fn", lambda: True) + # This test pins disabled_toolsets SUBTRACTION, not deferral policy — + # assemble with the legacy everything-eager override so the expected + # list stays deferral-independent (#97979 defers process_manage by + # default, which would swap it for the three bridge tools here). + from tools.tool_search import ToolSearchConfig + _legacy = ToolSearchConfig.from_raw({"enabled": "on", "defer": []}) + monkeypatch.setattr("tools.tool_search.load_config", lambda: _legacy) + monkeypatch.setattr("tools.tool_search.load_config_readonly", lambda: _legacy) from hermes_cli.tools_config import _get_platform_tools cfg = {} _blank_slate_minimal_toolsets(cfg) @@ -67,7 +75,7 @@ class TestBlankSlateMinimalToolsets: names = sorted( {(d.get("function") or {}).get("name") or d.get("name") for d in defs} ) - assert names == ["patch", "process", "read_file", "search_files", + assert names == ["patch", "process_manage", "read_file", "search_files", "skill_manage", "skill_view", "skills_list", "terminal", "vision_analyze", "write_file"] diff --git a/tests/hermes_cli/test_sqlite3_cli_salvage_gate.py b/tests/hermes_cli/test_sqlite3_cli_salvage_gate.py new file mode 100644 index 0000000000..ec999aa4e5 --- /dev/null +++ b/tests/hermes_cli/test_sqlite3_cli_salvage_gate.py @@ -0,0 +1,384 @@ +"""#100368 regression: the corruption guidance must not direct a WAL-reset- +vulnerable sqlite3 CLI at a live Hermes database. + +Field forensics (issue #100368, maintainer round 2 + the isolated reproducer +in its comments): when a shell with SQLite's WAL-reset opener bug (fixed +3.51.3+ / backports 3.50.7 / 3.44.6 — Debian/Ubuntu system CLIs 3.45.1 / +3.46.1 are in the vulnerable band) opens a live state.db whose writer's DMS +lock has been cancelled, it unlinks the live -wal/-shm pair and splits the +store into two concurrent generations. Both generations report +``integrity_check ok`` while an old-generation acknowledged write is lost. + +Hermes' own corruption banners used to instruct exactly that command +(`sqlite3 ~/.hermes/state.db ".recover"`). The fix routes operators to +`hermes sessions recover --source ...`, whose lane snapshots the damaged +bundle before any shell touches it, and refuses a WAL-reset-vulnerable +sqlite3 CLI for the page-level salvage lane even on the snapshot. +""" + +from __future__ import annotations + +import argparse +import inspect +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest + +from hermes_cli.session_lost_and_found import ( + _parse_sqlite3_cli_version, + _wal_reset_vulnerable, + find_sqlite3_cli, + find_sqlite3_cli_refusal, +) +from hermes_cli.sqlite_runtime import is_sqlite_wal_reset_vulnerable + + +LIVE_DB_SALVAGE_COMMAND = 'sqlite3 ~/.hermes/state.db ".recover"' + + +# --------------------------------------------------------------------------- +# The version gate itself +# --------------------------------------------------------------------------- + + +class TestWalResetVersionGate: + @pytest.mark.parametrize( + "version", + [(3, 45, 1), (3, 46, 1), (3, 44, 5), (3, 50, 4), (3, 51, 2), (3, 8, 0)], + ) + def test_vulnerable_versions(self, version): + assert _wal_reset_vulnerable(version) is True + + @pytest.mark.parametrize( + "version", + [ + (3, 44, 6), + (3, 44, 7), + (3, 50, 7), + (3, 50, 8), + (3, 51, 3), + (3, 51, 4), + (3, 52, 0), + (3, 53, 1), + (4, 0, 0), + ], + ) + def test_fixed_versions(self, version): + assert _wal_reset_vulnerable(version) is False + + def test_gate_mirrors_library_gate(self): + """The salvage gate must agree with the shared runtime gate so the + embedded library and the salvage shell can never disagree.""" + versions = [ + (3, 44, 5), + (3, 44, 6), + (3, 45, 1), + (3, 50, 4), + (3, 50, 7), + (3, 51, 2), + (3, 51, 3), + (3, 53, 1), + ] + for version in versions: + assert _wal_reset_vulnerable(version) == ( + is_sqlite_wal_reset_vulnerable(version) + ), f"salvage gate disagrees with the runtime gate at {version}" + + +# --------------------------------------------------------------------------- +# find_sqlite3_cli refuses unsafe shells and explains why +# --------------------------------------------------------------------------- + + +class TestFindSqlite3CliRefusal: + def test_missing_binary_refusal(self, monkeypatch): + monkeypatch.setattr( + "hermes_cli.session_lost_and_found.shutil.which", lambda _: None + ) + assert find_sqlite3_cli() is None + assert find_sqlite3_cli_refusal()["reason"] == "missing" + + def test_no_dbpage_refusal(self, monkeypatch): + monkeypatch.setattr( + "hermes_cli.session_lost_and_found.shutil.which", + lambda _: "/usr/bin/sqlite3", + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._cli_supports_recover", + lambda _: False, + ) + assert find_sqlite3_cli() is None + assert find_sqlite3_cli_refusal()["reason"] == "no_dbpage" + + def test_wal_reset_vulnerable_refusal(self, monkeypatch): + """A .recover-capable but WAL-reset-vulnerable CLI must be refused. + + This is the Debian/Ubuntu shape from the #100368 incident: the + system sqlite3 (3.45.1) has sqlite_dbpage, so the capability probe + passes, while the WAL-reset opener bug is still present. + """ + monkeypatch.setattr( + "hermes_cli.session_lost_and_found.shutil.which", + lambda _: "/usr/bin/sqlite3", + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._cli_supports_recover", + lambda _: True, + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._parse_sqlite3_cli_version", + lambda _: (3, 45, 1), + ) + assert find_sqlite3_cli() is None + refusal = find_sqlite3_cli_refusal() + assert refusal["reason"] == "wal_reset_vulnerable" + assert refusal["version"] == "3.45.1" + assert "WAL-reset" in refusal["detail"] + + def test_fixed_capable_cli_accepted(self, monkeypatch): + monkeypatch.setattr( + "hermes_cli.session_lost_and_found.shutil.which", + lambda _: "/usr/local/bin/sqlite3", + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._cli_supports_recover", + lambda _: True, + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._parse_sqlite3_cli_version", + lambda _: (3, 51, 3), + ) + assert find_sqlite3_cli() == "/usr/local/bin/sqlite3" + assert find_sqlite3_cli_refusal() == {} + + def test_unparsable_version_still_usable(self, monkeypatch): + """A CLI whose version line cannot be parsed is not refused on + version grounds alone (the salvage lane runs against a snapshot + copy, not the live file).""" + monkeypatch.setattr( + "hermes_cli.session_lost_and_found.shutil.which", + lambda _: "/usr/bin/sqlite3", + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._cli_supports_recover", + lambda _: True, + ) + monkeypatch.setattr( + "hermes_cli.session_lost_and_found._parse_sqlite3_cli_version", + lambda _: None, + ) + assert find_sqlite3_cli() == "/usr/bin/sqlite3" + + +class TestParseSqlite3CliVersion: + def test_parses_modern_output(self): + class Probe: + returncode = 0 + stdout = b"3.51.4 2026-XX-XX 12:34:56\n" + + with patch( + "hermes_cli.session_lost_and_found.subprocess.run", + return_value=Probe(), + ): + assert _parse_sqlite3_cli_version("x") == (3, 51, 4) + + def test_unexecutable_returns_none(self): + with patch( + "hermes_cli.session_lost_and_found.subprocess.run", + side_effect=OSError("no such file"), + ): + assert _parse_sqlite3_cli_version("x") is None + + +# --------------------------------------------------------------------------- +# The operator-facing guidance never names the live DB +# --------------------------------------------------------------------------- + + +class TestGuidanceNeverNamesLiveDb: + def test_gateway_corruption_banner(self): + """The gateway broadcast must route to the two-stage `sessions + recover` contract and must warn against pointing a raw sqlite3 + shell at the live file.""" + import gateway.run as gateway_run + + body = inspect.getsource( + gateway_run.GatewayRunner._send_session_db_warning_notifications + ) + assert LIVE_DB_SALVAGE_COMMAND not in body + assert "sessions recover --source" in body + assert "--inspect-only" in body + assert "--output" in body + assert "do NOT" in body + + def test_run_agent_corrupt_explanation(self): + from run_agent import AIAgent + + explanation = AIAgent._format_turn_completion_explanation( + "session_persistence_failed", "corrupt" + ) + assert LIVE_DB_SALVAGE_COMMAND not in explanation + assert "hermes sessions recover --source" in explanation + assert "--inspect-only" in explanation + assert "--output recovered-state.db" in explanation + assert ".recover" in explanation # the warning still names the hazard + + def test_repair_budget_error_names_safe_lane(self, tmp_path: Path): + import hermes_state + + message = hermes_state._persistent_repair_exhausted_error( + tmp_path / "state.db" + ) + assert "Manual recovery required" in message + assert "sessions recover --source" in message + assert "--inspect-only" in message + assert "--output recovered-state.db" in message + # The old shape embedded the live path straight into a raw sqlite3 + # command: `sqlite3 {db_path} ".recover"`. + assert ".recover\"`" not in message + assert "do NOT" in message + + def test_forensic_backup_refusals_name_safe_lane(self): + """The low-disk and stat-failure forensic backup refusal strings + must not embed a raw sqlite3 command against the live path.""" + import hermes_state + + body = inspect.getsource(hermes_state._backup_db_file) + assert ".recover\"`" not in body + assert "sessions recover --source" in body + assert "--inspect-only" in body + + def test_kanban_manual_recovery_warns_about_live_db(self): + import hermes_cli.kanban as kanban + + source = inspect.getsource(kanban) + assert '`sqlite3 kanban.db ".recover"`' not in source + assert "copy kanban.db aside FIRST" in source + + +# --------------------------------------------------------------------------- +# The emitted command satisfies the real CLI contract +# --------------------------------------------------------------------------- +# The reviewer's blocker on the first iteration of this fix: the banners +# printed `hermes sessions recover --source ` — which cmd_sessions +# rejects with exit 2 ("--output is required unless --inspect-only is +# used") before any snapshot is taken. These tests dispatch the EXACT argv +# shapes the banners emit through the real parser + cmd_sessions, so a +# guidance string can never again pass a source-substring test while the +# command it prints deterministically fails. + + +class TestEmittedCommandsSatisfyCliContract: + """Every `sessions recover` argv the guidance prints must be accepted + by the real CLI contract — the reviewer's blocker on the first + iteration of this fix was exactly this: the banners printed + `hermes sessions recover --source `, which cmd_sessions rejects + with exit 2 ("--output is required unless --inspect-only is used") + before any snapshot is taken. + + These tests dispatch the EXACT argv shapes the banners emit through + the real `cmd_sessions` (the same function `hermes` main() hands the + parsed namespace to), so a guidance string can never again pass a + source-substring test while the command it prints deterministically + fails. + """ + + @staticmethod + def _namespace(source: Path, **overrides) -> "argparse.Namespace": + """The namespace hermes main() produces for `sessions recover`. + + Mirrors the registrations in hermes_cli/main.py (sessions_recover + subparser): --source, --output, --inspect-only, --work-dir, + --chunk-size (default 1000), --allow-partial, --report. + """ + fields = dict( + sessions_action="recover", + source=source, + output=None, + inspect_only=False, + work_dir=None, + chunk_size=1000, + allow_partial=False, + report=None, + ) + fields.update(overrides) + return argparse.Namespace(**fields) + + def test_old_v1_shape_is_still_rejected(self, tmp_path): + """Guard the test's own premise: the bare `--source ` shape the + v1 banner printed (neither --inspect-only nor --output) is rejected + with rc 2 by the real dispatcher.""" + import hermes_cli.sessions_cmd as sc + + rc = sc.cmd_sessions(self._namespace(tmp_path / "state.db")) + assert rc == 2 + + def test_inspect_stage_dispatches_past_gate(self, tmp_path): + """`--inspect-only` (stage 1 of the emitted sequence) must pass + the contract gate and reach actual inspection work (rc 0/1, not + the gate's 2).""" + import hermes_cli.sessions_cmd as sc + + source = tmp_path / "state.db" + conn = sqlite3.connect(str(source)) + try: + conn.execute("CREATE TABLE t (x)") + conn.commit() + finally: + conn.close() + + rc = sc.cmd_sessions( + self._namespace(source, inspect_only=True) + ) + assert rc != 2, "--inspect-only shape must pass the contract gate" + + def test_output_stage_dispatches_past_gate(self, tmp_path): + """`--output recovered-state.db` (stage 2) must pass the contract + gate and reach actual recovery work (rc 0/1, not the gate's 2).""" + import hermes_cli.sessions_cmd as sc + + source = tmp_path / "state.db" + conn = sqlite3.connect(str(source)) + try: + conn.execute("CREATE TABLE t (x)") + conn.commit() + finally: + conn.close() + + rc = sc.cmd_sessions( + self._namespace(source, output=tmp_path / "recovered-state.db") + ) + assert rc != 2, "--output shape must pass the contract gate" + + def test_banner_strings_emit_only_contract_valid_argv(self, tmp_path): + """The exact argv shapes embedded in the guidance strings, when + parsed and dispatched, must never return the contract-gate 2. + + Extracts each `sessions recover` invocation printed by the + banners' code and runs its flag set through the real dispatcher. + """ + import hermes_cli.sessions_cmd as sc + + source = tmp_path / "state.db" + conn = sqlite3.connect(str(source)) + try: + conn.execute("CREATE TABLE t (x)") + conn.commit() + finally: + conn.close() + + # Every emitted flag-set from the five guidance sites. Stage 1 + # (inspect) and stage 2 (output) as printed by the banners: + emitted_shapes = [ + {"inspect_only": True}, # --inspect-only + {"output": tmp_path / "recovered-state.db"}, # --output + ] + for overrides in emitted_shapes: + rc = sc.cmd_sessions(self._namespace(source, **overrides)) + assert rc != 2, ( + f"emitted shape {overrides} must pass the cmd_sessions " + "contract gate — the banner is printing a command the CLI " + "rejects before doing anything" + ) diff --git a/tests/hermes_cli/test_terminal_notify.py b/tests/hermes_cli/test_terminal_notify.py new file mode 100644 index 0000000000..fce8b90d5e --- /dev/null +++ b/tests/hermes_cli/test_terminal_notify.py @@ -0,0 +1,50 @@ +"""display.bell_on_prompt / bell_on_complete also drive OSC 9 + Warp OSC 777 via _ring_bell.""" + +import json + +from cli import HermesCLI +from hermes_cli import terminal_notify + +_WARP_OK = { + "TERM_PROGRAM": "WarpTerminal", + "WARP_CLI_AGENT_PROTOCOL_VERSION": "1", + "WARP_CLIENT_VERSION": "v0.2026.08.01.00.00.stable_01", +} + + +def _ring(monkeypatch, *, flag_on, env, **kwargs): + for key in _WARP_OK: + monkeypatch.delenv(key, raising=False) + for key, value in env.items(): + monkeypatch.setenv(key, value) + written = [] + monkeypatch.setattr(terminal_notify, "_write_tty", written.append) + cli = HermesCLI.__new__(HermesCLI) + cli.bell_on_prompt = flag_on + cli.session_id = "sess-1" + cli._ring_bell(prompt=True, **kwargs) + return "".join(written) + + +def test_osc9_body_emitted_and_sanitized_only_when_flag_on(monkeypatch): + out = _ring(monkeypatch, flag_on=True, env={}, context="approval\x1b\x07\x00\x7f!") + assert out == "\x1b]9;Hermes: approval!\x07" + assert _ring(monkeypatch, flag_on=False, env={}, context="approval") == "" + + +def test_warp_osc777_only_under_supported_warp_build(monkeypatch): + out = _ring(monkeypatch, flag_on=True, env=_WARP_OK, context="approval", detail="rm -rf build") + prefix = "\x1b]777;notify;warp://cli-agent;" + assert out.count(prefix) == 1 + payload = json.loads(out.split(prefix, 1)[1].rstrip("\x07")) + assert payload["agent"] == "hermes" + assert payload["event"] == "permission_request" + assert payload["summary"] == "rm -rf build" + assert payload["session_id"] == "sess-1" + assert payload["v"] == 1 + # Broken build (advertises the protocol var but can't render) → OSC 9 only. + broken = dict(_WARP_OK, WARP_CLIENT_VERSION="v0.2026.03.25.08.24.stable_05") + assert prefix not in _ring(monkeypatch, flag_on=True, env=broken, context="approval") + # Not Warp at all → OSC 9 only. + not_warp = dict(_WARP_OK, TERM_PROGRAM="ghostty") + assert prefix not in _ring(monkeypatch, flag_on=True, env=not_warp, context="approval") diff --git a/tests/hermes_cli/test_update_autostash.py b/tests/hermes_cli/test_update_autostash.py index 07994d518e..028bbb5ed7 100644 --- a/tests/hermes_cli/test_update_autostash.py +++ b/tests/hermes_cli/test_update_autostash.py @@ -74,7 +74,7 @@ def _setup_update_mocks(monkeypatch, tmp_path): monkeypatch.setattr(hermes_main, "_restore_stashed_changes", lambda *a, **kw: True) monkeypatch.setattr(hermes_config, "get_missing_env_vars", lambda required_only=True: []) monkeypatch.setattr(hermes_config, "get_missing_config_fields", lambda: []) - monkeypatch.setattr(hermes_config, "check_config_version", lambda: (5, 5)) + monkeypatch.setattr(hermes_config, "check_config_version", lambda **_kwargs: (5, 5)) monkeypatch.setattr(hermes_config, "migrate_config", lambda **kw: {"env_added": [], "config_added": []}) monkeypatch.setattr(hermes_main, "_refresh_active_lazy_features", lambda *a, **kw: True) diff --git a/tests/hermes_cli/test_update_check.py b/tests/hermes_cli/test_update_check.py index 21c7251c49..22c34bd621 100644 --- a/tests/hermes_cli/test_update_check.py +++ b/tests/hermes_cli/test_update_check.py @@ -23,7 +23,10 @@ def test_check_for_updates_uses_cache(tmp_path, monkeypatch): (repo_dir / ".git").mkdir() cache_file = tmp_path / ".update_check" - cache_file.write_text(json.dumps({"ts": time.time(), "behind": 3, "ver": __version__})) + cache_file.write_text( + json.dumps({"ts": time.time(), "behind": 3, "ver": __version__}), + encoding="utf-8", + ) monkeypatch.setenv("HERMES_HOME", str(tmp_path)) with patch("hermes_cli.banner.subprocess.run") as mock_run: @@ -58,6 +61,21 @@ def test_prefetch_non_blocking(): assert banner._update_result == 5 +def test_upstream_main_sha_disables_git_prompts(monkeypatch): + """The passive HTTPS probe must never inherit the interactive terminal.""" + from hermes_cli import banner + + completed = MagicMock(returncode=1, stdout="", stderr="auth required") + run = MagicMock(return_value=completed) + monkeypatch.setattr(banner.subprocess, "run", run) + + assert banner._upstream_main_sha() is None + kwargs = run.call_args.kwargs + assert kwargs["stdin"] is banner.subprocess.DEVNULL + assert kwargs["env"]["GIT_TERMINAL_PROMPT"] == "0" + assert kwargs["env"]["GCM_INTERACTIVE"] == "Never" + + def test_check_via_local_git_fetch_failure_returns_none(tmp_path, monkeypatch): """When git fetch fails and the stale origin/main ref is not ahead, _check_via_local_git must return None (#82166). @@ -90,8 +108,12 @@ def test_check_via_local_git_fetch_failure_returns_none(tmp_path, monkeypatch): stale_zero_proc.returncode = 0 stale_zero_proc.stdout = "0" + fetch_kwargs = None + def mock_run(args, **kwargs): + nonlocal fetch_kwargs if args[:2] == ["git", "fetch"]: + fetch_kwargs = kwargs return failed_proc if args[:2] == ["git", "rev-list"]: return stale_zero_proc @@ -104,6 +126,10 @@ def test_check_via_local_git_fetch_failure_returns_none(tmp_path, monkeypatch): assert result is None, ( "Fetch failure with stale 0-behind must return None, not 'up to date'" ) + assert fetch_kwargs is not None + assert fetch_kwargs["stdin"] is banner.subprocess.DEVNULL + assert fetch_kwargs["env"]["GIT_TERMINAL_PROMPT"] == "0" + assert fetch_kwargs["env"]["GCM_INTERACTIVE"] == "Never" def test_check_via_local_git_fetch_failure_keeps_positive_stale_count(tmp_path, monkeypatch): diff --git a/tests/hermes_cli/test_update_fetch_failure_classifier.py b/tests/hermes_cli/test_update_fetch_failure_classifier.py index f5e270c82c..dfc6e1bcb5 100644 --- a/tests/hermes_cli/test_update_fetch_failure_classifier.py +++ b/tests/hermes_cli/test_update_fetch_failure_classifier.py @@ -51,6 +51,16 @@ class TestClassifyFetchFailure: ) assert msg.startswith("✗ Network error") + def test_username_prompt_401_reports_github_not_user_credentials(self): + # What GitHub's HTTP 401 looks like once the terminal prompt is + # disabled — must NOT be blamed on the user's credentials. + msg = update_cmd._classify_fetch_failure( + "fatal: could not read Username for 'https://github.com':" + " terminal prompts disabled" + ) + assert "GitHub" in msg and "outage" in msg + assert "check your git credentials" not in msg + def test_auth_failure(self): msg = update_cmd._classify_fetch_failure( "fatal: Authentication failed for 'https://github.com/x.git/'" @@ -75,3 +85,37 @@ class TestPrintFetchFailure: update_cmd._print_fetch_failure("") out = capsys.readouterr().out.strip().splitlines() assert out == ["✗ Failed to fetch updates from origin."] + + +def test_update_network_git_calls_never_prompt_for_credentials(): + """Every `git fetch`/`pull`/`push` in the updater runs with prompts disabled. + + Live incident (Sep 2026): a GitHub-side 401 made `hermes update` sit on + ``Username for 'https://github.com':`` instead of failing with a diagnosis. + """ + import inspect + import os + import re + import subprocess + + kw = update_cmd._no_prompt_git_kwargs() + assert kw["stdin"] is subprocess.DEVNULL + assert kw["env"]["GIT_TERMINAL_PROMPT"] == "0" + # Only the prompt is disabled — credential helpers / askpass stay + # configured so a private-fork origin still authenticates. + assert "GIT_CONFIG_COUNT" not in kw["env"] or kw["env"]["GIT_CONFIG_COUNT"] == os.environ.get("GIT_CONFIG_COUNT") + + src = inspect.getsource(update_cmd) + # Every subprocess.run(...) whose argv is a fetch/pull must spread the kwargs. + calls = [] + for m in re.finditer(r"subprocess\.run\(", src): + depth, i = 1, m.end() + while depth: + depth += {"(": 1, ")": -1}.get(src[i], 0) + i += 1 + call = src[m.start():i] + if re.search(r'git_cmd \+ \["(fetch|pull|push)"', call): + calls.append(call) + assert calls, "expected network git calls in update_cmd" + missing = [c for c in calls if "_no_prompt_git_kwargs()" not in c] + assert not missing, missing diff --git a/tests/hermes_cli/test_update_fleet_restart_pending.py b/tests/hermes_cli/test_update_fleet_restart_pending.py index bdc9945f76..e22f3a0af9 100644 --- a/tests/hermes_cli/test_update_fleet_restart_pending.py +++ b/tests/hermes_cli/test_update_fleet_restart_pending.py @@ -139,11 +139,13 @@ def _patch_update_deps(monkeypatch, tmp_path, run_side_effect): update_cmd, "_venv_core_imports_healthy", lambda: (True, "") ) monkeypatch.setattr(update_cmd, "_update_node_dependencies", lambda: []) + monkeypatch.setattr(update_cmd, "_purge_stale_hermes_modules", lambda: None) + monkeypatch.setattr(hermes_main, "_purge_stale_hermes_modules", lambda: None) import hermes_cli.gateway as hermes_gateway monkeypatch.setattr( - hermes_gateway, "find_gateway_pids", lambda all_profiles=False: [] + hermes_gateway, "find_gateway_pids", lambda **_kwargs: [] ) monkeypatch.setattr(hermes_gateway, "supports_systemd_services", lambda: False) monkeypatch.setattr( @@ -345,6 +347,97 @@ def test_marker_written_after_pull_cleared_after_successful_restart( assert "✓ Code updated!" in out +def test_clean_update_warns_about_surviving_pre_update_serve_runtime( + monkeypatch, tmp_path, capsys +): + """The successful update path must surface an inventoried stale serve.""" + args = _update_args() + _patch_update_deps(monkeypatch, tmp_path, _make_head_moved_side_effect()) + monkeypatch.setattr( + update_cmd, + "_surviving_pre_update_serve_runtimes", + lambda _plan: [ + { + "pid": 5555, + "kind": "serve", + "profile": "default", + "supervisor": "manual-serve", + } + ], + ) + + hermes_main.cmd_update(args) + + out = capsys.readouterr().out + assert "pid 5555" in out + assert "serve" in out + assert "pre-update code" in out + + +def test_clean_update_escalates_surviving_serve_as_unaccounted( + monkeypatch, tmp_path, capsys +): + """#100479 end to end: the plan inventoried a gateway (restarted through + ``hermes-gateway.service``) and an unmanaged ``serve`` on the same + default profile. The serve survives the update as the SAME process, so + the update must (1) warn, (2) reconcile it as ``unaccounted`` instead of + borrowing the gateway's restart, and (3) exit 1 with a ``partial`` + receipt — not print a clean success.""" + from hermes_cli.update_inventory import ( + RuntimeRecord, UpdatePlan, _restart_mechanism, + ) + import hermes_cli.update_inventory as ui + + args = _update_args() + _patch_update_deps(monkeypatch, tmp_path, _make_head_moved_side_effect()) + + plan = UpdatePlan() + plan.runtimes = [ + RuntimeRecord(kind="gateway", profile="default", pid=4444, + supervisor="systemd", + restart_via=_restart_mechanism("systemd", "default")), + RuntimeRecord(kind="serve", profile="default", pid=5555, + supervisor="manual-serve", + restart_via=_restart_mechanism("manual-serve", "default"), + detail={"create_time": 1000.0}), + ] + monkeypatch.setattr(ui, "collect_runtime_inventory", lambda: plan) + # The restart phase's own bookkeeping says the gateway unit restarted + # (systemd branch is stubbed off in _patch_update_deps, so feed it here). + real_match = ui.match_runtime_outcomes + + def _match(p, **kw): + kw["restarted_services"] = list(kw.get("restarted_services") or []) + [ + "hermes-gateway.service" + ] + return real_match(p, **kw) + + monkeypatch.setattr(ui, "match_runtime_outcomes", _match) + # Real survivor probe semantics against a fake ledger: pid 5555 is still + # the same incarnation the plan recorded. + import hermes_cli.process_identity as pi + + monkeypatch.setattr( + pi, "ledger_entries", + lambda **_k: [{"pid": 5555, "purpose": "serve", "create_time": 1000.0}], + ) + + with pytest.raises(SystemExit) as excinfo: + hermes_main.cmd_update(args) + assert excinfo.value.code == 1 + + out = capsys.readouterr().out + assert "pid 5555" in out and "pre-update code" in out + assert "Planned runtimes the restart phase never touched" in out + assert "serve [default] pid 5555" in out + + latest = get_hermes_home() / "logs" / "update_receipts" / "latest.json" + receipt = json.loads(latest.read_text(encoding="utf-8")) + assert receipt["outcome"] == "partial" + by_pid = {o["pid"]: o["outcome"] for o in receipt["runtime_outcomes"]} + assert by_pid == {4444: "restarted", 5555: "unaccounted"} + + def test_interrupt_between_pull_and_restart_leaves_marker( monkeypatch, tmp_path ): diff --git a/tests/hermes_cli/test_update_wedged_gateway.py b/tests/hermes_cli/test_update_wedged_gateway.py index a2e1bf77c2..91f884c3f0 100644 --- a/tests/hermes_cli/test_update_wedged_gateway.py +++ b/tests/hermes_cli/test_update_wedged_gateway.py @@ -14,6 +14,7 @@ import json import os import shutil import socket +import sys import tempfile import threading import time @@ -30,6 +31,19 @@ from gateway.shutdown_watchdog import ( write_loop_heartbeat, ) +# Native Windows exposes neither ``socket.AF_UNIX`` nor an asyncio UNIX +# server, so the witness cases that create real socket nodes +# (``_silent_socket_node``) or run the real producer +# (``loop_heartbeat_forever``) cannot execute there. Only those cases are +# skipped: the witness-absent contracts (mocked probes, file-only +# heartbeats) are platform-independent and keep running on Windows, per +# the Windows behavior pinned alongside the product-side guarantee. +_NEEDS_UNIX_SOCKETS = pytest.mark.skipif( + sys.platform == "win32", + reason="requires real UNIX-domain sockets " + "(socket.AF_UNIX / asyncio.start_unix_server), unavailable on native Windows", +) + @pytest.fixture() def tmp_path(): @@ -486,6 +500,7 @@ class TestLoopTickWitness: witnesses agree the loop stopped scheduling. """ + @_NEEDS_UNIX_SOCKETS def test_stalled_heartbeat_write_never_escalates_a_running_loop( self, tmp_path, monkeypatch ): @@ -635,6 +650,7 @@ class TestLoopTickWitness: thread.join(timeout=5.0) assert not errors, errors + @_NEEDS_UNIX_SOCKETS def test_off_loop_completion_cannot_manufacture_fresh_liveness(self, tmp_path): """A write landing after the loop froze must not look alive. @@ -654,6 +670,7 @@ class TestLoopTickWitness: == gateway_cli.GATEWAY_LOOP_UNKNOWN ) + @_NEEDS_UNIX_SOCKETS def test_true_wedge_requires_sustained_witness_silence(self, tmp_path): """Stale file + armed socket silent across the whole window: WEDGED. @@ -812,10 +829,16 @@ class TestLoopTickWitness: gateway_cli.probe_gateway_loop_liveness(pid, home=tmp_path) == gateway_cli.GATEWAY_LOOP_WEDGED ) - # And a fresh legacy file stays safe even if a dead-listener node - # exists for the PID (leftover from a newer process): the silent - # socket denies ALIVE, and UNKNOWN never escalates — the drain path - # keeps the full budget either way. + + @_NEEDS_UNIX_SOCKETS + def test_legacy_fresh_file_with_dead_node_is_unknown(self, tmp_path): + """A fresh legacy file stays safe under a dead-listener node. + + A dead-listener node for the PID (leftover from a newer process): + the silent socket denies ALIVE, and UNKNOWN never escalates — the + drain path keeps the full budget either way. + """ + pid = 4242 _write_heartbeat(tmp_path, pid, age_s=5.0) _silent_socket_node(get_loop_tick_socket_path(tmp_path, pid)) assert ( @@ -825,6 +848,7 @@ class TestLoopTickWitness: == gateway_cli.GATEWAY_LOOP_UNKNOWN ) + @_NEEDS_UNIX_SOCKETS @pytest.mark.asyncio async def test_producer_rebinds_over_stale_socket_node(self, tmp_path): """A leftover node from a dead process must not disarm the witness. @@ -866,6 +890,7 @@ class TestLoopTickWitness: except asyncio.CancelledError: pass + @_NEEDS_UNIX_SOCKETS def test_transient_stall_below_wedge_budget_never_escalates( self, tmp_path, monkeypatch ): @@ -918,6 +943,7 @@ class TestLoopTickWitness: state["thread"].join(timeout=5.0) assert not errors, errors + @_NEEDS_UNIX_SOCKETS def test_sustained_stop_above_wedge_budget_still_escalates( self, tmp_path ): @@ -964,6 +990,113 @@ class TestLoopTickWitness: assert not errors, errors +class TestLoopTickTcpWitness: + """Non-POSIX arm: the producer publishes ``loop_tick_tcp_port`` and the + consumer probes 127.0.0.1: instead of the AF_UNIX node. The + two-witness contract must hold identically over TCP.""" + + @staticmethod + def _tcp_answerer(): + """A loopback listener that answers b"1" — the armed, dispatching loop.""" + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind(("127.0.0.1", 0)) + srv.listen(8) + stop = threading.Event() + + def serve(): + srv.settimeout(0.1) + while not stop.is_set(): + try: + conn, _ = srv.accept() + except socket.timeout: + continue + try: + conn.sendall(b"1") + finally: + conn.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + return srv.getsockname()[1], stop, srv + + @staticmethod + def _write_tcp_heartbeat(home, pid, port, age_s=0.0): + write_loop_heartbeat( + pid=pid, + home=home, + extra={"loop_tick_socket": True, "loop_tick_tcp_port": port}, + ) + if age_s: + path = get_loop_heartbeat_path(home) + stamp = time.time() - age_s + os.utime(path, (stamp, stamp)) + + def test_stale_file_with_answering_tcp_witness_is_alive(self, tmp_path): + """#90502 shape over TCP: a stalled write must not kill a live loop.""" + port, stop, srv = self._tcp_answerer() + try: + self._write_tcp_heartbeat(tmp_path, 4343, port, age_s=600.0) + assert ( + gateway_cli.probe_gateway_loop_liveness(4343, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_ALIVE + ) + finally: + stop.set() + srv.close() + + def test_stale_file_with_silent_tcp_witness_is_wedged(self, tmp_path): + """Armed TCP witness that never answers across the window: WEDGED.""" + silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + silent.bind(("127.0.0.1", 0)) + silent.listen(1) # accepts but never sends + try: + port = silent.getsockname()[1] + self._write_tcp_heartbeat(tmp_path, 4344, port, age_s=600.0) + assert ( + gateway_cli.probe_gateway_loop_liveness( + 4344, home=tmp_path, tick_timeout=0.2, tick_gap_s=0.05 + ) + == gateway_cli.GATEWAY_LOOP_WEDGED + ) + finally: + silent.close() + + def test_fresh_file_with_silent_tcp_witness_is_unknown(self, tmp_path): + """Fresh file + silent TCP witness: an off-loop write landed after a + freeze — not proof of liveness, never destructive authority.""" + silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + silent.bind(("127.0.0.1", 0)) + silent.listen(1) + try: + port = silent.getsockname()[1] + self._write_tcp_heartbeat(tmp_path, 4345, port) + assert ( + gateway_cli.probe_gateway_loop_liveness( + 4345, home=tmp_path, tick_timeout=0.2 + ) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + finally: + silent.close() + + def test_garbage_tcp_port_falls_back_to_socket_contract(self, tmp_path): + """A non-numeric port must not be treated as an armed witness.""" + write_loop_heartbeat( + pid=4346, + home=tmp_path, + extra={"loop_tick_socket": False, "loop_tick_tcp_port": "nope"}, + ) + path = get_loop_heartbeat_path(tmp_path) + stamp = time.time() - 600.0 + os.utime(path, (stamp, stamp)) + # loop_tick_socket=False + no usable TCP port: witness could not be + # armed, staleness is not proof -> UNKNOWN, never WEDGED. + assert ( + gateway_cli.probe_gateway_loop_liveness(4346, home=tmp_path) + == gateway_cli.GATEWAY_LOOP_UNKNOWN + ) + + def test_default_probe_budget_stays_inside_query_tier(): """The module doc pins the worst-case wedge-suspected probe at ~3.4s, 'far inside the 10s query tier'. Assert the strike-count math so diff --git a/tests/hermes_cli/test_web_profile_soul_writes.py b/tests/hermes_cli/test_web_profile_soul_writes.py index 06f209d7f3..bbf8d64fda 100644 --- a/tests/hermes_cli/test_web_profile_soul_writes.py +++ b/tests/hermes_cli/test_web_profile_soul_writes.py @@ -12,6 +12,7 @@ small and focused on this one endpoint pair. from __future__ import annotations +import asyncio import os import stat import sys @@ -33,7 +34,11 @@ def client(tmp_path, monkeypatch): from hermes_cli import web_server with TestClient(web_server.app, raise_server_exceptions=False) as c: - c.headers["Authorization"] = "Bearer soul-test-token" + # web_server resolves _SESSION_TOKEN once, at import. Read it back from + # the module instead of assuming the env var above won the race — any + # test file that imports web_server earlier in the session fixes the + # token before this fixture runs. + c.headers["Authorization"] = f"Bearer {web_server._SESSION_TOKEN}" yield c @@ -129,3 +134,81 @@ class TestSoulWriteDurability: assert r.status_code == 200, r.text mode = stat.S_IMODE(soul.stat().st_mode) assert mode == 0o644, f"first save created SOUL.md as {oct(mode)}" + + +class TestSoulIoIsOffTheEventLoop: + """Neither half of the persona editor may run its I/O on the ASGI loop. + + The durability the tests above buy comes from ``atomic_write_text``, which + fsyncs before replacing — so the save blocks for as long as the filesystem + takes to commit. These handlers sit in the same router as the profile + delete and describe-auto paths; the rest of that sweep is covered by + ``tests/hermes_cli/test_web_profiles_off_loop.py``. + """ + + @staticmethod + def _probe(seen, tag): + """Record whether the caller's thread is running an event loop.""" + try: + asyncio.get_running_loop() + seen.append((tag, True)) + except RuntimeError: + seen.append((tag, False)) + + def test_get_soul_reads_off_loop(self, client, profile_dir: Path, monkeypatch): + (profile_dir / "SOUL.md").write_text(SOUL, encoding="utf-8") + seen: list[tuple[str, bool]] = [] + real_read_text = Path.read_text + + def probing_read_text(self, *args, **kwargs): + if self.name == "SOUL.md": + TestSoulIoIsOffTheEventLoop._probe(seen, "read") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", probing_read_text) + + r = client.get("/api/profiles/demo/soul") + + assert r.status_code == 200, r.text + assert r.json()["content"] == SOUL + assert ("read", False) in seen, ( + f"SOUL.md must be read off the event loop; proof: {seen}" + ) + + def test_put_soul_writes_off_loop(self, client, profile_dir: Path, monkeypatch): + seen: list[tuple[str, bool]] = [] + import utils + + real_write = utils.atomic_write_text + + def probing_write(*args, **kwargs): + TestSoulIoIsOffTheEventLoop._probe(seen, "write") + return real_write(*args, **kwargs) + + monkeypatch.setattr(utils, "atomic_write_text", probing_write) + + r = client.put("/api/profiles/demo/soul", json={"content": SOUL}) + + assert r.status_code == 200, r.text + assert (profile_dir / "SOUL.md").read_text(encoding="utf-8") == SOUL + assert ("write", False) in seen, ( + f"SOUL.md must be written off the event loop; proof: {seen}" + ) + + def test_missing_soul_is_still_reported_absent(self, client, profile_dir: Path): + """The offloaded reader must keep distinguishing "no file" from + "empty file" — the whole point of the durability tests above.""" + assert not (profile_dir / "SOUL.md").exists() + + r = client.get("/api/profiles/demo/soul") + + assert r.status_code == 200, r.text + assert r.json() == {"content": "", "exists": False} + + def test_empty_soul_is_still_reported_present(self, client, profile_dir: Path): + (profile_dir / "SOUL.md").write_text("", encoding="utf-8") + + r = client.get("/api/profiles/demo/soul") + + assert r.status_code == 200, r.text + assert r.json() == {"content": "", "exists": True} diff --git a/tests/hermes_cli/test_web_profiles_off_loop.py b/tests/hermes_cli/test_web_profiles_off_loop.py new file mode 100644 index 0000000000..0b19e56b97 --- /dev/null +++ b/tests/hermes_cli/test_web_profiles_off_loop.py @@ -0,0 +1,422 @@ +"""Regression tests: ``/api/profiles`` handlers must not block the event loop. + +``hermes_cli/web_routers/profiles.py`` holds handler bodies that were extracted +verbatim from ``web_server.py``, so the blocking library calls they inherited +run inline on the ASGI event loop. The worst of them are unbounded from the +dashboard's point of view: deleting a profile whose gateway is up sleeps up to +10 s in ``profiles._stop_gateway_process``, and ``describe-auto`` makes a +provider round-trip with a 60 s ceiling. While the loop is parked, the process +serves nothing else — including the ``/api/ws`` probes the desktop app and the +dashboard's own Chat tab depend on. + +Two complementary assertions per site: + +* a **loop probe** — the stubbed callee records whether an event loop is + running in its own thread, mirroring + ``tests/hermes_cli/test_cron_dashboard_off_loop.py``; and +* a **concurrency proof** — the stubbed callee blocks on a ``threading.Event`` + while an unrelated request is timed, which fails if the loop is parked. + +The block is bounded by a timeout so a regression costs the suite a few +seconds rather than hanging it. +""" + +from __future__ import annotations + +import asyncio +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +pytest.importorskip("fastapi") +from fastapi.testclient import TestClient # noqa: E402 + +# How long a stubbed blocking call holds its thread when nobody releases it. +BLOCK_SECONDS = 5.0 +# A concurrently-served request has to land well inside that window. The gap is +# large (a served request takes milliseconds) so the bound is not timing-fragile. +CONCURRENT_BUDGET = BLOCK_SECONDS / 2 + + +@pytest.fixture() +def profile_dir(tmp_path, monkeypatch) -> Path: + """A real profile directory under a throwaway HERMES_HOME.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from hermes_cli import profiles as profiles_mod + + d = profiles_mod.get_profile_dir("demo") + d.mkdir(parents=True, exist_ok=True) + return d + + +@pytest.fixture() +def client(profile_dir): + """One ``TestClient`` context == one portal == one event loop. + + This matters: entering the context manager pins a single blocking portal + for every request, so a handler that parks the loop really does starve the + concurrent request below. A bare ``TestClient(app)`` spins up a fresh loop + per request and would pass these tests even unfixed. + """ + from hermes_cli import web_server + + with TestClient(web_server.app, raise_server_exceptions=False) as c: + # web_server resolves _SESSION_TOKEN once, at import, so read it back + # from the module rather than pinning a value from this file. + c.headers["Authorization"] = f"Bearer {web_server._SESSION_TOKEN}" + yield c + + +@pytest.fixture() +def loop_probe(): + """Collect ``(tag, on_loop)`` proof from stubbed blocking callees.""" + seen: list[tuple[str, bool]] = [] + + def probe(tag: str) -> None: + try: + asyncio.get_running_loop() + seen.append((tag, True)) + except RuntimeError: + seen.append((tag, False)) + + return seen, probe + + +def assert_off_loop(seen, tag: str) -> None: + assert (tag, False) in seen, ( + f"{tag} must run off the event loop; proof: {seen}" + ) + + +class _Blocker: + """A stand-in for a slow library call, released by the test.""" + + def __init__(self, result=None): + self.entered = threading.Event() + self.release = threading.Event() + self._result = result + + def __call__(self, *args, **kwargs): + self.entered.set() + # Bounded on purpose: a regression must not hang the suite. + self.release.wait(timeout=BLOCK_SECONDS) + return self._result + + +def assert_serves_concurrently(client, blocker: _Blocker, fire) -> None: + """Fire a request that blocks inside its handler, then time another one. + + ``fire`` issues the blocking request from a worker thread. Once the stub + has been entered, an unrelated cheap route is timed on this thread: it can + only answer quickly if the blocking work left the event loop. + """ + with ThreadPoolExecutor(max_workers=1) as pool: + blocked = pool.submit(fire) + try: + assert blocker.entered.wait(timeout=BLOCK_SECONDS), ( + "the blocking stub was never reached" + ) + start = time.monotonic() + probe = client.get("/api/profiles/demo/setup-command") + elapsed = time.monotonic() - start + finally: + blocker.release.set() + blocked.result(timeout=BLOCK_SECONDS * 2) + + assert probe.status_code == 200, probe.text + assert elapsed < CONCURRENT_BUDGET, ( + f"a concurrent request waited {elapsed:.2f}s while the handler was " + f"busy — the event loop was parked by the blocking call" + ) + + +# ── DELETE /api/profiles/{name} — the 10 s gateway-stop sleep ──────────────── + + +def test_delete_profile_runs_off_loop(client, monkeypatch, loop_probe, tmp_path): + seen, probe = loop_probe + from hermes_cli import profiles as profiles_mod + + def fake_delete(name, yes=False): + probe("delete_profile") + return tmp_path / "profiles" / name + + monkeypatch.setattr(profiles_mod, "delete_profile", fake_delete) + + resp = client.delete("/api/profiles/demo") + + assert resp.status_code == 200, resp.text + assert_off_loop(seen, "delete_profile") + + +def test_delete_profile_does_not_block_the_dashboard(client, monkeypatch, tmp_path): + """``delete_profile`` stops a running gateway by polling for up to 10 s. + + That is longer than the desktop's WebSocket ready-probe tolerates, so it + must not hold the loop. + """ + from hermes_cli import profiles as profiles_mod + + blocker = _Blocker(result=tmp_path / "profiles" / "demo") + monkeypatch.setattr(profiles_mod, "delete_profile", blocker) + + assert_serves_concurrently( + client, blocker, lambda: client.delete("/api/profiles/demo") + ) + + +# ── POST /api/profiles/{name}/describe-auto — the 60 s LLM round-trip ──────── + + +def _outcome(ok=True, reason="described", description="a demo profile"): + from hermes_cli.profile_describer import DescribeOutcome + + return DescribeOutcome("demo", ok, reason, description=description) + + +def test_describe_auto_runs_off_loop(client, monkeypatch, loop_probe): + seen, probe = loop_probe + from hermes_cli import profile_describer + + def fake_describe(name, overwrite=False, timeout=None): + probe("describe_profile") + return _outcome() + + monkeypatch.setattr(profile_describer, "describe_profile", fake_describe) + + resp = client.post("/api/profiles/demo/describe-auto", json={"overwrite": True}) + + assert resp.status_code == 200, resp.text + assert resp.json()["ok"] is True + assert_off_loop(seen, "describe_profile") + + +def test_describe_auto_does_not_block_the_dashboard(client, monkeypatch): + """The auxiliary provider call has a 60 s ceiling — six times the + desktop's disconnect threshold.""" + from hermes_cli import profile_describer + + blocker = _Blocker(result=_outcome()) + monkeypatch.setattr(profile_describer, "describe_profile", blocker) + + assert_serves_concurrently( + client, + blocker, + lambda: client.post( + "/api/profiles/demo/describe-auto", json={"overwrite": True} + ), + ) + + +# ── PATCH /api/profiles/{name} — rename walks and rewrites the profile tree ── + + +def test_rename_profile_runs_off_loop(client, monkeypatch, loop_probe, tmp_path): + seen, probe = loop_probe + from hermes_cli import profiles as profiles_mod + + def fake_rename(old, new): + probe("rename_profile") + return tmp_path / "profiles" / new + + monkeypatch.setattr(profiles_mod, "rename_profile", fake_rename) + + resp = client.patch("/api/profiles/demo", json={"new_name": "renamed"}) + + assert resp.status_code == 200, resp.text + assert_off_loop(seen, "rename_profile") + + +def test_rename_profile_does_not_block_the_dashboard(client, monkeypatch, tmp_path): + from hermes_cli import profiles as profiles_mod + + blocker = _Blocker(result=tmp_path / "profiles" / "renamed") + monkeypatch.setattr(profiles_mod, "rename_profile", blocker) + + assert_serves_concurrently( + client, + blocker, + lambda: client.patch("/api/profiles/demo", json={"new_name": "renamed"}), + ) + + +# ── /api/profiles/active — sticky active-profile state file ────────────────── + + +def test_get_active_profile_runs_off_loop(client, monkeypatch, loop_probe): + seen, probe = loop_probe + from hermes_cli import profiles as profiles_mod + + def fake_get_active(): + probe("get_active_profile") + return "demo" + + def fake_get_current(): + probe("get_active_profile_name") + return "default" + + monkeypatch.setattr(profiles_mod, "get_active_profile", fake_get_active) + monkeypatch.setattr(profiles_mod, "get_active_profile_name", fake_get_current) + + resp = client.get("/api/profiles/active") + + assert resp.status_code == 200, resp.text + assert resp.json() == {"active": "demo", "current": "default"} + assert_off_loop(seen, "get_active_profile") + assert_off_loop(seen, "get_active_profile_name") + + +def test_set_active_profile_runs_off_loop(client, monkeypatch, loop_probe): + seen, probe = loop_probe + from hermes_cli import profiles as profiles_mod + + def fake_set_active(name): + probe("set_active_profile") + + monkeypatch.setattr(profiles_mod, "set_active_profile", fake_set_active) + + resp = client.post("/api/profiles/active", json={"name": "demo"}) + + assert resp.status_code == 200, resp.text + assert resp.json()["active"] == "demo" + assert_off_loop(seen, "set_active_profile") + + +# ── PUT /api/profiles/{name}/description — profile.yaml read/modify/write ──── + + +def test_update_description_runs_off_loop(client, monkeypatch, loop_probe): + seen, probe = loop_probe + from hermes_cli import profiles as profiles_mod + + def fake_write_meta(profile_dir, **kwargs): + probe("write_profile_meta") + + monkeypatch.setattr(profiles_mod, "write_profile_meta", fake_write_meta) + + resp = client.put( + "/api/profiles/demo/description", json={"description": "a demo profile"} + ) + + assert resp.status_code == 200, resp.text + assert resp.json()["description_auto"] is False + assert_off_loop(seen, "write_profile_meta") + + +# ── GET /api/profiles/{name}/desktop-overlay — reads desktop.json ──────────── + + +def test_desktop_overlay_read_runs_off_loop(client, monkeypatch, loop_probe, profile_dir): + seen, probe = loop_probe + (profile_dir / "desktop.json").write_text('{"theme": "dark"}', encoding="utf-8") + + real_read_text = Path.read_text + + def probing_read_text(self, *args, **kwargs): + if self.name == "desktop.json": + probe("desktop.json read") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", probing_read_text) + + resp = client.get("/api/profiles/demo/desktop-overlay") + + assert resp.status_code == 200, resp.text + assert resp.json() == {"exists": True, "desktop": {"theme": "dark"}} + assert_off_loop(seen, "desktop.json read") + + +def test_desktop_overlay_absent_still_reports_missing(client): + """The offloaded read must keep distinguishing "no file" from "no data".""" + resp = client.get("/api/profiles/demo/desktop-overlay") + + assert resp.status_code == 200, resp.text + assert resp.json() == {"exists": False, "desktop": None} + + +def test_desktop_overlay_null_document_still_reports_present(client, profile_dir): + """A ``desktop.json`` holding literal ``null`` exists, it is just empty — + it must not be reported the same as a missing file.""" + (profile_dir / "desktop.json").write_text("null", encoding="utf-8") + + resp = client.get("/api/profiles/demo/desktop-overlay") + + assert resp.status_code == 200, resp.text + assert resp.json() == {"exists": True, "desktop": None} + + +def test_desktop_overlay_unreadable_document_is_a_500(client, profile_dir): + """A malformed overlay still surfaces as a 500, not a silent success.""" + (profile_dir / "desktop.json").write_text("{not json", encoding="utf-8") + + resp = client.get("/api/profiles/demo/desktop-overlay") + + assert resp.status_code == 500, resp.text + + +# ── Status-code mapping must survive the move into a worker thread ─────────── + + +def test_delete_missing_profile_is_still_404(client, monkeypatch): + from hermes_cli import profiles as profiles_mod + + def fake_delete(name, yes=False): + raise FileNotFoundError(f"Profile '{name}' does not exist.") + + monkeypatch.setattr(profiles_mod, "delete_profile", fake_delete) + + assert client.delete("/api/profiles/demo").status_code == 404 + + +def test_rename_to_existing_profile_is_still_400(client, monkeypatch): + from hermes_cli import profiles as profiles_mod + + def fake_rename(old, new): + raise FileExistsError(f"Profile '{new}' already exists.") + + monkeypatch.setattr(profiles_mod, "rename_profile", fake_rename) + + resp = client.patch("/api/profiles/demo", json={"new_name": "taken"}) + assert resp.status_code == 400, resp.text + + +def test_set_active_missing_profile_is_still_404(client, monkeypatch): + from hermes_cli import profiles as profiles_mod + + def fake_set_active(name): + raise FileNotFoundError(f"Profile '{name}' does not exist.") + + monkeypatch.setattr(profiles_mod, "set_active_profile", fake_set_active) + + assert client.post("/api/profiles/active", json={"name": "demo"}).status_code == 404 + + +def test_describe_auto_unknown_profile_is_still_404(client): + """``_resolve_profile_dir`` still runs before the worker hop, so an + unknown profile is a 404 rather than a 500 from the wrapped call.""" + assert ( + client.post("/api/profiles/nope/describe-auto", json={}).status_code == 404 + ) + + +# ── PUT /api/profiles/{name}/model — config.yaml read-modify-write ─────────── + + +def test_update_profile_model_runs_off_loop(client, monkeypatch, loop_probe): + seen, probe = loop_probe + from hermes_cli.web_routers import profiles as router_mod + + def fake_write_model(profile_dir, provider, model): + probe("write_profile_model") + + monkeypatch.setattr(router_mod, "_write_profile_model", fake_write_model) + + resp = client.put( + "/api/profiles/demo/model", json={"provider": "openrouter", "model": "x/y"} + ) + + assert resp.status_code == 200, resp.text + assert_off_loop(seen, "write_profile_model") diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index d8cf0a6467..ea36b52151 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -319,6 +319,29 @@ class TestWebServerEndpoints: monitor.close() writer.close() + def test_get_sessions_transient_ioerr_is_503(self, monkeypatch): + """Busy store, not a gone store: the desktop keeps the list it has.""" + import sqlite3 + + from hermes_cli import web_server + + def boom(*_args, **_kwargs): + raise sqlite3.OperationalError("disk I/O error") + + monkeypatch.setattr(web_server, "_open_session_db_for_profile", boom) + assert self.client.get("/api/sessions?limit=1&offset=0").status_code == 503 + + def test_get_sessions_non_transient_operational_error_is_500(self, monkeypatch): + import sqlite3 + + from hermes_cli import web_server + + def boom(*_args, **_kwargs): + raise sqlite3.OperationalError("no such table: sessions") + + monkeypatch.setattr(web_server, "_open_session_db_for_profile", boom) + assert self.client.get("/api/sessions?limit=1&offset=0").status_code == 500 + def test_get_status_loads_gateway_config_off_event_loop(self, monkeypatch): """Cold gateway config loading must not block the WebSocket loop. @@ -417,6 +440,9 @@ class TestWebServerEndpoints: legacy = sqlite3.connect(str(db_path)) try: + # SQLite refuses DROP COLUMN while an index references the + # column; a pre-column legacy store has neither. + legacy.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity") legacy.execute(f"ALTER TABLE sessions DROP COLUMN {missing_column}") legacy.commit() finally: @@ -463,6 +489,7 @@ class TestWebServerEndpoints: legacy = sqlite3.connect(str(db_path)) try: + legacy.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity") legacy.execute("ALTER TABLE sessions DROP COLUMN last_activity_at") legacy.commit() finally: diff --git a/tests/hermes_cli/test_web_server_tts_lease.py b/tests/hermes_cli/test_web_server_tts_lease.py new file mode 100644 index 0000000000..be32621997 --- /dev/null +++ b/tests/hermes_cli/test_web_server_tts_lease.py @@ -0,0 +1,157 @@ +"""``POST /api/audio/tts-lease`` — desktop speech toggles as TTS warm-up/release. + +The desktop's "Read replies aloud" and voice-conversation toggles call this so +the backend can pre-load the configured TTS engine when speech is about to be +needed and unload resident local models once no surface holds a lease. +""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +def isolated_profiles(tmp_path, monkeypatch, _isolate_hermes_home): + from hermes_constants import get_hermes_home + from hermes_cli import profiles + + default_home = get_hermes_home() + profiles_root = default_home / "profiles" + worker_home = profiles_root / "worker_beta" + for home in (default_home, worker_home): + home.mkdir(parents=True, exist_ok=True) + (home / "config.yaml").write_text("{}\n", encoding="utf-8") + (worker_home / ".env").write_text("", encoding="utf-8") + + monkeypatch.setattr(profiles, "_get_default_hermes_home", lambda: default_home) + monkeypatch.setattr(profiles, "_get_profiles_root", lambda: profiles_root) + return {"default": default_home, "worker_beta": worker_home} + + +@pytest.fixture +def client(monkeypatch, isolated_profiles): + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_constants import get_hermes_home + from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN + + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + c = TestClient(app) + c.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + return c + + +@pytest.fixture(autouse=True) +def _clean_leases(): + from tools import tts_tool + + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + yield + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + + +def test_active_acquires_and_warms(client, monkeypatch): + from tools import tts_tool + + warmed = [] + monkeypatch.setattr( + tts_tool, + "warm_tts_provider", + lambda cfg=None, provider=None: warmed.append(1) or {"provider": "piper", "warmed": True, "action": "loaded"}, + ) + + resp = client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["lease"] == "desktop:read-aloud" + assert body["active"] is True + assert body["leases"] == 1 + assert body["action"] == "loaded" + assert warmed == [1] + assert tts_tool.tts_lease_holders() == ["desktop:read-aloud"] + + +def test_inactive_releases_and_unloads_when_last(client, monkeypatch): + from tools import tts_tool + + monkeypatch.setattr(tts_tool, "warm_tts_provider", lambda cfg=None, provider=None: {"action": "noop", "warmed": False, "provider": "piper"}) + client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": True}) + client.post("/api/audio/tts-lease", json={"lease": "desktop:conversation:abc", "active": True}) + tts_tool._piper_voice_cache["voice"] = object() + + first = client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": False}).json() + assert first["leases"] == 1 + assert first["released"] == 0 + assert len(tts_tool._piper_voice_cache) == 1 + + last = client.post("/api/audio/tts-lease", json={"lease": "desktop:conversation:abc", "active": False}).json() + assert last["leases"] == 0 + assert last["released"] == 1 + assert tts_tool._piper_voice_cache == {} + + +def test_warm_failure_is_reported_not_an_http_error(client, monkeypatch): + from tools import tts_tool + + def _boom(cfg=None, provider=None): + raise RuntimeError("engine exploded") + + monkeypatch.setattr(tts_tool, "warm_tts_provider", _boom) + resp = client.post("/api/audio/tts-lease", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["action"] == "error" + assert "engine exploded" in body["error"] + + +def test_blank_lease_rejected(client): + resp = client.post("/api/audio/tts-lease", json={"lease": " ", "active": True}) + assert resp.status_code == 400 + + +def test_active_default_true(client, monkeypatch): + from tools import tts_tool + + monkeypatch.setattr(tts_tool, "warm_tts_provider", lambda cfg=None, provider=None: {"action": "noop", "warmed": False, "provider": "x"}) + resp = client.post("/api/audio/tts-lease", json={"lease": "tui:x"}) + assert resp.json()["active"] is True + assert tts_tool.tts_lease_holders() == ["tui:x"] + + +def test_acquire_resolves_provider_inside_target_profile(client, isolated_profiles, monkeypatch): + """Warm-up must read the REQUESTING profile's tts config, like /api/audio/speak.""" + import yaml + from tools import tts_tool + + (isolated_profiles["worker_beta"] / "config.yaml").write_text( + yaml.safe_dump({"tts": {"provider": "kittentts"}}), encoding="utf-8" + ) + seen = {} + + def _fake_warm(cfg=None, provider=None): + from hermes_constants import get_hermes_home + + seen["home"] = str(get_hermes_home()) + seen["provider"] = tts_tool._get_provider(tts_tool._load_tts_config()) + return {"action": "noop", "warmed": False, "provider": seen["provider"]} + + monkeypatch.setattr(tts_tool, "warm_tts_provider", _fake_warm) + resp = client.post("/api/audio/tts-lease?profile=worker_beta", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 200 + assert seen["home"] == str(isolated_profiles["worker_beta"]) + assert seen["provider"] == "kittentts" + + +def test_unknown_profile_404(client): + resp = client.post("/api/audio/tts-lease?profile=ghost", json={"lease": "desktop:read-aloud", "active": True}) + assert resp.status_code == 404 diff --git a/tests/hermes_state/test_bounded_recent_sessions.py b/tests/hermes_state/test_bounded_recent_sessions.py new file mode 100644 index 0000000000..6211ff7de0 --- /dev/null +++ b/tests/hermes_state/test_bounded_recent_sessions.py @@ -0,0 +1,240 @@ +"""Regression coverage for latency-bounded recent-session browsing.""" + +import sqlite3 +import time + +import pytest + +from hermes_state import SessionDB + + +@pytest.fixture +def db(tmp_path): + return SessionDB(tmp_path / "state.db") + + +def _set_activity(db, session_id, when): + db._conn.execute( + "UPDATE sessions SET last_activity_at = ? WHERE id = ?", + (when, session_id), + ) + db._conn.commit() + + +def test_bounded_recent_uses_effective_activity_index(db): + indexes = { + row[0] + for row in db._conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'index'" + ).fetchall() + } + assert "idx_sessions_effective_activity" in indexes + + +def test_writable_startup_reconciles_legacy_activity_column_before_index(tmp_path): + """A pre-last_activity_at store must heal through the real startup path.""" + path = tmp_path / "legacy-state.db" + original = SessionDB(path) + original.close() + + conn = sqlite3.connect(path) + try: + conn.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity") + conn.execute("ALTER TABLE sessions DROP COLUMN last_activity_at") + conn.commit() + finally: + conn.close() + + healed = SessionDB(path) + try: + columns = { + row[1] for row in healed._conn.execute("PRAGMA table_info(sessions)") + } + indexes = { + row[0] + for row in healed._conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'index'" + ) + } + assert "last_activity_at" in columns + assert "idx_sessions_effective_activity" in indexes + assert healed.list_recent_sessions_bounded(limit=1) == [] + finally: + healed.close() + + +def test_bounded_recent_orders_by_durable_activity_and_shapes_preview(db): + now = time.time() + db.create_session("older", source="cli") + db.append_message("older", role="user", content="older preview") + db.create_session("newer", source="cli") + db.append_message("newer", role="user", content="newer preview") + _set_activity(db, "older", now - 20) + _set_activity(db, "newer", now - 10) + + rows = db.list_recent_sessions_bounded(limit=2) + + assert [row["id"] for row in rows] == ["newer", "older"] + assert rows[0]["preview"] == "newer preview" + + +def test_bounded_recent_maps_recent_compression_tip_to_logical_root(db): + now = time.time() + db.create_session("root", source="cli") + db.append_message("root", role="user", content="root preview") + db.end_session("root", "compression") + db.create_session("tip", source="cli", parent_session_id="root") + db.append_message("tip", role="user", content="tip preview") + _set_activity(db, "root", now - 1000) + _set_activity(db, "tip", now) + + rows = db.list_recent_sessions_bounded(limit=1) + + assert rows[0]["id"] == "tip" + assert rows[0]["_lineage_root_id"] == "root" + assert rows[0]["preview"] == "tip preview" + + +def test_bounded_recent_keeps_reset_child_user_visible(db): + now = time.time() + db.create_session("before-reset", source="cli", session_key="cli:one") + db.end_session("before-reset", "session_reset") + db.create_session( + "after-reset", + source="cli", + parent_session_id="before-reset", + session_key="cli:one", + ) + db.append_message("after-reset", role="user", content="fresh conversation") + _set_activity(db, "after-reset", now) + + rows = db.list_recent_sessions_bounded(limit=5) + + assert "after-reset" in [row["id"] for row in rows] + + +def test_bounded_recent_keeps_branch_separate_from_compression_parent(db): + now = time.time() + db.create_session("branch-parent", source="cli") + db.end_session("branch-parent", "compression") + db.create_session( + "branch-child", + source="cli", + parent_session_id="branch-parent", + model_config={"_branched_from": "branch-parent"}, + ) + db.append_message("branch-child", role="user", content="branch preview") + _set_activity(db, "branch-child", now) + + rows = db.list_recent_sessions_bounded(limit=5) + + branch = next(row for row in rows if row["id"] == "branch-child") + assert branch.get("_lineage_root_id") is None + + +def test_bounded_recent_excludes_delegated_children_and_sources(db): + now = time.time() + db.create_session("visible", source="cli") + db.append_message("visible", role="user", content="visible") + _set_activity(db, "visible", now - 1) + db.create_session( + "delegated", + source="cli", + model_config={"_delegate_from": "parent"}, + ) + db.append_message("delegated", role="user", content="hidden delegate") + _set_activity(db, "delegated", now) + db.create_session("hidden-source", source="cron") + db.append_message("hidden-source", role="user", content="hidden source") + _set_activity(db, "hidden-source", now + 1) + + rows = db.list_recent_sessions_bounded( + limit=5, + exclude_sources=["cron"], + ) + + assert [row["id"] for row in rows] == ["visible"] + + +def test_bounded_recent_omits_deep_lineage_when_traversal_cap_is_reached(db): + now = time.time() + parent = None + for i in range(40): + sid = f"deep-{i}" + db.create_session(sid, source="cli", parent_session_id=parent) + if parent is not None: + db.end_session(parent, "compression") + _set_activity(db, sid, now + i) + parent = sid + db.create_session("visible-deep-peer", source="cli") + _set_activity(db, "visible-deep-peer", now + 100) + + rows = db.list_recent_sessions_bounded( + limit=5, + candidate_limit=8, + lineage_limit=8, + ) + + assert [row["id"] for row in rows] == ["visible-deep-peer"] + + +def test_bounded_recent_omits_branching_lineage_at_total_row_cap(db): + now = time.time() + db.create_session("fanout-root", source="cli") + db.end_session("fanout-root", "compression") + for i in range(40): + sid = f"fanout-{i}" + db.create_session(sid, source="cli", parent_session_id="fanout-root") + _set_activity(db, sid, now + i) + db.create_session("visible-fanout-peer", source="cli") + _set_activity(db, "visible-fanout-peer", now + 100) + + rows = db.list_recent_sessions_bounded( + limit=5, + candidate_limit=8, + lineage_limit=8, + ) + + assert [row["id"] for row in rows] == ["visible-fanout-peer"] + + +def test_bounded_recent_cycle_is_deduplicated_and_omitted(db): + now = time.time() + db.create_session("cycle-a", source="cli") + db.create_session("cycle-b", source="cli", parent_session_id="cycle-a") + db.end_session("cycle-a", "compression") + db.end_session("cycle-b", "compression") + db._conn.execute( + "UPDATE sessions SET parent_session_id = ? WHERE id = ?", + ("cycle-b", "cycle-a"), + ) + _set_activity(db, "cycle-a", now) + _set_activity(db, "cycle-b", now + 1) + db.create_session("visible-cycle-peer", source="cli") + _set_activity(db, "visible-cycle-peer", now + 2) + + rows = db.list_recent_sessions_bounded( + limit=5, + candidate_limit=8, + lineage_limit=8, + ) + + assert [row["id"] for row in rows] == ["visible-cycle-peer"] + + +def test_bounded_recent_deadline_interrupts_sqlite(db): + for i in range(300): + sid = f"session-{i}" + db.create_session(sid, source="cli") + db.append_message(sid, role="user", content=f"message {i}") + + with pytest.raises(TimeoutError, match="recent-session browse exceeded"): + db.list_recent_sessions_bounded( + limit=20, + candidate_limit=300, + timeout_seconds=0.0, + ) + + # The progress handler is removed in finally: the same connection remains + # usable after cancellation instead of poisoning subsequent gateway reads. + assert db.get_session("session-0")["id"] == "session-0" \ No newline at end of file diff --git a/tests/hermes_state/test_canonical_title_guard.py b/tests/hermes_state/test_canonical_title_guard.py index b315ff785d..784825c25e 100644 --- a/tests/hermes_state/test_canonical_title_guard.py +++ b/tests/hermes_state/test_canonical_title_guard.py @@ -69,3 +69,43 @@ def test_auto_titler_still_cannot_touch_the_canonical_row(db): assert not db.set_auto_title(sid, "Chat about groceries", source=SessionDB.TITLE_SOURCE_LLM) row = db.get_session_by_title(SessionDB.CANONICAL_BOT_CHAT_TITLE) assert row and row["id"] == sid + + +def test_auto_titler_cannot_rename_derived_canonical_bot_chat(db): + # #99517: the guard must be provenance-blind. A derived (rank 0) canonical + # title loses to an llm (rank 1) auto-title on precedence alone, so the + # identity check — not precedence — has to stop the write. + db.create_session("derived", source="desktop") + assert db._set_session_title( + "derived", + SessionDB.CANONICAL_BOT_CHAT_TITLE, + source=SessionDB.TITLE_SOURCE_DERIVED, + ) + assert db.set_session_hidden("derived", True) + + assert not db.set_auto_title( + "derived", + "Renamed by titler", + source=SessionDB.TITLE_SOURCE_LLM, + ) + row = db.get_session("derived") + assert row["title"] == SessionDB.CANONICAL_BOT_CHAT_TITLE + assert row["title_source"] == SessionDB.TITLE_SOURCE_DERIVED + + +def test_auto_titler_can_rename_visible_derived_bot_chat(db): + # Control: hidden is still the discriminator — a visible session that + # merely carries the text "Bot Chat" upgrades derived -> llm as usual. + db.create_session("visible", source="desktop") + assert db._set_session_title( + "visible", + SessionDB.CANONICAL_BOT_CHAT_TITLE, + source=SessionDB.TITLE_SOURCE_DERIVED, + ) + + assert db.set_auto_title( + "visible", + "Renamed by titler", + source=SessionDB.TITLE_SOURCE_LLM, + ) + assert db.get_session("visible")["title"] == "Renamed by titler" diff --git a/tests/hermes_state/test_deleted_wal_generation_guard.py b/tests/hermes_state/test_deleted_wal_generation_guard.py new file mode 100644 index 0000000000..cdbd201220 --- /dev/null +++ b/tests/hermes_state/test_deleted_wal_generation_guard.py @@ -0,0 +1,198 @@ +"""Refuse SessionDB open/write when a deleted WAL generation is still held. + +A live writer that keeps the unlinked ``state.db-wal`` inode while a second +opener would mint a fresh WAL is the split-brain that produces intermittent +``database disk image is malformed`` / ``disk I/O error``. The store must +fail closed on both the open and write paths instead of creating the second +generation. +""" + +import os +import sqlite3 +import sys +from pathlib import Path + +import pytest + +import hermes_state +from hermes_state import ( + DeletedWalGenerationError, + SessionDB, + classify_persistence_error, + iter_deleted_sqlite_sidecar_holders, + refuse_deleted_wal_generation, +) + + +@pytest.fixture +def force_wal(monkeypatch): + """Pin WAL so this host's vulnerable SQLite still matches production topology.""" + monkeypatch.setattr( + hermes_state, "is_sqlite_wal_reset_vulnerable", lambda version_info=None: False + ) + monkeypatch.setattr(hermes_state, "resolve_journal_mode", lambda: "wal") + + +def _make_db(path: Path, session_id: str, content: str) -> SessionDB: + db = SessionDB(db_path=path) + db.create_session(session_id, "cli") + db.append_message(session_id, role="user", content=content) + return db + + +def _require_wal(db: SessionDB) -> Path: + if not db._wal_active: + db.close() + pytest.skip("WAL not active on this filesystem") + wal = Path(os.fspath(db.db_path) + "-wal") + if not wal.exists(): + db.close() + pytest.skip("WAL sidecar missing after first write") + return wal + + +def _unlink_sidecars(db_path: Path) -> None: + for suffix in ("-wal", "-shm"): + sidecar = Path(os.fspath(db_path) + suffix) + if sidecar.exists(): + os.unlink(sidecar) + + +def test_classify_deleted_wal_is_replaced_not_disk(): + err = DeletedWalGenerationError( + "FATAL: a live process holds a deleted state.db-wal or state.db-shm " + "inode while the path names a different (or missing) generation." + ) + assert classify_persistence_error(err) == "replaced" + assert classify_persistence_error(str(err)) == "replaced" + + +def test_iter_holders_empty_on_non_linux(monkeypatch, tmp_path): + monkeypatch.setattr(hermes_state.sys, "platform", "win32") + assert iter_deleted_sqlite_sidecar_holders(tmp_path / "state.db") == [] + + +def test_clean_open_and_second_open_still_work(tmp_path, force_wal): + path = tmp_path / "state.db" + db = _make_db(path, "s1", "hello") + _require_wal(db) + db.close() + reopened = SessionDB(db_path=path) + try: + reopened.append_message("s1", role="user", content="second-open") + rows = reopened.get_messages("s1") + assert any(m["content"] == "second-open" for m in rows) + finally: + reopened.close() + + +def test_delete_journal_two_writers_still_work(tmp_path, monkeypatch): + monkeypatch.setattr(hermes_state, "resolve_journal_mode", lambda: "delete") + monkeypatch.setattr( + hermes_state, "is_sqlite_wal_reset_vulnerable", lambda version_info=None: False + ) + path = tmp_path / "state.db" + a = _make_db(path, "s", "from-a") + try: + assert not Path(os.fspath(path) + "-wal").exists() + b = SessionDB(db_path=path) + try: + b.append_message("s", role="user", content="from-b") + contents = [m["content"] for m in b.get_messages("s")] + assert "from-a" in contents + assert "from-b" in contents + finally: + b.close() + finally: + a.close() + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="deleted-WAL /proc scan is Linux-only", +) +def test_iter_finds_self_after_wal_unlink(tmp_path, force_wal): + path = tmp_path / "state.db" + db = _make_db(path, "s", "held") + wal = _require_wal(db) + inode_before = wal.stat().st_ino + _unlink_sidecars(path) + holders = iter_deleted_sqlite_sidecar_holders(path) + try: + assert holders, "expected this process to still hold the deleted WAL inode" + assert any("(deleted)" in target for _pid, target in holders) + assert any( + target.removesuffix(" (deleted)").endswith(("-wal", "-shm")) + for _pid, target in holders + ) + assert not wal.exists() or wal.stat().st_ino != inode_before + finally: + db.close() + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="deleted-WAL /proc scan is Linux-only", +) +def test_second_sessiondb_open_refuses_and_does_not_mint_wal(tmp_path, force_wal): + path = tmp_path / "state.db" + writer = _make_db(path, "s", "before-unlink") + wal = _require_wal(writer) + inode_before = wal.stat().st_ino + _unlink_sidecars(path) + assert not wal.exists() + + with pytest.raises(DeletedWalGenerationError, match="deleted state.db-wal"): + SessionDB(db_path=path) + + assert not wal.exists(), "open must refuse before sqlite3.connect mints a WAL" + # If a WAL somehow reappeared it must not be a new generation. + if wal.exists(): + assert wal.stat().st_ino == inode_before + writer.close() + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="deleted-WAL write halt uses Linux unlink semantics", +) +def test_writer_halts_after_own_wal_unlinked(tmp_path, force_wal): + path = tmp_path / "state.db" + db = _make_db(path, "s", "before") + _require_wal(db) + recorded = db._db_sidecar_identity.get("-wal") + assert recorded is not None + _unlink_sidecars(path) + + with pytest.raises(DeletedWalGenerationError, match="deleted state.db-wal"): + db.append_message("s", role="user", content="after-unlink") + assert db._db_wal_generation_lost is True + + with pytest.raises(DeletedWalGenerationError): + db.append_message("s", role="user", content="second-after-halt") + db.close() + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="deleted-WAL /proc scan is Linux-only", +) +def test_refuse_helper_raises_while_deleted_wal_held(tmp_path, force_wal): + path = tmp_path / "state.db" + raw = sqlite3.connect(str(path)) + try: + raw.execute("PRAGMA journal_mode=WAL") + raw.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + raw.execute("INSERT INTO t VALUES (1, 'held')") + raw.commit() + wal = Path(str(path) + "-wal") + assert wal.exists() + os.unlink(wal) + shm = Path(str(path) + "-shm") + if shm.exists(): + os.unlink(shm) + with pytest.raises(DeletedWalGenerationError): + refuse_deleted_wal_generation(path) + assert not wal.exists() + finally: + raw.close() diff --git a/tests/hermes_state/test_display_projection_parity.py b/tests/hermes_state/test_display_projection_parity.py new file mode 100644 index 0000000000..c34c64c17e --- /dev/null +++ b/tests/hermes_state/test_display_projection_parity.py @@ -0,0 +1,213 @@ +"""Every display projection of a compacted session must agree. + +In-place compaction archives earlier turns as ``active=0, compacted=1`` rows. +They are durable display history — the user's own conversation, still on disk. +#80680 taught the REST transcript read to include them, but three GATEWAY +display projections kept filtering ``active = 1``: + +- ``get_resume_conversations()`` — what ``session.resume`` ships +- ``get_ancestor_display_prefix()`` — the ancestor lineage prefix +- ``get_messages_as_conversation()`` — the warm-session payload on tab switch + +So the same conversation read four ways gave two different answers: REST showed +everything, the gateway cut the transcript off at the compaction boundary. The +user sees their chat "vanish" down to a summary plus a couple of carried-forward +turns, and a resumed agent that cannot see its own completed work starts it over +(#92080, #93618, #68321). + +These tests assert the INVARIANT — all display reads of one session return the +same transcript — rather than any particular row count, and pin the two things +that must NOT grow with it: the model-fed projection stays compressed, and +soft-deleted Undo/Rewind rows stay hidden. +""" + +import pytest + +from hermes_state import SessionDB + + +@pytest.fixture +def db(tmp_path): + return SessionDB(tmp_path / "state.db") + + +def _compact_in_place(db, sid, *, epochs=3, turns=4, tail_count=2): + """Drive *sid* through repeated in-place compaction, like a long chat.""" + db.create_session(sid, source="desktop") + for epoch in range(epochs): + for i in range(turns): + db.append_message(sid, "user", f"e{epoch} user {i}") + db.append_message(sid, "assistant", f"e{epoch} assistant {i}") + live = db.get_messages_as_conversation(sid) + db.archive_and_compact( + sid, + [{"role": "user", "content": f"[summary {epoch}]"}] + live[-tail_count:], + tail_count=tail_count, + ) + return sid + + +def _texts(messages): + return [(m["role"], m["content"]) for m in messages] + + +def _rest_display(db, sid): + """The read that was already correct — the parity reference.""" + return [ + {"role": m["role"], "content": m["content"]} + for m in db.get_messages(sid, include_compacted=True) + ] + + +class TestDisplayProjectionParity: + def test_resume_display_matches_the_rest_transcript(self, db): + sid = _compact_in_place(db, "chat") + + _, display = db.get_resume_conversations(sid) + + assert _texts(display) == _texts(_rest_display(db, sid)) + + def test_warm_session_display_matches_the_rest_transcript(self, db): + """The read behind ``_live_visible_history`` (switching back to a tab).""" + sid = _compact_in_place(db, "chat") + + warm = db.get_messages_as_conversation( + sid, include_ancestors=True, include_row_ids=True, include_compacted=True + ) + + assert _texts(warm) == _texts(_rest_display(db, sid)) + + def test_pre_compaction_turns_survive_in_the_resume_transcript(self, db): + """The user's own first turn is still there after several compactions.""" + sid = _compact_in_place(db, "chat") + + _, display = db.get_resume_conversations(sid) + + assert ("user", "e0 user 0") in _texts(display) + assert ("assistant", "e0 assistant 0") in _texts(display) + + def test_display_read_dedupes_carried_forward_tail(self, db): + """Each logical message appears once, not once per compaction epoch.""" + sid = _compact_in_place(db, "chat", epochs=4, tail_count=2) + + _, display = db.get_resume_conversations(sid) + seen = _texts(display) + + assert len(seen) == len(set(seen)) + + +class TestModelProjectionStaysCompressed: + def test_model_history_excludes_archived_rows(self, db): + """Compaction must still do its job: the model gets the compressed set.""" + sid = _compact_in_place(db, "chat") + + model, display = db.get_resume_conversations(sid) + + assert len(model) < len(display) + assert ("user", "e0 user 0") not in _texts(model) + + def test_model_history_matches_the_active_only_read(self, db): + sid = _compact_in_place(db, "chat") + + model, _ = db.get_resume_conversations(sid) + active_only = db.get_messages_as_conversation(sid, repair_alternation=True) + + assert _texts(model) == _texts(active_only) + + +class TestSoftDeletedRowsStayHidden: + def test_rewound_rows_are_excluded_from_the_display_projections(self, db): + """Undo/Rewind rows (active=0, compacted=0) are NOT display history.""" + sid = "chat" + db.create_session(sid, source="desktop") + db.append_message(sid, "user", "kept") + db.append_message(sid, "assistant", "kept reply") + db.append_message(sid, "user", "taken back") + db.append_message(sid, "assistant", "taken back reply") + + rewind_target = next( + m for m in reversed(db.get_messages(sid)) if m["role"] == "user" + ) + db.rewind_to_message(sid, rewind_target["id"]) + + _, display = db.get_resume_conversations(sid) + warm = db.get_messages_as_conversation( + sid, include_ancestors=True, include_compacted=True + ) + + for projection in (display, warm): + contents = [c for _, c in _texts(projection)] + assert "taken back" not in contents + assert "kept" in contents + + +class TestAncestorPrefix: + def test_prefix_includes_a_compacted_ancestor_s_archived_rows(self, db): + """A compression ROTATION's parent still shows its pre-compaction turns.""" + parent, child = "parent", "child" + db.create_session(parent, source="desktop") + for i in range(3): + db.append_message(parent, "user", f"P user {i}") + db.append_message(parent, "assistant", f"P assistant {i}") + db.archive_and_compact(parent, [{"role": "user", "content": "[parent summary]"}]) + + db.create_session(child, source="desktop", parent_session_id=parent) + db.append_message(child, "user", "C user 0") + db.append_message(child, "assistant", "C assistant 0") + + prefix = db.get_ancestor_display_prefix(child) + _, display = db.get_resume_conversations(child) + + assert ("user", "P user 0") in _texts(prefix) + assert ("user", "P user 0") in _texts(display) + # The child's own turns belong to the tip, never the ancestor prefix. + assert ("user", "C user 0") not in _texts(prefix) + + def test_explicit_branch_has_no_ancestor_prefix(self, db): + """A /branch copy owns its transcript; the live parent must not leak in.""" + sid = _compact_in_place(db, "chat") + db.create_session( + "branch", + source="desktop", + parent_session_id=sid, + model_config={"_branched_from": sid}, + ) + db.append_message("branch", "user", "branch turn") + + assert db.get_ancestor_display_prefix("branch") == [] + + _, display = db.get_resume_conversations("branch") + assert _texts(display) == [("user", "branch turn")] + + +class TestResumeGuardBoundsWhatResumeLoads: + def test_guard_counts_the_rows_the_display_read_materializes(self, db): + """The guard must not undercount: it bounds an in-memory materialization.""" + sid = _compact_in_place(db, "chat", epochs=4) + + _, display = db.get_resume_conversations(sid) + + assert db.get_resume_message_count(sid) >= len(display) + + def test_guard_rejects_a_lineage_over_the_limit(self, db): + from hermes_state import SessionResumeTooLargeError + + sid = _compact_in_place(db, "chat", epochs=4) + + with pytest.raises(SessionResumeTooLargeError): + db.assert_resume_safe(sid, max_messages=2) + + def test_tip_only_guard_still_bounds_only_the_live_tip(self, db): + """The #4130 carve-out: a healthy compacted chat must stay resumable. + + A well-compressed conversation is exactly the shape compression is + meant to produce. Counting its archive against a tip-sized budget is + what stranded Bot Chats on "Waking up…"; ``tip_only`` callers never + materialize the archive, so they keep the active-only bound. + """ + sid = _compact_in_place(db, "chat", epochs=4) + + tip_count = db.get_resume_message_count(sid, tip_only=True) + + assert tip_count < db.get_resume_message_count(sid) + assert db.assert_resume_safe(sid, max_messages=tip_count, tip_only=True) diff --git a/tests/hermes_state/test_shared_session_db_registry.py b/tests/hermes_state/test_shared_session_db_registry.py index ea279c9647..58d4837625 100644 --- a/tests/hermes_state/test_shared_session_db_registry.py +++ b/tests/hermes_state/test_shared_session_db_registry.py @@ -35,10 +35,12 @@ def _clean_registry(): registry.close_all() registry._generations.clear() registry._retired.clear() + registry._opening.clear() yield registry.close_all() registry._generations.clear() registry._retired.clear() + registry._opening.clear() def _replace_file_preserving_schema(src: Path, dst: Path) -> None: @@ -178,6 +180,127 @@ def stats_live_for(path: Path): class TestTeardownOutsideLock: + def test_concurrent_cold_acquire_opens_one_writer(self, tmp_path, monkeypatch): + """Concurrent first callers must not construct redundant writers. + + Returning one winning object is not enough: every losing constructor + has already opened its own writable SQLite connection by then. Hold + the first construction so peer callers overlap deterministically and + assert the registry single-flights the open itself. + """ + db_path = tmp_path / "state.db" + callers = 6 + ready = threading.Barrier(callers + 1) + release_open = threading.Event() + count_lock = threading.Lock() + open_calls = 0 + results = [] + errors = [] + + class _FakeDB: + def __init__(self, path): + self.db_path = path + self._shared_registry_owned = False + self.closed = False + + def close(self): + self.closed = True + + def _blocked_open(path): + nonlocal open_calls + with count_lock: + open_calls += 1 + assert release_open.wait(5.0) + return _FakeDB(path) + + monkeypatch.setattr(registry, "_open_session_db", _blocked_open) + + def _acquire(): + try: + ready.wait() + results.append(registry.acquire(db_path)) + except BaseException as exc: # pragma: no cover - failure path + errors.append(exc) + + threads = [threading.Thread(target=_acquire) for _ in range(callers)] + for thread in threads: + thread.start() + ready.wait() + time.sleep(0.1) + release_open.set() + for thread in threads: + thread.join(10.0) + assert not thread.is_alive(), "concurrent acquire deadlocked" + + assert errors == [] + assert open_calls == 1 + assert len({id(db) for db in results}) == 1 + for db in results: + assert registry.release(db) is True + + def test_waiter_retries_after_cold_open_failure(self, tmp_path, monkeypatch): + """A failed elected opener must wake a peer to retry the path.""" + db_path = tmp_path / "state.db" + first_entered = threading.Event() + release_failure = threading.Event() + open_calls = 0 + results = [] + errors = [] + + class _FakeDB: + def __init__(self, path): + self.db_path = path + self._shared_registry_owned = False + + def close(self): + pass + + def _fail_then_open(path): + nonlocal open_calls + open_calls += 1 + if open_calls == 1: + first_entered.set() + assert release_failure.wait(5.0) + raise OSError("transient open failure") + return _FakeDB(path) + + monkeypatch.setattr(registry, "_open_session_db", _fail_then_open) + + def _acquire(): + try: + results.append(registry.acquire(db_path)) + except BaseException as exc: + errors.append(exc) + + first = threading.Thread(target=_acquire) + second = threading.Thread(target=_acquire) + first.start() + assert first_entered.wait(5.0) + second.start() + time.sleep(0.1) + release_failure.set() + first.join(10.0) + second.join(10.0) + + assert not first.is_alive() + assert not second.is_alive() + assert open_calls == 2 + assert len(errors) == 1 + assert isinstance(errors[0], OSError) + assert len(results) == 1 + assert registry.release(results[0]) is True + + def test_equivalent_path_spellings_share_generation(self, tmp_path): + """Registry identity is the resolved file, not caller spelling.""" + db_path = tmp_path / "nested" / "state.db" + equivalent = tmp_path / "nested" / ".." / "nested" / "state.db" + + first = registry.acquire(db_path) + second = registry.acquire(equivalent) + assert first is second + assert registry.release(first) is True + assert registry.release(second) is True + def test_final_release_does_not_hold_registry_lock_during_close(self, tmp_path, monkeypatch): """A final release's teardown (token-writer stop, WAL checkpoint, read-pool drain) must run OUTSIDE the registry lock — otherwise @@ -231,10 +354,16 @@ class TestTeardownOutsideLock: def _worker(n): try: - for _ in range(20): + for index in range(20): db = registry.acquire(db_path) try: - db.get_session("nonexistent") + db.create_session( + session_id=f"worker-{n}-{index}", + source="test", + model="test-model", + model_config={}, + system_prompt=None, + ) finally: registry.release(db) except Exception as exc: # pragma: no cover - failure path @@ -248,6 +377,12 @@ class TestTeardownOutsideLock: assert not t.is_alive(), "worker deadlocked" assert errors == [] + verifier = registry.acquire(db_path) + try: + with verifier._lock: + assert verifier._conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + finally: + registry.release(verifier) stats = registry.stats() assert stats["live_generations"] == 0 assert stats["retired_generations"] == 0 diff --git a/tests/hermes_state/test_state_db_corrupt_quarantine.py b/tests/hermes_state/test_state_db_corrupt_quarantine.py new file mode 100644 index 0000000000..2acac22939 --- /dev/null +++ b/tests/hermes_state/test_state_db_corrupt_quarantine.py @@ -0,0 +1,236 @@ +"""Quarantine of a live SessionDB handle after structural (non-FTS) corruption. + +Field evidence (the #90837 lost/reordered-page-write class): a gateway kept +retrying writes for ~50 minutes after ``gateway_routing`` reported +``database disk image is malformed``; on SIGTERM the close-time +``PRAGMA wal_checkpoint(PASSIVE)`` then wrote 15 pages to the wrong page +numbers (page 1 received a ``messages_fts_trigram_data`` leaf) and the file +stopped opening at all. Once structural corruption is observed on a handle +the only safe policy is to stop touching the file. +""" + +import sqlite3 + +import pytest + +from hermes_state import SessionDB, StateDbCorruptError + + +class _MalformedConn: + """Connection proxy whose every execute reports bare SQLITE_CORRUPT.""" + + def __init__(self, real_conn): + self._real = real_conn + + def execute(self, *args, **kwargs): + raise sqlite3.DatabaseError("database disk image is malformed") + + def __getattr__(self, name): + return getattr(self._real, name) + + +class TestQuarantineAfterStructuralCorruption: + def test_structural_corruption_sets_sticky_flag_and_raises_typed(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + real_conn = db._conn + try: + db.create_session(session_id="s1", source="cli", model="test") + db._conn = _MalformedConn(real_conn) + with pytest.raises(StateDbCorruptError, match="malformed") as excinfo: + db.create_session(session_id="s2", source="cli", model="test") + assert isinstance(excinfo.value.__cause__, sqlite3.DatabaseError) + assert db._db_corrupt is True + # Structural damage must never be mistaken for FTS-scoped damage. + assert db._fts_stale is False + finally: + db._conn = real_conn + db.close() + + +class _RecordingConn: + """Connection proxy that records every SQL text and delegates.""" + + def __init__(self, real_conn): + self._real = real_conn + self.recorded = [] + + def execute(self, sql, *args, **kwargs): + self.recorded.append(str(sql)) + return self._real.execute(sql, *args, **kwargs) + + def __getattr__(self, name): + return getattr(self._real, name) + + +def _quarantined_db(tmp_path): + """A SessionDB whose first corrupt write already tripped the quarantine.""" + db = SessionDB(db_path=tmp_path / "state.db") + real_conn = db._conn + db.create_session(session_id="s1", source="cli", model="test") + db._conn = _MalformedConn(real_conn) + with pytest.raises(StateDbCorruptError): + db.create_session(session_id="s2", source="cli", model="test") + db._conn = real_conn + assert db._db_corrupt is True + return db, real_conn + + +class TestQuarantinedHandleStopsTouchingTheFile: + def test_subsequent_writes_fail_fast_without_touching_connection(self, tmp_path): + db, real_conn = _quarantined_db(tmp_path) + recorder = _RecordingConn(real_conn) + db._conn = recorder + try: + with pytest.raises(StateDbCorruptError): + db.create_session(session_id="s3", source="cli", model="test") + assert recorder.recorded == [] + finally: + db._conn = real_conn + db.close() + + def test_close_skips_wal_checkpoint_when_quarantined(self, tmp_path, caplog): + db, real_conn = _quarantined_db(tmp_path) + recorder = _RecordingConn(real_conn) + db._conn = recorder + with caplog.at_level("WARNING", logger="hermes_state"): + db.close() + assert not any("wal_checkpoint" in sql for sql in recorder.recorded) + assert db._conn is None + assert any( + "Skipping the close-time WAL checkpoint" in rec.getMessage() + and "hermes sessions recover" in rec.getMessage() + for rec in caplog.records + ) + + def test_close_disables_sqlite_internal_checkpoint_on_py312(self, tmp_path): + """Quarantine must also stop SQLite's own last-connection checkpoint. + + Skipping the explicit PRAGMA is not enough: sqlite3.Connection.close() + runs an internal PASSIVE checkpoint and unlinks -wal/-shm unless + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE is set (Connection.setconfig, + Python 3.12+). On 3.11 the switch is unavailable — skip there. + """ + flag = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None) + db = SessionDB(db_path=tmp_path / "state.db") + if flag is None or not hasattr(db._conn, "setconfig"): + db.close() + pytest.skip("SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE needs Python 3.12+") + real_conn = db._conn + db.create_session(session_id="s1", source="cli", model="test") + assert real_conn.getconfig(flag) is False + db._conn = _MalformedConn(real_conn) + with pytest.raises(StateDbCorruptError): + db.create_session(session_id="s2", source="cli", model="test") + db._conn = real_conn + # _halt_db_corrupt armed the no-checkpoint-on-close switch. + assert real_conn.getconfig(flag) is True + db.close() + + def test_reopen_after_close_refused_when_quarantined(self, tmp_path, monkeypatch): + from unittest.mock import MagicMock + + db, real_conn = _quarantined_db(tmp_path) + db.close() + reopen = MagicMock() + monkeypatch.setattr("hermes_state._connect_tracked_db", reopen) + with pytest.raises(StateDbCorruptError, match="structural corruption"): + db.create_session(session_id="s4", source="cli", model="test") + reopen.assert_not_called() + # The read fallback after close() goes through the same reopen path. + with pytest.raises(StateDbCorruptError, match="refusing to reopen"): + db.get_session("s1") + reopen.assert_not_called() + + +class TestQuarantineScope: + def test_fts_scoped_corruption_does_not_trip_flag(self, tmp_path): + """Corrupt FTS shadow tables keep the existing fail-open detach path.""" + path = tmp_path / "state.db" + db = SessionDB(db_path=path) + db.create_session(session_id="s1", source="cli", model="test") + db.append_message("s1", role="user", content="hello world") + raw = sqlite3.connect(str(path)) + raw.execute( + "UPDATE messages_fts_data SET block = X'DEADBEEFDEADBEEFDEADBEEFDEADBEEF'" + ) + raw.commit() + raw.close() + try: + db.append_message("s1", role="user", content="healed append") + assert db._db_corrupt is False + assert db._fts_stale is True + assert db._fts_enabled is False + finally: + db.close() + + def test_replaced_file_takes_precedence_over_corrupt(self, tmp_path): + import os + + from hermes_state import StateDbReplacedError + + live = tmp_path / "state.db" + other = tmp_path / "other.db" + db = SessionDB(db_path=live) + real_conn = db._conn + try: + db.create_session(session_id="s1", source="cli", model="test") + if db._db_file_identity is None: + pytest.skip("filesystem does not expose st_dev/st_ino") + alt = SessionDB(db_path=other) + alt.create_session("other", "cli") + alt.close() + os.replace(other, live) + db._conn = _MalformedConn(real_conn) + with pytest.raises(StateDbReplacedError): + db.create_session(session_id="s2", source="cli", model="test") + assert db._db_replaced is True + assert db._db_corrupt is False + finally: + db._conn = real_conn + db.close() + + def test_classify_persistence_error_maps_quarantine_to_corrupt(self): + from hermes_state import _STATE_DB_CORRUPT_MSG, classify_persistence_error + + assert classify_persistence_error(StateDbCorruptError("x")) == "corrupt" + # The stringified form (RPC boundaries) must classify the same way. + assert classify_persistence_error(_STATE_DB_CORRUPT_MSG) == "corrupt" + + +@pytest.fixture +def _clean_registry(): + import hermes_state_registry as registry + + registry.close_all() + registry._generations.clear() + registry._retired.clear() + yield registry + registry.close_all() + registry._generations.clear() + registry._retired.clear() + + +class TestSharedRegistry: + def test_holders_share_quarantine_and_close_all_skips_checkpoint( + self, tmp_path, _clean_registry + ): + registry = _clean_registry + path = tmp_path / "state.db" + holder_a = registry.acquire(path) + holder_b = registry.acquire(path) + assert holder_a is holder_b + real_conn = holder_a._conn + holder_a.create_session(session_id="s1", source="cli", model="test") + + holder_a._conn = _MalformedConn(real_conn) + with pytest.raises(StateDbCorruptError): + holder_a.create_session(session_id="s2", source="cli", model="test") + recorder = _RecordingConn(real_conn) + holder_b._conn = recorder + + with pytest.raises(StateDbCorruptError): + holder_b.create_session(session_id="s3", source="cli", model="test") + + registry.close_all() + assert not any("wal_checkpoint" in sql for sql in recorder.recorded) + assert holder_a._conn is None diff --git a/tests/hermes_state/test_sweep_orphaned_sessions.py b/tests/hermes_state/test_sweep_orphaned_sessions.py index 4554513f2f..8d56985d61 100644 --- a/tests/hermes_state/test_sweep_orphaned_sessions.py +++ b/tests/hermes_state/test_sweep_orphaned_sessions.py @@ -16,6 +16,7 @@ to be older than the cutoff: actively producing messages. """ +import threading import time import pytest @@ -44,6 +45,15 @@ def _set_message_timestamps(db: SessionDB, session_id: str, ts: float) -> None: db._conn.commit() +def _set_last_activity(db: SessionDB, session_id: str, ts: float) -> None: + conn = db._conn + assert conn is not None + conn.execute( + "UPDATE sessions SET last_activity_at = ? WHERE id = ?", (ts, session_id) + ) + conn.commit() + + def _make_session( db: SessionDB, session_id: str, @@ -99,6 +109,17 @@ class TestSweepOrphanedSessions: assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == [] assert db.get_session("active")["ended_at"] is None + def test_recent_heartbeat_spares_old_session(self, db): + """A turn heartbeat is activity even before its next message lands.""" + stale = time.time() - 48 * 3600 + _make_session( + db, "active-heartbeat", source="tui", started_at=stale, message_at=stale + ) + _set_last_activity(db, "active-heartbeat", time.time()) + + assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == [] + assert db.get_session("active-heartbeat")["ended_at"] is None + def test_fresh_session_with_old_copied_messages_spared(self, db): """Compression/branch children copy history — old message timestamps on a just-created row must not get it swept.""" @@ -163,9 +184,398 @@ class TestSweepOrphanedSessions: assert db.get_session("stale-cli")["end_reason"] == "startup_orphan_reap" assert db.get_session("stale-tui")["ended_at"] is None + def test_explicit_source_scope_spares_gateway_sessions(self, db): + stale = time.time() - 8 * 3600 + _make_session( + db, "stale-cron", source="cron", started_at=stale, message_at=stale + ) + for sid, session_key in ( + ("keyed-telegram", "telegram:chat:1"), + ("unkeyed-telegram", None), + ): + db.create_session(sid, source="telegram", session_key=session_key) + db.append_message(sid, role="user", content="hello") + _set_message_timestamps(db, sid, stale) + _backdate_session(db, sid, stale) + + assert db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, sources=("cron",) + ) == ["stale-cron"] + assert db.get_session("stale-cron")["end_reason"] == "startup_orphan_reap" + assert db.get_session("keyed-telegram")["ended_at"] is None + assert db.get_session("unkeyed-telegram")["ended_at"] is None + + def test_automatic_source_scope_spares_pinned_session(self, db): + stale = time.time() - 8 * 3600 + _make_session( + db, "pinned", source="cli", started_at=stale, message_at=stale + ) + db.set_session_pinned("pinned", True) + + assert db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, + sources=("cli",), + exclude_pinned=True, + ) == [] + assert db.get_session("pinned")["ended_at"] is None + + def test_live_turn_lease_on_compression_lineage_spares_session(self, db): + stale = time.time() - 8 * 3600 + _make_session(db, "root", source="cli", started_at=stale, message_at=stale) + db.end_session("root", "compression") + db.create_session("tip", source="cli", parent_session_id="root") + db.append_message("tip", role="user", content="continued") + _set_message_timestamps(db, "tip", stale) + _backdate_session(db, "tip", stale) + assert db.try_acquire_session_turn_lease( + "tip", "external-turn", ttl_seconds=300 + ) + + assert db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, sources=("cli",) + ) == [] + assert db.get_session("tip")["ended_at"] is None + + def test_active_compression_lock_spares_and_expiry_fences_owner(self, db): + stale = time.time() - 8 * 3600 + _make_session( + db, "compressing", source="cli", started_at=stale, message_at=stale + ) + assert db.try_acquire_compression_lock( + "compressing", "compressor", ttl_seconds=300 + ) + + assert db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, sources=("cli",) + ) == [] + + conn = db._conn + assert conn is not None + conn.execute( + "UPDATE compression_locks SET expires_at = ? WHERE session_id = ?", + (time.time() - 1, "compressing"), + ) + conn.commit() + + assert db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, sources=("cli",) + ) == ["compressing"] + assert db.get_compression_lock_holder("compressing") is None + assert db.refresh_compression_lock("compressing", "compressor") is False + + def test_expired_turn_lease_does_not_block_sweep(self, db): + stale = time.time() - 8 * 3600 + _make_session( + db, "expired", source="cli", started_at=stale, message_at=stale + ) + assert db.try_acquire_session_turn_lease( + "expired", "expired-turn", ttl_seconds=300 + ) + db._conn.execute( + "UPDATE session_turn_leases SET expires_at = ? WHERE conversation_id = ?", + (time.time() - 1, "expired"), + ) + db._conn.commit() + + assert db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, sources=("cli",) + ) == ["expired"] + assert db.get_session("expired")["end_reason"] == "startup_orphan_reap" + assert db.refresh_session_turn_lease("expired", "expired-turn") is False + + def test_auto_prune_closes_stale_state_owned_rows_but_spares_live_turns(self, db): + stale = time.time() - 100 * 86400 + recent = time.time() - 86400 + for sid, source in ( + ("orphan", "cli"), + ("live-turn", "cli"), + ("stale-cron", "cron"), + ("runtime-owned-ui", "tui"), + ): + _make_session(db, sid, source=source, started_at=stale, message_at=stale) + _set_last_activity(db, sid, stale) + _make_session( + db, + "recent-orphan", + source="cli", + started_at=recent, + message_at=recent, + ) + _set_last_activity(db, "recent-orphan", recent) + db.create_session( + "keyed", source="telegram", session_key="telegram:chat:1" + ) + _backdate_session(db, "keyed", stale) + db.create_session("unkeyed-gateway", source="telegram") + _backdate_session(db, "unkeyed-gateway", stale) + _set_last_activity(db, "unkeyed-gateway", stale) + assert db.try_acquire_session_turn_lease( + "live-turn", "external-turn", ttl_seconds=300 + ) + db.register_backend_heartbeat( + backend_id="unrelated-dashboard", + pid=12345, + started_at=time.time(), + last_heartbeat=time.time(), + ) + + first = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert first["pruned"] == 0 + assert db.get_session("orphan")["end_reason"] == "startup_orphan_reap" + assert db.get_session("stale-cron")["end_reason"] == "startup_orphan_reap" + assert db.get_session("live-turn")["ended_at"] is None + assert db.get_session("recent-orphan")["ended_at"] is None + assert db.get_session("runtime-owned-ui")["ended_at"] is None + assert db.get_session("keyed")["ended_at"] is None + assert db.get_session("unkeyed-gateway")["ended_at"] is None + + second = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert second["pruned"] == 0 + assert db.get_session("orphan") is not None + assert db.get_session("stale-cron") is not None + + db._conn.execute( + "UPDATE sessions SET ended_at = ? WHERE id IN (?, ?)", + (stale, "orphan", "stale-cron"), + ) + db._conn.commit() + third = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert third["pruned"] == 2 + assert db.get_session("orphan") is None + assert db.get_session("stale-cron") is None + + def test_failed_maintenance_marker_keeps_newly_swept_row_recoverable( + self, db, monkeypatch + ): + stale = time.time() - 100 * 86400 + _make_session( + db, + "recoverable", + source="cli", + started_at=stale, + message_at=stale, + ) + _set_last_activity(db, "recoverable", stale) + set_meta = db.set_meta + fail_once = True + + def flaky_set_meta(key, value): + nonlocal fail_once + if key == "last_auto_prune" and fail_once: + fail_once = False + raise RuntimeError("injected marker failure") + return set_meta(key, value) + + monkeypatch.setattr(db, "set_meta", flaky_set_meta) + + first = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + retry = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert first["error"] == "injected marker failure" + assert retry["pruned"] == 0 + assert db.get_session("recoverable")["end_reason"] == "startup_orphan_reap" + + def test_concurrent_auto_maintenance_preserves_the_recovery_window( + self, db, monkeypatch + ): + stale = time.time() - 100 * 86400 + _make_session(db, "concurrent", source="cli", started_at=stale, message_at=stale) + _set_last_activity(db, "concurrent", stale) + peer = SessionDB(db.db_path) + read_barrier = threading.Barrier(2) + second_done = threading.Event() + release_first_prune = threading.Event() + errors = [] + results = {} + + for instance in (db, peer): + get_meta = instance.get_meta + + def synchronized_get_meta(key, *, _get_meta=get_meta): + value = _get_meta(key) + if key == "last_auto_prune": + try: + read_barrier.wait(timeout=1) + except threading.BrokenBarrierError: + pass + return value + + monkeypatch.setattr(instance, "get_meta", synchronized_get_meta) + + prune_sessions = db.prune_sessions + + def delayed_prune(*args, **kwargs): + assert release_first_prune.wait(timeout=5) + return prune_sessions(*args, **kwargs) + + monkeypatch.setattr(db, "prune_sessions", delayed_prune) + + def run(name, instance, *, done=None): + try: + results[name] = instance.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=24, + vacuum=False, + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + finally: + if done is not None: + done.set() + + first = threading.Thread(target=run, args=("first", db)) + second = threading.Thread( + target=run, args=("second", peer), kwargs={"done": second_done} + ) + try: + first.start() + second.start() + assert second_done.wait(timeout=5) + release_first_prune.set() + finally: + release_first_prune.set() + first.join(timeout=5) + second.join(timeout=5) + peer.close() + + assert not first.is_alive() + assert not second.is_alive() + assert errors == [] + assert sum(bool(result["skipped"]) for result in results.values()) == 1 + assert sum(int(result["pruned"]) for result in results.values()) == 0 + assert db.get_session("concurrent")["end_reason"] == "startup_orphan_reap" + + def test_auto_prune_spares_compression_root_of_live_turn(self, db): + stale = time.time() - 100 * 86400 + _make_session(db, "root", source="cli", started_at=stale, message_at=stale) + db.end_session("root", "compression") + db.create_session("tip", source="cli", parent_session_id="root") + db.append_message("tip", role="user", content="continued") + _set_message_timestamps(db, "tip", stale) + _backdate_session(db, "tip", stale) + _set_last_activity(db, "tip", stale) + assert db.try_acquire_session_turn_lease( + "tip", "external-turn", ttl_seconds=300 + ) + + result = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert result["pruned"] == 0 + assert db.get_session("root") is not None + assert db.get_session("tip")["ended_at"] is None + + def test_auto_prune_spares_prior_sweep_row_with_new_turn_lease(self, db): + stale = time.time() - 100 * 86400 + _make_session(db, "racy", source="cli", started_at=stale, message_at=stale) + _set_last_activity(db, "racy", stale) + db.end_session("racy", "startup_orphan_reap") + assert db.try_acquire_session_turn_lease( + "racy", "arriving-turn", ttl_seconds=300 + ) + + result = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert result["pruned"] == 0 + assert db.get_session("racy") is not None + + def test_auto_prune_spares_prior_sweep_row_with_new_compression_lock(self, db): + stale = time.time() - 100 * 86400 + _make_session( + db, + "racy-compression", + source="cli", + started_at=stale, + message_at=stale, + ) + _set_last_activity(db, "racy-compression", stale) + db.end_session("racy-compression", "startup_orphan_reap") + assert db.try_acquire_compression_lock( + "racy-compression", "arriving-compressor", ttl_seconds=300 + ) + + result = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=False, + ) + + assert result["pruned"] == 0 + assert db.get_session("racy-compression") is not None + def test_returns_empty_on_empty_db(self, db): assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == [] + def test_auto_prune_reports_closed_count_and_deletes_after_second_window( + self, db + ): + """#54189 end-to-end: leaky producers (cron/kanban/subagent) never set + ``ended_at``; pass 1 closes them (reported via ``closed``), pass 2 — + after a further retention window — deletes them, and a messaging row + is never touched by either pass.""" + stale = time.time() - 200 * 86400 + for sid, source in ( + ("cron-0", "cron"), + ("kanban-1", "kanban"), + ("subagent-2", "subagent"), + ("telegram-3", "telegram"), + ): + _make_session(db, sid, source=source, started_at=stale, message_at=stale) + _set_last_activity(db, sid, stale) + + first = db.maybe_auto_prune_and_vacuum( + retention_days=90, min_interval_hours=0, vacuum=False + ) + assert first["closed"] == 3 + assert first["pruned"] == 0 + for sid in ("cron-0", "kanban-1", "subagent-2"): + assert db.get_session(sid)["end_reason"] == "startup_orphan_reap" + assert db.get_session("telegram-3")["ended_at"] is None + + # Simulate the next maintenance pass after another retention window. + db._conn.execute( + "UPDATE sessions SET ended_at = ended_at - 91 * 86400 " + "WHERE end_reason = 'startup_orphan_reap'" + ) + db._conn.commit() + second = db.maybe_auto_prune_and_vacuum( + retention_days=90, min_interval_hours=0, vacuum=False + ) + assert second["closed"] == 0 + assert second["pruned"] == 3 + remaining = [r["id"] for r in db._conn.execute("SELECT id FROM sessions")] + assert remaining == ["telegram-3"] + def test_zero_ttl_is_noop(self, db): stale = time.time() - 8 * 3600 _make_session(db, "stale-tui", source="tui", started_at=stale, message_at=stale) diff --git a/tests/plugins/dashboard_auth/test_opaque_bearer_not_unreachable.py b/tests/plugins/dashboard_auth/test_opaque_bearer_not_unreachable.py new file mode 100644 index 0000000000..8b62421484 --- /dev/null +++ b/tests/plugins/dashboard_auth/test_opaque_bearer_not_unreachable.py @@ -0,0 +1,156 @@ +"""#94558 — a non-JWT bearer must not be reported as "Auth provider unreachable". + +Hosted agents answered every opaque/peer bearer on the gated API with a fast +HTTP 503 ``{"detail": "Auth provider 'nous' unreachable"}`` while Portal was +perfectly healthy: ``NousDashboardAuthProvider._verify_jwt`` folded *every* +``PyJWKClient`` failure — including ``DecodeError('Not enough segments')`` for +a token that is not a JWT at all — into ``ProviderError``. Only a transport +failure fetching the JWKS is "unreachable"; anything else means "not my +token" (``verify_session`` -> None -> 401 / next provider). + +Real ``NousDashboardAuthProvider`` + real ``SelfHostedOIDCProvider`` JWKS path, +a real local HTTP JWKS server (reachable case) or a closed port (unreachable), +and the real gated web_server app for the HTTP-level assertion. +""" +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import jwt +import pytest +from starlette.testclient import TestClient + +from hermes_cli import web_server +from hermes_cli.dashboard_auth import ( + InvalidCodeError, + ProviderError, + classify_jwks_lookup_error, + clear_providers, + register_provider, +) +from hermes_cli.dashboard_auth.cookies import SESSION_AT_COOKIE +import plugins.dashboard_auth.nous as nous_plugin + +OPAQUE_PEER_KEY = "hk_live_opaque_peer_key_0123456789abcdef" +# Well-formed RS256 JWT header with an unknown kid, bogus payload/signature. +FOREIGN_KID_JWT = "eyJhbGciOiJSUzI1NiIsImtpZCI6Inp6eiJ9.e30.sig" + + +@pytest.fixture(scope="module") +def empty_jwks_server(): + """A reachable JWKS endpoint that knows no keys.""" + + class _H(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + self.send_response(200) + self.send_header("content-type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"keys": []}).encode()) + + def log_message(self, *a): # silence + pass + + srv = HTTPServer(("127.0.0.1", 0), _H) + t = threading.Thread(target=srv.serve_forever, daemon=True) + t.start() + yield f"http://127.0.0.1:{srv.server_address[1]}" + srv.shutdown() + + +def _nous(portal_url: str) -> nous_plugin.NousDashboardAuthProvider: + return nous_plugin.NousDashboardAuthProvider(client_id="agent:test-instance", portal_url=portal_url) + + +# ── classifier ──────────────────────────────────────────────────────────── + +def test_classifier_maps_transport_failure_to_provider_error(): + exc = jwt.PyJWKClientConnectionError("Fail to fetch data from the url") + assert isinstance(classify_jwks_lookup_error(exc), ProviderError) + + +@pytest.mark.parametrize( + "exc", + [ + jwt.DecodeError("Not enough segments"), + jwt.PyJWKSetError("The JWK Set did not contain any keys"), + jwt.InvalidTokenError("bad"), + ], +) +def test_classifier_maps_unverifiable_token_to_invalid_code(exc): + assert isinstance(classify_jwks_lookup_error(exc), InvalidCodeError) + + +def test_classifier_keeps_bare_jwk_client_error_as_provider_fault(): + assert isinstance(classify_jwks_lookup_error(jwt.PyJWKClientError("weird JWKS shape")), ProviderError) + + +# ── Nous provider ───────────────────────────────────────────────────────── + +def test_opaque_bearer_with_healthy_portal_is_not_unreachable(empty_jwks_server): + provider = _nous(empty_jwks_server) + assert provider.verify_session(access_token=OPAQUE_PEER_KEY) is None + + +def test_foreign_kid_jwt_with_healthy_portal_is_not_unreachable(empty_jwks_server): + provider = _nous(empty_jwks_server) + assert provider.verify_session(access_token=FOREIGN_KID_JWT) is None + + +def test_real_jwt_with_unreachable_portal_still_raises_provider_error(): + provider = _nous("http://127.0.0.1:9") # discard port: connection refused + with pytest.raises(ProviderError): + provider.verify_session(access_token=FOREIGN_KID_JWT) + + +def test_opaque_bearer_with_unreachable_portal_is_still_just_not_ours(): + """No network call is even needed to know an opaque string is not our JWT.""" + provider = _nous("http://127.0.0.1:9") + assert provider.verify_session(access_token=OPAQUE_PEER_KEY) is None + + +# ── self-hosted OIDC provider (sibling site of the same hunk) ────────────── + +def test_self_hosted_provider_shares_the_classification(empty_jwks_server, monkeypatch): + import plugins.dashboard_auth.self_hosted as sh + + provider = object.__new__(sh.SelfHostedOIDCProvider) + provider._jwks_client = None + provider._client_id = "hermes" + monkeypatch.setattr( + provider, "_get_discovery", + lambda: {"jwks_uri": f"{empty_jwks_server}/jwks", "issuer": empty_jwks_server}, + ) + with pytest.raises(InvalidCodeError): + provider._verify_id_token(OPAQUE_PEER_KEY) + + +# ── HTTP level: the gated API answers 401, not 503 ──────────────────────── + +@pytest.fixture +def _gated_nous(empty_jwks_server): + clear_providers() + prev = {k: getattr(web_server.app.state, k, None) for k in ("bound_host", "bound_port", "auth_required")} + web_server.app.state.bound_host = "agent.example.test" + web_server.app.state.bound_port = 443 + web_server.app.state.auth_required = True + register_provider(_nous(empty_jwks_server)) + yield TestClient(web_server.app, base_url="https://agent.example.test") + clear_providers() + for k, v in prev.items(): + setattr(web_server.app.state, k, v) + + +def test_gated_api_rejects_opaque_bearer_with_401_not_503(_gated_nous): + r = _gated_nous.get("/api/auth/me", headers={"Authorization": f"Bearer {OPAQUE_PEER_KEY}"}) + assert r.status_code != 503, r.text + assert r.status_code == 401 + assert "unreachable" not in r.text.lower() + + +def test_gated_api_rejects_opaque_cookie_with_401_not_503(_gated_nous): + _gated_nous.cookies.set(SESSION_AT_COOKIE, OPAQUE_PEER_KEY) + r = _gated_nous.get("/api/auth/me") + assert r.status_code != 503, r.text + assert "unreachable" not in r.text.lower() diff --git a/tests/plugins/image_gen/test_meta_ai_provider.py b/tests/plugins/image_gen/test_meta_ai_provider.py new file mode 100644 index 0000000000..3ff129a539 --- /dev/null +++ b/tests/plugins/image_gen/test_meta_ai_provider.py @@ -0,0 +1,316 @@ +"""Tests for the bundled Meta Model API image_gen plugin (muse-image).""" + +from __future__ import annotations + +import importlib +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +# The plugin directory uses a hyphen, which is not a valid Python identifier +# for the dotted-import form. Load it via importlib so tests don't need to +# touch sys.path or rename the directory. +meta_plugin = importlib.import_module("plugins.image_gen.meta-ai") + + +# 1×1 transparent PNG — valid bytes for save_b64_image() +_PNG_HEX = ( + "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4" + "890000000d49444154789c6300010000000500010d0a2db40000000049454e44" + "ae426082" +) + + +def _b64_png() -> str: + import base64 + + return base64.b64encode(bytes.fromhex(_PNG_HEX)).decode() + + +def _fake_response(*, b64=None, url=None, revised_prompt=None): + item = SimpleNamespace(b64_json=b64, url=url, revised_prompt=revised_prompt) + return SimpleNamespace(data=[item]) + + +@pytest.fixture(autouse=True) +def _tmp_hermes_home(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + # Clear every auth + override env var so tests start from a clean slate. + for env in ( + "MODEL_API_KEY", + "META_API_KEY", + "META_MODEL_API_KEY", + "META_BASE_URL", + "META_IMAGE_MODEL", + ): + monkeypatch.delenv(env, raising=False) + yield tmp_path + + +@pytest.fixture +def provider(monkeypatch): + monkeypatch.setenv("META_MODEL_API_KEY", "test-key") + return meta_plugin.MetaImageGenProvider() + + +def _patched_openai(fake_client: MagicMock): + fake_openai = MagicMock() + fake_openai.OpenAI.return_value = fake_client + return patch.dict("sys.modules", {"openai": fake_openai}) + + +# ── Metadata ──────────────────────────────────────────────────────────────── + + +class TestMetadata: + def test_name(self, provider): + assert provider.name == "meta-ai" + + def test_display_name(self, provider): + assert provider.display_name == "Meta Model API" + + def test_default_model(self, provider): + assert provider.default_model() == "muse-image-1.0" + + def test_list_models(self, provider): + ids = [m["id"] for m in provider.list_models()] + assert ids == ["muse-image-1.0"] + + def test_catalog_entries_have_display_speed_strengths_price(self, provider): + for entry in provider.list_models(): + assert entry["display"] + assert entry["speed"] + assert entry["strengths"] + assert entry["price"] + + def test_text_only_capabilities(self, provider): + caps = provider.capabilities() + assert caps["modalities"] == ["text"] + assert caps["max_reference_images"] == 0 + + +# ── Availability ──────────────────────────────────────────────────────────── + + +class TestAvailability: + def test_no_api_key_unavailable(self): + assert meta_plugin.MetaImageGenProvider().is_available() is False + + @pytest.mark.parametrize( + "env", ["MODEL_API_KEY", "META_API_KEY", "META_MODEL_API_KEY"] + ) + def test_each_auth_alias_makes_available(self, monkeypatch, env): + monkeypatch.setenv(env, "test") + assert meta_plugin.MetaImageGenProvider().is_available() is True + + +# ── Auth / base-url resolution ──────────────────────────────────────────────── + + +class TestResolution: + def test_api_key_priority_order(self, monkeypatch): + # MODEL_API_KEY wins over the aliases. + monkeypatch.setenv("META_MODEL_API_KEY", "third") + monkeypatch.setenv("META_API_KEY", "second") + monkeypatch.setenv("MODEL_API_KEY", "first") + assert meta_plugin._resolve_api_key() == "first" + + def test_default_base_url(self): + assert meta_plugin._resolve_base_url() == "https://api.meta.ai/v1" + + def test_base_url_override(self, monkeypatch): + monkeypatch.setenv("META_BASE_URL", "https://proxy.internal/v1") + assert meta_plugin._resolve_base_url() == "https://proxy.internal/v1" + + +# ── Model resolution ────────────────────────────────────────────────────────── + + +class TestModelResolution: + def test_default(self): + model_id, _meta = meta_plugin._resolve_model() + assert model_id == "muse-image-1.0" + + def test_env_var_override_ignores_unknown(self, monkeypatch): + monkeypatch.setenv("META_IMAGE_MODEL", "not-a-real-model") + model_id, _meta = meta_plugin._resolve_model() + # Unknown id is ignored; falls through to the default. + assert model_id == "muse-image-1.0" + + def test_caller_model_kwarg_wins(self, monkeypatch): + # The dispatcher forwards top-level image_gen.model as the `model` + # kwarg; it must beat the env override (#55893 bug class). + monkeypatch.setitem( + meta_plugin._MODELS, + "muse-image-test", + dict(meta_plugin._MODELS["muse-image-1.0"]), + ) + monkeypatch.setenv("META_IMAGE_MODEL", "muse-image-1.0") + model_id, _meta = meta_plugin._resolve_model("muse-image-test") + assert model_id == "muse-image-test" + + def test_caller_model_unknown_falls_through(self): + model_id, _meta = meta_plugin._resolve_model("not-a-real-model") + assert model_id == "muse-image-1.0" + + +# ── Generate ────────────────────────────────────────────────────────────────── + + +class TestGenerate: + def test_model_kwarg_reaches_payload(self, provider, monkeypatch): + monkeypatch.setitem( + meta_plugin._MODELS, + "muse-image-test", + dict(meta_plugin._MODELS["muse-image-1.0"]), + ) + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + with _patched_openai(fake_client): + result = provider.generate("a cat", model="muse-image-test") + assert result["success"] is True + assert ( + fake_client.images.generate.call_args.kwargs["model"] == "muse-image-test" + ) + + def test_badge_is_standard_paid(self, provider): + assert provider.get_setup_schema()["badge"] == "paid" + + def test_empty_prompt_rejected(self, provider): + result = provider.generate("", aspect_ratio="square") + assert result["success"] is False + assert result["error_type"] == "invalid_argument" + assert result["provider"] == "meta-ai" + + def test_missing_api_key(self): + result = meta_plugin.MetaImageGenProvider().generate("a cat") + assert result["success"] is False + assert result["error_type"] == "auth_required" + + def test_b64_saves_to_cache(self, provider, tmp_path): + png_bytes = bytes.fromhex(_PNG_HEX) + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + + with _patched_openai(fake_client): + result = provider.generate("a cat", aspect_ratio="landscape") + + assert result["success"] is True + assert result["model"] == "muse-image-1.0" + assert result["aspect_ratio"] == "landscape" + assert result["provider"] == "meta-ai" + assert result["modality"] == "text" + + saved = Path(result["image"]) + assert saved.exists() + assert saved.parent == tmp_path / "cache" / "images" + assert saved.read_bytes() == png_bytes + + call_kwargs = fake_client.images.generate.call_args.kwargs + assert call_kwargs["model"] == "muse-image-1.0" + assert call_kwargs["size"] == "1536x1024" + assert call_kwargs["n"] == 1 + + def test_client_uses_meta_base_url(self, provider): + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + fake_openai = MagicMock() + fake_openai.OpenAI.return_value = fake_client + + with patch.dict("sys.modules", {"openai": fake_openai}): + provider.generate("a cat") + + assert ( + fake_openai.OpenAI.call_args.kwargs["base_url"] == "https://api.meta.ai/v1" + ) + + def test_base_url_override_reaches_client(self, provider, monkeypatch): + monkeypatch.setenv("META_BASE_URL", "https://proxy.internal/v1") + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + fake_openai = MagicMock() + fake_openai.OpenAI.return_value = fake_client + + with patch.dict("sys.modules", {"openai": fake_openai}): + provider.generate("a cat") + + assert ( + fake_openai.OpenAI.call_args.kwargs["base_url"] + == "https://proxy.internal/v1" + ) + + @pytest.mark.parametrize( + "aspect,expected_size", + [ + ("landscape", "1536x1024"), + ("square", "1024x1024"), + ("portrait", "1024x1536"), + ], + ) + def test_aspect_ratio_mapping(self, provider, aspect, expected_size): + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=_b64_png()) + + with _patched_openai(fake_client): + provider.generate("a cat", aspect_ratio=aspect) + + assert fake_client.images.generate.call_args.kwargs["size"] == expected_size + + def test_revised_prompt_passed_through(self, provider): + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response( + b64=_b64_png(), + revised_prompt="A photo of a cat", + ) + + with _patched_openai(fake_client): + result = provider.generate("a cat") + + assert result["revised_prompt"] == "A photo of a cat" + + def test_url_response_is_cached_locally(self, provider): + """A URL response is materialized locally (symmetric to the openai/xai + providers) so ephemeral signed URLs can't expire mid-flight.""" + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response( + b64=None, + url="https://example.com/img.webp", + ) + + with ( + _patched_openai(fake_client), + patch.object( + meta_plugin, + "save_url_image", + return_value=Path("/tmp/meta_20260524_000000_deadbeef.webp"), + ) as mock_save_url, + ): + result = provider.generate("a cat") + + assert result["success"] is True + assert result["image"].startswith("/") + assert "example.com" not in result["image"] + mock_save_url.assert_called_once() + + def test_empty_response_errors(self, provider): + fake_client = MagicMock() + fake_client.images.generate.return_value = _fake_response(b64=None, url=None) + + with _patched_openai(fake_client): + result = provider.generate("a cat") + + assert result["success"] is False + assert result["error_type"] == "empty_response" + + def test_api_error_surfaced(self, provider): + fake_client = MagicMock() + fake_client.images.generate.side_effect = RuntimeError("boom") + + with _patched_openai(fake_client): + result = provider.generate("a cat") + + assert result["success"] is False + assert result["error_type"] == "api_error" + assert "boom" in result["error"] diff --git a/tests/plugins/memory/test_hindsight_provider.py b/tests/plugins/memory/test_hindsight_provider.py index b8fe91a7b4..f681af6d6e 100644 --- a/tests/plugins/memory/test_hindsight_provider.py +++ b/tests/plugins/memory/test_hindsight_provider.py @@ -10,6 +10,7 @@ import os import re import stat import sys +import threading import time from datetime import datetime from pathlib import Path @@ -32,6 +33,7 @@ from plugins.memory.hindsight import ( _normalize_retain_tags, _resolve_bank_id_template, _sanitize_bank_segment, + _WRITER_SENTINEL, ) @@ -1640,3 +1642,68 @@ class TestClientAutoUpgradeRoutesThroughPm: assert len(calls) == 1 # attempted exactly once, init still completed assert any("runtime installs are disabled" in r.getMessage() for r in caplog.records) + + + +class TestMultiplexBackgroundScope: + """Under multiplex_profiles get_secret fails closed on an unscoped thread; + the writer / daemon-start threads are spawned from a scoped context and + must carry it along (#92608, #94933).""" + + @pytest.fixture() + def scoped_embedded(self, tmp_path, monkeypatch): + from agent.secret_scope import ( + build_profile_secret_scope, reset_secret_scope, set_multiplex_active, set_secret_scope, + ) + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + created = [] + + class FakeHindsightEmbedded: + def __init__(self, **kwargs): + created.append(kwargs["llm_api_key"]) + self._manager = SimpleNamespace(is_running=lambda profile: False, stop=lambda profile: None) + self._ensure_started = lambda: None + + dem = SimpleNamespace(console=None) + monkeypatch.setitem(sys.modules, "hindsight", SimpleNamespace(HindsightEmbedded=FakeHindsightEmbedded)) + monkeypatch.setitem(sys.modules, "hindsight_embed", SimpleNamespace(daemon_embed_manager=dem)) + monkeypatch.setitem(sys.modules, "hindsight_embed.daemon_embed_manager", dem) + monkeypatch.setattr("plugins.memory.hindsight._check_local_runtime", lambda: (True, "")) + + home = tmp_path / "profiles" / "p1" + (home / "hindsight").mkdir(parents=True) + (home / ".env").write_text("HINDSIGHT_LLM_API_KEY=p1-secret\n") + (home / "hindsight" / "config.json").write_text(json.dumps( + {"mode": "local_embedded", "llm_provider": "openai", "llm_model": "m", "memory_mode": "hybrid"} + )) + # Enter the profile scope the way gateway _profile_runtime_scope does. + set_multiplex_active(True) + monkeypatch.setattr("plugins.memory.hindsight.get_hermes_home", lambda: home) + home_tok = set_hermes_home_override(str(home)) + scope_tok = set_secret_scope(build_profile_secret_scope(home)) + yield created, home + set_multiplex_active(False) + reset_secret_scope(scope_tok) + reset_hermes_home_override(home_tok) + + def test_writer_thread_resolves_profile_secret(self, scoped_embedded): + created, home = scoped_embedded + p = HindsightMemoryProvider() + p._mode = "local_embedded" + p._config = {"profile": "hermes", "llm_provider": "openai", "llm_model": "m"} + p._ensure_writer() + p._retain_queue.put(p._get_client) # real body: get_secret(HINDSIGHT_LLM_API_KEY) + p._retain_queue.put(_WRITER_SENTINEL) + p._writer_thread.join(timeout=5) + assert created == ["p1-secret"] + + def test_daemon_start_thread_resolves_profile_secret(self, scoped_embedded): + created, home = scoped_embedded + p = HindsightMemoryProvider() + p.initialize(session_id="s1", hermes_home=str(home), platform="cli") + for t in threading.enumerate(): + if t.name == "hindsight-daemon-start": + t.join(timeout=5) + assert created == ["p1-secret"] + assert "Daemon started successfully" in (home / "logs" / "hindsight-embed.log").read_text() diff --git a/tests/plugins/platforms/photon/test_multiplex_profile_scope.py b/tests/plugins/platforms/photon/test_multiplex_profile_scope.py new file mode 100644 index 0000000000..5b32327b1e --- /dev/null +++ b/tests/plugins/platforms/photon/test_multiplex_profile_scope.py @@ -0,0 +1,122 @@ +"""Multiplex secondary-profile scope tests for the Photon adapter + auth module. + +__init__'s project_id, check_requirements'/validate_config's node_bin/ +project_id, _env_enablement's home_channel, _reactions_enabled's +PHOTON_REACTIONS, __init__'s require_mention, and _standalone_send's +sidecar_port, plus auth.py's load_project_credentials/ +load_dashboard_project_id, all previously read raw os.getenv +unconditionally (only PHOTON_PROJECT_SECRET/PHOTON_SIDECAR_TOKEN were +already scoped via _get_scoped_secret). Under gateway.multiplex_profiles, +os.environ holds the DEFAULT profile's YAML-to-env bridge output -- a +secondary profile with its own (different or absent) Photon config could +silently authenticate against the default profile's Spectrum project, or +have its mention-gating/reaction behavior driven by the default profile's +settings. + +Notably project_id was a stronger variant of the bug (like the IRC fix in +this series): __init__'s original +`os.getenv("PHOTON_PROJECT_ID") or extra.get("project_id") or stored_id` +ordering let a raw env read override even an explicitly configured +config.yaml extra. + +Mirrors the LINE/DingTalk/IRC/Mattermost fix for #98738. +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from gateway.config import PlatformConfig +from plugins.platforms.photon import auth as photon_auth +from plugins.platforms.photon.adapter import PhotonAdapter + +_PHOTON_ENV = ( + "PHOTON_PROJECT_ID", + "PHOTON_PROJECT_SECRET", + "PHOTON_DASHBOARD_PROJECT_ID", + "PHOTON_REQUIRE_MENTION", + "PHOTON_REACTIONS", + "PHOTON_HOME_CHANNEL", + "PHOTON_HOME_CHANNEL_NAME", + "PHOTON_SIDECAR_PORT", +) + + +@pytest.fixture +def tmp_hermes_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Isolate from the real ~/.hermes/auth.json fallback in load_project_credentials().""" + home = tmp_path / "hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + for key in _PHOTON_ENV: + monkeypatch.delenv(key, raising=False) + yield home + for key in _PHOTON_ENV: + os.environ.pop(key, None) + + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + tokens = [] + + def install(scope=None): + from agent.secret_scope import set_multiplex_active, set_secret_scope + + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + return tokens[-1] + + yield install + + from agent.secret_scope import reset_secret_scope, set_multiplex_active + + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("PHOTON_PROJECT_ID", "default-project-id") + monkeypatch.setenv("PHOTON_PROJECT_SECRET", "default-project-secret") + monkeypatch.setenv("PHOTON_REQUIRE_MENTION", "true") + monkeypatch.setenv("PHOTON_REACTIONS", "true") + + +class TestAuthMultiplexProfileScope: + """load_project_credentials / load_dashboard_project_id (auth.py).""" + + def test_scoped_miss_does_not_leak_default_project_id( + self, tmp_hermes_home, multiplex_scope, default_profile_env + ): + multiplex_scope({"SOMETHING_ELSE": "x"}) + sid, secret = photon_auth.load_project_credentials() + assert sid is None + assert secret is None + adapter = PhotonAdapter(PlatformConfig(enabled=True, extra={})) + assert adapter._project_id == "" + assert adapter.require_mention is False + assert adapter._reactions_enabled() is False + +class TestAdapterMultiplexProfileScope: + """PhotonAdapter.__init__ / _env_enablement / _reactions_enabled (adapter.py).""" + + def test_secondary_extra_wins_over_default_profile_env( + self, tmp_hermes_home, multiplex_scope, default_profile_env + ): + """A secondary profile's own config.yaml extra project_id must be + authoritative -- not the default profile's bridged env value. The + pre-fix ordering (raw os.getenv checked BEFORE extra) meant even an + explicit extra config was silently overridden.""" + multiplex_scope({"PHOTON_PROJECT_SECRET": "profile-secret"}) + cfg = PlatformConfig( + enabled=True, + extra={"project_id": "profile-project-id"}, + ) + adapter = PhotonAdapter(cfg) + assert adapter._project_id == "profile-project-id" + diff --git a/tests/plugins/test_a2a_plugin.py b/tests/plugins/test_a2a_plugin.py index 678d558e7e..fedee65eb1 100644 --- a/tests/plugins/test_a2a_plugin.py +++ b/tests/plugins/test_a2a_plugin.py @@ -910,7 +910,9 @@ def _make_live_adapter(monkeypatch, reply_fn=None): port = _free_port() monkeypatch.setenv("A2A_PORT", str(port)) - adapter = A2AAdapter(PlatformConfig(enabled=True)) + # A scoped secondary profile ignores the process env (#100382); pass the + # port through config.extra so both construction paths bind the same port. + adapter = A2AAdapter(PlatformConfig(enabled=True, extra={"port": port})) async def fake_handle_message(event): if reply_fn is None: @@ -1223,6 +1225,54 @@ class TestInboundRoundTrip: asyncio.run(run()) + def test_multiplex_adapter_keeps_profile_scoped_peer_tokens(self, monkeypatch): + """A secondary listener must not authenticate with the default profile's tokens.""" + from agent.secret_scope import ( + reset_secret_scope, + set_multiplex_active, + set_secret_scope, + ) + + monkeypatch.setenv("A2A_PEER_TOKENS", "default:default-token") + monkeypatch.delenv("A2A_BEARER_TOKEN", raising=False) + monkeypatch.setenv("A2A_HOST", "127.0.0.1") + + set_multiplex_active(True) + scope_token = set_secret_scope( + {"A2A_PEER_TOKENS": "secondary:secondary-token"} + ) + try: + adapter, base = _make_live_adapter(monkeypatch) + finally: + reset_secret_scope(scope_token) + + async def run(): + try: + assert await adapter.connect() is True + response = await asyncio.to_thread( + _post_json, + base + "/", + _send_body("profile-scoped auth"), + {"Authorization": "Bearer secondary-token"}, + ) + assert response["result"]["status"]["state"] == "TASK_STATE_COMPLETED" + + with pytest.raises(urllib.error.HTTPError) as exc_info: + await asyncio.to_thread( + _post_json, + base + "/", + _send_body("wrong profile"), + {"Authorization": "Bearer default-token"}, + ) + assert exc_info.value.code == 401 + finally: + await adapter.disconnect() + + try: + asyncio.run(run()) + finally: + set_multiplex_active(False) + # -------------------------------------------------------------------------- # Push notifications end-to-end (inline config in message/send) @@ -1619,3 +1669,101 @@ print('fake reply') title = con.execute("SELECT title FROM sessions WHERE id='sess-1'").fetchone()[0] con.close() assert title == "a2a-dev-ctx-unsafe-value" + + +# -------------------------------------------------------------------------- +# Multiplex secondary-profile scope (construction-time config leak) +# -------------------------------------------------------------------------- +# +# __init__'s port/advertised-toolsets reads and _load_served_agents's +# description default all previously read raw A2A_* env vars unconditionally. +# Under a multiplexed secondary profile, os.environ holds the DEFAULT +# profile's YAML-to-env bridge output — a secondary profile with its own +# (different, or absent) A2A config would silently borrow the default +# profile's port, toolset advertisement, agent name, or Agent Card +# description. Mirrors the Buzz/SimpleX fix for #98738. + +_A2A_ENV_VARS = ( + "A2A_PORT", + "A2A_AGENT_NAME", + "A2A_ADVERTISED_TOOLSETS", + "A2A_AGENT_DESCRIPTION", +) + + +@pytest.fixture(autouse=True) +def _clean_a2a_construction_env(monkeypatch): + """Keep the new multiplex tests hermetic regardless of ambient env.""" + for var in _A2A_ENV_VARS: + monkeypatch.delenv(var, raising=False) + yield + + +@pytest.fixture +def multiplex_scope(): + """Install multiplex + a secondary-profile secret scope; restore after.""" + tokens = [] + + def install(scope=None): + from agent.secret_scope import set_multiplex_active, set_secret_scope + + set_multiplex_active(True) + tokens.append(set_secret_scope(scope or {})) + return tokens[-1] + + yield install + + from agent.secret_scope import reset_secret_scope, set_multiplex_active + + for token in reversed(tokens): + reset_secret_scope(token) + set_multiplex_active(False) + + +@pytest.fixture +def default_profile_env(monkeypatch): + """The default profile's YAML-to-env bridge output in os.environ.""" + monkeypatch.setenv("A2A_PORT", "9111") + monkeypatch.setenv("A2A_AGENT_NAME", "default-profile-agent") + monkeypatch.setenv("A2A_ADVERTISED_TOOLSETS", "default-only-toolset") + monkeypatch.setenv("A2A_AGENT_DESCRIPTION", "Default profile's own agent.") + + +class TestMultiplexConstructionScope: + + def test_secondary_profile_never_borrows_default_profile_env( + self, multiplex_scope, default_profile_env + ): + """The secondary profile's own config is authoritative; keys absent + from it fall to the module defaults, never to the default profile's + bridged A2A_* env values.""" + from plugins.platforms.a2a.adapter import A2AAdapter, _DEFAULT_PORT + from gateway.config import PlatformConfig + + multiplex_scope() + assert A2AAdapter(PlatformConfig(enabled=True, extra={"port": 9222})).port == 9222 + + adapter = A2AAdapter(PlatformConfig(enabled=True, extra={})) + assert adapter.port == _DEFAULT_PORT + assert adapter.agent_name != "default-profile-agent" + assert adapter._agents[""]["description"] == ( + "Hermes Agent — a general-purpose agent reachable over A2A." + ) + + def test_default_profile_unscoped_keeps_env_precedence( + self, monkeypatch, default_profile_env + ): + """Multiplex ON but no scope (the DEFAULT profile constructs + unscoped): env is its own bridge output and still wins.""" + from agent.secret_scope import set_multiplex_active + from plugins.platforms.a2a.adapter import A2AAdapter + from gateway.config import PlatformConfig + + set_multiplex_active(True) + try: + adapter = A2AAdapter(PlatformConfig(enabled=True, extra={})) + finally: + set_multiplex_active(False) + assert adapter.port == 9111 + assert adapter.agent_name == "default-profile-agent" + assert adapter._agents[""]["description"] == "Default profile's own agent." diff --git a/tests/plugins/test_a2a_schema_registration.py b/tests/plugins/test_a2a_schema_registration.py index 6068b62fc7..76f9a3819d 100644 --- a/tests/plugins/test_a2a_schema_registration.py +++ b/tests/plugins/test_a2a_schema_registration.py @@ -35,7 +35,8 @@ def test_a2a_call_schema_round_trips_through_tool_describe(monkeypatch): monkeypatch.setattr( tool_search, "is_deferrable_tool_name", - lambda name: name == "a2a_call", + # #97979 added the defer_tools positional (curated-set override). + lambda name, defer_tools=None: name == "a2a_call", ) described = json.loads( diff --git a/tests/providers/test_installed_plugin_discovery.py b/tests/providers/test_installed_plugin_discovery.py new file mode 100644 index 0000000000..985aaf8b53 --- /dev/null +++ b/tests/providers/test_installed_plugin_discovery.py @@ -0,0 +1,78 @@ +"""A provider installed by ``hermes plugins install`` must actually be found. + +The installer clones into ``$HERMES_HOME/plugins//`` (flat), provider +discovery only scanned ``plugins/model-providers//``, and PluginManager +skips ``kind: model-provider`` on purpose — so the documented install path +reported success and registered nothing. These tests pin the join, and that +discovery keeps its hands off every other plugin in that directory. +""" + +from __future__ import annotations + +import sys +import textwrap +from pathlib import Path + +import pytest + +_PROFILE_SOURCE = textwrap.dedent( + """ + from providers import register_provider + from providers.base import ProviderProfile + + register_provider(ProviderProfile(name="{name}", aliases=("{name}-alias",), + base_url="acp://{name}", auth_type="external_process")) + """ +) + + +def _clear_provider_caches(): + import providers as _pkg + + _pkg._REGISTRY.clear() + _pkg._ALIASES.clear() + _pkg._PROVIDER_LIST_CACHE = None + _pkg._discovered = False + for mod in list(sys.modules): + if mod.startswith(("plugins.model_providers", "_hermes_user_provider")): + del sys.modules[mod] + + +def _write_plugin(directory: Path, *, name: str, manifest: str | None): + directory.mkdir(parents=True, exist_ok=True) + if manifest is not None: + (directory / "plugin.yaml").write_text(manifest, encoding="utf-8") + # Registers a provider on import, so an unwanted import is *visible*. + (directory / "__init__.py").write_text(_PROFILE_SOURCE.format(name=name), encoding="utf-8") + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + _clear_provider_caches() + yield tmp_path + _clear_provider_caches() + + +def test_flat_installed_model_provider_plugins_are_discovered_alongside_nested_ones(hermes_home): + _write_plugin(hermes_home / "plugins" / "installed-acp", name="installed-acp", + manifest='name: installed-acp\nkind: "model-provider"\n') + _write_plugin(hermes_home / "plugins" / "model-providers" / "nested-acp", name="nested-acp", + manifest="name: nested-acp\nkind: model-provider\n") + from providers import get_provider_profile + + assert get_provider_profile("installed-acp").base_url == "acp://installed-acp" + assert get_provider_profile("installed-acp-alias") is not None + assert get_provider_profile("nested-acp") is not None + + +def test_other_plugins_in_the_flat_directory_are_left_to_the_plugin_manager(hermes_home): + _write_plugin(hermes_home / "plugins" / "other-standalone", name="other-standalone", + manifest="name: other-standalone\nkind: standalone\n") + _write_plugin(hermes_home / "plugins" / "manifestless", name="manifestless", manifest=None) + _write_plugin(hermes_home / "plugins" / "broken-manifest", name="broken-manifest", + manifest="kind: [this is: not valid\n") + from providers import get_provider_profile, list_providers + + assert not [p for p in list_providers() if p.name in ("other-standalone", "manifestless", "broken-manifest")] + assert get_provider_profile("copilot-acp") is not None # bundled set still intact diff --git a/tests/providers/test_meta_ai_profile.py b/tests/providers/test_meta_ai_profile.py index 91ecdcdd6e..5a0ea074ce 100644 --- a/tests/providers/test_meta_ai_profile.py +++ b/tests/providers/test_meta_ai_profile.py @@ -8,6 +8,7 @@ bundled, so profiles resolve through normal registry discovery. import pytest from providers import get_provider_profile +from providers.base import ProviderProfile def _profile(): @@ -27,10 +28,29 @@ class TestMetaAIProfile: assert p.api_mode == "codex_responses" assert "MODEL_API_KEY" in p.env_vars assert p.supports_vision is True + # Images are accepted on user turns only; tool-result envelopes 400 (#101668). + assert p.supports_vision_tool_messages is False assert p.default_aux_model == "muse-spark-1.2-contributor" assert p.default_max_tokens == 16384 - assert "muse-spark-1.2-contributor" in p.fallback_models - assert "muse-spark-1.2" in p.fallback_models + assert p.fallback_models == ("muse-spark-1.2",) + + def test_live_catalog_filters_non_chat_models(self, monkeypatch): + p = _profile() + seen = [] + + def fake_fetch_models(_self, **_kwargs): + seen.append(True) + return [ + "muse-voice-transcribe-1.0", + "muse-spark-latest", + "muse-image-1.0-eval", + "muse-nova-test", + ] + + monkeypatch.setattr(ProviderProfile, "fetch_models", fake_fetch_models) + + assert p.fetch_models() == ["muse-spark-latest", "muse-nova-test"] + assert seen @pytest.mark.parametrize("alias", ["meta", "muse", "muse-spark", "model-api", "msl"]) def test_aliases_resolve(self, alias): diff --git a/tests/run_agent/test_413_compression.py b/tests/run_agent/test_413_compression.py index a91a585823..2c99ff03a0 100644 --- a/tests/run_agent/test_413_compression.py +++ b/tests/run_agent/test_413_compression.py @@ -979,6 +979,156 @@ class TestPreflightCompression: assert result["final_response"] == "Recovered after overflow" assert mock_compress.call_count == 2 + def test_provider_overflow_rechecks_complete_request_before_retry(self, agent): + """Provider-proven overflow bypasses post-compaction estimate deferral. + + The first recovery pass drops message rows but rebuilds a larger + request. The compressor then awaits real usage, so the old path sent + that oversized request back to llama.cpp, which may silently truncate + instead of returning another overflow error. Recovery must run another + bounded preflight pass first. + """ + agent.compression_enabled = True + agent.max_compression_attempts = 2 + agent.context_compressor.context_length = 65_536 + agent.context_compressor.threshold_tokens = 34_078 + + overflow = Exception( + "request (70000 tokens) exceeds the available context size " + "(65536 tokens)" + ) + overflow.status_code = 400 + agent.client.chat.completions.create.side_effect = [overflow] + + history = [ + {"role": "user", "content": "earlier question"}, + {"role": "assistant", "content": "earlier answer"}, + ] + compress_calls = 0 + + def _request_pressure(*_args, **_kwargs): + if agent.client.chat.completions.create.call_count == 0: + return 30_000 + return 70_000 + + def _compress(_messages, *_args, **_kwargs): + nonlocal compress_calls + compress_calls += 1 + return ( + [ + {"role": "user", "content": f"summary {compress_calls}"}, + {"role": "assistant", "content": "summary acknowledged"}, + ], + "rebuilt prompt remains oversized", + ) + + with ( + patch( + "agent.turn_context.estimate_request_tokens_rough", + return_value=30_000, + ), + patch( + "agent.conversation_loop._midturn_request_pressure_tokens", + side_effect=_request_pressure, + ), + patch.object( + agent.context_compressor, + "should_defer_preflight_to_real_usage", + return_value=True, + ), + patch.object(agent, "_compress_context", side_effect=_compress) as mock_compress, + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation( + "continue", + conversation_history=history, + ) + + assert result["completed"] is False + assert result["compression_exhausted"] is True + assert mock_compress.call_count == 2 + assert agent.client.chat.completions.create.call_count == 1 + + def test_long_context_tier_recovery_rechecks_complete_request_before_retry(self, agent): + """The Anthropic long-context 429 handler is the same recovery class. + + It compacts and restarts on row count alone, exactly like the generic + overflow handler. The rebuilt request must be measured against the + (now-reduced) window before the provider is retried, so a compaction + that drops rows but stays oversized fails closed instead of being + sent again. + """ + agent.compression_enabled = True + agent.max_compression_attempts = 2 + agent.context_compressor.context_length = 1_000_000 + agent.context_compressor.threshold_tokens = 500_000 + + tier_error = Exception( + "Extra usage is required for long context requests." + ) + tier_error.status_code = 429 + agent.client.chat.completions.create.side_effect = [tier_error] + + history = [ + {"role": "user", "content": "earlier question"}, + {"role": "assistant", "content": "earlier answer"}, + ] + compress_calls = 0 + + def _request_pressure(*_args, **_kwargs): + if agent.client.chat.completions.create.call_count == 0: + return 30_000 + return 250_000 + + def _compress(_messages, *_args, **_kwargs): + nonlocal compress_calls + compress_calls += 1 + return ( + [ + {"role": "user", "content": f"summary {compress_calls}"}, + {"role": "assistant", "content": "summary acknowledged"}, + ], + "rebuilt prompt remains oversized", + ) + + def _update_model(*, context_length, **_kwargs): + agent.context_compressor.context_length = context_length + agent.context_compressor.threshold_tokens = context_length // 2 + + with ( + patch( + "agent.turn_context.estimate_request_tokens_rough", + return_value=30_000, + ), + patch( + "agent.conversation_loop._midturn_request_pressure_tokens", + side_effect=_request_pressure, + ), + patch.object( + agent.context_compressor, + "should_defer_preflight_to_real_usage", + return_value=True, + ), + patch.object( + agent.context_compressor, "update_model", side_effect=_update_model + ), + patch.object(agent, "_compress_context", side_effect=_compress) as mock_compress, + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation( + "continue", + conversation_history=history, + ) + + assert result["completed"] is False + assert result["compression_exhausted"] is True + assert mock_compress.call_count == 2 + assert agent.client.chat.completions.create.call_count == 1 + def test_interrupt_before_first_provider_call_restores_preflight_display_seed(self, agent): """Interrupted turns must not keep a speculative preflight display seed. diff --git a/tests/run_agent/test_background_review.py b/tests/run_agent/test_background_review.py index ca2ebea949..7a10ef16e3 100644 --- a/tests/run_agent/test_background_review.py +++ b/tests/run_agent/test_background_review.py @@ -461,6 +461,27 @@ def test_background_review_registers_before_start_runs_and_cleans_up(monkeypatch assert agent._active_children == [] +def test_background_review_snapshot_isolated_from_live_nested_messages(): + """A review must not mutate the persisted/live transcript through aliases.""" + original = [{ + "role": "assistant", + "content": [{"type": "text", "text": "answer"}], + "tool_calls": [{ + "id": "call-1", + "function": {"name": "read_file", "arguments": '{"path":"x"}'}, + }], + }] + + from agent.turn_finalizer import _clone_background_review_messages + + snapshot = _clone_background_review_messages(original) + snapshot[0]["content"][0]["text"] = "review mutation" + snapshot[0]["tool_calls"][0]["function"]["arguments"] = "{}" + + assert original[0]["content"][0]["text"] == "answer" + assert original[0]["tool_calls"][0]["function"]["arguments"] == '{"path":"x"}' + + def test_live_turn_waits_for_review_exit_before_relay_and_turn_context(monkeypatch): """The outer production wrapper waits before same-session instrumentation.""" review_entered = threading.Event() diff --git a/tests/run_agent/test_corruption_recovery_guidance.py b/tests/run_agent/test_corruption_recovery_guidance.py index e4b5063ff7..155d326e20 100644 --- a/tests/run_agent/test_corruption_recovery_guidance.py +++ b/tests/run_agent/test_corruption_recovery_guidance.py @@ -31,6 +31,27 @@ def test_format_turn_completion_corrupt_includes_recovery_options(): assert "Freeing disk space will not help" in explanation +def test_format_turn_completion_corrupt_never_names_the_live_db(): + """The 'corrupt' cause must not direct a raw sqlite3 shell at the live DB. + + #100368 forensics: the system sqlite3 CLI on Debian/Ubuntu (3.45.1/ + 3.46.1, below the 3.51.x WAL-reset fix) unlinks the live WAL/SHM pair + when pointed at a live state.db, splitting the store into two + generations whose acknowledged writes vanish. The guidance that ships + in the corruption banner must be the snapshot-copying + `hermes sessions recover` lane. + """ + from run_agent import AIAgent + + explanation = AIAgent._format_turn_completion_explanation( + "session_persistence_failed", "corrupt" + ) + assert "sessions recover" in explanation + assert 'sqlite3 ~/.hermes/state.db ".recover"' not in explanation + # The replacement guidance names the safe command. + assert "hermes sessions recover --source" in explanation + + def test_format_turn_completion_disk_still_advises_space(): """The 'disk' cause still gives disk-space advice (unchanged).""" from run_agent import AIAgent diff --git a/tests/run_agent/test_direct_contexts_stream_inline.py b/tests/run_agent/test_direct_contexts_stream_inline.py new file mode 100644 index 0000000000..22720f09f0 --- /dev/null +++ b/tests/run_agent/test_direct_contexts_stream_inline.py @@ -0,0 +1,243 @@ +"""Delegated children and cron turns stream on the wire (#90202, #100260). + +``should_use_direct_api_call`` contexts (gateway cron turns, delegate_task +children) must not spawn the interrupt worker — it wedges inside their nested +thread pools (#62151, #60203). The original fix short-circuited them onto the +NON-streaming wire, which silently dropped every liveness property streaming +provides: edge proxies killed the silent POST (z.ai HTTP 524, #90202), and the +non-stream stale watchdog could not tell a reasoning model's thinking phase +from a hung provider (#100260 — children died at exactly ``stale_timeout``). + +These tests pin the replacement contract: those contexts stay on the streaming +path, issue ``stream=True`` on the calling thread (no worker), and keep the +stale detector + cross-thread interrupt abort working from the monitor thread. +""" + +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace + +import pytest + +import run_agent +from agent import chat_completion_helpers as helpers +from agent.chat_completion_helpers import ( + interruptible_streaming_api_call, + should_use_direct_api_call, +) + + +# --------------------------------------------------------------------------- +# Real OpenAI-wire SSE server: records the wire ``stream`` flag per request. +# --------------------------------------------------------------------------- + + +class _Wire: + def __init__(self, *, stall_after_first_chunk: bool = False): + self.requests: list[dict] = [] + self.stall = stall_after_first_chunk + self.hits = threading.Semaphore(0) + wire = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_a): + pass + + def do_POST(self): + n = int(self.headers.get("content-length", 0)) + body = json.loads(self.rfile.read(n) or b"{}") + if not self.path.endswith("/chat/completions"): + # Local-endpoint capability probes (/api/show etc.) — + # answer fast so agent construction never waits on the + # stalling stream below. + self.send_response(404) + self.end_headers() + return + wire.requests.append(body) + wire.hits.release() + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.end_headers() + first = { + "id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hello"}, + "finish_reason": None}], + } + self.wfile.write(f"data: {json.dumps(first)}\n\n".encode()) + self.wfile.flush() + if wire.stall: + try: + for _ in range(400): + time.sleep(0.05) + self.wfile.write(b": keepalive\n\n") + self.wfile.flush() + except Exception: + pass + return + second = { + "id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {"content": " world"}, "finish_reason": None}], + } + fin = { + "id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + for c in (second, fin): + self.wfile.write(f"data: {json.dumps(c)}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.base_url = f"http://127.0.0.1:{self.server.server_address[1]}/v1" + + def close(self): + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def wire(): + w = _Wire() + yield w + w.close() + + +@pytest.fixture +def stalling_wire(): + w = _Wire(stall_after_first_chunk=True) + yield w + w.close() + + +def _make_agent(base_url: str, *, platform: str): + return run_agent.AIAgent( + api_key="test-key", + base_url=base_url, + model="m", + provider="custom", + platform=platform, + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + enabled_toolsets=[], + max_iterations=1, + ) + + +_KW = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} + + +@pytest.mark.parametrize("platform", ["subagent", "cron"]) +def test_direct_contexts_stream_on_the_wire_and_on_the_calling_thread(wire, platform): + agent = _make_agent(wire.base_url, platform=platform) + assert should_use_direct_api_call(agent) is True + + issued_on = {} + real_create = agent._create_request_openai_client + + def spy(*a, **k): + issued_on["tid"] = threading.get_ident() + return real_create(*a, **k) + + agent._create_request_openai_client = spy + + response = interruptible_streaming_api_call(agent, dict(_KW)) + + completions = [r for r in wire.requests if "messages" in r] + assert completions, "no chat completion reached the wire" + assert completions[-1].get("stream") is True, ( + f"{platform} turn went out non-streaming: stream={completions[-1].get('stream')!r}" + ) + # No interrupt worker: the request was dispatched from the caller's thread + # (the #62151 / #60203 deadlock class needs the request on a spawned worker). + assert issued_on["tid"] == threading.get_ident() + assert response.choices[0].message.content == "hello world" + assert response.choices[0].finish_reason == "stop" + + +def test_interactive_platform_still_uses_the_worker_thread(wire): + """Regression guard for the refactor: non-direct contexts keep the + interrupt worker (interactive /stop responsiveness relies on it).""" + agent = _make_agent(wire.base_url, platform="cli") + assert should_use_direct_api_call(agent) is False + + issued_on = {} + real_create = agent._create_request_openai_client + + def spy(*a, **k): + issued_on["tid"] = threading.get_ident() + return real_create(*a, **k) + + agent._create_request_openai_client = spy + response = interruptible_streaming_api_call(agent, dict(_KW)) + + assert issued_on["tid"] != threading.get_ident() + assert response.choices[0].message.content == "hello world" + + +def test_inline_stream_stale_detector_still_fires_from_monitor_thread( + stalling_wire, monkeypatch +): + """The stale-stream detector moved onto a monitor thread for inline + mode; a stream that sends one chunk then only keep-alives must still be + killed at the stale budget instead of hanging until the socket dies.""" + monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "1.0") + monkeypatch.setenv("HERMES_STREAM_RETRIES", "0") + agent = _make_agent(stalling_wire.base_url, platform="subagent") + + started = time.time() + response = interruptible_streaming_api_call(agent, dict(_KW)) + elapsed = time.time() - started + + assert elapsed < 6.0, f"inline stream was not bounded by the stale detector ({elapsed:.1f}s)" + # A partial delta was delivered → the loop gets the length-truncated + # partial-stream stub (same contract as the worker path). + assert getattr(response, "id", None) == helpers.PARTIAL_STREAM_STUB_ID + assert response.choices[0].finish_reason == helpers.FINISH_REASON_LENGTH + + +def test_inline_stream_cross_thread_interrupt_aborts_promptly(stalling_wire, monkeypatch): + """``AIAgent.interrupt()`` from another thread (cron watchdog, delegation + stall monitor) must abort the inline stream and surface InterruptedError + — the property the direct_api_call path guaranteed via + ``_active_request_abort``.""" + monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "60") + monkeypatch.setenv("HERMES_STREAM_RETRIES", "0") + agent = _make_agent(stalling_wire.base_url, platform="cron") + box: dict = {} + + def _run(): + t0 = time.time() + try: + interruptible_streaming_api_call(agent, dict(_KW)) + box["outcome"] = "returned" + except BaseException as exc: # noqa: BLE001 — record whatever surfaces + box["outcome"] = type(exc).__name__ + box["elapsed"] = time.time() - t0 + + worker = threading.Thread(target=_run, daemon=True) + worker.start() + assert stalling_wire.hits.acquire(timeout=5.0), "request never reached the wire" + time.sleep(0.3) # let the first chunk land + agent.interrupt("test interrupt") + worker.join(timeout=10.0) + + assert not worker.is_alive(), "inline stream did not unwind after interrupt" + assert box["outcome"] == "InterruptedError" + assert box["elapsed"] < 5.0 + + +def test_should_use_direct_api_call_gate_is_unchanged(): + """The routing predicate itself is untouched — only what it routes to.""" + def mk(platform, api_mode="chat_completions", provider="openrouter"): + return SimpleNamespace(platform=platform, api_mode=api_mode, provider=provider) + + assert should_use_direct_api_call(mk("cron")) is True + assert should_use_direct_api_call(mk("subagent")) is True + assert should_use_direct_api_call(mk("cli")) is False + assert should_use_direct_api_call(mk("cron", api_mode="anthropic_messages")) is False + assert should_use_direct_api_call(mk("cron", provider="moa")) is False diff --git a/tests/run_agent/test_flush_diverts_on_corrupt_state_db.py b/tests/run_agent/test_flush_diverts_on_corrupt_state_db.py new file mode 100644 index 0000000000..26e941c495 --- /dev/null +++ b/tests/run_agent/test_flush_diverts_on_corrupt_state_db.py @@ -0,0 +1,70 @@ +"""Agent flush path: a quarantined (structurally corrupt) SessionDB diverts to JSONL. + +Mirrors the replaced-file contract: the batch that SQLite will never take +again is kept on disk under ``sessions/.jsonl`` instead of only in RAM, +the flush fails closed (no retry loop), and the turn-end explanation gets the +``corrupt`` cause. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from hermes_state import SessionDB, StateDbCorruptError +from run_agent import AIAgent + + +def _flush_agent(db, session_id): + agent = SimpleNamespace( + _session_db=db, + _session_db_created=True, + _persist_disabled=False, + session_id=session_id, + _session_persist_lock=None, + _flushed_db_message_ids=set(), + _flushed_db_message_session_id=None, + _last_flushed_db_idx=0, + _db_flush_scan_prefix=None, + _persist_user_message_idx=None, + _persist_user_message_override=None, + _persist_user_message_timestamp=None, + _pending_cli_user_message=None, + _active_session_turn_lease_holder=None, + _last_persistence_error_cause=None, + _compression_adoption_failed=False, + ) + agent._ensure_db_session = lambda: None + agent._flush_messages_to_session_db = ( + AIAgent._flush_messages_to_session_db.__get__(agent, AIAgent) + ) + agent._flush_messages_to_session_db_unlocked = ( + AIAgent._flush_messages_to_session_db_unlocked.__get__(agent, AIAgent) + ) + return agent + + +def test_flush_diverts_batch_to_jsonl_when_handle_is_quarantined( + tmp_path: Path, monkeypatch +) -> None: + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + db = SessionDB(db_path=tmp_path / "state.db") + try: + db.create_session("live", source="cli") + agent = _flush_agent(db, "live") + + def _quarantined(self, *, session_id, messages, **kwargs): + raise StateDbCorruptError("database disk image is malformed (quarantined)") + + monkeypatch.setattr(SessionDB, "append_messages_batch", _quarantined) + + messages = [{"role": "user", "content": "kept-on-disk-after-corruption"}] + result = agent._flush_messages_to_session_db(messages, []) + + assert result is False + assert agent._last_persistence_error_cause == "corrupt" + jsonl = tmp_path / "sessions" / "live.jsonl" + assert jsonl.is_file() + assert "kept-on-disk-after-corruption" in jsonl.read_text(encoding="utf-8") + finally: + db.close() diff --git a/tests/run_agent/test_infinite_compaction_loop.py b/tests/run_agent/test_infinite_compaction_loop.py index 79c6b734be..9eb0a84de5 100644 --- a/tests/run_agent/test_infinite_compaction_loop.py +++ b/tests/run_agent/test_infinite_compaction_loop.py @@ -262,3 +262,52 @@ class TestCodexSparkShortSessionBoundary: f"This would cause the silent context wipe described in #48621." ) assert comp.has_content_to_compress(messages) is True + + +class TestPressureRealFloor: + """Regression: Cyrillic-heavy sessions under-count in the rough estimate, + letting real prompts ride the provider window (64,842→64,995 observed) + while the pre-API gate saw sub-threshold pressure.""" + + def _compressor(self, last_real, awaiting=False): + class _C: + last_real_prompt_tokens = last_real + awaiting_real_usage_after_compression = awaiting + return _C() + + def test_real_floor_lifts_undercounted_rough(self): + from agent.conversation_loop import _pressure_with_real_floor + assert _pressure_with_real_floor(self._compressor(64_842), 45_000) == 64_842 + + def test_rough_wins_when_larger(self): + from agent.conversation_loop import _pressure_with_real_floor + assert _pressure_with_real_floor(self._compressor(30_000), 45_000) == 45_000 + + def test_stale_real_ignored_right_after_compaction(self): + from agent.conversation_loop import _pressure_with_real_floor + compressor = self._compressor(64_842, awaiting=True) + assert _pressure_with_real_floor(compressor, 20_000) == 20_000 + + def test_zero_and_missing_real_are_safe(self): + from agent.conversation_loop import _pressure_with_real_floor + assert _pressure_with_real_floor(self._compressor(0), 10_000) == 10_000 + assert _pressure_with_real_floor(object(), 10_000) == 10_000 + + def test_anchored_pressure_is_never_floored(self): + """A valid usage anchor is provider-exact and wins as-is. + + On MoA turns the anchor deliberately uses the pre-fold aggregator + usage while ``last_real_prompt_tokens`` holds the folded figure; + flooring the anchored value would re-add the advisor fan-out tokens + the anchor exists to exclude. Pin the wiring shape: the floor is + applied only on the ``else`` (rough fallback) branch. + """ + import inspect + from agent import conversation_loop + + src = inspect.getsource(conversation_loop.run_conversation) + i = src.index("if _anchored_pressure is not None:") + window = src[i : i + 400] + assert "request_pressure_tokens = _anchored_pressure" in window + assert "else:" in window + assert window.index("else:") < window.index("_pressure_with_real_floor(") diff --git a/tests/run_agent/test_length_continuation_thinking_exhaustion.py b/tests/run_agent/test_length_continuation_thinking_exhaustion.py new file mode 100644 index 0000000000..983d726e88 --- /dev/null +++ b/tests/run_agent/test_length_continuation_thinking_exhaustion.py @@ -0,0 +1,327 @@ +"""Regression tests for thinking-only length truncations. + +GLM-5.3-flash on ollama-cloud with reasoning_effort=high can burn the ENTIRE +output cap on reasoning delivered in a separate field and return +finish_reason="length" with NO visible content (verified live: max_tokens=4096 +→ completion_tokens=4096, reasoning ~18.5KB, content empty). + +The old continuation flow handled this badly: + 1. the empty response was appended as an interim assistant fragment, + poisoning the transcript until the pre-call sanitizer "healed" it + (observed 3+ healings per turn); + 2. every continuation re-ran with thinking ON, re-deriving — and re-burning + — the whole thinking budget against a growing context, so 4 attempts + still produced nothing and the turn died with + "Response remained truncated after 4 continuation attempts". + +The fix: skip empty interim fragments, and issue the continuation with a +one-shot reasoning-off override so the budget goes to writing the answer. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from hermes_constants import FINISH_REASON_LENGTH + + +class _AgentStandIn: + """Minimal agent surface _reasoning_config_for_wire needs.""" + + def __init__(self, reasoning_config): + self.reasoning_config = reasoning_config + + +class TestReasoningOffOneShotOverride: + def test_flag_consumed_exactly_once(self): + from agent.chat_completion_helpers import _reasoning_config_for_wire + + agent = _AgentStandIn({"enabled": True, "effort": "high"}) + # Without the flag the reasoning config passes through untouched. + assert _reasoning_config_for_wire(agent) == { + "enabled": True, + "effort": "high", + } + + agent._ephemeral_reasoning_off = True + cfg = _reasoning_config_for_wire(agent) + assert cfg["enabled"] is False + assert cfg["effort"] == "none" + assert agent._ephemeral_reasoning_off is False, ( + "The one-shot override must be consumed by the first call." + ) + + # Subsequent calls keep the user's own reasoning config. + assert _reasoning_config_for_wire(agent) == { + "enabled": True, + "effort": "high", + } + + def test_flag_with_no_user_reasoning_config(self): + from agent.chat_completion_helpers import _reasoning_config_for_wire + + agent = _AgentStandIn(None) + agent._ephemeral_reasoning_off = True + cfg = _reasoning_config_for_wire(agent) + assert cfg == {"enabled": False, "effort": "none"} + + def test_rejected_disable_resends_users_config_verbatim(self): + """After a 'reasoning is mandatory' 400 the retry must land on the + SAME provider cache key as every prior request: the ephemeral + continuation override is discarded and the user's own config goes + out unchanged. A config that is itself a disable is omitted.""" + from agent.chat_completion_helpers import _reasoning_config_for_wire + + agent = _AgentStandIn({"enabled": True, "effort": "high"}) + agent._reasoning_disable_rejected = True + agent._ephemeral_reasoning_off = True + assert _reasoning_config_for_wire(agent) == {"enabled": True, "effort": "high"} + assert agent._ephemeral_reasoning_off is False + + agent.reasoning_config = {"enabled": False} + assert _reasoning_config_for_wire(agent) is None + + +@pytest.fixture() +def loop_agent(): + from run_agent import AIAgent + + with ( + patch("run_agent.get_tool_definitions", return_value=[]), + patch("run_agent.check_toolset_requirements", return_value={}), + patch("run_agent.OpenAI"), + ): + a = AIAgent( + api_key="test-key-1234567890", + base_url="https://openrouter.ai/api/v1", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + a.client = MagicMock() + a._cached_system_prompt = "You are helpful." + a._use_prompt_caching = False + a.compression_enabled = False + a.save_trajectories = False + return a + + +def _thinking_only_length_response(): + """finish_reason='length' with reasoning but zero visible content — the + live GLM-5.3-flash-on-ollama-cloud shape (normal response id, NOT the + partial-stream stub).""" + from tests.run_agent.test_run_agent import _mock_assistant_msg + + return SimpleNamespace( + id="chatcmpl-thinking-exhausted", + model="test/model", + choices=[SimpleNamespace( + index=0, + message=_mock_assistant_msg(content=""), + finish_reason=FINISH_REASON_LENGTH, + )], + usage=None, + ) + + +def _full_response(content): + from tests.run_agent.test_run_agent import _mock_response + + return _mock_response(content=content, finish_reason="stop") + + +def _truncated_text_response(content): + from tests.run_agent.test_run_agent import _mock_response + + return _mock_response(content=content, finish_reason=FINISH_REASON_LENGTH) + + +def _run(agent, message, history=None): + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + return agent.run_conversation(message, conversation_history=history) + + +def _no_empty_assistant_rows(messages): + return [ + m for m in messages + if m.get("role") == "assistant" + and not (m.get("content") or "").strip() + and not m.get("tool_calls") + ] + + +class TestThinkingOnlyTruncation: + def test_retry_after_thinking_only_truncation_completes(self, loop_agent): + """One thinking-only truncation, then a normal answer: the retry must + drop thinking (one-shot), boost the output cap, and finish the turn.""" + loop_agent.client.chat.completions.create.side_effect = [ + _thinking_only_length_response(), + _full_response("Here is the full answer."), + ] + result = _run(loop_agent, "write me a long report") + + assert result["completed"] is True + assert "full answer" in (result["final_response"] or "") + assert _no_empty_assistant_rows(result["messages"]) == [], ( + "An empty (thinking-only) truncated response must never be " + "appended to the transcript." + ) + + calls = loop_agent.client.chat.completions.create.call_args_list + assert len(calls) == 2 + # Continuation retry boosts the output cap (2^1 × 4096 base floor). + assert calls[1].kwargs.get("max_tokens") == 8192, ( + "The continuation retry must request a larger output budget than " + "the request that truncated." + ) + assert loop_agent._ephemeral_reasoning_off is False, ( + "The one-shot reasoning-off override must be consumed by the " + "continuation call." + ) + + def test_thinking_only_truncation_sets_reasoning_off(self, loop_agent): + from tests.run_agent.test_run_agent import _mock_response + + loop_agent.client.chat.completions.create.side_effect = [ + _thinking_only_length_response(), + _mock_response( + content="done", finish_reason=FINISH_REASON_LENGTH + ), + _full_response("finally complete."), + ] + _run(loop_agent, "write me a long report") + + calls = loop_agent.client.chat.completions.create.call_args_list + assert len(calls) == 3 + # The thinking-only fragment set the flag; it was consumed by the + # next call, and the SECOND truncated fragment (which had visible + # text) does not set it again — so the third call sees thinking ON. + assert loop_agent._ephemeral_reasoning_off is False + + def test_full_ceiling_with_empty_fragments_still_settles(self, loop_agent): + """All four attempts thinking-only: the turn must exit through the + ceiling with an actionable final_response, no poisoned transcript, + and no leaked reasoning-off flag.""" + loop_agent.client.chat.completions.create.side_effect = [ + _thinking_only_length_response() for _ in range(4) + ] + result = _run(loop_agent, "write me a long report") + + assert result["completed"] is False + assert result["partial"] is True + assert "truncated after 4 continuation attempts" in (result.get("error") or "") + assert result["final_response"], ( + "An all-empty ceiling exit must still surface a user-facing " + "message instead of an invisible None." + ) + assert "reasoning" in (result["final_response"] or "").lower() + assert _no_empty_assistant_rows(result["messages"]) == [] + assert loop_agent._ephemeral_reasoning_off is False, ( + "The ceiling exit must clear the pending one-shot override so the " + "next turn does not silently lose thinking." + ) + + def test_mixed_fragments_keep_visible_text(self, loop_agent): + """A visible fragment followed by a thinking-only one: the visible + text must be stitched, the empty one skipped.""" + loop_agent.client.chat.completions.create.side_effect = [ + _truncated_text_response("visible part one. "), + _thinking_only_length_response(), + _full_response("and the ending."), + ] + result = _run(loop_agent, "write me a long report") + + assert result["completed"] is True + assert "visible part one." in (result["final_response"] or "") + assert "and the ending." in (result["final_response"] or "") + assert _no_empty_assistant_rows(result["messages"]) == [] + +class TestReasoningOffReachesTheWire: + def test_continuation_request_carries_reasoning_off_on_the_wire(self, loop_agent): + """The flag is only useful if the continuation REQUEST goes out with + thinking disabled — assert the OpenRouter extra_body, not the flag.""" + loop_agent.reasoning_config = {"enabled": True, "effort": "high"} + loop_agent._supports_reasoning_extra_body = lambda: True + loop_agent.client.chat.completions.create.side_effect = [ + _thinking_only_length_response(), + _full_response("Here is the full answer."), + ] + result = _run(loop_agent, "write me a long report") + assert result["completed"] is True + + calls = loop_agent.client.chat.completions.create.call_args_list + assert len(calls) == 2 + first = (calls[0].kwargs.get("extra_body") or {}).get("reasoning") + second = (calls[1].kwargs.get("extra_body") or {}).get("reasoning") + assert first == {"enabled": True, "effort": "high"}, first + assert second is not None and second.get("enabled") is False, ( + f"continuation must be sent with thinking off, got {second!r}" + ) + + def test_reasoning_off_is_exactly_one_request_and_prefix_stays_stable(self, loop_agent): + """Prompt-cache invariant for the override. + + The reasoning parameter is part of the provider's cache key on + config-sensitive providers (Anthropic renders thinking/effort into + the prompt; OpenAI lists reasoning.effort as a prefix-affecting + setting), so the reasoning-off request is a deliberate one-request + cache miss. It must stay exactly one request: the request AFTER it + (a second, visible-text continuation) must go out with the + configured reasoning again, and the system prompt must be + byte-identical on every request so the miss never compounds into a + rebuilt prefix. + """ + loop_agent.reasoning_config = {"enabled": True, "effort": "high"} + loop_agent._supports_reasoning_extra_body = lambda: True + loop_agent.client.chat.completions.create.side_effect = [ + _thinking_only_length_response(), + _truncated_text_response("PART ONE of the answer"), + _full_response(" and PART TWO, done."), + ] + result = _run(loop_agent, "write me a long report") + assert result["completed"] is True + assert "PART ONE" in result["final_response"] + assert "PART TWO" in result["final_response"] + + calls = loop_agent.client.chat.completions.create.call_args_list + assert len(calls) == 3 + wire = [ + (c.kwargs.get("extra_body") or {}).get("reasoning") for c in calls + ] + assert wire[0] == {"enabled": True, "effort": "high"}, wire + assert wire[1] == {"enabled": False, "effort": "none"}, wire + assert wire[2] == {"enabled": True, "effort": "high"}, ( + f"reasoning must be restored on the very next request; got {wire!r}" + ) + system_prompts = { + c.kwargs["messages"][0]["content"] for c in calls + if c.kwargs["messages"][0].get("role") == "system" + } + assert len(system_prompts) == 1, ( + "system prompt must be byte-identical across the retry sequence " + "(the override may only change request parameters, never the prefix)" + ) + assert loop_agent._ephemeral_reasoning_off is False + + def test_stale_flag_does_not_leak_into_next_turn(self, loop_agent): + """A flag armed by a previous turn that never reached build_api_kwargs + (interrupt/error between arm and consume) must not silently strip + thinking from the next turn's first request.""" + loop_agent.reasoning_config = {"enabled": True, "effort": "high"} + loop_agent._supports_reasoning_extra_body = lambda: True + loop_agent._ephemeral_reasoning_off = True # stale from a prior turn + loop_agent.client.chat.completions.create.side_effect = [ + _full_response("fresh turn answer."), + ] + result = _run(loop_agent, "hello") + assert result["completed"] is True + calls = loop_agent.client.chat.completions.create.call_args_list + first = (calls[0].kwargs.get("extra_body") or {}).get("reasoning") + assert first == {"enabled": True, "effort": "high"}, first diff --git a/tests/run_agent/test_model_streaming_config.py b/tests/run_agent/test_model_streaming_config.py new file mode 100644 index 0000000000..1ed5073f12 --- /dev/null +++ b/tests/run_agent/test_model_streaming_config.py @@ -0,0 +1,151 @@ +"""``model.streaming`` config seeds the session's streaming decision (#72901). + +The conversation loop prefers ``stream=True`` for every turn — subagents +included — for liveness health-checking (#3120). Self-hosted OpenAI-compatible +backends with broken streaming tool-call paths (e.g. vLLM +``--tool-call-parser qwen3_xml`` + reasoning parser) can leak tool-call markup +into plain text and return zero ``tool_calls``, silently no-oping delegated +tasks. ``model.streaming: false`` must seed ``_disable_streaming`` at agent +init so the whole session (parent and subagents) uses the non-streaming path. +""" +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +from run_agent import AIAgent + +_BASE = { + "model": { + "default": "test/model", + "provider": "custom", + "base_url": "http://127.0.0.1:9999/v1", + "api_key": "x", + } +} + + +def _build_agent(config): + with patch("hermes_cli.config.load_config_readonly", return_value=config): + return AIAgent( + api_key="x", + base_url="http://127.0.0.1:9999/v1", + model="test/model", + provider="custom", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + + +@patch("run_agent.OpenAI") +def test_streaming_false_seeds_disable_streaming(mock_openai): + mock_openai.return_value = MagicMock() + agent = _build_agent({"model": {**_BASE["model"], "streaming": False}}) + + assert agent._disable_streaming is True + + +@patch("run_agent.OpenAI") +def test_streaming_absent_keeps_streaming_enabled(mock_openai): + mock_openai.return_value = MagicMock() + agent = _build_agent(_BASE) + + assert agent._disable_streaming is False + + +@patch("run_agent.OpenAI") +def test_streaming_true_keeps_streaming_enabled(mock_openai): + mock_openai.return_value = MagicMock() + agent = _build_agent({"model": {**_BASE["model"], "streaming": True}}) + + assert agent._disable_streaming is False + + +@patch("run_agent.OpenAI") +def test_streaming_string_false_seeds_disable_streaming(mock_openai): + """String falsy values ('false', '0') must also disable streaming — + YAML users commonly quote booleans.""" + mock_openai.return_value = MagicMock() + agent = _build_agent({"model": {**_BASE["model"], "streaming": "false"}}) + + assert agent._disable_streaming is True + + +@patch("run_agent.OpenAI") +def test_streaming_zero_seeds_disable_streaming(mock_openai): + mock_openai.return_value = MagicMock() + agent = _build_agent({"model": {**_BASE["model"], "streaming": 0}}) + + assert agent._disable_streaming is True + + +@patch("run_agent.OpenAI") +def test_streaming_invalid_value_keeps_streaming_enabled(mock_openai): + """Unrecognized values warn and keep the safe default (streaming on), + rather than silently disabling or crashing init.""" + mock_openai.return_value = MagicMock() + agent = _build_agent({"model": {**_BASE["model"], "streaming": "flase"}}) + + assert agent._disable_streaming is False + + +@patch("run_agent.OpenAI") +def test_missing_model_section_keeps_streaming_enabled(mock_openai): + mock_openai.return_value = MagicMock() + agent = _build_agent({}) + + assert agent._disable_streaming is False + + +@patch("run_agent.OpenAI") +def test_legacy_string_model_section_does_not_crash(mock_openai): + """The top-level ``model`` key is a legacy string; init must not crash.""" + mock_openai.return_value = MagicMock() + agent = _build_agent({"model": "test/model"}) + + assert agent._disable_streaming is False + + +@patch("run_agent.OpenAI") +def test_streaming_false_applies_to_every_agent_built_from_config(mock_openai): + """Delegate children are constructed through the same init, so any agent + (parent or subagent) built under this config gets the escape hatch — + covering the reported failure surface.""" + mock_openai.return_value = MagicMock() + cfg = {"model": {**_BASE["model"], "streaming": False}} + + first = _build_agent(cfg) + second = _build_agent(cfg) + + assert first._disable_streaming is True + assert second._disable_streaming is True + + +@patch("run_agent.OpenAI") +def test_streaming_false_read_from_real_config_file(mock_openai): + """End-to-end: a real config.yaml in HERMES_HOME (sandboxed per-test by + conftest) with ``model.streaming: false`` must seed the flag through the + actual config loader — not just the patched function.""" + mock_openai.return_value = MagicMock() + home = Path(os.environ["HERMES_HOME"]) + (home / "config.yaml").write_text( + "model:\n" + " default: \"test/model\"\n" + " provider: \"custom\"\n" + " base_url: \"http://127.0.0.1:9999/v1\"\n" + " api_key: \"x\"\n" + " streaming: false\n", + encoding="utf-8", + ) + + agent = AIAgent( + api_key="x", + base_url="http://127.0.0.1:9999/v1", + model="test/model", + provider="custom", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + + assert agent._disable_streaming is True diff --git a/tests/run_agent/test_proactive_prune_loop_wiring.py b/tests/run_agent/test_proactive_prune_loop_wiring.py index 957799f466..259bbbb25b 100644 --- a/tests/run_agent/test_proactive_prune_loop_wiring.py +++ b/tests/run_agent/test_proactive_prune_loop_wiring.py @@ -189,6 +189,32 @@ class TestProactivePruneLoopWiring: assert tool_rows, "expected tool rows in the final transcript" assert all(m["content"] == marker for m in tool_rows) + def test_should_compress_true_but_skipped_is_warned(self, agent): + """``should_compress_info`` says RUN (``(True, None)``) yet this branch + was taken — the per-turn compression budget is spent. Over threshold + with no reclamation running must not be swallowed silently (#101889). + + Faithful to the real engine: ``should_compress()`` is + ``should_compress_info()[0]``, so the only way into this branch with + ``(True, None)`` is an exhausted per-turn budget.""" + agent.max_compression_attempts = 0 # budget already spent this turn + agent.context_compressor.should_compress.return_value = True + agent.context_compressor.should_compress_info.return_value = (True, None) + agent.context_compressor.prune_tool_results_only = ( + lambda messages, current_tokens=None: (messages, 0) + ) + warned = [] + with patch.object( + agent, + "_warn_context_overflow_blocked", + side_effect=lambda reason, tokens, threshold: warned.append(reason), + ): + result = _run_tool_loop(agent, n_tool_iterations=1) + + assert result["completed"] is True + assert warned, "over-threshold turn with no compaction ran silently" + assert all(r.startswith("attempts_exhausted") for r in warned) + def test_noop_input_object_commits_nothing(self, agent): """Engine returns the INPUT object with a (bogus) non-zero count — the caller's ``result is not input`` gate must refuse the commit.""" diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 5a8f067a28..77128fd092 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2253,7 +2253,7 @@ class TestConcurrentToolExecution: def test_invoke_tool_handles_agent_level_tools(self, agent): """_invoke_tool should handle todo tool directly.""" with patch("tools.todo_tool.todo_tool", return_value='{"ok":true}') as mock_todo: - result = agent._invoke_tool("todo", {"todos": []}, "task-1") + result = agent._invoke_tool("todo_list", {"todos": []}, "task-1") mock_todo.assert_called_once() assert "ok" in result @@ -2345,7 +2345,7 @@ class TestConcurrentToolExecution: """Sequential and concurrent agent-level paths share post-hook ownership.""" from agent.agent_runtime_helpers import agent_runtime_owns_post_tool_hook - for tool_name in ("todo", "session_search", "memory", "clarify", "delegate_task"): + for tool_name in ("todo_list", "session_search", "memory", "clarify", "delegate_task"): assert agent_runtime_owns_post_tool_hook(agent, tool_name) is True agent._context_engine_tool_names = {"context_query"} @@ -2491,7 +2491,7 @@ class TestAgentRuntimePostHookOwnershipSync: """Exercise post-hook ownership through both agent-runtime tool paths.""" _CASES = ( - ("todo", {"todos": []}), + ("todo_list", {"todos": []}), ("session_search", {"query": "needle"}), ("memory", {"action": "view", "target": "memory"}), ("clarify", {"question": "Continue?"}), @@ -2501,7 +2501,7 @@ class TestAgentRuntimePostHookOwnershipSync: ("annotate_preview", {"action": "clear"}), ("read_window_below", {}), ("setup_mcp", {"server": "linear", "action": "install"}), - ("tour", {"action": "stop"}), + ("gui_tour", {"action": "stop"}), ("delegate_task", {"goal": "Check the child path"}), ) @@ -3513,12 +3513,12 @@ class TestRunConversation: assert result["api_calls"] == 6 # 1 original + 2 prefill + 3 retries - def test_truly_empty_response_retries_3_times_then_empty(self, agent): - """Truly empty response (no content, no reasoning) retries 3 times then falls through to (empty).""" + def test_truly_empty_response_stops_after_repeated_empty(self, agent): + """Repeated empty responses stop after one retry and return an explanation.""" self._setup_agent(agent) agent.base_url = "http://127.0.0.1:1234/v1" empty_resp = _mock_response(content=None, finish_reason="stop") - # 4 responses: 1 original + 3 nudge retries, all empty + # Extra responses prove the guard stops consuming after repetition. agent.client.chat.completions.create.side_effect = [ empty_resp, empty_resp, empty_resp, empty_resp, ] @@ -3532,7 +3532,7 @@ class TestRunConversation: # #34452: explanation replaces the bare "(empty)" sentinel. assert result["final_response"] != "(empty)" assert "No reply:" in result["final_response"] - assert result["api_calls"] == 4 # 1 original + 3 retries + assert result["api_calls"] == 2 # 1 original + 1 retry def test_deterministic_empty_stops_retries_early(self, agent): """NS-503: consecutive zero-output-token empties with identical @@ -3588,10 +3588,11 @@ class TestRunConversation: assert result["completed"] is True assert result["api_calls"] == 4 # legacy: 1 original + 3 retries - def test_empty_without_usage_keeps_full_retry_budget(self, agent): - """NS-503 fail-open: no usage data means no evidence of a - deterministic empty — legacy 3-retry behaviour must be preserved - (this is the flaky-provider case retries exist for).""" + def test_empty_without_usage_stops_after_one_retry_and_logs_calls( + self, agent, caplog + ): + """Two complete empty responses are enough evidence to stop even when + the provider omits usage; both attempts remain observable.""" self._setup_agent(agent) agent.base_url = "http://127.0.0.1:1234/v1" empty_resp = _mock_response(content=None, finish_reason="stop") @@ -3600,10 +3601,13 @@ class TestRunConversation: patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"), patch.object(agent, "_cleanup_task_resources"), + caplog.at_level(logging.INFO, logger="agent.conversation_loop"), ): result = agent.run_conversation("answer me") assert result["completed"] is True - assert result["api_calls"] == 4 # unchanged: 1 original + 3 retries + assert result["api_calls"] == 2 + assert agent.session_api_calls == 2 + assert caplog.text.count("usage=unavailable") == 2 def test_truly_empty_response_succeeds_on_nudge(self, agent): """Model produces content after being nudged for empty response.""" @@ -4340,11 +4344,11 @@ class TestRunConversation: assert requested_caps == [65536, 65536] def test_ollama_glm_stop_after_tools_without_terminal_boundary_requests_continuation(self, agent): - """Ollama-hosted GLM responses can misreport truncated output as stop.""" + """Local Ollama-hosted GLM (no :cloud suffix) misreports truncated output as stop.""" self._setup_agent(agent) agent.base_url = "http://localhost:11434/v1" agent._base_url_lower = agent.base_url.lower() - agent.model = "glm-5.1:cloud" + agent.model = "glm-4-9b" # local GLM — no :cloud suffix tool_turn = _mock_response( content="", @@ -4384,11 +4388,18 @@ class TestRunConversation: assert third_call_messages[-1]["role"] == "user" assert "truncated by the output length limit" in third_call_messages[-1]["content"] - - - - - + @pytest.mark.parametrize("base_url, model", [ + ("https://ollama.com/v1", "glm-5.3-flash"), # Ollama Cloud host (#72316) + ("http://localhost:11434/v1", "glm-5.1:cloud"), # :cloud via local proxy (#98406) + ]) + def test_ollama_cloud_glm_stop_is_never_rewritten(self, agent, base_url, model): + """Ollama Cloud reports finish_reason faithfully — an unpunctuated stop stays stop.""" + self._setup_agent(agent) + agent.base_url = base_url + agent._base_url_lower = base_url.lower() + agent.model = model + unpunctuated = SimpleNamespace(content="Based on the results the best next step is to update the config", tool_calls=None) + assert agent._should_treat_stop_as_truncated("stop", unpunctuated, [{"role": "tool", "content": "r"}]) is False def test_length_thinking_exhausted_skips_continuation(self, agent): """When finish_reason='length' but content is only thinking, skip retries.""" @@ -4414,7 +4425,7 @@ class TestRunConversation: # Should have a user-friendly response (not None) assert result["final_response"] is not None assert "Thinking Budget Exhausted" in result["final_response"] - assert "/thinkon" in result["final_response"] + assert "/reasoning" in result["final_response"] def test_length_with_tool_calls_returns_partial_without_executing_tools(self, agent): diff --git a/tests/run_agent/test_run_agent_codex_responses.py b/tests/run_agent/test_run_agent_codex_responses.py index 9c0165e066..26228f0b21 100644 --- a/tests/run_agent/test_run_agent_codex_responses.py +++ b/tests/run_agent/test_run_agent_codex_responses.py @@ -2502,3 +2502,126 @@ def test_codex_first_compaction_continuation_is_still_a_bare_retry(monkeypatch): if m.get("role") == "user" and m.get("content") == _CODEX_INCOMPLETE_NUDGE ] + + +class _LazyCreateStream: + """Lazy iterable fake — events are produced during consumption, not upfront. + + ``_FakeCreateStream`` materializes its events with ``list(events)`` in + __init__, which would run any side effect a generator encodes (such as + retiring the request token) before consumption starts. Retirement tests + need the side effect to land *between* two consumed frames. + """ + + def __init__(self, event_factory): + self._event_factory = event_factory + self.closed = False + + def __iter__(self): + return iter(self._event_factory()) + + def close(self): + self.closed = True + + +def _retiring_stream(agent, deltas, *, retire_after): + """Yield ``deltas`` lazily, clearing the request token mid-stream. + + The token is cleared just before yielding delta index ``retire_after``, + mimicking a watchdog (TTFB / stream-idle / stale-call) retiring the + in-flight request while the worker thread is still draining SSE frames. + """ + + def _events(): + yield SimpleNamespace(type="response.created") + for index, delta in enumerate(deltas): + if index == retire_after: + agent._active_codex_stream_request_token = None + yield SimpleNamespace(type="response.output_text.delta", delta=delta) + # A retired stream never reaches a terminal frame on the wire; the + # connection is force-closed under it. + + return _LazyCreateStream(_events) + + +def test_run_codex_stream_retired_request_raises_instead_of_partial_final(monkeypatch): + """A retired request must not be normalized into a completed response. + + ``_consume_codex_event_stream`` returns ``status=terminal_status`` which + defaults to ``"completed"``, and its only guard is + ``if not saw_terminal and not output``. A watchdog kill mid-stream leaves + ``saw_terminal=False`` but ``output``/text non-empty, so the partial text + used to come back as a ``finish_reason=stop`` response and get persisted as + a complete assistant turn (a long reply would just stop mid-sentence). + + Retirement must surface as a retryable ``TimeoutError`` instead. + """ + agent = _build_agent(monkeypatch) + token = object() + agent._active_codex_stream_request_token = token + + def _fake_create(**kwargs): + assert kwargs.get("stream") is True + return _retiring_stream( + agent, ["1. Create ", "(6/6)", " [END-BILLING"], retire_after=2 + ) + + agent.client = SimpleNamespace(responses=SimpleNamespace(create=_fake_create)) + + with pytest.raises(TimeoutError, match="retired"): + agent._run_codex_stream(_codex_request_kwargs()) + + +def test_run_codex_stream_without_token_keeps_partial_tolerance(monkeypatch): + """No token installed (non-watchdog callers) keeps the existing behavior. + + ``_active_codex_stream_request_token`` is only set by + ``interruptible_api_call``. Auxiliary callers (compression summaries, + title generation) drive ``_run_codex_stream`` directly with no token and + must keep tolerating a stream that ends without a terminal frame. + """ + agent = _build_agent(monkeypatch) + agent._active_codex_stream_request_token = None + output_item = SimpleNamespace( + type="message", + status="completed", + content=[SimpleNamespace(type="output_text", text="no terminal frame")], + ) + + def _fake_create(**kwargs): + return _FakeCreateStream([ + SimpleNamespace(type="response.created"), + SimpleNamespace(type="response.output_item.done", item=output_item), + ]) + + agent.client = SimpleNamespace(responses=SimpleNamespace(create=_fake_create)) + + response = agent._run_codex_stream(_codex_request_kwargs()) + assert response.status == "completed" + assert response.output == [output_item] + + +def test_run_codex_stream_retired_request_stops_firing_callbacks(monkeypatch): + """Deltas that arrive after retirement must not reach the UI callbacks. + + The gateway caches AIAgent instances per session, so a retired worker that + keeps draining frames would otherwise stream tokens from an abandoned + attempt into the live turn's bubble alongside the retry's output. + """ + agent = _build_agent(monkeypatch) + token = object() + agent._active_codex_stream_request_token = token + + streamed: list[str] = [] + monkeypatch.setattr(agent, "_fire_stream_delta", streamed.append) + + def _fake_create(**kwargs): + return _retiring_stream(agent, ["keep", "DROPPED"], retire_after=1) + + agent.client = SimpleNamespace(responses=SimpleNamespace(create=_fake_create)) + + with pytest.raises(TimeoutError): + agent._run_codex_stream(_codex_request_kwargs()) + + assert streamed == ["keep"] + assert "DROPPED" not in streamed diff --git a/tests/run_agent/test_streamed_text_accumulation.py b/tests/run_agent/test_streamed_text_accumulation.py new file mode 100644 index 0000000000..12c903b22a --- /dev/null +++ b/tests/run_agent/test_streamed_text_accumulation.py @@ -0,0 +1,173 @@ +"""Tests for how a turn's streamed assistant text is built up. + +The text used to be grown with ``+=`` on an attribute. Python cannot grow a +string in place there, so every delta copied the whole thing again and a long +reply cost the square of its length in copying. The text is now held as a list +of pieces and joined when something reads it. + +These tests cover the behaviour callers depend on, plus a check on the stored +pieces that fails if the copying ever comes back. +""" +from unittest.mock import patch + +import pytest + + +def _make_agent(): + from run_agent import AIAgent + + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "chat_completions" + agent._interrupt_requested = False + return agent + + +class TestStreamedTextValue: + """The value callers read must not change.""" + + def test_starts_empty(self): + agent = _make_agent() + assert agent._current_streamed_assistant_text == "" + + def test_deltas_join_in_order(self): + agent = _make_agent() + for piece in ["Hello", ", ", "world", "!"]: + agent._record_streamed_assistant_text(piece) + assert agent._current_streamed_assistant_text == "Hello, world!" + + def test_reading_twice_gives_the_same_answer(self): + agent = _make_agent() + agent._record_streamed_assistant_text("one ") + agent._record_streamed_assistant_text("two") + first = agent._current_streamed_assistant_text + second = agent._current_streamed_assistant_text + assert first == second == "one two" + + def test_reading_does_not_stop_later_deltas(self): + agent = _make_agent() + agent._record_streamed_assistant_text("before ") + assert agent._current_streamed_assistant_text == "before " + agent._record_streamed_assistant_text("after") + assert agent._current_streamed_assistant_text == "before after" + + def test_direct_assignment_still_works(self): + # Several call sites set this attribute straight, both to seed a value + # and to clear it between turns. + agent = _make_agent() + agent._record_streamed_assistant_text("thrown away") + agent._current_streamed_assistant_text = "set by hand" + assert agent._current_streamed_assistant_text == "set by hand" + agent._record_streamed_assistant_text(" plus more") + assert agent._current_streamed_assistant_text == "set by hand plus more" + + def test_clearing_resets_to_empty(self): + agent = _make_agent() + agent._record_streamed_assistant_text("left over") + agent._current_streamed_assistant_text = "" + assert agent._current_streamed_assistant_text == "" + agent._record_streamed_assistant_text("new turn") + assert agent._current_streamed_assistant_text == "new turn" + + def test_empty_and_non_string_deltas_are_ignored(self): + agent = _make_agent() + agent._record_streamed_assistant_text("keep") + agent._record_streamed_assistant_text("") + agent._record_streamed_assistant_text(None) # type: ignore[arg-type] + agent._record_streamed_assistant_text(12345) # type: ignore[arg-type] + assert agent._current_streamed_assistant_text == "keep" + + def test_superseded_writer_is_still_fenced_out(self): + # The single-writer guard (#65991) must keep working now that the + # text is stored as pieces. + agent = _make_agent() + agent._record_streamed_assistant_text("allowed") + with patch.object(agent, "_stream_writer_superseded", return_value=True): + agent._record_streamed_assistant_text("blocked") + assert agent._current_streamed_assistant_text == "allowed" + + +class TestStreamedTextCost: + """Adding a delta must not touch the text already collected. + + Checked by looking at the stored pieces rather than by timing, so the + test gives the same answer on a busy CI box as it does on a quiet one. + """ + + def test_each_delta_is_stored_as_its_own_piece(self): + agent = _make_agent() + for i in range(500): + agent._record_streamed_assistant_text(f"delta-{i} ") + # One piece per delta means nothing joined or copied the text that was + # already there. If a delta ever rebuilds the whole string again, this + # collapses to a single piece and the test fails. + assert len(agent._streamed_assistant_text_parts) == 500 + + def test_reading_the_text_does_not_collapse_the_pieces(self): + # Collapsing on read would drop any delta that lands between the join + # and the write back, so reading has to leave the pieces alone. + agent = _make_agent() + for i in range(10): + agent._record_streamed_assistant_text(str(i)) + assert agent._current_streamed_assistant_text == "0123456789" + assert len(agent._streamed_assistant_text_parts) == 10 + + def test_a_long_reply_is_assembled_correctly(self): + agent = _make_agent() + delta = "x" * 8 + for _ in range(20000): + agent._record_streamed_assistant_text(delta) + assert agent._current_streamed_assistant_text == delta * 20000 + assert len(agent._streamed_assistant_text_parts) == 20000 + + +def _agent_with_sink(): + agent = _make_agent() + delivered = [] + agent.stream_delta_callback = delivered.append + agent._stream_callback = None + return agent, delivered + + +class TestFireStreamDeltaEmptiness: + """_fire_stream_delta used to join the whole reply on every token just + to decide whether to strip leading newlines. That check now looks at + the parts list. + """ + + def test_first_delta_strips_leading_newlines(self): + agent, delivered = _agent_with_sink() + agent._fire_stream_delta("\n\nhello") + assert delivered == ["hello"] + assert agent._current_streamed_assistant_text == "hello" + + def test_later_delta_keeps_leading_newlines(self): + agent, delivered = _agent_with_sink() + agent._fire_stream_delta("hello") + agent._fire_stream_delta("\n\nworld") + assert delivered == ["hello", "\n\nworld"] + assert agent._current_streamed_assistant_text == "hello\n\nworld" + + def test_after_clear_the_next_delta_strips_again(self): + agent, delivered = _agent_with_sink() + agent._fire_stream_delta("hello") + agent._current_streamed_assistant_text = "" + agent._fire_stream_delta("\n\nagain") + assert delivered[-1] == "again" + assert agent._current_streamed_assistant_text == "again" + + def test_fire_path_stores_one_piece_per_delta(self): + agent, _delivered = _agent_with_sink() + for i in range(200): + agent._fire_stream_delta(f"d{i} ") + assert len(agent._streamed_assistant_text_parts) == 200 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/run_agent/test_streaming.py b/tests/run_agent/test_streaming.py index e5b0471e64..7a04ab0e68 100644 --- a/tests/run_agent/test_streaming.py +++ b/tests/run_agent/test_streaming.py @@ -305,8 +305,171 @@ class TestStreamingAccumulator: assert tc[0].function.name == "terminal" assert tc[0].function.arguments == '{"command": "ls"}' + @patch("run_agent.AIAgent._create_request_openai_client") + @patch("run_agent.AIAgent._close_request_openai_client") + def test_tool_argument_deltas_are_collected_without_concatenating_each_chunk( + self, mock_close, mock_create + ): + """Large tool arguments must not rebuild the accumulated string per delta.""" + from run_agent import AIAgent + class AppendOnlyChunk(str): + def __radd__(self, other): + raise AssertionError("tool argument delta was concatenated eagerly") + chunks = [ + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta( + index=0, tc_id="call_123", name="write_file" + ) + ]), + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta( + index=0, arguments=AppendOnlyChunk('{"path":"out.txt",') + ) + ]), + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta( + index=0, arguments=AppendOnlyChunk('"content":"hello"}') + ) + ]), + _make_stream_chunk(finish_reason="tool_calls"), + ] + mock_client = MagicMock() + mock_client.chat.completions.create.return_value = iter(chunks) + mock_create.return_value = mock_client + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "chat_completions" + agent._interrupt_requested = False + + response = agent._interruptible_streaming_api_call({}) + + tool_call = response.choices[0].message.tool_calls[0] + assert tool_call.function.arguments == ( + '{"path":"out.txt","content":"hello"}' + ) + + @patch("run_agent.AIAgent._create_request_openai_client") + @patch("run_agent.AIAgent._close_request_openai_client") + @patch("agent.relay_llm.stream") + def test_relay_finalizer_emits_joined_tool_arguments( + self, mock_relay_stream, mock_close, mock_create + ): + """Relay receives the public string shape, not buffered fragments.""" + from run_agent import AIAgent + + captured = {} + fake_stream = MagicMock() + fake_stream.final_response = None + fake_stream.__iter__.return_value = iter([ + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta( + index=0, + tc_id="call_123", + name="search", + arguments='{"q":', + ) + ]), + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta(index=0, arguments='"hello"}') + ]), + _make_stream_chunk(finish_reason="tool_calls"), + ]) + + def relay_stream_impl(*args, **kwargs): + captured["finalizer"] = kwargs["finalizer"] + return fake_stream + + mock_relay_stream.side_effect = relay_stream_impl + mock_client = MagicMock() + mock_client.chat.completions.create.return_value = iter([]) + mock_create.return_value = mock_client + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "chat_completions" + agent._interrupt_requested = False + + agent._interruptible_streaming_api_call({}) + + payload = captured["finalizer"]() + tool_calls = payload["choices"][0]["message"]["tool_calls"] + assert len(tool_calls) == 1 + assert tool_calls[0]["function"] == { + "name": "search", + "arguments": '{"q":"hello"}', + } + + @patch("run_agent.AIAgent._create_request_openai_client") + @patch("run_agent.AIAgent._close_request_openai_client") + def test_tool_argument_assembly_is_chunk_boundary_invariant( + self, mock_close, mock_create + ): + """Argument bytes are identical across ASCII and Unicode fragment sizes.""" + import json + + from run_agent import AIAgent + + payload = json.dumps( + {"path": "/tmp/x", "content": "héllo wörld 日本語 " * 50}, + ensure_ascii=False, + ) + + def assemble(fragment_size): + fragments = [ + payload[i : i + fragment_size] + for i in range(0, len(payload), fragment_size) + ] + chunks = [ + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta( + index=0, + tc_id="call_123", + name="write_file", + arguments=fragments[0], + ) + ]) + ] + chunks.extend( + _make_stream_chunk(tool_calls=[ + _make_tool_call_delta(index=0, arguments=fragment) + ]) + for fragment in fragments[1:] + ) + chunks.append(_make_stream_chunk(finish_reason="tool_calls")) + + mock_client = MagicMock() + mock_client.chat.completions.create.return_value = iter(chunks) + mock_create.return_value = mock_client + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent.api_mode = "chat_completions" + agent._interrupt_requested = False + + response = agent._interruptible_streaming_api_call({}) + return response.choices[0].message.tool_calls[0].function.arguments + + for fragment_size in (len(payload), 64, 7, 3, 1): + arguments = assemble(fragment_size) + assert arguments.encode("utf-8") == payload.encode("utf-8") # ── Test: Streaming Callbacks ──────────────────────────────────────────── diff --git a/tests/run_agent/test_strip_reasoning_tags_cli.py b/tests/run_agent/test_strip_reasoning_tags_cli.py index 60525990c5..7d32c538cd 100644 --- a/tests/run_agent/test_strip_reasoning_tags_cli.py +++ b/tests/run_agent/test_strip_reasoning_tags_cli.py @@ -7,8 +7,21 @@ AIAgent instance. It must stay in sync with run_agent.py::_strip_think_blocks for tool-call tag coverage.""" +from agent.agent_runtime_helpers import strip_think_blocks from cli import _strip_reasoning_tags +# GLM text-channel tool call cut mid-serialization by a stream drop (#101899): +# the first key and call name never arrived, only orphan argument markup. +_CUT_FRAGMENT = ( + "Both gates started.\n" + "wait\nsession_id\nabc\n" + "timeout\n59" +) +_COMPLETE_WITH_PROSE = ( + "Use in JS. The arg_key field maps to arg_value.\n" + "xa1\nDone." +) + class TestToolCallStripping: def test_tool_call_block_stripped(self): @@ -26,3 +39,16 @@ class TestToolCallStripping: def test_empty_string(self): assert _strip_reasoning_tags("") == "" + def test_cut_tool_call_stripped_to_visible_prefix(self): + """Both strippers drop the unrecoverable tail; only prose survives.""" + assert _strip_reasoning_tags(_CUT_FRAGMENT) == "Both gates started." + assert strip_think_blocks(None, _CUT_FRAGMENT).strip() == "Both gates started." + assert strip_think_blocks(None, "Waiting.\nprocess_manage").strip() == "Waiting." + + def test_complete_block_and_inline_prose_mentions_untouched(self): + for out in (_strip_reasoning_tags(_COMPLETE_WITH_PROSE), + strip_think_blocks(None, _COMPLETE_WITH_PROSE)): + assert "Use in JS. The arg_key field maps to arg_value." in out + assert out.rstrip().endswith("Done.") + assert "" not in out + diff --git a/tests/run_agent/test_tool_arg_coercion.py b/tests/run_agent/test_tool_arg_coercion.py index 4390c3e9a1..00dcb289b0 100644 --- a/tests/run_agent/test_tool_arg_coercion.py +++ b/tests/run_agent/test_tool_arg_coercion.py @@ -244,5 +244,5 @@ class TestCoerceToolArgsNested: """Against the real todo schema from the registry.""" import json as _json args = {"todos": [_json.dumps({"id": "1", "content": "x", "status": "pending"})]} - result = coerce_tool_args("todo", args) + result = coerce_tool_args("todo_list", args) assert result["todos"][0] == {"id": "1", "content": "x", "status": "pending"} diff --git a/tests/run_agent/test_tool_call_guardrail_runtime.py b/tests/run_agent/test_tool_call_guardrail_runtime.py index 83f327e2cb..1bb239e842 100644 --- a/tests/run_agent/test_tool_call_guardrail_runtime.py +++ b/tests/run_agent/test_tool_call_guardrail_runtime.py @@ -6,6 +6,8 @@ from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch +import pytest + from run_agent import AIAgent @@ -37,7 +39,12 @@ def _mock_response(content="Hello", finish_reason="stop", tool_calls=None): return SimpleNamespace(choices=[choice], model="test/model", usage=None) -def _make_agent(*tool_names: str, max_iterations: int = 10, config: dict | None = None) -> AIAgent: +def _make_agent( + *tool_names: str, + max_iterations: int = 10, + config: dict | None = None, + platform: str | None = None, +) -> AIAgent: with ( patch("run_agent.get_tool_definitions", return_value=_make_tool_defs(*tool_names)), patch("run_agent.check_toolset_requirements", return_value={}), @@ -52,6 +59,7 @@ def _make_agent(*tool_names: str, max_iterations: int = 10, config: dict | None quiet_mode=True, skip_context_files=True, skip_memory=True, + platform=platform or "cli", ) agent.client = MagicMock() agent._cached_system_prompt = "You are helpful." @@ -87,6 +95,29 @@ def _hard_stop_config(**overrides) -> dict: return cfg +def test_gateway_platform_uses_hard_stop_default_without_cli_opt_in(): + agent = _make_agent("web_search", platform="telegram") + args = {"query": "same"} + + _seed_exact_failures(agent, "web_search", args, count=5) + + decision = getattr(agent, "_tool_guardrails").before_call("web_search", args) + assert decision.action == "block" + assert decision.code == "repeated_exact_failure_block" + + +@pytest.mark.parametrize("platform", ["desktop", "acp"]) +def test_interactive_platforms_keep_warning_only_default(platform): + agent = _make_agent("web_search", platform=platform) + args = {"query": "same"} + + _seed_exact_failures(agent, "web_search", args, count=5) + + decision = getattr(agent, "_tool_guardrails").before_call("web_search", args) + assert decision.action == "allow" + assert decision.code == "allow" + + def test_default_sequential_path_warns_repeated_exact_failure_without_blocking_execution(): agent = _make_agent("web_search") args = {"query": "same"} diff --git a/tests/run_agent/test_turn_completion_explainer.py b/tests/run_agent/test_turn_completion_explainer.py index 99b052c7f9..3a7f6b62e0 100644 --- a/tests/run_agent/test_turn_completion_explainer.py +++ b/tests/run_agent/test_turn_completion_explainer.py @@ -438,3 +438,8 @@ def test_run_conversation_partial_stream_recovery_surfaces_explanation(): assert result["response_previewed"] is False +def test_classify_persistence_error_quarantined_handle_is_corrupt() -> None: + """A quarantined SessionDB raises the typed error; it stays in the corrupt bucket.""" + from hermes_state import StateDbCorruptError, classify_persistence_error + + assert classify_persistence_error(StateDbCorruptError("quarantined")) == "corrupt" diff --git a/tests/scripts/test_contributor_map.py b/tests/scripts/test_contributor_map.py index 40fd3567a2..b6082ccfe5 100644 --- a/tests/scripts/test_contributor_map.py +++ b/tests/scripts/test_contributor_map.py @@ -116,3 +116,66 @@ def test_cli_entrypoint_end_to_end(tmp_path): assert proc.returncode == 0, proc.stderr out = (tmp_path / "contributors" / "emails" / "cli@example.com").read_text(encoding="utf-8") assert out.splitlines()[0] == "cliperson" + + +# ── case-insensitive filename collisions ────────────────────────────── +# +# The mapping key IS the filename, so two emails differing only in case are the +# same file on Windows and on default macOS. When both exist, git writes one and +# then reports the other as modified in a FRESH clone, permanently: the repo can +# never be checked out clean on those platforms. +# +# The historical agent@Agents-Mac-mini.local / agent@agents-Mac-mini.local pair +# was removed from the tree (fcdae2cf0b), so there is no allowlist: any pair +# is a regression. scripts/check-case-collisions.py enforces the same +# invariant repo-wide in CI; this test keeps it visible next to the writer. +EMAILS_DIR = REPO_ROOT / "contributors" / "emails" + + +def test_no_case_insensitive_mapping_collisions(): + groups: dict[str, set[str]] = {} + for entry in EMAILS_DIR.iterdir(): + if entry.is_file(): + groups.setdefault(entry.name.casefold(), set()).add(entry.name) + + collisions = {frozenset(names) for names in groups.values() if len(names) > 1} + + assert not collisions, ( + "contributor mappings differing only in case cannot coexist on " + "case-insensitive filesystems (Windows, default macOS) — a fresh clone " + f"there is permanently dirty: {sorted(sorted(c) for c in collisions)}" + ) + + +def test_add_contributor_refuses_a_case_collision(tmp_path, monkeypatch): + d = tmp_path / "emails" + d.mkdir() + (d / "agent@Example-Host.local").write_text("someone\n") + + import add_contributor as mod + + monkeypatch.setattr(mod, "EMAILS_DIR", d) + + assert mod.add_contributor("agent@example-host.local", "otherperson") == 1 + assert not (d / "agent@example-host.local").exists() + + +def test_add_contributor_refuses_case_collision_even_for_same_login(emails_dir, capsys): + # Same login, different spelling: still refused — the problem is the + # filename pair, not the login. The exact spelling is what's "present". + emails_dir.mkdir(parents=True) + (emails_dir / "Foo@Example.com").write_text("foouser\n") + + assert add_contributor("foo@example.com", "foouser") == 1 + assert "Foo@Example.com" in capsys.readouterr().err + assert sorted(p.name for p in emails_dir.iterdir()) == ["Foo@Example.com"] + # Exact-case re-add is the ordinary idempotent path. + assert add_contributor("Foo@Example.com", "foouser") == 0 + + +def test_case_collision_uses_casefold(emails_dir): + # casefold, not lower: matches how macOS/Windows fold non-ASCII (ß ~ ss). + emails_dir.mkdir(parents=True) + (emails_dir / "strasse@example.com").write_text("someone\n") + assert add_contributor("STRASSE@example.com", "someone") == 1 + assert add_contributor("straße@example.com", "someone") == 1 diff --git a/tests/security/test_gitspawn_config_injection.py b/tests/security/test_gitspawn_config_injection.py new file mode 100644 index 0000000000..bd212f9e3c --- /dev/null +++ b/tests/security/test_gitspawn_config_injection.py @@ -0,0 +1,200 @@ +"""GitSpawn / GHSA-7x36-8jrh-v4pw regression suite. + +A repository delivered as files (zip, sync folder, USB) can carry a +``.git/config`` that names a command in an execution-sink git setting — +``core.fsmonitor``, ``core.hooksPath`` hooks, or an attribute-scoped +``[diff "x"] command=/textconv=`` driver. Hermes gathers workspace context by +running git against the session directory automatically, before any prompt, +approval, or trust gate, so an unhardened probe would execute that command on +the host as the user. + +These tests build a real malicious repo and assert that every automatic +context-gathering git path Hermes runs neutralizes every sink. They use a real +``git`` and skip if it is unavailable. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from hermes_cli._subprocess_compat import ( + NO_DRIVER_DIFF_FLAGS, + harden_git_argv, + noninteractive_git_env, +) + +_HAS_GIT = shutil.which("git") is not None +pytestmark = pytest.mark.skipif(not _HAS_GIT, reason="git not installed") + + +# --------------------------------------------------------------------------- +# 1. harden_git_argv unit contract +# --------------------------------------------------------------------------- + + +class TestHardenGitArgv: + def test_diff_gets_flags_after_subcommand(self): + assert harden_git_argv(["diff", "HEAD"]) == [ + "diff", *NO_DRIVER_DIFF_FLAGS, "HEAD", + ] + + def test_show_log_blame_are_hardened(self): + for sub in ("show", "log", "blame"): + out = harden_git_argv([sub, "x"]) + assert out[0] == sub + assert out[1:3] == list(NO_DRIVER_DIFF_FLAGS) + + def test_status_is_not_touched(self): + # status rejects --no-ext-diff (`unknown option`), so it must pass through. + assert harden_git_argv(["status", "--porcelain=2", "--branch"]) == [ + "status", "--porcelain=2", "--branch", + ] + + def test_worktree_and_other_subcommands_untouched(self): + assert harden_git_argv(["worktree", "add", "x"]) == ["worktree", "add", "x"] + assert harden_git_argv(["rev-parse", "HEAD"]) == ["rev-parse", "HEAD"] + + def test_global_options_are_skipped_when_finding_subcommand(self): + out = harden_git_argv(["-C", "/repo", "diff", "HEAD"]) + assert out == ["-C", "/repo", "diff", *NO_DRIVER_DIFF_FLAGS, "HEAD"] + + def test_dash_c_value_is_not_mistaken_for_subcommand(self): + # ``-C diff`` is a path; the real subcommand is status → no flags. + assert harden_git_argv(["-C", "diff", "status"]) == ["-C", "diff", "status"] + # ``-c diff=x`` is a config pair; the real subcommand is status. + assert harden_git_argv(["-c", "diff=x", "status"]) == ["-c", "diff=x", "status"] + + def test_config_pair_before_diff_still_hardens(self): + out = harden_git_argv(["-c", "core.quotePath=false", "diff", "--numstat"]) + assert out == [ + "-c", "core.quotePath=false", "diff", *NO_DRIVER_DIFF_FLAGS, "--numstat", + ] + + +# --------------------------------------------------------------------------- +# 2. Real-git E2E: every automatic path neutralizes every sink +# --------------------------------------------------------------------------- + + +def _make_malicious_repo(tmp: Path) -> tuple[Path, Path]: + """Build a repo whose .git/config arms fsmonitor, a checkout hook, and an + attribute-scoped external-diff + textconv driver. Returns (repo, marker_stem): + a fired sink leaves ``.`` on disk.""" + repo = tmp / "poc" + clean = { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + } + subprocess.run(["git", "init", "-q", str(repo)], check=True, env=clean) + (repo / "README").write_text("hi\n") + ident = ["-c", "user.email=a@b", "-c", "user.name=a"] + subprocess.run(["git", "-C", str(repo), *ident, "add", "."], check=True, env=clean) + subprocess.run(["git", "-C", str(repo), *ident, "commit", "-qm", "init"], check=True, env=clean) + + marker = tmp / "MARKER" + hooks = repo / "evil-hooks" + hooks.mkdir() + hook = hooks / "post-checkout" + hook.write_text(f"#!/bin/sh\ntouch {marker}.hook\n") + hook.chmod(0o755) + with (repo / ".git" / "config").open("a") as f: + f.write(f'[core]\n\tfsmonitor = "touch {marker}.fsmonitor"\n\thooksPath = {hooks}\n') + f.write(f'[diff "evil"]\n\tcommand = "touch {marker}.extdiff"\n') + f.write(f'\ttextconv = "sh -c \'touch {marker}.textconv; cat\'"\n') + (repo / ".gitattributes").write_text("* diff=evil\n") + (repo / "README").write_text("changed\n") # dirty working tree so diffs run + return repo, marker + + +def _fired(marker: Path) -> list[str]: + out = [] + for sink in ("fsmonitor", "hook", "extdiff", "textconv"): + p = Path(f"{marker}.{sink}") + if p.exists(): + out.append(sink) + p.unlink() + return out + + +@pytest.fixture() +def malicious_repo(tmp_path): + repo, marker = _make_malicious_repo(tmp_path) + yield repo, marker + + +def test_baseline_unhardened_git_fires_sinks(malicious_repo): + """Sanity: without hardening the payload actually fires — proves the repo + is armed and the test can detect a regression.""" + repo, marker = malicious_repo + subprocess.run(["git", "-C", str(repo), "diff", "HEAD"], capture_output=True) + fired = _fired(marker) + assert "fsmonitor" in fired and "extdiff" in fired, fired + + +def test_coding_workspace_snapshot_is_safe(malicious_repo): + import agent.coding_context as cc + repo, marker = malicious_repo + cc.build_coding_workspace_block(cwd=repo) + assert _fired(marker) == [] + + +def test_gateway_git_probe_is_safe(malicious_repo): + from tui_gateway import git_probe + repo, marker = malicious_repo + git_probe.branch(str(repo)) + git_probe.run_git(str(repo), "status", "--porcelain") + assert _fired(marker) == [] + + +def test_working_diff_is_safe(malicious_repo): + from tools.working_diff import collect_working_diff + repo, marker = malicious_repo + collect_working_diff(str(repo), "working") + assert _fired(marker) == [] + + +def test_goals_fingerprint_is_safe(malicious_repo): + from hermes_cli.goals import workspace_fingerprint + repo, marker = malicious_repo + workspace_fingerprint(str(repo)) + assert _fired(marker) == [] + + +def test_web_git_diff_is_safe(malicious_repo): + from hermes_cli import web_git + repo, marker = malicious_repo + web_git._git(str(repo), ["status", "--porcelain=v2", "-z"]) + web_git._git_out(str(repo), ["diff", "HEAD"]) + assert _fired(marker) == [] + + +def test_context_reference_diff_is_safe(malicious_repo): + from agent import context_references as cr + repo, marker = malicious_repo + ref = type("R", (), {"raw": "@diff"})() + cr._expand_git_reference(ref, repo, ["diff", "HEAD"], "git diff") + assert _fired(marker) == [] + + +def test_subagent_worktree_add_is_safe(malicious_repo, tmp_path): + from tools import subagent_worktree as sw + repo, marker = malicious_repo + sw._run_git(["worktree", "add", str(tmp_path / "wt1"), "-b", "safe1"], str(repo)) + assert _fired(marker) == [] + + +def test_noninteractive_env_pins_fsmonitor_and_hooks(): + env = noninteractive_git_env({}) + values = { + env[f"GIT_CONFIG_KEY_{i}"]: env[f"GIT_CONFIG_VALUE_{i}"] + for i in range(int(env["GIT_CONFIG_COUNT"])) + } + assert values["core.fsmonitor"] == "false" + assert values["core.hooksPath"] == os.devnull diff --git a/tests/state/test_fts_rebuild_admission.py b/tests/state/test_fts_rebuild_admission.py index 96f7cd6523..cac3ef1690 100644 --- a/tests/state/test_fts_rebuild_admission.py +++ b/tests/state/test_fts_rebuild_admission.py @@ -17,9 +17,11 @@ prove nothing. """ import contextlib +import errno import subprocess import sqlite3 import sys +import time from pathlib import Path import pytest @@ -363,3 +365,308 @@ os._exit(1) finally: with contextlib.suppress(OSError): os.kill(grandchild, signal.SIGKILL) + + +class TestNonContentionErrnoFailsFast: + def test_non_contention_oserror_does_not_wait_out_timeout( + self, tmp_path, monkeypatch + ): + import fcntl + + monkeypatch.setattr( + hermes_state_common, "_FTS_REBUILD_LOCK_TIMEOUT_SECONDS", 30.0 + ) + + def _flock(*_args, **_kwargs): + raise OSError(getattr(errno, "ESTALE", errno.EIO), "stale handle") + + monkeypatch.setattr(fcntl, "flock", _flock) + db_path = tmp_path / "state.db" + t0 = time.monotonic() + with hermes_state_common.fts_rebuild_admission(db_path) as admitted: + assert admitted is False + assert time.monotonic() - t0 < 2.0 + + def test_retry_deferred_fts_recovery_rebuilds_same_instance( + self, tmp_path, monkeypatch + ): + """Gateway-shaped: same SessionDB stays open and retries after deferral.""" + import hermes_state_schema + + monkeypatch.setattr(hermes_state_schema, "_FTS_STALE_RETRY_SECONDS", 0.0) + db_path = tmp_path / "state.db" + d = SessionDB(db_path=db_path) + if not d._fts_enabled: + d.close() + pytest.skip("FTS5 unavailable in this build") + d.create_session("s1", source="test") + d.append_message("s1", "user", "hello recovery path") + d.close() + + raw = sqlite3.connect(str(db_path)) + raw.execute( + "INSERT OR REPLACE INTO state_meta(key, value) VALUES (?, '1')", + (FTS_STALE_KEY,), + ) + for trig in _FTS_TRIGGERS: + raw.execute(f"DROP TRIGGER IF EXISTS {trig}") + raw.commit() + raw.close() + + holders = [(4242, str(db_path))] + monkeypatch.setattr( + SessionDB, "_foreign_state_db_holders", lambda self: list(holders) + ) + d2 = SessionDB(db_path=db_path) + try: + assert d2._fts_stale is True + d2._fts_stale_retry_after = 0.0 + assert d2.retry_deferred_fts_recovery() is False + holders.clear() + d2._fts_stale_retry_after = 0.0 + assert d2.retry_deferred_fts_recovery() is True + assert d2._fts_stale is False + finally: + d2.close() + assert _meta_value(db_path, FTS_STALE_KEY) is None + assert _base_fts_triggers(db_path) == set(_FTS_TRIGGERS) + + def test_non_contention_errno_skips_holder_warning( + self, tmp_path, monkeypatch, caplog + ): + """The fast-fail must not ALSO log the misleading 'held by another + process for more than Ns' line — there is no holder.""" + import fcntl + import logging + + monkeypatch.setattr( + hermes_state_common, "_FTS_REBUILD_LOCK_TIMEOUT_SECONDS", 30.0 + ) + + def _flock(*_args, **_kwargs): + raise OSError(errno.ENOTSUP, "no locks on this fs") + + monkeypatch.setattr(fcntl, "flock", _flock) + with caplog.at_level(logging.INFO, logger="hermes_state"): + with hermes_state_common.fts_rebuild_admission( + tmp_path / "state.db" + ) as admitted: + assert admitted is False + messages = [r.getMessage() for r in caplog.records] + assert any("non-contention error" in m for m in messages) + assert not any("held by another process" in m for m in messages) + + def test_repair_lock_non_contention_errno_fails_fast( + self, tmp_path, monkeypatch + ): + """Sibling site: the state.db repair lock shares the errno filter.""" + import fcntl + + import hermes_state + + monkeypatch.setattr(hermes_state, "_REPAIR_LOCK_TIMEOUT_SECONDS", 30.0) + + def _flock(*_args, **_kwargs): + raise OSError(errno.EIO, "i/o error") + + monkeypatch.setattr(fcntl, "flock", _flock) + t0 = time.monotonic() + with hermes_state._cross_process_repair_lock(tmp_path / "state.db") as ok: + assert ok is False + assert time.monotonic() - t0 < 2.0 + + @pytest.mark.parametrize( + "exc, expected", + [ + (BlockingIOError(errno.EAGAIN, "x"), True), + (OSError(errno.EWOULDBLOCK, "x"), True), + (OSError(errno.EACCES, "x"), True), + (OSError(errno.ESTALE, "x"), False), + (OSError(errno.ENOTSUP, "x"), False), + (OSError(errno.ENOLCK, "x"), False), + (OSError(errno.EIO, "x"), False), + (ValueError("not an oserror"), False), + ], + ) + def test_is_advisory_lock_contention_table(self, exc, expected): + assert hermes_state_common.is_advisory_lock_contention(exc) is expected + + +class TestDeferredFtsRetryInProcess: + """Gateway shape (#100108): one SessionDB stays open for days. A deferral + at open must be recoverable from an in-process periodic tick, with the + REAL rebuild lock held by a REAL child process at open time.""" + + @staticmethod + def _mark_stale(db_path: Path) -> None: + raw = sqlite3.connect(str(db_path)) + raw.execute( + "INSERT OR REPLACE INTO state_meta(key, value) VALUES (?, '1')", + (FTS_STALE_KEY,), + ) + for trig in _FTS_TRIGGERS: + raw.execute(f"DROP TRIGGER IF EXISTS {trig}") + raw.commit() + raw.close() + + def test_retry_is_non_blocking_while_live_holder_and_backs_off( + self, tmp_path, fast_timeout, monkeypatch + ): + import hermes_state_schema + + db_path = tmp_path / "state.db" + d = SessionDB(db_path=db_path) + if not d._fts_enabled: + d.close() + pytest.skip("FTS5 unavailable in this build") + d.create_session("s1", source="test") + d.append_message("s1", "user", "hello gateway retry") + d.close() + self._mark_stale(db_path) + + with _rebuild_lock_held_by_other_process(db_path): + gw = SessionDB(db_path=db_path) # long-lived "gateway" open + try: + assert gw._fts_stale is True + # Live holder: the retry must return quickly (timeout=0), + # not wait out any admission budget. + monkeypatch.setattr( + hermes_state_common, "_FTS_REBUILD_LOCK_TIMEOUT_SECONDS", 30.0 + ) + t0 = time.monotonic() + assert gw.retry_deferred_fts_recovery() is False + assert time.monotonic() - t0 < 2.0 + assert gw._fts_stale is True + # Rate limit engaged: an immediate second call is a no-op. + assert gw.retry_deferred_fts_recovery() is False + # Backoff doubled (60s -> 120s) but capped at the max. + assert gw._fts_stale_retry_interval == min( + 2 * hermes_state_schema._FTS_STALE_RETRY_SECONDS, + hermes_state_schema._FTS_STALE_RETRY_MAX_SECONDS, + ) + assert gw._fts_stale_retry_after > time.monotonic() + except BaseException: + gw.close() + raise + # Holder gone. Same instance recovers on the next eligible tick. + try: + gw._fts_stale_retry_after = 0.0 + assert gw.retry_deferred_fts_recovery() is True + assert gw._fts_stale is False + assert gw._fts_enabled is True + # Search actually works again on this very instance. + gw.append_message("s1", "user", "needle-after-holder-gone") + assert gw.retry_deferred_fts_recovery() is False # nothing stale + finally: + gw.close() + assert _meta_value(db_path, FTS_STALE_KEY) is None + assert _base_fts_triggers(db_path) == set(_FTS_TRIGGERS) + + def test_gateway_housekeeping_tick_drives_the_retry( + self, tmp_path, fast_timeout, monkeypatch + ): + """The retry hangs off the EXISTING housekeeping loop (no new thread) + and reaches shared-registry instances.""" + import threading + + import hermes_state_registry + import hermes_state_schema + import gateway.run as grun + + monkeypatch.setattr(hermes_state_schema, "_FTS_STALE_RETRY_SECONDS", 0.0) + db_path = tmp_path / "state.db" + d = SessionDB(db_path=db_path) + if not d._fts_enabled: + d.close() + pytest.skip("FTS5 unavailable in this build") + d.create_session("s1", source="test") + d.append_message("s1", "user", "hello housekeeping") + d.close() + self._mark_stale(db_path) + + with _rebuild_lock_held_by_other_process(db_path): + gw = hermes_state_registry.acquire(db_path) + try: + assert gw._fts_stale is True + assert gw in hermes_state_registry.live_shared_session_dbs() + stop = threading.Event() + th = threading.Thread( + target=grun._start_gateway_housekeeping, + args=(stop,), + kwargs={"interval": 0.05}, + daemon=True, + ) + th.start() + deadline = time.monotonic() + 10.0 + while gw._fts_stale and time.monotonic() < deadline: + time.sleep(0.05) + stop.set() + th.join(timeout=5) + assert gw._fts_stale is False + assert gw._fts_enabled is True + finally: + hermes_state_registry.release_or_close(gw) + assert _meta_value(db_path, FTS_STALE_KEY) is None + + def test_retry_noop_when_not_stale_or_read_only(self, tmp_path): + db_path = tmp_path / "state.db" + d = SessionDB(db_path=db_path) + try: + assert d._fts_stale is False + assert d.retry_deferred_fts_recovery() is False + finally: + d.close() + ro = SessionDB(db_path=db_path, read_only=True) + try: + ro._fts_stale = True + assert ro.retry_deferred_fts_recovery() is False + finally: + ro.close() + + def test_retry_skips_quarantined_handle(self, tmp_path, fast_timeout): + """A structurally corrupt handle must never run a full FTS rebuild — + the housekeeping tick calls this unconditionally for the life of a + long-running gateway process, so a stale-FTS flag left set on a + now-corrupt handle must not retry the rebuild forever against the + damaged image (real DDL/DML the quarantine exists to prevent).""" + db_path = tmp_path / "state.db" + d = SessionDB(db_path=db_path) + if not d._fts_enabled: + d.close() + pytest.skip("FTS5 unavailable in this build") + d.create_session("s1", source="test") + d.append_message("s1", "user", "hello quarantine") + d.close() + self._mark_stale(db_path) + + # Force the open-time recovery to defer (foreign rebuild-lock + # holder) so _fts_stale is still True once the handle is open — + # mirrors test_retry_is_non_blocking_while_live_holder_and_backs_off. + with _rebuild_lock_held_by_other_process(db_path): + gw = SessionDB(db_path=db_path) + try: + assert gw._fts_stale is True + gw._db_corrupt = True + gw._db_corrupt_reason = "database disk image is malformed" + # A retry that is DUE (backoff already elapsed) on a handle that + # had been backing off before it tripped quarantine. Seeding the + # deadline in the past matters: a future deadline would make the + # unguarded code short-circuit on the backoff check and this test + # would pass without the quarantine guard ever being exercised. + gw._fts_stale_retry_after = time.monotonic() - 1.0 + gw._fts_stale_retry_interval = 900.0 + assert gw.retry_deferred_fts_recovery() is False + # Untouched: still marked stale, triggers still absent — no + # rebuild ran against the "damaged" handle. + assert gw._fts_stale is True + # The backoff bookkeeping is reset too, mirroring the success + # path's own reset — a doubled interval left behind a flag + # nothing currently clears would otherwise make the next real + # retry (if this handle is ever un-quarantined) start from a + # stale multi-minute backoff instead of the default. + assert gw._fts_stale_retry_after == 0.0 + assert gw._fts_stale_retry_interval == 0.0 + finally: + gw.close() + assert _meta_value(db_path, FTS_STALE_KEY) == "1" + assert _base_fts_triggers(db_path) == set() diff --git a/tests/state/test_fts_runtime_rebuild.py b/tests/state/test_fts_runtime_rebuild.py index f8dec25d34..0709aee29f 100644 --- a/tests/state/test_fts_runtime_rebuild.py +++ b/tests/state/test_fts_runtime_rebuild.py @@ -16,11 +16,11 @@ rebuild later, outside the failed live write/search operation. import json import os import sqlite3 -from types import SimpleNamespace import pytest import hermes_state +import hermes_state_holders import hermes_state_schema from hermes_state import ( FTS_REBUILD_DEFERRAL_KEY, @@ -140,48 +140,53 @@ class TestRuntimeFtsRebuild: } ) - def test_foreign_holder_detection_includes_deleted_wal( - self, db, tmp_path, monkeypatch - ): - db_path = tmp_path / "state.db" + @pytest.mark.parametrize( + "argv", + ( + ("journalctl", "-u", "hermes-agent.service"), + ("grep", "hermes-agent", "/var/log/syslog"), + ( + "/usr/sbin/tailscaled", + "be-child", + "ssh", + "--cmd=python -m hermes_cli.main gateway", + ), + ("tmux", "new-session", "/opt/hermes-agent/.venv/bin/hermes gateway"), + ("python3", "/opt/hermes-agent/tools/check_state.py"), + ("hermes-monitor", "gateway"), + ("hermesctl", "serve"), + ("python3", "worker.py", "hermes_cli.main"), + ("python3", "-m", "other.module", "hermes_cli.main"), + ("python3", "-c", "hermes_cli.main"), + ("python3", "-Icprint('hermes_cli.main')", "hermes_cli/main.py"), + ), + ) + def test_uninspectable_non_hermes_process_is_not_a_holder(self, argv): + assert not hermes_state_holders._looks_like_hermes(argv) - class FakePsutil: - @staticmethod - def process_iter(_attrs): - return iter( - ( - SimpleNamespace( - info={ - "pid": 111, - "open_files": [SimpleNamespace(path=str(db_path))], - } - ), - SimpleNamespace( - info={ - "pid": 222, - "open_files": [ - SimpleNamespace(path=f"{db_path}-wal (deleted)") - ], - } - ), - SimpleNamespace( - info={ - "pid": 333, - "open_files": [SimpleNamespace(path=str(tmp_path / "other.db"))], - } - ), - ) - ) - - monkeypatch.setattr(hermes_state, "psutil", FakePsutil) - monkeypatch.setattr(hermes_state, "_IS_WINDOWS", False) - monkeypatch.setattr(hermes_state.os, "getpid", lambda: 111) - # Force the macOS/psutil path even on Linux test runners - monkeypatch.setattr(hermes_state.sys, "platform", "darwin") - - assert db._foreign_state_db_holders() == [ - (222, f"{db_path}-wal (deleted)") - ] + @pytest.mark.parametrize( + "argv", + ( + ("/usr/local/bin/hermes", "gateway"), + ("/usr/local/bin/hermes-agent", "serve"), + ("/usr/local/bin/hermes-acp", "--stdio"), + ("/usr/bin/python3", "-m", "hermes_cli.main", "gateway"), + ("/usr/bin/python3", "-m", "acp_adapter"), + ("/usr/bin/python3", "-Im", "hermes_cli.main", "gateway"), + ("/usr/bin/python3", "-mhermes_cli.main", "gateway"), + ("/usr/bin/python3", "-W", "ignore", "-m", "hermes_cli.main"), + ("/usr/bin/python3", "-Xdev", "-m", "hermes_cli.main"), + ( + "/opt/hermes-agent/.venv/bin/python", + "/opt/hermes-agent/hermes_cli/main.py", + "gateway", + ), + ("python.exe", "--", "hermes_cli/main.py", "gateway"), + ("python3", "/opt/hermes-agent/run_agent.py", "--query", "hello"), + ), + ) + def test_uninspectable_hermes_process_remains_a_holder(self, argv): + assert hermes_state_holders._looks_like_hermes(argv) @pytest.mark.platforms("linux") def test_foreign_holder_detection_proc_readlink_deleted_wal( @@ -210,22 +215,32 @@ class TestRuntimeFtsRebuild: other.touch() os.symlink(str(other), str(proc_root / "333" / "fd" / "3")) - monkeypatch.setattr(hermes_state, "_IS_WINDOWS", False) - monkeypatch.setattr(hermes_state.os, "getpid", lambda: 111) - monkeypatch.setattr(hermes_state.sys, "platform", "linux") + monkeypatch.setattr(hermes_state_holders.os, "getpid", lambda: 111) real_listdir = os.listdir def _listdir(path): if isinstance(path, str): path = path.replace("/proc", str(proc_root)) return real_listdir(path) - monkeypatch.setattr(hermes_state.os, "listdir", _listdir) + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) real_readlink = os.readlink def _readlink(path): path = path.replace("/proc", str(proc_root)) return real_readlink(path) - monkeypatch.setattr(hermes_state.os, "readlink", _readlink) + monkeypatch.setattr(hermes_state_holders.os, "readlink", _readlink) + real_stat = os.stat + def _stat(path, *args, **kwargs): + path_s = str(path).replace("/proc", str(proc_root)) + if path_s.endswith("/222/fd/3"): + # A real /proc fd remains statable after unlink and retains + # the deleted sidecar's filesystem identity: same device as + # state.db, but an inode no live watched path can reach. + fields = list(real_stat(db_path)) + fields[1] += 1000 + return os.stat_result(fields) + return real_stat(path_s, *args, **kwargs) + monkeypatch.setattr(hermes_state_holders.os, "stat", _stat) - holders = db._foreign_state_db_holders() + holders = hermes_state_holders.foreign_state_db_holders(db_path) assert holders == [(222, db_path_wal + " (deleted)")] @pytest.mark.platforms("linux") @@ -243,32 +258,34 @@ class TestRuntimeFtsRebuild: os.chmod(proc_root / "222" / "fd", 0o000) # PID 222's cmdline is world-readable and looks like Hermes cmdline_path = proc_root / "222" / "cmdline" - cmdline_path.write_bytes(b"python3\x00hermes_cli.main\x00chat\x00") + cmdline_path.write_bytes( + b"python3\x00-m\x00hermes_cli.main\x00chat\x00" + ) - monkeypatch.setattr(hermes_state, "_IS_WINDOWS", False) - monkeypatch.setattr(hermes_state.os, "getpid", lambda: 111) - monkeypatch.setattr(hermes_state.sys, "platform", "linux") + monkeypatch.setattr(hermes_state_holders.os, "getpid", lambda: 111) real_listdir = os.listdir def _listdir(path): if isinstance(path, str): + if path == "/proc/222/fd": + raise PermissionError(path) path = path.replace("/proc", str(proc_root)) return real_listdir(path) - monkeypatch.setattr(hermes_state.os, "listdir", _listdir) - # _read_proc_cmdline opens /proc//cmdline directly; redirect + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + # _read_proc_argv opens /proc//cmdline directly; redirect # it to our fake proc tree. - def _fake_cmdline(pid): + def _fake_argv(pid): fake_path = str(proc_root / str(pid) / "cmdline") try: with open(fake_path, "rb") as f: raw = f.read() if not raw: return None - return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() + return raw.decode("utf-8", "replace").rstrip("\x00").split("\x00") except OSError: return None - monkeypatch.setattr(hermes_state, "_read_proc_cmdline", _fake_cmdline) + monkeypatch.setattr(hermes_state_holders, "_read_proc_argv", _fake_argv) - holders = db._foreign_state_db_holders() + holders = hermes_state_holders.foreign_state_db_holders(db_path) # Should include PID 222 with the cmdline info assert len(holders) == 1 assert holders[0][0] == 222 @@ -334,7 +351,15 @@ class TestRuntimeFtsRebuild: with pytest.raises(sqlite3.DatabaseError) as caught: db._execute_write(lambda _conn: (_ for _ in ()).throw(structural)) - assert caught.value is structural + # Structural corruption quarantines the handle: the typed error wraps + # the original (cause preserved, SQLite result code copied) and the + # sticky flag is set, so later writes fail fast. + from hermes_state import StateDbCorruptError + + assert isinstance(caught.value, StateDbCorruptError) + assert caught.value.__cause__ is structural + assert caught.value.sqlite_errorcode == sqlite3.SQLITE_CORRUPT + assert db._db_corrupt is True assert rebuild_called is False assert db._fts_stale is False assert _meta_value(tmp_path / "state.db", FTS_STALE_KEY) is None @@ -833,6 +858,19 @@ class TestPhysicalCorruptionAcceptance: # The misdiagnosis message from the field incident must be gone. assert "canonical message rows are preserved" not in caplog.text assert "attempting one-shot in-place FTS rebuild" not in caplog.text + # Structural damage quarantines the handle: typed error, sticky + # flag, later writes fail fast, and close() must not checkpoint + # the WAL over a damaged page image (the #90950 page-1 clobber). + from hermes_state import StateDbCorruptError + + assert isinstance(caught.value, StateDbCorruptError) + assert db._db_corrupt is True + with pytest.raises(StateDbCorruptError): + db.append_message("s1", "user", "second write after corruption") + caplog.clear() + with caplog.at_level("WARNING", logger="hermes_state"): + db.close() + assert "Skipping the close-time WAL checkpoint" in caplog.text finally: db.close() diff --git a/tests/state/test_fts_trigram_cron_exclusion.py b/tests/state/test_fts_trigram_cron_exclusion.py new file mode 100644 index 0000000000..5e115cc67d --- /dev/null +++ b/tests/state/test_fts_trigram_cron_exclusion.py @@ -0,0 +1,254 @@ +"""Cron-source exclusion from the external-content trigram FTS index.""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from hermes_state import FTS_TRIGRAM_SQL, SCHEMA_VERSION, SessionDB + + +@pytest.fixture +def db(tmp_path): + session_db = SessionDB(db_path=tmp_path / "state.db") + if not session_db._trigram_available: + session_db.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + yield session_db + session_db.close() + + +def _trigram_rowids(db: SessionDB) -> set[int]: + return { + row[0] + for row in db._conn.execute( + "SELECT id FROM messages_fts_trigram_docsize ORDER BY id" + ).fetchall() + } + + +def _install_pre_v27_trigram(db: SessionDB, *, with_tool_calls: bool = False) -> None: + """Recreate the pre-cron-exclusion external-content trigram boundary. + + ``with_tool_calls=True`` reproduces the FTS_STORAGE_VERSION 1 vtable + (``tool_calls`` projected) that installs upgraded before #88217 carry; + the default is the v2 column set with only the view/trigger predicates + behind, which is what the in-place v29 migration handles. + """ + cols = "content, tool_name" + (", tool_calls" if with_tool_calls else "") + vals = "new.content, new.tool_name" + (", new.tool_calls" if with_tool_calls else "") + db._conn.executescript( + f""" + DROP TRIGGER messages_fts_trigram_insert; + DROP TRIGGER messages_fts_trigram_delete; + DROP TRIGGER messages_fts_trigram_update; + DROP TABLE messages_fts_trigram; + DROP VIEW messages_fts_trigram_src; + CREATE VIEW messages_fts_trigram_src AS + SELECT id, role, content, tool_name, tool_calls + FROM messages WHERE role <> 'tool'; + CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( + {cols}, + content='messages_fts_trigram_src', + content_rowid='id', + tokenize='trigram' + ); + CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages + WHEN new.role <> 'tool' + BEGIN + INSERT INTO messages_fts_trigram(rowid, {cols}) + VALUES (new.id, {vals}); + END; + """ + ) + + +def test_fresh_trigram_indexes_conversations_but_not_cron(db: SessionDB): + db.create_session("cli", source="cli") + db.create_session("cron", source="cron") + cli_id = db.append_message("cli", role="user", content="交付状态正常") + cron_id = db.append_message("cron", role="user", content="定时任务状态正常") + + assert _trigram_rowids(db) == {cli_id} + assert cron_id not in _trigram_rowids(db) + assert db._conn.execute( + "SELECT id FROM messages_fts_docsize WHERE id = ?", (cron_id,) + ).fetchone() is not None + + +def test_cron_remains_searchable_via_standard_fts_and_explicit_cjk_fallback( + db: SessionDB, +): + db.create_session("cron", source="cron") + db.append_message( + "cron", role="assistant", content="quarterly archive 大别山项目 complete" + ) + + assert [row["session_id"] for row in db.search_messages("quarterly")] == [ + "cron" + ] + assert [ + row["session_id"] + for row in db.search_messages("大别山项目", source_filter=["cron"]) + ] == ["cron"] + + +def test_deferred_rebuild_does_not_reintroduce_cron(db: SessionDB): + db.create_session("cli", source="cli") + db.create_session("cron", source="cron") + cli_id = db.append_message("cli", role="assistant", content="交互会话内容") + db.append_message("cron", role="assistant", content="定时会话内容") + + with db._lock: + db._reset_fts_index_to_empty(db._conn) + db._seed_fts_rebuild_markers(db._conn, force=True) + db._conn.commit() + while db.fts_rebuild_step(): + pass + + assert _trigram_rowids(db) == {cli_id} + + +def test_existing_external_layout_rebuilds_trigram_on_upgrade(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + # The virtual table keeps referring to the view by name, so this recreates + # the exact old external-content boundary without reading source text. + _install_pre_v27_trigram(old) + old.create_session("cli", source="cli") + old.create_session("cron", source="cron") + cli_id = old.append_message("cli", role="user", content="交互迁移内容") + cron_id = old.append_message("cron", role="user", content="定时迁移内容") + assert _trigram_rowids(old) == {cli_id, cron_id} + old._conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION - 1,)) + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == {cli_id} + view_sql = migrated._conn.execute( + "SELECT sql FROM sqlite_master " + "WHERE type = 'view' AND name = 'messages_fts_trigram_src'" + ).fetchone()[0] + assert "sessions" in view_sql + assert "cron" in view_sql + migrated._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) + finally: + migrated.close() + + +def test_install_already_at_v28_still_gets_the_cron_exclusion_migration(tmp_path): + """The migration gate must fire for installs that were on main's v28. + + The original PR gated on ``current_version < 27``; main had meanwhile + reached SCHEMA_VERSION 28 via column-reconciliation bumps, so a v28 + database would have skipped the rebuild and kept cron rows in the trigram + index forever. Pin the gate against the version main actually shipped. + """ + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + _install_pre_v27_trigram(old) + old.create_session("cli", source="cli") + old.create_session("cron", source="cron") + cli_id = old.append_message("cli", role="user", content="交互迁移内容") + cron_id = old.append_message("cron", role="user", content="定时迁移内容") + assert _trigram_rowids(old) == {cli_id, cron_id} + old._conn.execute("UPDATE schema_version SET version = 28") + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == {cli_id}, ( + "a v28 database kept cron rows in the trigram index: the migration gate did not fire" + ) + finally: + migrated.close() + + +def test_v1_tool_calls_layout_is_left_for_optimize_storage(tmp_path): + """A FTS_STORAGE_VERSION 1 trigram vtable (``tool_calls`` projected) must + survive the v29 startup migration untouched and be finished by the opt-in + ``optimize_fts_storage`` path — not half-migrated into a view/vtable + column mismatch (which used to fail the rebuild with + ``no such column: T.tool_calls``).""" + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + _install_pre_v27_trigram(old, with_tool_calls=True) + old.create_session("cli", source="cli") + old.create_session("cron", source="cron") + cli_id = old.append_message("cli", role="user", content="交互迁移内容") + cron_id = old.append_message("cron", role="user", content="定时迁移内容") + assert _trigram_rowids(old) == {cli_id, cron_id} + old._conn.execute("UPDATE schema_version SET version = 28") + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) # must not raise + try: + # Startup left the v1 layout alone (cron row still there) … + assert _trigram_rowids(migrated) == {cli_id, cron_id} + assert migrated.fts_optimize_available() is True + # … and the opt-in path completes the transition: v2 columns, + # cron-filtered view, cron row purged. + migrated.optimize_fts_storage() + cols = [r[1] for r in migrated._conn.execute("PRAGMA table_info(messages_fts_trigram)")] + assert "tool_calls" not in cols + assert _trigram_rowids(migrated) == {cli_id} + finally: + migrated.close() + + +def test_partial_upgrade_view_does_not_skip_historical_rebuild(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + _install_pre_v27_trigram(old) + old.create_session("cron", source="cron") + cron_id = old.append_message("cron", role="assistant", content="迁移中断内容") + assert _trigram_rowids(old) == {cron_id} + + # Simulate a crash after new DDL landed but before the rebuild/schema stamp. + for name in ( + "messages_fts_trigram_insert", + "messages_fts_trigram_delete", + "messages_fts_trigram_update", + ): + old._conn.execute(f"DROP TRIGGER IF EXISTS {name}") + old._conn.execute("DROP VIEW messages_fts_trigram_src") + old._conn.executescript(FTS_TRIGRAM_SQL) + old._conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION - 1,)) + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == set() + finally: + migrated.close() + + +def test_delete_of_unindexed_cron_row_keeps_trigram_consistent(db: SessionDB): + db.create_session("cron", source="cron") + cron_id = db.append_message("cron", role="user", content="不会进入索引") + assert cron_id not in _trigram_rowids(db) + + db._conn.execute("DELETE FROM messages WHERE id = ?", (cron_id,)) + db._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) diff --git a/tests/state/test_fts_trigram_subagent_exclusion.py b/tests/state/test_fts_trigram_subagent_exclusion.py new file mode 100644 index 0000000000..00d1a63fa6 --- /dev/null +++ b/tests/state/test_fts_trigram_subagent_exclusion.py @@ -0,0 +1,161 @@ +"""Delegate-child (subagent) transcripts stay out of the trigram FTS index (v30). + +Mirrors ``test_fts_trigram_cron_exclusion.py``: children are canonical rows +in ``messages`` and stay searchable through the standard ``messages_fts`` +word index; only the trigram (CJK substring) shadow index skips them. +""" + +from __future__ import annotations + +import pytest + +from hermes_state import SCHEMA_VERSION, SessionDB +from hermes_state_common import FTS_TRIGRAM_EXCLUDED_SOURCES, fts_trigram_session_sql + + +@pytest.fixture +def db(tmp_path): + session_db = SessionDB(db_path=tmp_path / "state.db") + if not session_db._trigram_available: + session_db.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + yield session_db + session_db.close() + + +def _trigram_rowids(db: SessionDB) -> set[int]: + return { + row[0] + for row in db._conn.execute("SELECT id FROM messages_fts_trigram_docsize").fetchall() + } + + +def _fts_rowids(db: SessionDB) -> set[int]: + return { + row[0] for row in db._conn.execute("SELECT id FROM messages_fts_docsize").fetchall() + } + + +def _seed(db: SessionDB) -> dict[str, int]: + db.create_session("root", source="cli") + # delegate_tool children: source='subagent' via platform, plus the + # _delegate_from creation marker. + db.create_session( + "kid", source="subagent", parent_session_id="root", + model_config={"_delegate_from": "root"}, + ) + # A child spawned under a gateway turn inherits the gateway's source but + # still carries the marker. + db.create_session( + "gw-kid", source="telegram", parent_session_id="root", + model_config={"_delegate_from": "root"}, + ) + # Compression continuation: parent_session_id but NO marker -> indexed. + db.create_session("cont", source="cli", parent_session_id="root") + return { + "root": db.append_message("root", role="user", content="交付状态正常 root-word"), + "kid": db.append_message("kid", role="assistant", content="子任务状态正常 kid-word"), + "gw-kid": db.append_message("gw-kid", role="assistant", content="网关子任务 gwkid-word"), + "cont": db.append_message("cont", role="assistant", content="继续会话内容 cont-word"), + } + + +def test_subagent_rows_skip_trigram_but_stay_in_standard_fts(db: SessionDB): + ids = _seed(db) + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + assert _fts_rowids(db) >= set(ids.values()) + + +def test_subagent_rows_remain_word_searchable(db: SessionDB): + _seed(db) + assert [r["session_id"] for r in db.search_messages("kid-word")] == ["kid"] + assert [r["session_id"] for r in db.search_messages("gwkid-word")] == ["gw-kid"] + # Explicit CJK search scoped to the excluded source falls back to LIKE. + assert [ + r["session_id"] + for r in db.search_messages("子任务状态", source_filter=["subagent"]) + ] == ["kid"] + # Top-level CJK substring search unaffected. + assert [r["session_id"] for r in db.search_messages("交付状态")] == ["root"] + + +def test_update_and_delete_of_unindexed_child_row_keep_trigram_consistent(db: SessionDB): + ids = _seed(db) + db._conn.execute( + "UPDATE messages SET content = ? WHERE id = ?", ("改写后的内容", ids["kid"]) + ) + db._conn.execute("DELETE FROM messages WHERE id = ?", (ids["kid"],)) + db._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + + +def test_deferred_rebuild_does_not_reintroduce_children(db: SessionDB): + ids = _seed(db) + with db._lock: + db._reset_fts_index_to_empty(db._conn) + db._seed_fts_rebuild_markers(db._conn, force=True) + db._conn.commit() + while db.fts_rebuild_step(): + pass + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + assert _fts_rowids(db) >= set(ids.values()) + + +def test_full_rebuild_honours_exclusion(db: SessionDB): + ids = _seed(db) + db.rebuild_fts() + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + + +def test_v29_install_purges_child_rows_on_upgrade(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + # Recreate the v29 (cron-only) view/trigger boundary. + old._conn.executescript( + """ + DROP TRIGGER messages_fts_trigram_insert; + DROP TRIGGER messages_fts_trigram_delete; + DROP TRIGGER messages_fts_trigram_update; + DROP VIEW messages_fts_trigram_src; + CREATE VIEW messages_fts_trigram_src AS + SELECT m.id, m.role, m.content, m.tool_name + FROM messages AS m JOIN sessions AS s ON s.id = m.session_id + WHERE m.role <> 'tool' AND s.source <> 'cron'; + CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages + WHEN new.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions WHERE id = new.session_id AND source <> 'cron') + BEGIN + INSERT INTO messages_fts_trigram(rowid, content, tool_name) + VALUES (new.id, new.content, new.tool_name); + END; + """ + ) + ids = _seed(old) + assert _trigram_rowids(old) == set(ids.values()) + old._conn.execute("UPDATE schema_version SET version = 29") + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == {ids["root"], ids["cont"]} + assert migrated._conn.execute( + "SELECT version FROM schema_version" + ).fetchone()[0] == SCHEMA_VERSION + migrated._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) + finally: + migrated.close() + + +def test_predicate_constants_agree(): + assert "subagent" in FTS_TRIGRAM_EXCLUDED_SOURCES + assert "cron" in FTS_TRIGRAM_EXCLUDED_SOURCES + sql = fts_trigram_session_sql("s") + assert sql.startswith("s.source NOT IN (") and "s.model_config" in sql diff --git a/tests/state/test_no_locked_readers_gate.py b/tests/state/test_no_locked_readers_gate.py index e4db17d5db..0664669a9a 100644 --- a/tests/state/test_no_locked_readers_gate.py +++ b/tests/state/test_no_locked_readers_gate.py @@ -17,6 +17,13 @@ convoying on the writer lock. Methods that write under the lock are the lock's legitimate users and pass. New violations fail with the method name and the fix (route through ``_read_ctx()``). +``SessionDB`` itself is declared in ``hermes_state.py`` as +``class SessionDB(SessionSearchMixin, SessionSchemaMixin, +SessionPortabilityMixin)`` — its actual methods live across four files. +A gate that only opens ``hermes_state.py`` never sees a locked reader +declared in one of the three mixin files, so ``_ALL_STATE_SOURCES`` scans +each of them under their own class name. + Deliberately NOT flagged: - methods that INSERT/UPDATE/DELETE/REPLACE under the lock (writers); - read-modify-write methods (the read is ordered against its own write); @@ -32,7 +39,18 @@ from pathlib import Path import pytest -_STATE_PY = Path(__file__).resolve().parents[2] / "hermes_state.py" +_REPO_ROOT = Path(__file__).resolve().parents[2] +_STATE_PY = _REPO_ROOT / "hermes_state.py" + +# SessionDB's own class body lives in hermes_state.py; the rest of its +# methods come from these mixins (see module docstring). Each entry is +# (source file, class name to scan in that file). +_ALL_STATE_SOURCES: list[tuple[Path, str]] = [ + (_STATE_PY, "SessionDB"), + (_REPO_ROOT / "hermes_state_search.py", "SessionSearchMixin"), + (_REPO_ROOT / "hermes_state_schema.py", "SessionSchemaMixin"), + (_REPO_ROOT / "hermes_state_portability.py", "SessionPortabilityMixin"), +] _WRITE_RE = re.compile( r"^\s*(INSERT|UPDATE|DELETE|REPLACE|CREATE|DROP|ALTER|VACUUM|BEGIN|COMMIT|ANALYZE)\b", @@ -124,17 +142,19 @@ def _is_self_lock_with(item: ast.withitem) -> bool: ) -def _scan_locked_readers(state_py: "Path | None" = None) -> list[str]: +def _scan_locked_readers( + state_py: "Path | None" = None, class_name: str = "SessionDB" +) -> list[str]: target = state_py if state_py is not None else _STATE_PY tree = ast.parse(target.read_text(encoding="utf-8")) violations: list[str] = [] session_db = None for node in tree.body: - if isinstance(node, ast.ClassDef) and node.name == "SessionDB": + if isinstance(node, ast.ClassDef) and node.name == class_name: session_db = node break - assert session_db is not None, "SessionDB class not found" + assert session_db is not None, f"{class_name} class not found in {target}" for method in session_db.body: if not isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)): @@ -197,9 +217,22 @@ def _scan_locked_readers(state_py: "Path | None" = None) -> list[str]: return violations +def _scan_all_state_sources() -> list[str]: + """Run ``_scan_locked_readers`` over every file that contributes methods + to ``SessionDB`` — the class body in ``hermes_state.py`` plus each mixin + it inherits from (see module docstring). Violations are prefixed with + their source filename since methods can share names across mixins. + """ + violations: list[str] = [] + for path, class_name in _ALL_STATE_SOURCES: + for v in _scan_locked_readers(path, class_name): + violations.append(f"{path.name}: {v}") + return violations + + class TestNoPureReadersUnderWriterLock: def test_no_locked_pure_readers(self): - violations = _scan_locked_readers() + violations = _scan_all_state_sources() assert violations == [], ( "Pure-read SessionDB methods holding the writer lock " "(Pattern C — every concurrent turn's persistence convoys " @@ -235,3 +268,24 @@ class TestNoPureReadersUnderWriterLock: assert flagged == { "guilty_reader", "guilty_alias_reader", "guilty_variable_sql" }, violations + + def test_scan_all_state_sources_visits_every_mixin_file(self, tmp_path): + """Sabotage self-check for the multi-file scope itself: a locked + reader planted in a MIXIN file (not hermes_state.py) must still be + caught. Guards against the gate's scope silently narrowing back to + one file — exactly how the real 2026-08 gap (9 locked readers across + three mixin files, invisible to the single-file scanner) happened. + """ + mixin_sabotage = ( + "class FakeMixin:\n" + " def guilty_mixin_reader(self):\n" + " with self._lock:\n" + " return self._conn.execute(\"SELECT 1\").fetchone()\n" + ) + p = tmp_path / "fake_mixin.py" + p.write_text(mixin_sabotage, encoding="utf-8") + + violations = [ + f"{p.name}: {v}" for v in _scan_locked_readers(p, "FakeMixin") + ] + assert any("guilty_mixin_reader" in v for v in violations), violations diff --git a/tests/state/test_session_git_metadata_generation.py b/tests/state/test_session_git_metadata_generation.py index 2e1cb30988..b97727725c 100644 --- a/tests/state/test_session_git_metadata_generation.py +++ b/tests/state/test_session_git_metadata_generation.py @@ -237,7 +237,7 @@ def test_legacy_sessions_table_reconciles_generation_column(tmp_path): assert "git_metadata_generation" in columns assert reopened._conn.execute( "SELECT version FROM schema_version" - ).fetchone()[0] == SCHEMA_VERSION == 26 + ).fetchone()[0] == SCHEMA_VERSION reopened.create_session("session", "desktop", cwd="/repo") assert reopened.update_session_cwd("session", "/repo") == 1 finally: diff --git a/tests/state/test_state_db_holders.py b/tests/state/test_state_db_holders.py new file mode 100644 index 0000000000..4fb5c24f38 --- /dev/null +++ b/tests/state/test_state_db_holders.py @@ -0,0 +1,50 @@ +"""Behavioral tests for the state-holder and repair-admission authority.""" + +import os + +import pytest + +import hermes_state_holders + + +@pytest.mark.linux_only +def test_foreign_holder_accepts_same_inode_reached_through_an_alias( + tmp_path, monkeypatch +): + """Descriptor identity is authoritative even when /proc spells another path.""" + db_path = tmp_path / "state.db" + db_path.touch() + alias_path = tmp_path / "namespace-alias" / "state.db" + + proc_root = tmp_path / "proc" + for pid in (111, 222): + (proc_root / str(pid) / "fd").mkdir(parents=True) + os.symlink(db_path, proc_root / "222" / "fd" / "3") + + monkeypatch.setattr(hermes_state_holders.os, "getpid", lambda: 111) + real_listdir = os.listdir + + def _listdir(path): + if isinstance(path, str): + path = path.replace("/proc", str(proc_root)) + return real_listdir(path) + + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + + def _readlink(path): + if path == "/proc/222/fd/3": + return str(alias_path) + return os.readlink(path.replace("/proc", str(proc_root))) + + monkeypatch.setattr(hermes_state_holders.os, "readlink", _readlink) + real_stat = os.stat + + def _stat(path, *args, **kwargs): + path = str(path).replace("/proc", str(proc_root)) + return real_stat(path, *args, **kwargs) + + monkeypatch.setattr(hermes_state_holders.os, "stat", _stat) + + assert hermes_state_holders.foreign_state_db_holders(db_path) == [ + (222, str(alias_path)) + ] diff --git a/tests/state/test_state_db_lock_fail_closed.py b/tests/state/test_state_db_lock_fail_closed.py new file mode 100644 index 0000000000..a98a23f7cf --- /dev/null +++ b/tests/state/test_state_db_lock_fail_closed.py @@ -0,0 +1,160 @@ +"""Unopenable admission lock files must fail CLOSED (#100368). + +`state.db` has two cross-process admission authorities that gate destructive +work on a file several Hermes processes share (gateway service, the Desktop +app's `hermes serve` backend, CLI sessions, the TUI slash worker): + +* `hermes_state_common.fts_rebuild_admission` — full structural FTS rebuilds +* `hermes_state._cross_process_repair_lock` — writable_schema surgery / VACUUM + +Both document themselves as fail-closed, and both honoured that only for a +*timed-out* acquire. When the lock file could not be `open()`ed at all they +yielded True and proceeded "with in-process serialisation only" — which is no +cross-process authority whatsoever. + +That inversion is reachable exactly when it does the most damage. Creating the +lock file needs a directory entry and an inode, so on a full disk `open()` +raises ENOSPC — while a sibling process that opened ITS handle before the disk +filled is still mid-rebuild or mid-surgery. Every process then ran concurrent +destructive work on the same live DB, i.e. the precise interleaving PR #93200 +added these locks to prevent. #100368 reports that shape: a disk-full trigger, +then a fresh corruption on every boot with other writers alive, and no +re-corruption on a boot with zero other writers. + +These tests drive a real unopenable lock path (a directory where the code +expects a file, so `open()` raises a genuine OSError from the kernel) rather +than monkeypatching the helpers, and assert the deferral is honoured at both +the primitive and the behavior level. +""" + +import sqlite3 +import sys +from pathlib import Path + +import pytest + +import hermes_state +import hermes_state_common +from hermes_state import SessionDB, repair_state_db_schema + + +def _make_unopenable(lock_path: Path) -> None: + """Make ``open(lock_path, "a+b")`` raise a real OSError. + + A directory standing where the code expects a regular file yields + IsADirectoryError on POSIX and PermissionError on Windows — both OSError, + both raised by the kernel. This stands in for the ENOSPC/EMFILE the field + reports hit, without needing to fill a real disk. + """ + lock_path.unlink(missing_ok=True) + lock_path.mkdir(parents=True, exist_ok=True) + with pytest.raises(OSError): + open(lock_path, "a+b").close() + + +# ── FTS rebuild authority ─────────────────────────────────────────────────── + + +def test_fts_admission_fails_closed_when_lock_file_is_unopenable(tmp_path): + """The primitive must refuse admission, not fall back to no authority.""" + db_path = tmp_path / "state.db" + _make_unopenable(db_path.with_name(db_path.name + ".fts_rebuild.lock")) + + with hermes_state_common.fts_rebuild_admission(db_path) as admitted: + assert admitted is False + + +def test_fts_admission_still_admits_a_pathless_db(tmp_path): + """Guardrail: an in-memory store has no cross-process surface at all. + + The fix must not turn the legitimate no-op case into a permanent deferral. + """ + with hermes_state_common.fts_rebuild_admission(None) as admitted: + assert admitted is True + + +def test_rebuild_fts_defers_when_lock_file_is_unopenable(tmp_path): + """Behavior: the rebuild entry point reports no progress and rebuilds nothing.""" + db = SessionDB(db_path=tmp_path / "state.db") + if not db._fts_enabled: + db.close() + pytest.skip("FTS5 unavailable in this build") + try: + db.create_session("s1", source="test") + db.append_message("s1", "user", "hello world") + + # Sanity: with an openable lock the rebuild really runs, so a 0 below + # is the deferral and not an unrelated no-op. + assert db.rebuild_fts() >= 1 + + _make_unopenable( + db.db_path.with_name(db.db_path.name + ".fts_rebuild.lock") + ) + assert db.rebuild_fts() == 0 + finally: + try: + db.close() + except Exception: + pass + + +# ── Schema-surgery authority ──────────────────────────────────────────────── + + +def _build_healthy_db(db_path: Path) -> None: + db = SessionDB(db_path=db_path) + db.create_session("s1", source="test") + db.append_message("s1", "user", "hello world") + db.close() + + +def _corrupt_duplicate_fts(db_path: Path) -> None: + """Inject a duplicate messages_fts row into sqlite_master. + + Reproduces 'malformed database schema (messages_fts) - table + messages_fts already exists'. + """ + conn = sqlite3.connect(str(db_path)) + conn.execute("PRAGMA writable_schema=ON") + conn.execute( + "INSERT INTO sqlite_master (type, name, tbl_name, rootpage, sql) " + "SELECT type, name, tbl_name, rootpage, sql FROM sqlite_master " + "WHERE name='messages_fts'" + ) + conn.commit() + conn.close() + + +def test_repair_lock_fails_closed_when_lock_file_is_unopenable(tmp_path): + """The primitive must refuse the repair authority.""" + db_path = tmp_path / "state.db" + _make_unopenable(db_path.with_name(db_path.name + ".repair.lock")) + + with hermes_state._cross_process_repair_lock(db_path) as holding: + assert holding is False + + +@pytest.mark.skipif(sys.platform == "win32", reason="writable_schema corruption harness") +def test_repair_skips_surgery_when_lock_file_is_unopenable(tmp_path): + """Behavior: no writable_schema surgery, no forensic backup, DB untouched. + + A full disk is the worst possible moment to start an unsynchronised + VACUUM on a live shared DB, and it is exactly when the lock file cannot + be created. + """ + db_path = tmp_path / "state.db" + _build_healthy_db(db_path) + _corrupt_duplicate_fts(db_path) + assert hermes_state._db_opens_cleanly(db_path) is not None + before = db_path.read_bytes() + + _make_unopenable(db_path.with_name(db_path.name + ".repair.lock")) + + report = repair_state_db_schema(db_path) + + assert report["repaired"] is False + assert "repair lock" in (report["error"] or "") + assert report["backup_path"] is None + assert not list(tmp_path.glob("state.db.malformed-backup-*")) + # The damaged image is left byte-identical for the next (authorised) pass. + assert db_path.read_bytes() == before diff --git a/tests/state/test_state_db_wal_unlink_race.py b/tests/state/test_state_db_wal_unlink_race.py new file mode 100644 index 0000000000..5f8bf5271c --- /dev/null +++ b/tests/state/test_state_db_wal_unlink_race.py @@ -0,0 +1,84 @@ +"""Regression coverage for WAL restoration during state.db repair (#101064). + +Journal-mode restoration used to open a NEW connection after the exclusive +repair guard had released the live database. In WAL mode a writer could still +hold the unlinked old WAL inode while that second connection created a fresh +``state.db-wal`` path — two generations of one store. The restore must run +through the guard connection, before the guard releases. +""" + +import sqlite3 + +import pytest + +import hermes_state +from hermes_state import repair_state_db_schema + + +def _make_db(path): + conn = sqlite3.connect(str(path), isolation_level=None) + conn.execute("CREATE TABLE sessions (name TEXT)") + conn.execute("INSERT INTO sessions VALUES ('seed')") + conn.close() + + +def test_wal_restoration_reuses_exclusive_repair_connection(tmp_path, monkeypatch): + """Unit contract: given the guard connection, no reopen happens.""" + db_path = tmp_path / "state.db" + conn = sqlite3.connect(db_path, isolation_level=None) + conn.execute("CREATE TABLE marker (value TEXT)") + + def fail_if_reopened(_path): + pytest.fail("WAL restoration reopened state.db outside the repair guard") + + monkeypatch.setattr(hermes_state, "_connect_repair_durable", fail_if_reopened) + + hermes_state._restore_journal_mode_after_repair(db_path, None, conn=conn) + # The mode itself is whatever apply_wal_with_fallback resolves on this + # runtime (WAL, or DELETE on WAL-reset-vulnerable SQLite builds); the + # contract under test is the connection reuse, asserted above. + assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() in ("wal", "delete") + conn.close() + + +def test_repair_never_reopens_after_the_guard_releases(tmp_path, monkeypatch): + """End to end through repair_state_db_schema: every connection the repair + opens is opened while the exclusive guard is still held, and none after.""" + db = tmp_path / "state.db" + _make_db(db) + monkeypatch.setattr(hermes_state, "_db_opens_cleanly", lambda path: "forced-unhealthy") + # The scratch-space pre-flight wants ~10GB headroom; irrelevant here. + monkeypatch.setattr(hermes_state, "_repair_scratch_space_error", lambda path: None) + + def fake_strategies(scratch_path, report): + report["repaired"] = True + report["strategy"] = "test_strategy" + return report + + monkeypatch.setattr(hermes_state, "_run_repair_strategies", fake_strategies) + + events: list[str] = [] + real_guard = hermes_state._exclusive_repair_db_guard + real_connect = hermes_state._connect_repair_durable + + from contextlib import contextmanager + + @contextmanager + def tracing_guard(path): + events.append("guard-enter") + with real_guard(path) as pair: + yield pair + events.append("guard-exit") + + def tracing_connect(path, *a, **kw): + events.append("connect") + return real_connect(path, *a, **kw) + + monkeypatch.setattr(hermes_state, "_exclusive_repair_db_guard", tracing_guard) + monkeypatch.setattr(hermes_state, "_connect_repair_durable", tracing_connect) + + report = repair_state_db_schema(db, backup=False) + assert report["repaired"] is True + assert "guard-exit" in events + after_release = events[events.index("guard-exit") + 1 :] + assert "connect" not in after_release, events diff --git a/tests/test_desktop_update_windows_progress.py b/tests/test_desktop_update_windows_progress.py index 38e1a7fa57..0f9118fb17 100644 --- a/tests/test_desktop_update_windows_progress.py +++ b/tests/test_desktop_update_windows_progress.py @@ -35,17 +35,24 @@ def _read_progress(url: str, deadline: float) -> dict[str, object]: ``urlopen(timeout=5)`` propagating TimeoutError was exactly the Aug 2026 flake (run 32440286339). Only a listener that stays unresponsive until the deadline fails the test. + + Per-attempt timeout is 1s, not 5s: a connection the kernel accepted into + the backlog before the runspace was serving never gets answered, and a 5s + wait on it burned half the readiness budget per attempt (two stale + attempts = red, run 33591547099). The script's own readiness handshake + now keeps that gap from reaching us, but the probe should not be able to + lose the whole budget to one dead socket either way. """ last_exc: Exception | None = None attempted = False while not attempted or time.monotonic() < deadline: attempted = True try: - with urlopen(f"{url}progress", timeout=5) as response: + with urlopen(f"{url}progress", timeout=1) as response: return json.loads(response.read().decode("utf-8")) except (TimeoutError, OSError) as exc: # transient stall — retry last_exc = exc - time.sleep(0.2) + time.sleep(0.1) raise AssertionError( f"/progress unresponsive until deadline (last error: {last_exc!r})" ) diff --git a/tests/test_env_loader_secret_sources.py b/tests/test_env_loader_secret_sources.py index 303ed92268..c2959144c9 100644 --- a/tests/test_env_loader_secret_sources.py +++ b/tests/test_env_loader_secret_sources.py @@ -174,6 +174,85 @@ def test_cold_profile_bitwarden_uses_profile_bootstrap_without_global_env( assert os.environ.get("ANTHROPIC_API_KEY") is None +def test_single_profile_scoped_load_keeps_override_behavior(tmp_path, monkeypatch): + """Without multiplex, a scoped load keeps its historical override behaviour. + + Ported from #77970 (@DonShelly): the guard must key on the multiplex flag, + not on the home override alone -- single-profile ``-p`` runs still load. + """ + from agent import secret_scope + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + monkeypatch.delenv("HERMES_TEST_SHARED_ADAPTER_CONFIG", raising=False) + other_home = tmp_path / "other" + other_home.mkdir() + (other_home / ".env").write_text("HERMES_TEST_SHARED_ADAPTER_CONFIG=second\n") + + was_active = secret_scope.is_multiplex_active() + secret_scope.set_multiplex_active(False) + home_token = set_hermes_home_override(other_home) + try: + loaded = env_loader.load_hermes_dotenv(hermes_home=other_home) + finally: + secret_scope.set_multiplex_active(was_active) + reset_hermes_home_override(home_token) + + try: + assert os.environ.get("HERMES_TEST_SHARED_ADAPTER_CONFIG") == "second" + assert (other_home / ".env") in loaded + finally: + os.environ.pop("HERMES_TEST_SHARED_ADAPTER_CONFIG", None) + + +def test_multiplex_dotenv_load_hydrates_sources_without_global_env( + tmp_path, monkeypatch +): + """The safe multiplex path must still refresh profile secret sources.""" + from agent import secret_scope + import agent.secret_sources.bitwarden as bw_module + from agent.secret_sources import registry as reg_module + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + monkeypatch.delenv("BWS_ACCESS_TOKEN", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + (tmp_path / ".env").write_text( + "BWS_ACCESS_TOKEN=profile-bootstrap\n", encoding="utf-8" + ) + (tmp_path / "config.yaml").write_text( + "secrets:\n" + " bitwarden:\n" + " enabled: true\n" + " project_id: test-project\n" + " access_token_env: BWS_ACCESS_TOKEN\n", + encoding="utf-8", + ) + monkeypatch.setattr(bw_module, "find_bws", lambda **_kw: Path("/fake/bws")) + monkeypatch.setattr( + bw_module, + "fetch_bitwarden_secrets", + lambda **_kw: ({"ANTHROPIC_API_KEY": "profile-provider-key"}, []), + ) + reg_module._reset_registry_for_tests() + + was_active = secret_scope.is_multiplex_active() + home_token = set_hermes_home_override(tmp_path) + secret_scope.set_multiplex_active(True) + try: + assert env_loader.load_hermes_dotenv(hermes_home=tmp_path) == [] + finally: + secret_scope.set_multiplex_active(was_active) + reset_hermes_home_override(home_token) + + assert env_loader.get_secret_source_values(tmp_path) == { + "ANTHROPIC_API_KEY": "profile-provider-key" + } + assert os.environ.get("BWS_ACCESS_TOKEN") is None + assert os.environ.get("ANTHROPIC_API_KEY") is None + + def test_cold_profile_hydration_seeds_op_env_bootstrap(tmp_path, monkeypatch): """The .op.env bootstrap file must feed cold-profile hydration. diff --git a/tests/test_fts_tool_write_bounds.py b/tests/test_fts_tool_write_bounds.py new file mode 100644 index 0000000000..fb38bd60d4 --- /dev/null +++ b/tests/test_fts_tool_write_bounds.py @@ -0,0 +1,204 @@ +import sqlite3 + +import pytest + +from hermes_state import SessionDB +from hermes_state_common import ( + FTS_TOOL_CONTENT_PREFIX_CHARS, + FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, + LEGACY_FTS_SQL, + _FTS_TRIGGERS, +) + + +def _long_message(prefix: str, tail: str) -> str: + padding = "padding " * (FTS_TOOL_CONTENT_PREFIX_CHARS // len("padding ") + 8) + return f"{prefix} {padding} {tail}" + + +@pytest.fixture +def db(tmp_path): + session_db = SessionDB(db_path=tmp_path / "state.db") + if not session_db._fts_enabled: + session_db.close() + pytest.skip("SQLite FTS5 unavailable") + session_db.create_session("session", source="cli") + try: + yield session_db + finally: + session_db.close() + + +def test_new_tool_rows_bound_fts_content_but_explicit_tool_search_is_complete(db): + tool_id = db.append_message( + "session", + role="tool", + content=_long_message("indexed-prefix-token", "tool-tail-token"), + tool_name="terminal", + ) + user_id = db.append_message( + "session", + role="user", + content=_long_message("user-prefix-token", "user-tail-token"), + ) + + assert [row["id"] for row in db.search_messages("indexed-prefix-token")] == [ + tool_id + ] + assert db.search_messages("tool-tail-token") == [] + assert [ + row["id"] + for row in db.search_messages("tool-tail-token", role_filter=["tool"]) + ] == [tool_id] + assert [row["id"] for row in db.search_messages("user-tail-token")] == [ + user_id + ] + + +def test_trigger_migration_preserves_historical_tool_tokens_without_rebuild(tmp_path): + path = tmp_path / "state.db" + first = SessionDB(db_path=path) + if not first._fts_enabled: + first.close() + pytest.skip("SQLite FTS5 unavailable") + first.create_session("session", source="cli") + + # Model the pre-migration trigger contract: every id through this artificial + # boundary receives full-content indexing. + first.set_meta(FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, str(2**62)) + old_id = first.append_message( + "session", + role="tool", + content=_long_message("old-prefix-token", "old-tail-token"), + ) + assert [row["id"] for row in first.search_messages("old-tail-token")] == [old_id] + first._conn.execute( + "DELETE FROM state_meta WHERE key = ?", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY,), + ) + first.close() + + migrated = SessionDB(db_path=path) + try: + assert int(migrated.get_meta(FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY)) == old_id + assert [ + row["id"] for row in migrated.search_messages("old-tail-token") + ] == [old_id] + + new_id = migrated.append_message( + "session", + role="tool", + content=_long_message("new-prefix-token", "new-tail-token"), + ) + assert migrated.search_messages("new-tail-token") == [] + assert [ + row["id"] + for row in migrated.search_messages( + "new-tail-token", role_filter=["tool"] + ) + ] == [new_id] + + # Historical rows still use their full old token stream for the FTS5 + # external-content delete command; redaction must remove the tail token. + migrated._execute_write( + lambda conn: conn.execute( + "UPDATE messages SET content = '' WHERE id = ?", (old_id,) + ) + ) + assert migrated.search_messages("old-tail-token") == [] + + # New bounded rows use the same prefix for delete as insert. A mismatch + # corrupts external-content FTS and makes this delete or later write fail. + migrated._execute_write( + lambda conn: conn.execute("DELETE FROM messages WHERE id = ?", (new_id,)) + ) + migrated.append_message("session", role="assistant", content="fts-still-healthy") + assert migrated.search_messages("fts-still-healthy") + finally: + migrated.close() + + +def test_full_rebuild_moves_boundary_before_future_tool_writes(db): + before_id = db.append_message( + "session", + role="tool", + content=_long_message("before-prefix-token", "before-tail-token"), + ) + assert db.search_messages("before-tail-token") == [] + + assert db.rebuild_fts() >= 1 + assert int(db.get_meta(FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY)) == before_id + assert [row["id"] for row in db.search_messages("before-tail-token")] == [ + before_id + ] + + after_id = db.append_message( + "session", + role="tool", + content=_long_message("after-prefix-token", "after-tail-token"), + ) + assert db.search_messages("after-tail-token") == [] + assert [ + row["id"] + for row in db.search_messages("after-tail-token", role_filter=["tool"]) + ] == [after_id] + + +def test_role_changes_switch_between_bounded_and_full_indexing(db): + message_id = db.append_message( + "session", + role="tool", + content=_long_message("role-prefix-token", "role-tail-token"), + ) + assert db.search_messages("role-tail-token") == [] + + db._execute_write( + lambda conn: conn.execute( + "UPDATE messages SET role = 'assistant' WHERE id = ?", (message_id,) + ) + ) + assert [row["id"] for row in db.search_messages("role-tail-token")] == [ + message_id + ] + + db._execute_write( + lambda conn: conn.execute( + "UPDATE messages SET role = 'tool' WHERE id = ?", (message_id,) + ) + ) + assert db.search_messages("role-tail-token") == [] + + +def test_legacy_inline_fts_also_bounds_new_tool_rows(tmp_path): + path = tmp_path / "legacy.db" + initial = SessionDB(db_path=path) + initial.create_session("session", source="cli") + for trigger in _FTS_TRIGGERS: + initial._conn.execute(f"DROP TRIGGER IF EXISTS {trigger}") + initial._conn.execute("DROP TABLE IF EXISTS messages_fts_trigram") + initial._conn.execute("DROP VIEW IF EXISTS messages_fts_trigram_src") + initial._conn.execute("DROP TABLE IF EXISTS messages_fts") + initial._conn.executescript(LEGACY_FTS_SQL) + initial._conn.execute( + "DELETE FROM state_meta WHERE key IN (?, 'fts_storage_version')", + (FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY,), + ) + initial.close() + + legacy = SessionDB(db_path=path) + try: + assert legacy._db_has_legacy_inline_fts(legacy._conn.cursor()) is True + message_id = legacy.append_message( + "session", + role="tool", + content=_long_message("legacy-prefix-token", "legacy-tail-token"), + ) + assert legacy.search_messages("legacy-tail-token") == [] + assert [ + row["id"] + for row in legacy.search_messages( + "legacy-tail-token", role_filter=["tool"] + ) + ] == [message_id] + finally: + legacy.close() diff --git a/tests/test_hermes_home_key_cache.py b/tests/test_hermes_home_key_cache.py new file mode 100644 index 0000000000..f421c188f8 --- /dev/null +++ b/tests/test_hermes_home_key_cache.py @@ -0,0 +1,164 @@ +"""Tests for the remembered results in `hermes_home_key`. + +`Path.resolve()` is a filesystem call. `hermes_home_key` sits under +`ToolRegistry.current_scope_key()`, which runs on every registry lookup, so +before the results were remembered the registry paid a syscall per lookup. + +The value this returns must not change, so most of these tests compare +against the plain uncached calculation. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +import hermes_constants as hc + + +def _uncached(path=None) -> str: + """The calculation as it was before results were remembered.""" + candidate = Path(path) if path is not None else hc.get_hermes_home() + return os.path.normcase(str(candidate.expanduser().resolve(strict=False))) + + +@pytest.fixture(autouse=True) +def _clear_cache(): + hc.reset_hermes_home_key_cache() + yield + hc.reset_hermes_home_key_cache() + + +class TestSameAnswerAsBefore: + """Remembering a result must not change what comes back.""" + + @pytest.mark.parametrize("case", ["real", "missing", "trailing_sep", "dot_dot"]) + def test_matches_the_uncached_calculation(self, tmp_path, case): + (tmp_path / "real").mkdir() + target = { + "real": str(tmp_path / "real"), + "missing": str(tmp_path / "not_there"), + "trailing_sep": str(tmp_path / "real") + os.sep, + "dot_dot": str(tmp_path / "real" / ".." / "real"), + }[case] + assert hc.hermes_home_key(target) == _uncached(target) + + def test_matches_for_the_default_home(self): + assert hc.hermes_home_key() == _uncached() + + def test_matches_for_a_tilde_path(self): + assert hc.hermes_home_key("~") == _uncached("~") + + def test_accepts_a_path_object(self, tmp_path): + (tmp_path / "real").mkdir() + assert hc.hermes_home_key(tmp_path / "real") == _uncached(tmp_path / "real") + + def test_second_call_returns_the_same_string(self, tmp_path): + (tmp_path / "real").mkdir() + first = hc.hermes_home_key(str(tmp_path / "real")) + second = hc.hermes_home_key(str(tmp_path / "real")) + assert first == second == _uncached(str(tmp_path / "real")) + + +class TestWhatGetsRemembered: + def test_an_existing_path_is_remembered(self, tmp_path): + (tmp_path / "real").mkdir() + hc.hermes_home_key(str(tmp_path / "real")) + assert len(hc._HOME_KEY_CACHE) == 1 + + def test_a_missing_path_is_not_remembered(self, tmp_path): + # The answer can change once the directory is created, for example + # when part of the path turns out to be a link, so it must not stick. + hc.hermes_home_key(str(tmp_path / "not_there")) + assert hc._HOME_KEY_CACHE == {} + + def test_a_path_created_later_picks_up_the_real_answer(self, tmp_path): + later = tmp_path / "later" + before = hc.hermes_home_key(str(later)) + later.mkdir() + after = hc.hermes_home_key(str(later)) + assert after == _uncached(str(later)) + assert hc._HOME_KEY_CACHE == {str(later): after} + # On a plain directory both answers agree anyway. The point is that + # the first one was never stored. + assert before == after + + def test_different_paths_get_their_own_entries(self, tmp_path): + for name in ("a", "b", "c"): + (tmp_path / name).mkdir() + hc.hermes_home_key(str(tmp_path / name)) + assert len(hc._HOME_KEY_CACHE) == 3 + + def test_reset_clears_everything(self, tmp_path): + (tmp_path / "real").mkdir() + hc.hermes_home_key(str(tmp_path / "real")) + assert hc._HOME_KEY_CACHE + hc.reset_hermes_home_key_cache() + assert hc._HOME_KEY_CACHE == {} + + +class TestHomeChanges: + def test_pointing_hermes_home_somewhere_else_gives_a_new_key( + self, tmp_path, monkeypatch, + ): + # A different home is a different input path, so it lands on its own + # entry rather than reusing the first one. + first = tmp_path / "home_one" + second = tmp_path / "home_two" + first.mkdir() + second.mkdir() + + monkeypatch.setenv("HERMES_HOME", str(first)) + key_one = hc.hermes_home_key() + monkeypatch.setenv("HERMES_HOME", str(second)) + key_two = hc.hermes_home_key() + + assert key_one != key_two + assert key_one == _uncached(str(first)) + assert key_two == _uncached(str(second)) + + +class TestSymlinks: + def test_a_link_resolves_to_its_target(self, tmp_path): + target = tmp_path / "target" + target.mkdir() + link = tmp_path / "link" + try: + link.symlink_to(target, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("this platform or account cannot create symlinks") + assert hc.hermes_home_key(str(link)) == _uncached(str(link)) + assert hc.hermes_home_key(str(link)) == hc.hermes_home_key(str(target)) + + +class TestRegistryLookupsDoNotHitTheDisk: + def test_scope_key_resolves_the_path_once(self, monkeypatch): + # The reason this cache exists. ToolRegistry.current_scope_key() runs + # on every registry lookup, so it must not resolve the home path on + # the filesystem every time. + from tools.registry import registry + + calls = {"n": 0} + real_resolve = Path.resolve + + def counting_resolve(self, *a, **kw): + calls["n"] += 1 + return real_resolve(self, *a, **kw) + + monkeypatch.setattr(Path, "resolve", counting_resolve) + + registry.current_scope_key() + after_first = calls["n"] + for _ in range(50): + registry.current_scope_key() + + assert calls["n"] == after_first, ( + f"current_scope_key resolved the path on the filesystem " + f"{calls['n'] - after_first} extra times across 50 calls" + ) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index bfc0c77629..facfcf2ae5 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -11,7 +11,13 @@ import pytest import hermes_state from agent.session_activity import ActivityProvenance -from hermes_state import SCHEMA_SQL, SCHEMA_VERSION, SessionDB +from hermes_state import ( + FTS_SQL, + FTS_STORAGE_VERSION, + SCHEMA_SQL, + SCHEMA_VERSION, + SessionDB, +) class _NoFtsCursor(sqlite3.Cursor): @@ -281,6 +287,83 @@ class TestConnectionLifecycle: healed.close() assert list(tmp_path.glob("*malformed-backup*")) + def test_read_only_open_retries_transient_wal_ioerr(self, tmp_path, monkeypatch): + """A transient SQLITE_IOERR on a read-only open must retry, not raise. + + A ``mode=ro`` connection cannot perform WAL recovery (recovery would + need to write the -shm index, which read-only mode refuses), so a + concurrent checkpoint / WAL reset / frame-flush on the writer side can + surface "disk I/O error" to a reader on a perfectly healthy database + (#100436). The transition window is millisecond-scale; a bounded retry + must let the open succeed instead of 500-ing the /api/sessions poll + and every other read-only opener. + """ + import sqlite3 + + from hermes_cli.sqlite_safe_read import has_live_connection + + db_path = tmp_path / "state.db" + writable = SessionDB(db_path=db_path) + writable.create_session("wal-race", source="cli") + writable.close() + + real_connect = hermes_state._connect_tracked_db + attempts = [] + + def flaky_connect(*args, **kwargs): + attempts.append(kwargs.get("uri")) + if len(attempts) == 1: + # First open lands inside the writer's WAL transition window. + raise sqlite3.OperationalError("disk I/O error") + return real_connect(*args, **kwargs) + + monkeypatch.setattr(hermes_state, "_connect_tracked_db", flaky_connect) + # Keep the test fast: one backoff tick is enough; the retry budget + # itself is exercised by the attempt count below. + monkeypatch.setattr(hermes_state, "_READ_ONLY_IOERR_RETRY_BACKOFF_S", 0.0) + + read_only = SessionDB(db_path=db_path, read_only=True) + try: + assert read_only._fts_enabled is True + matches = read_only.search_messages("wal-race") + finally: + read_only.close() + + assert len(attempts) >= 2, "the transient IOERR must be retried" + assert has_live_connection(db_path) is False # no leaked connections + + def test_read_only_open_exhausts_retry_budget_for_persistent_ioerr( + self, tmp_path, monkeypatch + ): + """A persistent SQLITE_IOERR must exhaust the budget and raise. + + The retry exists to ride out a millisecond WAL transition — a + storage layer that keeps failing after the full budget is genuinely + broken and must surface the error (and not loop forever). + """ + import sqlite3 + + db_path = tmp_path / "state.db" + writable = SessionDB(db_path=db_path) + writable.create_session("broken-disk", source="cli") + writable.close() + + attempts = [] + + def bad_connect(*args, **kwargs): + attempts.append(1) + raise sqlite3.OperationalError("disk I/O error") + + monkeypatch.setattr(hermes_state, "_connect_tracked_db", bad_connect) + monkeypatch.setattr(hermes_state, "_READ_ONLY_IOERR_RETRY_BACKOFF_S", 0.0) + budget = hermes_state._READ_ONLY_IOERR_RETRY_ATTEMPTS + + with pytest.raises(sqlite3.OperationalError, match="disk I/O error"): + SessionDB(db_path=db_path, read_only=True) + + # budget + 1 = the initial attempt plus `budget` retries. + assert len(attempts) == budget + 1 + # ========================================================================= # Session lifecycle @@ -1701,7 +1784,7 @@ class TestSchemaInit: assert binding["user_id"] == "208214988" assert binding["session_key"] == "telegram:dm:208214988:thread:17585" assert binding["session_id"] == "topic-session" - assert db.get_meta("telegram_dm_topic_schema_version") == "2" + assert db.get_meta("telegram_dm_topic_schema_version") == "3" db.close() @@ -2678,6 +2761,24 @@ class TestCompressionChainProjection: assert db.get_compression_tip("mid1") == "tip1" assert db.get_compression_tip("tip1") == "tip1" + def test_list_serves_full_lineage_ids_for_projected_rows(self, db): + """The projected tip row must carry every chain id. Root and tip + alone are not enough client-side: a persisted tile or route can hold + a MIDDLE segment's id (it was the tip when opened), and without the + intermediates that surface cannot prove it names this conversation — + which is how one chat ends up open twice after a compaction.""" + import time as _time + self._build_compression_chain(db, _time.time() - 3600) + db.create_session("solo", "cli") + db.append_message("solo", "user", "standalone") + db._conn.commit() + + sessions = db.list_sessions_rich(source="cli", limit=20) + tip_row = next(s for s in sessions if s["id"] == "tip1") + assert tip_row["_lineage_ids"] == ["root1", "mid1", "tip1"] + solo_row = next(s for s in sessions if s["id"] == "solo") + assert solo_row.get("_lineage_ids") is None + def test_list_surfaces_tip_for_compressed_root(self, db): @@ -2915,6 +3016,7 @@ class TestVacuum: def test_auto_maintenance_records_successful_vacuum(self, db, monkeypatch): monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 3) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.5) # ratio gate open vacuum_calls = [] monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) @@ -2926,6 +3028,7 @@ class TestVacuum: def test_auto_maintenance_skips_recent_vacuum(self, db, monkeypatch): monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 3) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.5) # ratio gate open db.set_meta("last_vacuum", str(time.time())) vacuum_calls = [] monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) @@ -2940,6 +3043,7 @@ class TestVacuum: def test_auto_maintenance_retries_after_vacuum_interval(self, db, monkeypatch): monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 3) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.5) # ratio gate open db.set_meta("last_vacuum", str(time.time() - 31 * 86400)) vacuum_calls = [] monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) @@ -2954,6 +3058,7 @@ class TestVacuum: def test_auto_maintenance_retries_after_failed_vacuum(self, db, monkeypatch): monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 3) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.5) # ratio gate open vacuum_calls = [] def fail_first_vacuum(): @@ -2974,6 +3079,97 @@ class TestVacuum: assert vacuum_calls == [True, True] assert db.get_meta("last_vacuum") is not None + # ── freelist-ratio gate (#54189) ───────────────────────────────────── + def test_auto_maintenance_skips_vacuum_below_freelist_ratio(self, db, monkeypatch): + """A prune that frees few pages on a dense DB must NOT trigger VACUUM.""" + monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 1) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.05) + vacuum_calls = [] + monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) + + result = db.maybe_auto_prune_and_vacuum(min_interval_hours=0) + + assert result["pruned"] == 1 + assert result["vacuumed"] is False + assert result["freelist_ratio"] == 0.05 + assert vacuum_calls == [] + assert db.get_meta("last_vacuum") is None + # The prune itself still counts as a maintenance run. + assert db.get_meta("last_auto_prune") is not None + + def test_auto_maintenance_vacuums_above_freelist_ratio(self, db, monkeypatch): + monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 1) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.40) + vacuum_calls = [] + monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) + + result = db.maybe_auto_prune_and_vacuum(min_interval_hours=0) + + assert result["vacuumed"] is True + assert result["freelist_ratio"] == 0.40 + assert vacuum_calls == [True] + + def test_auto_maintenance_freelist_ratio_exactly_at_threshold_skips(self, db, monkeypatch): + """Gate is strictly greater-than: 25.0% reclaimable does not VACUUM.""" + from hermes_state import AUTO_VACUUM_MIN_FREELIST_RATIO + + monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 1) + monkeypatch.setattr(db, "_freelist_ratio", lambda: AUTO_VACUUM_MIN_FREELIST_RATIO) + vacuum_calls = [] + monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) + + result = db.maybe_auto_prune_and_vacuum(min_interval_hours=0) + + assert result["vacuumed"] is False + assert vacuum_calls == [] + + def test_auto_maintenance_unknown_freelist_ratio_falls_back_to_time_throttle(self, db, monkeypatch): + """If the pragmas cannot be read, don't silently disable VACUUM forever.""" + monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 1) + monkeypatch.setattr(db, "_freelist_ratio", lambda: None) + vacuum_calls = [] + monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) + + result = db.maybe_auto_prune_and_vacuum(min_interval_hours=0) + + assert result["vacuumed"] is True + assert result["freelist_ratio"] is None + assert vacuum_calls == [True] + + def test_auto_maintenance_ratio_gate_threshold_is_overridable(self, db, monkeypatch): + monkeypatch.setattr(db, "prune_sessions", lambda **_kwargs: 1) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 0.10) + vacuum_calls = [] + monkeypatch.setattr(db, "vacuum", lambda: vacuum_calls.append(True)) + + result = db.maybe_auto_prune_and_vacuum( + min_interval_hours=0, min_vacuum_freelist_ratio=0.05 + ) + + assert result["vacuumed"] is True + assert vacuum_calls == [True] + + def test_freelist_ratio_reads_real_pragmas(self, db): + """Real-DB check: freeing most of the file pushes the ratio past the gate.""" + from hermes_state import AUTO_VACUUM_MIN_FREELIST_RATIO + + db.create_session(session_id="keep", source="cli") + db.append_message(session_id="keep", role="user", content="hi") + for i in range(6): + sid = f"bulk{i}" + db.create_session(session_id=sid, source="cli") + for _ in range(20): + db.append_message(session_id=sid, role="assistant", content="z" * 4000) + db._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + dense = db._freelist_ratio() + assert dense is not None and dense < AUTO_VACUUM_MIN_FREELIST_RATIO + + for i in range(6): + db.delete_session(f"bulk{i}") + db._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + sparse = db._freelist_ratio() + assert sparse is not None and sparse > AUTO_VACUUM_MIN_FREELIST_RATIO + def test_wal_size_limit_is_bounded(self, db): """journal_size_limit must be a finite bound, not SQLite's -1 default. @@ -3104,7 +3300,8 @@ class TestAutoMaintenance: ) db._conn.commit() - def test_first_run_prunes_and_vacuums(self, db): + def test_first_run_prunes_and_skips_vacuum_when_little_reclaimable(self, db): + """Pruning two empty sessions frees almost nothing → prune yes, VACUUM no.""" self._make_old_ended(db, "old1", days_old=100) self._make_old_ended(db, "old2", days_old=100) db.create_session(session_id="new", source="cli") # active, must survive @@ -3112,12 +3309,40 @@ class TestAutoMaintenance: result = db.maybe_auto_prune_and_vacuum(retention_days=90) assert result["skipped"] is False assert result["pruned"] == 2 - assert result["vacuumed"] is True + assert result["vacuumed"] is False # freelist ratio gate (#54189) + assert result["freelist_ratio"] is not None + assert result["freelist_ratio"] <= 0.25 assert result.get("error") is None assert db.get_session("old1") is None assert db.get_session("old2") is None assert db.get_session("new") is not None + def test_first_run_prunes_and_vacuums_when_mostly_reclaimable(self, db): + """Pruning the bulk of the file's pages crosses the 25% gate → VACUUM runs.""" + db.create_session(session_id="new", source="cli") # active, must survive + db.append_message(session_id="new", role="user", content="hi") + for i in range(6): + sid = f"old{i}" + self._make_old_ended(db, sid, days_old=100) + for _ in range(20): + db.append_message(session_id=sid, role="assistant", content="z" * 4000) + # Keep the row aged: append_message bumps activity, prune ages by + # latest message, so push the message timestamps back too. + db._conn.execute( + "UPDATE messages SET timestamp = ? WHERE session_id = ?", + (time.time() - 100 * 86400, sid), + ) + db._conn.commit() + + result = db.maybe_auto_prune_and_vacuum(retention_days=90) + assert result["skipped"] is False + assert result["pruned"] == 6 + assert result["freelist_ratio"] > 0.25 + assert result["vacuumed"] is True + assert result.get("error") is None + assert db.get_session("new") is not None + assert db.get_meta("last_vacuum") is not None + def test_second_call_within_interval_skips(self, db): self._make_old_ended(db, "old", days_old=100) first = db.maybe_auto_prune_and_vacuum( @@ -3418,6 +3643,160 @@ class TestFTSExternalContentMigration: finally: db.close() + @pytest.mark.parametrize("with_message", [False, True]) + def test_v23_rebuild_from_trigram_tool_calls_projection( + self, tmp_path, with_message + ): + """v23 installs built with historical trigram projection should be + repaired via optimize-storage: trigram must drop tool_calls while + standard messages_fts keeps indexing them.""" + db_path = tmp_path / "v23-toolcalls.db" + + # Build an external-content DB that is already at schema version 23, + # but with the old tool_calls-inclusive trigram projection. + conn = sqlite3.connect(str(db_path)) + conn.executescript(SCHEMA_SQL) + conn.executescript(FTS_SQL) + conn.executescript( + """ + DROP TRIGGER IF EXISTS messages_fts_trigram_insert; + DROP TRIGGER IF EXISTS messages_fts_trigram_delete; + DROP TRIGGER IF EXISTS messages_fts_trigram_update; + DROP TABLE IF EXISTS messages_fts_trigram; + DROP VIEW IF EXISTS messages_fts_trigram_src; + + CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS + SELECT id, role, content, tool_name, tool_calls + FROM messages + WHERE role <> 'tool'; + + CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( + content, + tool_name, + tool_calls, + content='messages_fts_trigram_src', + content_rowid='id', + tokenize='trigram' + ); + + CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages + WHEN new.role <> 'tool' + BEGIN + INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) + VALUES (new.id, new.content, new.tool_name, new.tool_calls); + END; + + CREATE TRIGGER messages_fts_trigram_delete AFTER DELETE ON messages + WHEN old.role <> 'tool' + BEGIN + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) + VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); + END; + + CREATE TRIGGER messages_fts_trigram_update + AFTER UPDATE OF content, tool_name, tool_calls, role ON messages + WHEN (old.content IS NOT new.content + OR old.tool_name IS NOT new.tool_name + OR old.tool_calls IS NOT new.tool_calls + OR old.role IS NOT new.role) + BEGIN + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) + SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls + WHERE old.role <> 'tool'; + INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) + SELECT new.id, new.content, new.tool_name, new.tool_calls + WHERE new.role <> 'tool'; + END; + """ + ) + # Simulate the historical v23 projection shipped before this fix. + conn.execute( + "INSERT OR REPLACE INTO state_meta (key, value) VALUES ('fts_storage_version', '1')" + ) + conn.execute( + "INSERT OR REPLACE INTO state_meta (key, value) VALUES ('fts_optimize_available', '1')" + ) + conn.commit() + conn.close() + + if with_message: + conn = sqlite3.connect(str(db_path)) + conn.execute( + "INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)", + ("s1", "cli", time.time()), + ) + conn.execute( + "INSERT INTO messages (session_id, timestamp, role, content, tool_name, tool_calls) " + "VALUES (?, ?, ?, ?, ?, ?)", + ( + "s1", + time.time(), + "assistant", + "部署完成 assistant content", + "legacyTool", + '{"name": "legacy", "arguments": "UNIQUE_TOOLCALL_TOKEN_43701"}', + ), + ) + conn.commit() + assert conn.execute( + "SELECT rowid FROM messages_fts_trigram WHERE messages_fts_trigram MATCH 'UNIQUE_TOOLCALL_TOKEN_43701'" + ).fetchall() + conn.close() + + db = SessionDB(db_path=db_path) + try: + assert db._conn is not None + assert db.fts_optimize_available() is True + assert db.get_meta("fts_storage_version") == "1" + + original_ensure = db._ensure_fts_schema + + def interrupt_after_demote(cursor, table_name, ddl): + if table_name == "messages_fts_trigram": + raise RuntimeError("injected trigram rebuild interruption") + return original_ensure(cursor, table_name, ddl) + + db._ensure_fts_schema = interrupt_after_demote + with pytest.raises(RuntimeError, match="injected trigram"): + db.optimize_fts_storage(vacuum=False) + + db.close() + db = SessionDB(db_path=db_path) + assert db._conn is not None + assert db.get_meta("fts_rebuild_high_water") is None + assert db.get_meta("fts_rebuild_progress") is None + assert db._has_fts_trash(db._conn) is True + assert db.fts_optimize_available() is True + assert db.get_meta("fts_storage_version") == "1" + if with_message: + assert db._conn.execute( + "SELECT 1 FROM messages_fts_trigram " + "WHERE messages_fts_trigram MATCH '部署完成' LIMIT 1" + ).fetchone() + + result = db.optimize_fts_storage(vacuum=False) + assert result["ok"] is True + + if with_message: + # messages_fts stays in the tool-calls search path. + assert len(db.search_messages("UNIQUE_TOOLCALL_TOKEN_43701")) == 1 + # New trigram schema excludes tool_calls from trigram projection. + assert not db._conn.execute( + "SELECT 1 FROM messages_fts_trigram WHERE messages_fts_trigram MATCH 'UNIQUE_TOOLCALL_TOKEN_43701' LIMIT 1" + ).fetchone() + assert db._conn.execute( + "SELECT 1 FROM messages_fts_trigram " + "WHERE messages_fts_trigram MATCH '部署完成' LIMIT 1" + ).fetchone() + trigger_sql = db._conn.execute( + "SELECT sql FROM sqlite_master " + "WHERE type = 'trigger' AND name = 'messages_fts_trigram_update'" + ).fetchone()[0] + assert "tool_calls" not in trigger_sql + assert db.get_meta("fts_storage_version") == str(FTS_STORAGE_VERSION) + finally: + db.close() + @@ -4263,6 +4642,63 @@ def test_gateway_session_recovery_does_not_cross_newer_reset_boundary( ) is None +def test_peer_fallback_never_adopts_a_sibling_profiles_row(tmp_path, monkeypatch): + """#74285: the peer-tuple fallback is fenced by the store's own profile. + + A Telegram DM peer tuple (chat_id == user_id, no thread) is identical for + every bot, so a legacy sibling-profile row sitting in this store — written + before the per-profile partition — must lose to the older own row, and + with no own row recovery must return nothing rather than the sibling's. + """ + import hermes_state + + root = tmp_path / "hermes" + root.mkdir() + monkeypatch.setenv("HERMES_HOME", str(root)) + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", hermes_state._IMPORT_DEFAULT_DB_PATH) + store = SessionDB(db_path=root / "state.db") # owner: default + try: + peer = {"user_id": "42", "chat_id": "42", "chat_type": "dm"} + store.create_session("sibling", "telegram", session_key="agent:bot2:telegram:dm:42", + profile_name="bot2", **peer) + store.append_message("sibling", "user", "bot2's conversation") + + def recover(): + return store.find_latest_gateway_session_for_peer( + source="telegram", session_key="agent:main:telegram:dm:42", **peer + ) + + assert recover() is None # only the sibling exists: fail closed + + store.create_session("own", "telegram", session_key="agent:main:telegram:dm:42:old", **peer) + store.append_message("own", "user", "default's conversation") + store._execute_write( + lambda c: c.execute("UPDATE sessions SET last_activity_at = 1 WHERE id = 'own'") + ) + assert recover()["id"] == "own" # older own row beats newer sibling row + finally: + store.close() + + +def test_child_inherits_parent_profile_only_within_its_key_namespace(db): + """#88381: parent→child ``profile_name`` COALESCE is fenced by ``agent::``. + + A default child (``agent:main:``) forked from a sibling profile's row must + not be durably mislabelled as that profile's; same-namespace and keyless + (CLI/subagent) children keep inheriting. + """ + db.create_session("parent", "telegram", session_key="agent:bot2:telegram:dm:42", + profile_name="bot2") + db.create_session("cross", "telegram", parent_session_id="parent", + session_key="agent:main:telegram:dm:42") + db.create_session("same", "telegram", parent_session_id="parent", + session_key="agent:bot2:telegram:dm:42:r2") + db.create_session("keyless", "cli", parent_session_id="parent") + assert db.get_session("cross")["profile_name"] is None + assert db.get_session("same")["profile_name"] == "bot2" + assert db.get_session("keyless")["profile_name"] == "bot2" + + @@ -4577,6 +5013,66 @@ class TestGetMessagesPagination: assert exc_info.value.message_count == 5 assert exc_info.value.limit == 4 + def test_resume_safety_tip_only_counts_the_tip_segment(self, db): + """A deep compression lineage behind a small tip resumes tip-only. + + The Desktop Bot Chat shape: many compaction segments (~29k rows of + lineage) and a small live tip. Callers that never materialize the + ancestors (deferred / omit_messages / lazy resume, tip-only model + restore) must be bounded by the tip alone, and the message must name + the scope it counted. + """ + prev = None + for i in range(6): + sid = f"seg-{i}" + kwargs = {"parent_session_id": prev} if prev else {} + db.create_session(session_id=sid, source="tui", **kwargs) + db.append_messages_batch( + sid, + [{"role": "user", "content": f"{sid}-{j}"} for j in range(4)], + ) + if i < 5: + db.end_session(sid, "compression") + prev = sid + + assert db.get_resume_message_count("seg-5") == 24 + assert db.get_resume_message_count("seg-5", tip_only=True) == 4 + with pytest.raises(hermes_state.SessionResumeTooLargeError) as full: + db.assert_resume_safe("seg-5", max_messages=10) + assert "across its lineage" in str(full.value) + assert db.assert_resume_safe("seg-5", max_messages=10, tip_only=True) == 4 + with pytest.raises(hermes_state.SessionResumeTooLargeError) as tip: + db.assert_resume_safe("seg-5", max_messages=3, tip_only=True) + assert tip.value.message_count == 4 + assert "in its tip segment" in str(tip.value) + + def test_resume_guard_counts_exactly_what_a_branch_resume_loads(self, db): + """An explicit /branch copy owns its transcript: the guard and the + resume readers must agree that its lineage is itself alone.""" + db.create_session(session_id="parent", source="tui") + db.append_messages_batch( + "parent", + [{"role": "user", "content": f"parent-{i}"} for i in range(6)], + ) + db.create_session( + session_id="branch", + source="tui", + parent_session_id="parent", + model_config={"_branched_from": "parent"}, + ) + db.append_messages_batch( + "branch", + [{"role": "user", "content": f"branch-{i}"} for i in range(2)], + ) + + _, display = db.get_resume_conversations("branch") + assert len(display) == 2 + assert db.get_ancestor_display_prefix("branch") == [] + # Before: the guard walked parent_session_id and counted 8, so a branch + # could be refused for rows a resume would never load. + assert db.get_resume_message_count("branch") == 2 + assert db.assert_resume_safe("branch", max_messages=5) == 2 + def test_export_safety_is_bounded_to_the_requested_active_segment(self, db): db.create_session(session_id="root", source="cli") db.append_messages_batch( diff --git a/tests/test_install_ps1_managed_python_provenance.py b/tests/test_install_ps1_managed_python_provenance.py index cefa8bc450..34eeca5737 100644 --- a/tests/test_install_ps1_managed_python_provenance.py +++ b/tests/test_install_ps1_managed_python_provenance.py @@ -18,6 +18,46 @@ pytestmark = pytest.mark.platforms("windows") _INSTALL_PS1 = Path(__file__).resolve().parents[1] / "scripts" / "install.ps1" +def test_fresh_install_manifest_orders_repo_before_checkout_scoped_python( + tmp_path: Path, +) -> None: + powershell = shutil.which("powershell") + if not powershell: + pytest.skip("Windows PowerShell is required") + + install_dir = tmp_path / "install" + run = subprocess.run( + [ + powershell, + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(_INSTALL_PS1), + "-Manifest", + "-HermesHome", + str(tmp_path / "hermes-home"), + "-InstallDir", + str(install_dir), + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + timeout=45, + ) + + assert run.returncode == 0, run.stdout + run.stderr + manifest = json.loads(run.stdout) + stages = [stage["name"] for stage in manifest["stages"]] + assert ( + stages.index("repository") + < stages.index("python") + < stages.index("venv") + ) + assert not install_dir.exists(), "manifest lookup must remain read-only" + + def _run_venv_stage( powershell: str, tmp_path: Path, diff --git a/tests/test_model_tools.py b/tests/test_model_tools.py index a967f61575..9e1fa9886e 100644 --- a/tests/test_model_tools.py +++ b/tests/test_model_tools.py @@ -227,7 +227,7 @@ class TestHandleFunctionCall: class TestAgentLoopTools: def test_expected_tools_in_set(self): - assert "todo" in _AGENT_LOOP_TOOLS + assert "todo_list" in _AGENT_LOOP_TOOLS assert "memory" in _AGENT_LOOP_TOOLS assert "session_search" in _AGENT_LOOP_TOOLS assert "delegate_task" in _AGENT_LOOP_TOOLS diff --git a/tests/test_schema_read_probe.py b/tests/test_schema_read_probe.py index 9770a8f050..a1c0c83978 100644 --- a/tests/test_schema_read_probe.py +++ b/tests/test_schema_read_probe.py @@ -62,6 +62,9 @@ class TestSchemaReadProbeStatements: """ conn = _fresh_schema_conn() try: + # Indexes that depend on the column must be removed before SQLite + # can emulate a pre-column legacy store via DROP COLUMN. + conn.execute("DROP INDEX IF EXISTS idx_sessions_effective_activity") conn.execute("ALTER TABLE sessions DROP COLUMN last_activity_at") # The failure must come from the sessions probe naming the exact # column — not incidentally from some other statement — so a diff --git a/tests/test_session_vacuum_config.py b/tests/test_session_vacuum_config.py index d231996b59..43adfdacba 100644 --- a/tests/test_session_vacuum_config.py +++ b/tests/test_session_vacuum_config.py @@ -8,6 +8,87 @@ def test_default_config_exposes_vacuum_interval(): assert DEFAULT_CONFIG["sessions"]["min_vacuum_interval_days"] == 30 +def test_default_config_auto_prune_on_with_90_day_retention(): + """#54189: state.db retention is ON by default (ended sessions, 90 days).""" + from hermes_cli.config import DEFAULT_CONFIG + + sessions = DEFAULT_CONFIG["sessions"] + assert sessions["auto_prune"] is True + assert sessions["retention_days"] == 90 + assert sessions["vacuum_after_prune"] is True + + +def test_fresh_config_runs_auto_prune_at_startup(monkeypatch, tmp_path: Path): + """A config.yaml with NO ``sessions:`` keys must reach the prune call with the + new defaults (the loader deep-merges DEFAULT_CONFIG).""" + import cli + import hermes_cli.config + import hermes_constants + from hermes_cli.config import DEFAULT_CONFIG + + session_db = MagicMock() + session_db.get_meta.return_value = "already-done" + # Simulate load_config() on a fresh home: only defaults for the section. + monkeypatch.setattr( + hermes_cli.config, + "load_config", + lambda: {"sessions": dict(DEFAULT_CONFIG["sessions"])}, + ) + monkeypatch.setattr(hermes_constants, "get_hermes_home", lambda: tmp_path) + + cli._run_state_db_auto_maintenance(session_db) + + session_db.maybe_auto_prune_and_vacuum.assert_called_once_with( + retention_days=90, + min_interval_hours=24, + min_vacuum_interval_days=30, + vacuum=True, + sessions_dir=tmp_path / "sessions", + ) + + +def test_explicit_auto_prune_false_is_respected(monkeypatch, tmp_path: Path): + """Migration guard: an install that explicitly opted out keeps its choice.""" + import cli + import hermes_cli.config + import hermes_constants + + session_db = MagicMock() + session_db.get_meta.return_value = "already-done" + monkeypatch.setattr( + hermes_cli.config, + "load_config", + lambda: {"sessions": {"auto_prune": False, "retention_days": 90}}, + ) + monkeypatch.setattr(hermes_constants, "get_hermes_home", lambda: tmp_path) + + cli._run_state_db_auto_maintenance(session_db) + + session_db.maybe_auto_prune_and_vacuum.assert_not_called() + + +def test_shipped_template_does_not_pin_sessions_keys(): + """Installers copy cli-config.yaml.example verbatim into config.yaml, so any + uncommented ``sessions:`` value there becomes an EXPLICIT user setting that + would freeze the retention defaults. The template must leave them commented + so code defaults (and future flips) apply.""" + import yaml + + template = Path(__file__).resolve().parents[1] / "cli-config.yaml.example" + data = yaml.safe_load(template.read_text(encoding="utf-8")) or {} + assert "sessions" not in data + + +def test_loader_yields_new_defaults_for_fresh_home(monkeypatch, tmp_path: Path): + """Real load_config() against an empty HERMES_HOME → auto_prune on, 90 days.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from hermes_cli.config import load_config + + sessions = load_config().get("sessions") or {} + assert sessions.get("auto_prune") is True + assert sessions.get("retention_days") == 90 + + def test_cli_auto_maintenance_forwards_vacuum_interval(monkeypatch, tmp_path: Path): import cli import hermes_cli.config diff --git a/tests/test_sqlite_wal_reset_gate.py b/tests/test_sqlite_wal_reset_gate.py index a717b6bb5e..1dafce8115 100644 --- a/tests/test_sqlite_wal_reset_gate.py +++ b/tests/test_sqlite_wal_reset_gate.py @@ -69,6 +69,7 @@ class TestApplyWalWalResetGate: assert mode == "delete" assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "delete" assert any("instead of enabling WAL" in r.getMessage() for r in caplog.records) + assert any(sys.executable in r.getMessage() for r in caplog.records) conn.close() def test_existing_wal_left_alone_when_vulnerable( diff --git a/tests/test_state_db_notadb_fail_closed.py b/tests/test_state_db_notadb_fail_closed.py index fbdf376fef..9fce19e3e1 100644 --- a/tests/test_state_db_notadb_fail_closed.py +++ b/tests/test_state_db_notadb_fail_closed.py @@ -1,9 +1,10 @@ """Tests for fail-closed state.db NOTADB handling and journal-mode EIO retries. -Covers the two independently-valuable pieces salvaged from the state.db -hardening rollup: +Covers: * fail closed when a live write connection reports ``file is not a database``; +* the write-path SQLITE_IOERR retry boundary: admitted only when the callback + has provably not run, never by closing and replaying; * transient ``disk i/o error`` retry in ``_on_disk_journal_mode`` so a one-shot EIO doesn't push callers onto the fail-closed unknown-mode branch. """ @@ -13,7 +14,7 @@ from unittest.mock import MagicMock import pytest -from hermes_state import SessionDB, _on_disk_journal_mode +from hermes_state import SessionDB, StateDbCorruptError, _on_disk_journal_mode class _NotADbOnce: @@ -41,14 +42,100 @@ class TestFailClosedAfterNotADb: reopen = MagicMock() monkeypatch.setattr("hermes_state._connect_tracked_db", reopen) db._conn = _NotADbOnce(real_conn) - with pytest.raises(sqlite3.DatabaseError, match="not a database"): + with pytest.raises(sqlite3.DatabaseError, match="not a database") as excinfo: db.create_session(session_id="s2", source="cli", model="test") reopen.assert_not_called() + # NOTADB on a live write is structural: the handle is quarantined. + assert isinstance(excinfo.value, StateDbCorruptError) + assert db._db_corrupt is True finally: db._conn = real_conn db.close() +class TestWriteIoerrRetryBoundary: + """IOERR retry is admitted by EFFECT POSITION, not error spelling. + + ``_execute_write`` owns non-idempotent transcript/counter mutations, so + replaying its callback is only safe when the first attempt provably did + nothing. SQLite does not define ``SQLITE_IOERR`` as pre-effect-only (an + IOERR at fsync/commit may or may not have landed), so the admission gate + is "did the callback start", not "does the message say disk I/O". + """ + + def test_ioerr_on_begin_retries_because_the_callback_never_ran(self, tmp_path): + db = SessionDB(db_path=tmp_path / "state.db") + real_conn = db._conn + try: + + class _BeginIoerrOnce: + def __init__(self, conn): + self._real = conn + self.begins = 0 + + def execute(self, sql, *args, **kwargs): + if str(sql).strip().upper().startswith("BEGIN") and self.begins == 0: + self.begins += 1 + raise sqlite3.OperationalError("disk I/O error") + return self._real.execute(sql, *args, **kwargs) + + def __getattr__(self, name): + return getattr(self._real, name) + + proxy = _BeginIoerrOnce(real_conn) + db._conn = proxy + db.create_session(session_id="s1", source="cli", model="test") + assert proxy.begins == 1 + finally: + db._conn = real_conn + + rows = db.list_sessions_rich(limit=10, compact_rows=True) + assert [row["id"] for row in rows] == ["s1"] + db.close() + + def test_ioerr_after_the_callback_mutates_does_not_replay(self, tmp_path): + """Settlement is unknown once the callback has run — surface, don't rerun.""" + db = SessionDB(db_path=tmp_path / "state.db") + try: + calls = [] + + def mutate_then_fail(conn): + calls.append(1) + conn.execute( + "INSERT INTO sessions (id, started_at, source) VALUES (?, ?, ?)", + (f"row-{len(calls)}", 1.0, "cli"), + ) + raise sqlite3.OperationalError("disk I/O error") + + with pytest.raises(sqlite3.OperationalError, match="disk I/O error"): + db._execute_write(mutate_then_fail) + + assert calls == [1], "a started write must not be replayed" + assert db.list_sessions_rich(limit=10, compact_rows=True) == [] + finally: + db.close() + + def test_write_ioerr_never_closes_the_connection(self, tmp_path, monkeypatch): + """close() cancels this process's POSIX locks for every sibling fd.""" + db = SessionDB(db_path=tmp_path / "state.db") + try: + closed = [] + monkeypatch.setattr( + type(db._conn), "close", lambda self: closed.append(1), raising=False + ) + + def always_ioerr(conn): + raise sqlite3.OperationalError("disk I/O error") + + with pytest.raises(sqlite3.OperationalError): + db._execute_write(always_ioerr) + + assert closed == [] + assert db._conn is not None + finally: + db.close() + + class TestOnDiskJournalModeEioRetry: def _conn_raising_then(self, failures, result_rows): conn = MagicMock() diff --git a/tests/test_state_db_repair_live_writer_guard.py b/tests/test_state_db_repair_live_writer_guard.py index 8288cb322b..161592e1cb 100644 --- a/tests/test_state_db_repair_live_writer_guard.py +++ b/tests/test_state_db_repair_live_writer_guard.py @@ -18,12 +18,18 @@ live-writer guard.) from __future__ import annotations +import errno +import select import sqlite3 +import subprocess +import sys import uuid from pathlib import Path import pytest +import hermes_state +import hermes_state_holders from hermes_state import ( SessionDB, repair_state_db_schema, @@ -58,18 +64,17 @@ def _make_wal_db(tmp_path: Path) -> Path: def test_repair_refuses_while_another_connection_holds_the_db(tmp_path): """Surgery under concurrent writers is what spread the corruption. - Gated on ``requires_wal``: ``_live_writer_holds_db`` detects an - out-of-process holder via ``PRAGMA locking_mode=EXCLUSIVE`` + a - ``BEGIN IMMEDIATE`` that a concurrent connection makes fail with - SQLITE_BUSY through the WAL index. On SQLite builds carrying the - WAL-reset bug (and on NFS/SMB) Hermes deliberately runs ``state.db`` in - ``journal_mode=DELETE``, where a held reader takes only a SHARED lock and - ``BEGIN IMMEDIATE`` can still acquire RESERVED — so the probe cannot see - the holder and the guard fails open. In DELETE mode repair is instead - serialised only by the cross-process repairer lock (see - ``_live_writer_holds_db``'s docstring). The conftest auto-skips this test - where WAL is unusable rather than assert a guarantee the runtime doesn't - make there. + Gated on ``requires_wal``: repair admission + (``hermes_state_holders.live_writer_holds_db``) first scans for foreign + holders — deleted sidecar generations, uninspectable Hermes processes — + and then probes SQLite with ``PRAGMA locking_mode=EXCLUSIVE`` + + ``BEGIN IMMEDIATE``, which a concurrent connection makes fail with + SQLITE_BUSY through the WAL index. This test exercises the probe leg: on + SQLite builds carrying the WAL-reset bug (and on NFS/SMB) Hermes runs + ``state.db`` in ``journal_mode=DELETE``, where a held reader takes only a + SHARED lock and ``BEGIN IMMEDIATE`` still acquires RESERVED, so the probe + alone cannot see the holder. The conftest auto-skips this test where WAL + is unusable rather than assert a guarantee the probe doesn't make there. """ db = _make_wal_db(tmp_path) @@ -84,6 +89,312 @@ def test_repair_refuses_while_another_connection_holds_the_db(tmp_path): assert "live writer" in (report["error"] or "").lower() +def test_repair_checks_foreign_holders_before_opening_sqlite(tmp_path, monkeypatch): + """A replacement pathname cannot expose locks on the deleted old inode.""" + db = _make_wal_db(tmp_path) + monkeypatch.setattr( + hermes_state_holders, + "foreign_state_db_holders", + lambda _path: [(4242, f"{db}-wal (deleted)")], + ) + + def _unexpected_probe(*_args, **_kwargs): + pytest.fail("repair opened SQLite before excluding foreign holders") + + monkeypatch.setattr(hermes_state, "_connect_repair_durable", _unexpected_probe) + + report = repair_state_db_schema(db, backup=False) + + assert report["repaired"] is False + assert "live writer" in (report["error"] or "").lower() + + +@pytest.mark.linux_only +def test_linux_holder_scan_does_not_require_psutil(tmp_path, monkeypatch): + """The Linux safety scan must not make psutil a repair dependency.""" + monkeypatch.setattr(hermes_state_holders, "psutil", None) + + holders = hermes_state_holders.foreign_state_db_holders( + tmp_path / "absent-state.db" + ) + + assert holders == [] + + +@pytest.mark.linux_only +def test_incomplete_holder_scan_keeps_unknown_sentinel(tmp_path, monkeypatch): + """A partial scan must not hide uncertainty behind an ordinary holder.""" + db = tmp_path / "state.db" + db.touch() + + def _listdir(path): + if path == "/proc": + return ["4242", "4343"] + if path == "/proc/4242/fd": + return ["7"] + if path == "/proc/4343/fd": + raise RuntimeError("scan interrupted") + raise AssertionError(f"unexpected scan path: {path}") + + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + monkeypatch.setattr(hermes_state_holders.os, "readlink", lambda _path: str(db)) + real_stat = hermes_state_holders.os.stat + + def _stat(path, *args, **kwargs): + if path == "/proc/4242/fd/7": + return real_stat(db) + return real_stat(path, *args, **kwargs) + + monkeypatch.setattr(hermes_state_holders.os, "stat", _stat) + + holders = hermes_state_holders.foreign_state_db_holders(db) + + assert (4242, str(db)) in holders + assert any(pid < 0 and "scan interrupted" in path for pid, path in holders) + + +@pytest.mark.linux_only +def test_uninspectable_watched_descriptor_blocks_repair_before_sqlite( + tmp_path, monkeypatch +): + """A watched fd whose identity cannot be read is not proven safe.""" + db = _make_wal_db(tmp_path) + + def _listdir(path): + if path == "/proc": + return ["4242"] + if path == "/proc/4242/fd": + return ["7"] + raise AssertionError(f"unexpected scan path: {path}") + + real_stat = hermes_state_holders.os.stat + + def _stat(path, *args, **kwargs): + if path == "/proc/4242/fd/7": + raise PermissionError(errno.EACCES, "descriptor denied", path) + return real_stat(path, *args, **kwargs) + + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + monkeypatch.setattr(hermes_state_holders.os, "readlink", lambda _path: str(db)) + monkeypatch.setattr(hermes_state_holders.os, "stat", _stat) + + def _unexpected_probe(*_args, **_kwargs): + pytest.fail("repair opened SQLite with unproven descriptor identity") + + monkeypatch.setattr(hermes_state, "_connect_repair_durable", _unexpected_probe) + + report = repair_state_db_schema(db, backup=False) + + assert report["repaired"] is False + assert "live writer" in (report["error"] or "").lower() + + +@pytest.mark.linux_only +@pytest.mark.parametrize( + ("argv", "should_block"), + ( + (["python3", "backup.py"], False), + (["python3", "-m", "hermes_cli.main", "gateway"], True), + ), +) +def test_uninspectable_unknown_descriptor_uses_hermes_identity_at_repair_boundary( + tmp_path, monkeypatch, argv, should_block +): + """An unknown fd target blocks only when argv identifies Hermes.""" + db = _make_wal_db(tmp_path) + + def _listdir(path): + if path == "/proc": + return ["4242"] + if path == "/proc/4242/fd": + return ["7"] + raise AssertionError(f"unexpected scan path: {path}") + + def _readlink(path): + raise PermissionError(errno.EACCES, "descriptor denied", path) + + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + monkeypatch.setattr(hermes_state_holders.os, "readlink", _readlink) + monkeypatch.setattr( + hermes_state_holders, + "_read_proc_argv", + lambda _pid: argv, + ) + + if should_block: + + def _unexpected_probe(*_args, **_kwargs): + pytest.fail("repair opened SQLite with an unproven Hermes descriptor") + + monkeypatch.setattr( + hermes_state, + "_connect_repair_durable", + _unexpected_probe, + ) + report = repair_state_db_schema(db, backup=False) + + assert report["repaired"] is False + assert "live writer" in (report["error"] or "").lower() + else: + real_connect = hermes_state._connect_repair_durable + probe_reached = False + + def _record_probe(*args, **kwargs): + nonlocal probe_reached + probe_reached = True + return real_connect(*args, **kwargs) + + monkeypatch.setattr( + hermes_state, + "_connect_repair_durable", + _record_probe, + ) + report = repair_state_db_schema(db, backup=False) + + assert probe_reached is True + assert "live writer" not in (report["error"] or "").lower() + + +@pytest.mark.linux_only +def test_uninspectable_watched_identity_blocks_alias_before_sqlite( + tmp_path, monkeypatch +): + """A non-disappearance stat error cannot prove an aliased holder safe.""" + db = _make_wal_db(tmp_path) + alias = tmp_path / "namespace-alias" / "state.db" + + def _listdir(path): + if path == "/proc": + return ["4242"] + if path == "/proc/4242/fd": + return ["7"] + raise AssertionError(f"unexpected scan path: {path}") + + real_stat = hermes_state_holders.os.stat + + def _stat(path, *args, **kwargs): + if str(path) == str(db) and not args and not kwargs: + raise PermissionError(errno.EACCES, "watched identity denied", path) + if path == "/proc/4242/fd/7": + return real_stat(db) + return real_stat(path, *args, **kwargs) + + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + monkeypatch.setattr( + hermes_state_holders.os, "readlink", lambda _path: str(alias) + ) + monkeypatch.setattr(hermes_state_holders.os, "stat", _stat) + + def _unexpected_probe(*_args, **_kwargs): + pytest.fail("repair opened SQLite with an unproven watched identity") + + monkeypatch.setattr(hermes_state, "_connect_repair_durable", _unexpected_probe) + + report = repair_state_db_schema(db, backup=False) + + assert report["repaired"] is False + assert "live writer" in (report["error"] or "").lower() + + +@pytest.mark.linux_only +def test_uninspectable_alias_descriptor_for_hermes_blocks_before_sqlite( + tmp_path, monkeypatch +): + """Hermes cannot make an aliased fd safe when its identity is unreadable.""" + db = _make_wal_db(tmp_path) + alias = tmp_path / "namespace-alias" / "state.db" + + def _listdir(path): + if path == "/proc": + return ["4242"] + if path == "/proc/4242/fd": + return ["7"] + raise AssertionError(f"unexpected scan path: {path}") + + real_stat = hermes_state_holders.os.stat + + def _stat(path, *args, **kwargs): + if path == "/proc/4242/fd/7": + raise PermissionError(errno.EACCES, "descriptor denied", path) + return real_stat(path, *args, **kwargs) + + monkeypatch.setattr(hermes_state_holders.os, "listdir", _listdir) + monkeypatch.setattr( + hermes_state_holders.os, "readlink", lambda _path: str(alias) + ) + monkeypatch.setattr(hermes_state_holders.os, "stat", _stat) + monkeypatch.setattr( + hermes_state_holders, + "_read_proc_argv", + lambda _pid: ["python3", "-m", "hermes_cli.main", "gateway"], + ) + + def _unexpected_probe(*_args, **_kwargs): + pytest.fail("repair opened SQLite with an unproven Hermes alias fd") + + monkeypatch.setattr(hermes_state, "_connect_repair_durable", _unexpected_probe) + + report = repair_state_db_schema(db, backup=False) + + assert report["repaired"] is False + assert "live writer" in (report["error"] or "").lower() + + +@pytest.mark.requires_wal +@pytest.mark.linux_only +def test_repair_refuses_while_foreign_process_holds_deleted_wal(tmp_path): + """Reproduce the inode split that a pathname lock probe cannot observe.""" + db = _make_wal_db(tmp_path) + holder_code = """ +import sqlite3 +import sys + +conn = sqlite3.connect(sys.argv[1]) +conn.execute("PRAGMA journal_mode=WAL") +conn.execute("BEGIN IMMEDIATE") +print("ready", flush=True) +sys.stdin.read(1) +conn.rollback() +conn.close() +""" + holder = subprocess.Popen( + [sys.executable, "-c", holder_code, str(db)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert holder.stdout is not None + readable, _, _ = select.select([holder.stdout], [], [], 10) + assert readable, "holder subprocess did not signal readiness" + assert holder.stdout.readline().strip() == "ready" + deleted = [] + for suffix in ("-wal", "-shm"): + sidecar = Path(f"{db}{suffix}") + if sidecar.exists(): + sidecar.unlink() + deleted.append(sidecar) + assert deleted + + report = repair_state_db_schema(db, backup=False) + + assert report["repaired"] is False + assert "live writer" in (report["error"] or "").lower() + finally: + if holder.poll() is None and holder.stdin is not None: + try: + holder.stdin.write("x") + holder.stdin.close() + except (BrokenPipeError, ValueError): + pass + try: + holder.wait(timeout=10) + except subprocess.TimeoutExpired: + holder.kill() + holder.wait(timeout=10) + + def test_repair_proceeds_once_the_database_is_quiescent(tmp_path): """The guard must not deadlock repair on an exclusively-held file.""" db = _make_wal_db(tmp_path) diff --git a/tests/test_state_db_repair_loop_cap.py b/tests/test_state_db_repair_loop_cap.py index 7293d7a641..0eafbcb7a3 100644 --- a/tests/test_state_db_repair_loop_cap.py +++ b/tests/test_state_db_repair_loop_cap.py @@ -82,6 +82,13 @@ class TestPersistentAttemptCap: assert report["repaired"] is False assert "Manual recovery required" in report["error"] assert ".recover" in report["error"] + assert "sessions recover" in report["error"] + assert "sqlite3" in report["error"] + # The terminal error must not direct a raw sqlite3 shell at the + # live database: a WAL-reset-vulnerable CLI (Debian/Ubuntu 3.45.x/ + # 3.46.x, pre-#100368 forensics) unlinks the live WAL/SHM pair + # and splits the store into two generations. + assert 'sqlite3 state.db ".recover"' not in report["error"] assert len(_existing_malformed_backups(db)) == backups_before def test_changed_file_resets_the_budget(self, tmp_path): diff --git a/tests/test_state_db_stats.py b/tests/test_state_db_stats.py index f64ce4821e..8e51cc73ac 100644 --- a/tests/test_state_db_stats.py +++ b/tests/test_state_db_stats.py @@ -236,6 +236,20 @@ def test_render_large_db_legacy_trigram_suggests_optimize(): assert "optimize-storage" in blob +def test_render_large_db_v1_trigram_suggests_optimize(): + from hermes_cli.doctor import STATE_DB_SIZE_WARN_BYTES, _render_state_db_stats + + lines = _render_state_db_stats( + _base_stats( + logical_size_bytes=STATE_DB_SIZE_WARN_BYTES + 1, + fts_storage_version=1, + ), + holders=None, + ) + blob = " ".join(" ".join(str(p) for p in line) for line in lines) + assert "optimize-storage" in blob + + def test_render_does_not_duplicate_legacy_wal_warning(): """A large WAL must NOT warn here: doctor's pre-existing WAL check (50 MB threshold, with a --fix checkpoint) already covers it, and a diff --git a/tests/test_toolsets.py b/tests/test_toolsets.py index 73ce7a9eab..8499335d6a 100644 --- a/tests/test_toolsets.py +++ b/tests/test_toolsets.py @@ -265,7 +265,7 @@ class TestResolveToolsetIncludeRegistry: finally: registry.deregister("__probe_registry_only_tool__") - assert static == {"terminal", "process"}, static + assert static == {"terminal", "process_manage"}, static # Registered into 'terminal' but not part of the static definition — it # must only appear in the merged view. assert "__probe_registry_only_tool__" in merged @@ -275,7 +275,7 @@ class TestResolveToolsetIncludeRegistry: def test_static_view_threads_through_includes(self): # 'debugging' has direct tools [terminal, process] and includes [web, file] static = set(resolve_toolset("debugging", include_registry=False)) - assert {"terminal", "process"} <= static + assert {"terminal", "process_manage"} <= static assert "web_search" in static assert "read_file" in static diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index 11f665f95c..b5ea0fccdc 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -633,7 +633,7 @@ def test_slash_exec_compress_flag_on_applies_host_control_mirror(monkeypatch): def __init__(self): self.controls = [] - def control(self, sid, *, route_name, payload=None, wait=True, timeout=30.0): + def control(self, sid, *, route_name, payload=None, wait=True, timeout=30.0, on_late_ack=None): self.controls.append((sid, route_name, dict(payload or {}), wait)) return { "type": "control.ack", @@ -5534,7 +5534,7 @@ def test_superseded_runtime_finalized_without_reclaimed_broadcast(monkeypatch): # mark it finalized-for-lookup via a different stored key is wrong — # instead simulate the mint race by removing it from lookup). old["_finalized"] = False - monkeypatch.setattr(server, "_find_live_session_by_key", lambda _k: None) + monkeypatch.setattr(server, "_find_live_session_by_key", lambda _k, *_a: None) result = server._claim_or_reuse_live("new-sid", "stored-super", fresh, None) @@ -8504,7 +8504,7 @@ def test_config_set_fast_updates_live_agent_session_scoped(monkeypatch): monkeypatch.setattr(server, "_emit", lambda *args: emits.append(args)) monkeypatch.setattr( "hermes_cli.models.resolve_fast_mode_overrides", - lambda _model_id: {"service_tier": "priority"}, + lambda _model_id, **_route: {"service_tier": "priority"}, ) try: @@ -8583,7 +8583,7 @@ def test_config_set_fast_rejects_unsupported_model(monkeypatch): ) monkeypatch.setattr( "hermes_cli.models.resolve_fast_mode_overrides", - lambda _model_id: None, + lambda _model_id, **_route: None, ) try: @@ -8903,7 +8903,7 @@ def test_enable_gateway_prompts_sets_gateway_env(monkeypatch): def test_setup_status_reports_provider_config(monkeypatch): - monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda: False) + monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda **_kw: False) resp = server.handle_request({"id": "1", "method": "setup.status", "params": {}}) @@ -8925,7 +8925,7 @@ def test_probe_credentials_allows_keyless_custom_runtime(): def test_setup_runtime_check_rejects_empty_runtime_key(monkeypatch): - monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda: True) + monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda **_kw: True) monkeypatch.setattr( "hermes_cli.runtime_provider.resolve_runtime_provider", lambda requested=None: { @@ -8947,7 +8947,7 @@ def test_setup_runtime_check_rejects_empty_runtime_key(monkeypatch): def test_setup_runtime_check_allows_no_key_custom_runtime(monkeypatch): - monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda: True) + monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda **_kw: True) monkeypatch.setattr( "hermes_cli.runtime_provider.resolve_runtime_provider", lambda requested=None: { @@ -8964,7 +8964,7 @@ def test_setup_runtime_check_allows_no_key_custom_runtime(monkeypatch): def test_setup_runtime_check_rejects_implicit_bedrock_when_unconfigured(monkeypatch): - monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda: False) + monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda **_kw: False) monkeypatch.setattr( "hermes_cli.runtime_provider.resolve_runtime_provider", lambda requested=None: { @@ -8982,7 +8982,7 @@ def test_setup_runtime_check_rejects_implicit_bedrock_when_unconfigured(monkeypa def test_setup_runtime_check_honors_requested_provider(monkeypatch): """Onboarding must be able to validate the provider the user just connected.""" - monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda: True) + monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda **_kw: True) def fake_resolve(requested=None, **kwargs): if requested == "nous": @@ -9013,6 +9013,67 @@ def test_setup_runtime_check_honors_requested_provider(monkeypatch): assert default["result"]["provider"] == "anthropic" +def test_setup_readiness_scopes_to_requested_profile(monkeypatch, tmp_path): + """#94071: the Desktop preflights a freshly created bot on its target + backend. ``profile`` binds THAT profile's home + .env — launch-process + credentials must not make an unconfigured bot look ready, and the bot's + own .env must be what the strict check sees.""" + from agent import secret_scope + from hermes_constants import get_hermes_home + + bot_home = tmp_path / "profiles" / "bot" + bot_home.mkdir(parents=True) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-launch-profile-secret-0000") + monkeypatch.setattr("hermes_cli.profiles.profile_exists", lambda name: name == "bot") + monkeypatch.setattr(server, "_profile_home", lambda profile: bot_home if profile == "bot" else None) + seen = {} + + def fake_resolve(requested=None, **kwargs): + seen["home"] = Path(str(get_hermes_home())).resolve() + seen["secret"] = secret_scope.get_secret("OPENROUTER_API_KEY") + return {"provider": "openrouter", "api_key": seen["secret"] or "", "source": "env"} + + monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", fake_resolve) + + secret_scope.set_multiplex_active(True) + try: + status = server.handle_request( + {"id": "1", "method": "setup.status", "params": {"profile": "bot"}} + ) + assert status["result"] == {"provider_configured": False, "profile": "bot"} + + (bot_home / ".env").write_text("OPENROUTER_API_KEY=sk-or-bot-profile-secret-00001\n") + status = server.handle_request( + {"id": "2", "method": "setup.status", "params": {"profile": "bot"}} + ) + runtime = server.handle_request( + {"id": "3", "method": "setup.runtime_check", "params": {"profile": "bot"}} + ) + finally: + secret_scope.set_multiplex_active(False) + + assert status["result"] == {"provider_configured": True, "profile": "bot"} + assert runtime["result"]["ok"] is True + assert runtime["result"]["profile"] == "bot" + assert seen == {"home": bot_home.resolve(), "secret": "sk-or-bot-profile-secret-00001"} + assert Path(str(get_hermes_home())).resolve() != bot_home.resolve() + + +def test_setup_readiness_unknown_profile_never_answers_for_launch_profile(monkeypatch): + monkeypatch.setattr("hermes_cli.main._has_any_provider_configured", lambda **_kw: True) + monkeypatch.setattr( + "hermes_cli.runtime_provider.resolve_runtime_provider", + lambda requested=None, **kw: {"provider": "openrouter", "api_key": "sk-or-launch-0000000000", "source": "env"}, + ) + monkeypatch.setattr("hermes_cli.profiles.profile_exists", lambda name: False) + + for method in ("setup.status", "setup.runtime_check"): + resp = server.handle_request({"id": "1", "method": method, "params": {"profile": "ghost"}}) + assert resp["result"]["ok"] is False + assert resp["result"]["profile"] == "ghost" + assert "does not exist" in resp["result"]["error"] + + def test_complete_slash_drops_removed_provider_alias(): # `/provider` was folded into a single `/model` command, so autocomplete # must no longer offer the dead alias... @@ -10500,7 +10561,7 @@ def test_session_compress_returns_compute_host_history(monkeypatch): } -def test_session_compress_forwards_120_second_budget_to_compute_host(monkeypatch): +def test_session_compress_forwards_config_ceiling_budget_to_compute_host(monkeypatch): session = _session(agent=None, _compute_host_active=True) server._sessions["sid"] = session calls = [] @@ -10519,6 +10580,9 @@ def test_session_compress_forwards_120_second_budget_to_compute_host(monkeypatch monkeypatch.setattr(server, "_session_uses_compute_host", lambda _session: True) monkeypatch.setattr(server, "_send_compute_host_control", send_control) + monkeypatch.setattr( + server, "_load_cfg", lambda: {"compression": {"context_total_ceiling_seconds": 300}} + ) try: resp = server.handle_request( @@ -10528,17 +10592,17 @@ def test_session_compress_forwards_120_second_budget_to_compute_host(monkeypatch server._sessions.pop("sid", None) assert resp["result"]["status"] == "compressed" - assert calls == [ - ( - ("sid",), - { - "route_name": "session.compress", - "command": "/compress", - "wait": True, - "timeout": 120.0, - }, - ) - ] + assert len(calls) == 1 + (sid_arg,), kwargs = calls[0] + assert sid_arg == "sid" + assert kwargs["route_name"] == "session.compress" + assert kwargs["command"] == "/compress" + assert kwargs["wait"] is True + # #97948: the waiter follows compression.context_total_ceiling_seconds + # (+30s slack) instead of a hard-coded 120s, and registers a late-ack + # handler so a compress that outlives it is still adopted. + assert kwargs["timeout"] == 330.0 + assert callable(kwargs["on_late_ack"]) def test_session_compress_preserves_compute_host_aborted_summary(monkeypatch): @@ -14986,6 +15050,73 @@ def test_session_most_recent_honors_params_profile(monkeypatch, tmp_path): assert resp["result"]["session_id"] == "ml-tip" +def test_handoff_request_uses_session_profile_home(monkeypatch, tmp_path): + """Handoff validation must read the owning session's gateway config.""" + import contextlib + + from gateway.config import GatewayConfig, HomeChannel, Platform, PlatformConfig + from hermes_cli.config import get_hermes_home + from tui_gateway import methods_session + + methods_session.register(server) + profile_home = tmp_path / "profiles" / "coder" + profile_home.mkdir(parents=True) + seen_homes = [] + + def load_config(): + home = get_hermes_home() + seen_homes.append(home) + config = GatewayConfig() + if home == profile_home: + config.platforms[Platform.DISCORD] = PlatformConfig( + enabled=True, + home_channel=HomeChannel( + platform=Platform.DISCORD, + chat_id="discord-home", + name="Hermes / #chat-coding", + ), + ) + return config + + class ProfileDB: + def get_session(self, _key): + return {"id": _key} + + def request_handoff(self, _key, platform): + return platform == "discord" + + @contextlib.contextmanager + def profile_db(_session): + yield ProfileDB() + + monkeypatch.setattr("gateway.config.load_gateway_config", load_config) + monkeypatch.setattr(server, "_ensure_session_db_row", lambda _session: None) + monkeypatch.setattr(server, "_session_db", profile_db) + server._sessions["handoff-profile"] = { + "running": False, + "session_key": "desktop-coder-session", + "profile_home": str(profile_home), + } + try: + resp = server.handle_request( + { + "id": "1", + "method": "handoff.request", + "params": { + "session_id": "handoff-profile", + "platform": "discord", + }, + } + ) + finally: + server._sessions.pop("handoff-profile", None) + + assert "result" in resp, resp + assert resp["result"]["queued"] is True + assert seen_homes == [profile_home] + assert get_hermes_home() != profile_home + + def test_session_create_reports_requested_profile_name(monkeypatch, tmp_path): """Issue #62503: session.create info.profile_name must not always be launch.""" profile_home = tmp_path / "profiles" / "mlperf" @@ -15423,6 +15554,312 @@ def test_session_branch_writes_to_parent_profile_db(monkeypatch, tmp_path): server._sessions.pop(k, None) +def test_session_create_persists_seeded_branch_child(monkeypatch): + """A desktop branch (session.create with parent_session_id + seeded + messages) must persist its row + transcript immediately (#93959). + + The renderer re-fetches the fresh child via REST and defer_history + hydration right after create; both read the DB. An unpersisted child + 404s/hydrates empty, the client fail-latch refuses to bind it, and the + user gets an infinite spinner whose optimistic row vanishes on restart. + """ + + class _FakeAgent: + def __init__(self): + self.model = "test-model" + + seen: dict = {} + + class _FakeDB: + def get_session_title(self, key): + seen["parent_title"] = key + return "My Parent Session" + + def get_next_title_in_lineage(self, current): + return f"{current} #2" + + def create_session(self, key, **kwargs): + seen["created"] = key + seen["parent"] = kwargs.get("parent_session_id") + seen["branched_from"] = (kwargs.get("model_config") or {}).get("_branched_from") + + def append_messages_batch(self, session_id, messages, **kwargs): + seen["messages"] = list(messages) + + def set_session_title(self, key, title): + seen["title"] = title + return True + + monkeypatch.setattr(server, "_get_db", lambda: _FakeDB()) + monkeypatch.setattr(server, "_make_agent", lambda sid, key, session_db=None, **_kw: _FakeAgent()) + monkeypatch.setattr(server, "_SlashWorker", lambda *a, **k: None) + monkeypatch.setattr(server, "_session_info", lambda _a, *a2: {"model": "x"}) + monkeypatch.setattr(server, "_probe_credentials", lambda _a: None) + monkeypatch.setattr(server, "_wire_callbacks", lambda _sid: None) + monkeypatch.setattr(server, "_emit", lambda *a, **kw: None) + + import tools.approval as _approval + + monkeypatch.setattr(_approval, "register_gateway_notify", lambda key, cb: None) + monkeypatch.setattr(_approval, "load_permanent_allowlist", lambda: None) + + seeded = [ + {"role": "user", "content": "hello from parent"}, + {"role": "assistant", "content": "parent reply"}, + ] + + resp = server.handle_request( + { + "id": "1", + "method": "session.create", + "params": { + "cols": 96, + "source": "desktop", + "parent_session_id": "20260823_084113_6de211", + "messages": seeded, + }, + } + ) + + assert "result" in resp, resp + key = resp["result"]["stored_session_id"] + + # Row persisted up front with lineage linkage and a lineage title — + # not deferred to the first prompt. + assert seen.get("created") == key + assert seen.get("parent") == "20260823_084113_6de211" + assert seen.get("branched_from") == "20260823_084113_6de211" + assert seen.get("title") == "My Parent Session #2" + + # Seeded transcript copied into the durable row so REST prefetch and + # defer_history hydration both find it immediately. + assert len(seen.get("messages") or []) == 2 + assert seen["messages"][0]["content"] == "hello from parent" + + # The live record no longer queues the title — the DB already holds it. + runtime_sid = resp["result"]["session_id"] + assert server._sessions[runtime_sid]["pending_title"] is None + + server._sessions.pop(runtime_sid, None) + + +def test_session_create_branch_seed_failure_does_not_break_create(monkeypatch): + """Best-effort persistence: a broken DB must not fail session.create.""" + + class _FakeAgent: + def __init__(self): + self.model = "test-model" + + class _BrokenDB: + def get_session_title(self, key): + raise RuntimeError("db down") + + monkeypatch.setattr(server, "_get_db", lambda: _BrokenDB()) + monkeypatch.setattr(server, "_make_agent", lambda sid, key, session_db=None, **_kw: _FakeAgent()) + monkeypatch.setattr(server, "_SlashWorker", lambda *a, **k: None) + monkeypatch.setattr(server, "_session_info", lambda _a, *a2: {"model": "x"}) + monkeypatch.setattr(server, "_probe_credentials", lambda _a: None) + monkeypatch.setattr(server, "_wire_callbacks", lambda _sid: None) + monkeypatch.setattr(server, "_emit", lambda *a, **kw: None) + + import tools.approval as _approval + + monkeypatch.setattr(_approval, "register_gateway_notify", lambda key, cb: None) + monkeypatch.setattr(_approval, "load_permanent_allowlist", lambda: None) + + resp = server.handle_request( + { + "id": "1", + "method": "session.create", + "params": { + "source": "desktop", + "parent_session_id": "parent-1", + "messages": [{"role": "user", "content": "seed"}], + }, + } + ) + + # Create itself still succeeds — the lazy first-prompt path remains as + # the fallback for the seed. + assert "result" in resp + + server._sessions.pop(resp["result"]["stored_session_id"], None) + + +def test_session_create_seed_failure_after_row_compensates(monkeypatch): + """Partial-failure compensation (#93959 review): if the row commits but + the transcript copy fails, the just-created child is DELETED so the lazy + first-prompt fallback can retry cleanly. Without this, a durable empty + row defeats _ensure_session_db_row's INSERT OR IGNORE and the renderer + fail-latches on a transcript-less session again.""" + + class _FakeAgent: + def __init__(self): + self.model = "test-model" + + seen: dict = {} + + class _FakeDB: + def get_session_title(self, key): + return "Parent" + + def get_next_title_in_lineage(self, current): + return f"{current} #2" + + def create_session(self, key, **kwargs): + seen["created"] = key + + def append_messages_batch(self, session_id, messages, **kwargs): + raise RuntimeError("transcript write failed") + + def delete_session(self, session_id): + seen["deleted"] = session_id + return True + + monkeypatch.setattr(server, "_get_db", lambda: _FakeDB()) + monkeypatch.setattr(server, "_make_agent", lambda sid, key, session_db=None, **_kw: _FakeAgent()) + monkeypatch.setattr(server, "_SlashWorker", lambda *a, **k: None) + monkeypatch.setattr(server, "_session_info", lambda _a, *a2: {"model": "x"}) + monkeypatch.setattr(server, "_probe_credentials", lambda _a: None) + monkeypatch.setattr(server, "_wire_callbacks", lambda _sid: None) + monkeypatch.setattr(server, "_emit", lambda *a, **kw: None) + + import tools.approval as _approval + + monkeypatch.setattr(_approval, "register_gateway_notify", lambda key, cb: None) + monkeypatch.setattr(_approval, "load_permanent_allowlist", lambda: None) + + resp = server.handle_request( + { + "id": "1", + "method": "session.create", + "params": { + "source": "desktop", + "parent_session_id": "parent-1", + "title": "My Branch", + "messages": [{"role": "user", "content": "seed"}], + }, + } + ) + + assert "result" in resp + key = resp["result"]["stored_session_id"] + # The half-written child was rolled back — no durable empty row left to + # shadow the lazy seed path. + assert seen.get("deleted") == key + # pending_title survived: it still lands via the lazy post-turn apply. + runtime_sid = resp["result"]["session_id"] + assert server._sessions[runtime_sid]["pending_title"] == "My Branch" + + server._sessions.pop(runtime_sid, None) + + +def test_session_create_seed_disk_full_keeps_row_for_retry(monkeypatch): + """Disk-full is NOT compensated: the row stays (deleting data on a full + disk can make things worse), create still succeeds, and the failure is + observable at warning level (#93959 review).""" + + import logging as _logging + + class _FakeAgent: + def __init__(self): + self.model = "test-model" + + class _FakeDB: + def get_session_title(self, key): + return "Parent" + + def get_next_title_in_lineage(self, current): + return f"{current} #2" + + def create_session(self, key, **kwargs): + pass + + def append_messages_batch(self, session_id, messages, **kwargs): + raise OSError(28, "No space left on device") + + monkeypatch.setattr(server, "_get_db", lambda: _FakeDB()) + monkeypatch.setattr(server, "_make_agent", lambda sid, key, session_db=None, **_kw: _FakeAgent()) + monkeypatch.setattr(server, "_SlashWorker", lambda *a, **k: None) + monkeypatch.setattr(server, "_session_info", lambda _a, *a2: {"model": "x"}) + monkeypatch.setattr(server, "_probe_credentials", lambda _a: None) + monkeypatch.setattr(server, "_wire_callbacks", lambda _sid: None) + monkeypatch.setattr(server, "_emit", lambda *a, **kw: None) + + import tools.approval as _approval + + monkeypatch.setattr(_approval, "register_gateway_notify", lambda key, cb: None) + monkeypatch.setattr(_approval, "load_permanent_allowlist", lambda: None) + + records: list = [] + + class _Capture(_logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=_logging.WARNING) + root = _logging.getLogger() + root.addHandler(handler) + try: + resp = server.handle_request( + { + "id": "1", + "method": "session.create", + "params": { + "source": "desktop", + "parent_session_id": "parent-1", + "messages": [{"role": "user", "content": "seed"}], + }, + } + ) + finally: + root.removeHandler(handler) + + assert "result" in resp + # The failure surfaced at WARNING (observable), not buried at debug. + warnings = [r for r in records if r.levelno >= _logging.WARNING] + assert any("seeded-branch persistence failed" in r.getMessage() for r in warnings) + + server._sessions.pop(resp["result"]["stored_session_id"], None) + + +def test_session_create_without_parent_still_defers_row(monkeypatch): + """Plain drafts keep the lazy-row contract: no parent + no explicit branch + intent means no eager persistence (the original draft-hygiene invariant).""" + + class _FakeAgent: + def __init__(self): + self.model = "test-model" + + calls: dict = {"create": 0} + + class _FakeDB: + def create_session(self, *a, **k): + calls["create"] += 1 + + monkeypatch.setattr(server, "_get_db", lambda: _FakeDB()) + monkeypatch.setattr(server, "_make_agent", lambda sid, key, session_db=None, **_kw: _FakeAgent()) + monkeypatch.setattr(server, "_SlashWorker", lambda *a, **k: None) + monkeypatch.setattr(server, "_session_info", lambda _a, *a2: {"model": "x"}) + monkeypatch.setattr(server, "_probe_credentials", lambda _a: None) + monkeypatch.setattr(server, "_wire_callbacks", lambda _sid: None) + monkeypatch.setattr(server, "_emit", lambda *a, **kw: None) + + import tools.approval as _approval + + monkeypatch.setattr(_approval, "register_gateway_notify", lambda key, cb: None) + monkeypatch.setattr(_approval, "load_permanent_allowlist", lambda: None) + + resp = server.handle_request( + {"id": "1", "method": "session.create", "params": {"cols": 80}} + ) + sid = resp["result"]["session_id"] + server._sessions[sid]["agent_ready"].wait(timeout=2.0) + + assert calls["create"] == 0, "plain drafts must not persist eagerly" + + server._sessions.pop(sid, None) + def test_session_branch_installs_parent_profile_secret_scope(monkeypatch, tmp_path): """The branched agent must be built under the parent profile's secrets. diff --git a/tests/tools/file_ops_fakes.py b/tests/tools/file_ops_fakes.py new file mode 100644 index 0000000000..4cbb2d992b --- /dev/null +++ b/tests/tools/file_ops_fakes.py @@ -0,0 +1,61 @@ +"""Fakes for ``ShellFileOperations``' compound shell probes. + +``read_file`` and ``write_file`` ask the shell everything in ONE command +whose stdout is split on a per-call random sentinel line. Test doubles that +script ``env.execute`` / ``_exec`` need to answer that command with exactly +the stream the shell would produce; these helpers build it. Match the +sentinel out of the command first (it is random), then compose: + + m = READ_SENTINEL_RE.search(command) + if m: + return {"output": compound_read_output(m.group(0), size=5, sample=b"hello", + content="hello\\n", total_lines=1), + "returncode": 0} +""" + +import base64 +import re +from typing import Optional + +READ_SENTINEL_RE = re.compile(r"__HERMES_RF_[0-9a-f]{32}__") +WRITE_SENTINEL_RE = re.compile(r"__HERMES_WF_[0-9a-f]{32}__") + + +def compound_read_output( + sentinel: str, + *, + size: int, + sample: Optional[bytes], + content: str, + total_lines: int, + trailing_newline: bool = True, + sample_rc: int = 0, + read_rc: int = 0, +) -> str: + """Stdout of ``_read_probe_cmd`` for a regular file. + + ``content`` is the ``sed | cut`` page exactly as the shell prints it: + every line newline-terminated (``cut`` always adds one), or ``""`` for a + page past EOF. ``sample`` is the raw first-1000-bytes slice (``None`` + emits an empty base64 segment, e.g. a shell without ``base64``). + """ + sample_seg = base64.b64encode(sample).decode() + "\n" if sample else "" + return ( + f"{size}\n{sentinel}\n" + f"{sample_seg}{sentinel}\n" + f"{content}{sentinel}\n" + f"{total_lines}\n{sentinel}\n" + f"{1 if trailing_newline else 0}\n{sentinel}\n" + f"{sample_rc} {read_rc}\n" + ) + + +def compound_write_probe_output(sentinel: str, *, head3: bytes, body: str) -> str: + """Stdout of ``_write_probe_cmd`` for an existing file. + + ``head3`` is the first three bytes on disk (BOM detection); ``body`` is + the second segment: the whole file when pre-content was wanted, else + the 4 KB line-ending sample. + """ + head_seg = base64.b64encode(head3).decode() + "\n" if head3 else "" + return f"{head_seg}{sentinel}\n{body}" diff --git a/tests/tools/test_bot_mode_dm.py b/tests/tools/test_bot_mode_dm.py index 670dc2062c..69eb9830c5 100644 --- a/tests/tools/test_bot_mode_dm.py +++ b/tests/tools/test_bot_mode_dm.py @@ -422,6 +422,29 @@ def test_delivery_runner_preserves_child_failure_and_unlinks(tmp_path): assert not dm_file.exists() +def test_delivery_runner_surfaces_live_owner_refusal(tmp_path, capsys): + """#100523: the CLI's single-owner lease refusal is a delivery FAILURE the + sender can read, not a raw exit-1 with the payload silently gone.""" + dm_file = tmp_path / "message.txt" + dm_file.write_text("hi", encoding="utf-8") + child = tmp_path / "owned.py" + child.write_text( + "import sys\n" + "print('Session abc already has a live owner (desktop, pid 1).', file=sys.stderr)\n" + "raise SystemExit(1)\n", + encoding="utf-8", + ) + + returncode = bot_mode_dm._run_delivery( + [sys.executable, str(child), "-p", "ops"], str(dm_file), stdin_file=False + ) + + assert returncode == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["reason"] == "target_busy" + assert "NOT delivered" in payload["error"] + + def test_query_file_delivery_closes_stdin_for_initial_attempt_and_retry( tmp_path, monkeypatch ): diff --git a/tests/tools/test_bot_relay.py b/tests/tools/test_bot_relay.py index b7b9f4a37f..5ec1322578 100644 --- a/tests/tools/test_bot_relay.py +++ b/tests/tools/test_bot_relay.py @@ -84,6 +84,9 @@ def test_resolve_remote_target_forms(root): assert bot_relay.resolve_remote_target("default", roster)["connection_id"] == "cloud-1" # exact connection-qualified form assert bot_relay.resolve_remote_target("hermes@cloud-1", roster)["profile"] == "default" + # profile@connection — the form Desktop's mention middleware annotates + # for remote bots (#97678); the UI alias form must not be required + assert bot_relay.resolve_remote_target("default@cloud-1", roster)["profile"] == "default" assert bot_relay.resolve_remote_target("hermes@nope", roster) is None assert bot_relay.resolve_remote_target("ghost", roster) is None @@ -151,6 +154,31 @@ def test_waiter_command_quotes_and_targets_reply_file(root): assert "rm -rf" not in cmd # sanity: single quoted -c payload +def test_waiter_picks_up_reply_within_a_sub_second_cadence(root): + """The reply file is written once; the waiter must notice it fast, not + on a multi-second sleep (dead air the sender's completion notification + inherits on every cross-machine reply).""" + import shlex + import subprocess + import threading + import time + + env = {"id": "c" * 32, "target_handle": "researcher", "target_connection": "ssh-vps"} + reply_path = bot_relay.relay_root(root) / bot_relay.REPLIES_DIR / f"{env['id']}.json" + reply_path.parent.mkdir(parents=True, exist_ok=True) + + def write_reply(): + time.sleep(0.3) + reply_path.write_text(json.dumps({"reply": "pong"}), encoding="utf-8") + + threading.Thread(target=write_reply, daemon=True).start() + started = time.monotonic() + proc = subprocess.run(shlex.split(bot_relay.waiter_command(root, env)), capture_output=True, text=True, timeout=10) + elapsed = time.monotonic() - started + assert proc.returncode == 0 and "pong" in proc.stdout + assert elapsed < 1.5, f"waiter took {elapsed:.2f}s to notice a reply written at 0.3s" + + def test_roster_rejects_connection_id_outside_handle_charset(root): bad = [ {"profile": "researcher", "handle": "researcher", "connection_id": "vps'); print(1)"}, diff --git a/tests/tools/test_browser_cleanup.py b/tests/tools/test_browser_cleanup.py index 6c929da628..b1f89b3c84 100644 --- a/tests/tools/test_browser_cleanup.py +++ b/tests/tools/test_browser_cleanup.py @@ -83,3 +83,106 @@ class TestBrowserCleanup: assert browser_tool._session_last_activity == {} assert browser_tool._recording_sessions == set() assert browser_tool._cleanup_done is True + + +class TestInactivityJanitorMultiplex: + """#86402 / #100738: the process-global janitor thread has no profile scope.""" + + def setup_method(self): + from agent import secret_scope + from tools import browser_tool + + self.bt = browser_tool + self.saved = { + name: getattr(browser_tool, name).copy() + for name in ( + "_active_sessions", "_session_last_activity", + "_session_owner_homes", "_cleanup_failures", "_recording_sessions", + ) + } + self.orig_timeout = browser_tool.BROWSER_SESSION_INACTIVITY_TIMEOUT + browser_tool.BROWSER_SESSION_INACTIVITY_TIMEOUT = 0 + for name in self.saved: + getattr(browser_tool, name).clear() + secret_scope.set_multiplex_active(True) + + def teardown_method(self): + from agent import secret_scope + + secret_scope.set_multiplex_active(False) + self.bt.BROWSER_SESSION_INACTIVITY_TIMEOUT = self.orig_timeout + for name, saved in self.saved.items(): + live = getattr(self.bt, name) + live.clear() + live.update(saved) + + def test_janitor_tears_down_under_owner_profile_scope(self, tmp_path, monkeypatch): + from agent import secret_scope + from hermes_constants import ( + get_hermes_home, reset_hermes_home_override, set_hermes_home_override, + ) + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("CAMOFOX_URL", raising=False) + monkeypatch.delenv("BROWSER_CDP_URL", raising=False) + p1 = tmp_path / "profiles" / "p1" + p1.mkdir(parents=True) + (p1 / ".env").write_text("CAMOFOX_URL=http://127.0.0.1:1\n") + + # Profile p1's turn opens the session; the janitor later runs unscoped. + home_tok = set_hermes_home_override(str(p1)) + scope_tok = secret_scope.set_secret_scope(secret_scope.build_profile_secret_scope(p1)) + try: + self.bt._update_session_activity("t1") + self.bt._active_sessions["t1"] = {"session_name": "s1", "bb_session_id": None} + finally: + secret_scope.reset_secret_scope(scope_tok) + reset_hermes_home_override(home_tok) + self.bt._session_last_activity["t1"] -= 10 + + seen = {} + + def fake_close(task_id, cmd, args, timeout=None): + seen["home"] = str(get_hermes_home()) + seen["url"] = secret_scope.get_secret("CAMOFOX_URL") + return {"success": True} + + with ( + patch("tools.browser_tool._run_browser_command", side_effect=fake_close), + patch("tools.browser_camofox._delete", return_value={}), + patch("tools.browser_tool.os.path.exists", return_value=False), + ): + self.bt._cleanup_inactive_browser_sessions() + + assert seen == {"home": str(p1), "url": "http://127.0.0.1:1"} + assert "t1" not in self.bt._session_last_activity + assert "t1" not in self.bt._active_sessions + assert "t1" not in self.bt._session_owner_homes + + def test_repeated_failures_force_reap_and_close_cloud_session(self): + from unittest.mock import MagicMock + + self.bt._active_sessions["t1"] = {"session_name": "s1", "bb_session_id": "bb-1"} + self.bt._session_last_activity["t1"] = 1.0 + provider = MagicMock() + + with ( + patch("tools.browser_tool.cleanup_browser", side_effect=RuntimeError("boom")), + patch("tools.browser_tool._get_cloud_provider", return_value=provider), + patch("tools.browser_tool.os.path.exists", return_value=False), + ): + for _ in range(self.bt.MAX_INACTIVITY_CLEANUP_FAILURES - 1): + self.bt._cleanup_inactive_browser_sessions() + # An activity touch must NOT reset the failure budget. + self.bt._update_session_activity("t1") + self.bt._session_last_activity["t1"] = 1.0 + assert self.bt._cleanup_failures["t1"] == self.bt.MAX_INACTIVITY_CLEANUP_FAILURES - 1 + assert "t1" in self.bt._active_sessions + provider.close_session.assert_not_called() + + self.bt._cleanup_inactive_browser_sessions() + + provider.close_session.assert_called_once_with("bb-1") + assert "t1" not in self.bt._active_sessions + assert "t1" not in self.bt._session_last_activity + assert "t1" not in self.bt._cleanup_failures diff --git a/tests/tools/test_browser_lightpanda_serve.py b/tests/tools/test_browser_lightpanda_serve.py index 73200bac14..db607ec77b 100644 --- a/tests/tools/test_browser_lightpanda_serve.py +++ b/tests/tools/test_browser_lightpanda_serve.py @@ -5,12 +5,17 @@ import json import os import stat import subprocess +from pathlib import Path from unittest.mock import patch import pytest import tools.browser_lightpanda as lp +# The autouse _isolate fixture swaps _binary_supports_http_cache for a lambda; +# the probe test needs the real (lru_cache-wrapped) function back. +_real_probe = lp._binary_supports_http_cache + class FakeProc: def __init__(self, pid=4242, exit_code=None): @@ -42,6 +47,8 @@ def _isolate(tmp_path, monkeypatch): # Never touch the developer's real ~/.local/bin/lightpanda. monkeypatch.setattr(lp, "_home_candidates", lambda: []) monkeypatch.setattr(lp, "_safe_start_time", lambda pid: 111) + lp._binary_supports_http_cache.cache_clear() + monkeypatch.setattr(lp, "_binary_supports_http_cache", lambda binary: True) with lp._servers_lock: lp._servers.clear() yield state @@ -112,6 +119,7 @@ class TestLaunch: assert err is None assert calls["argv"] == [ "/opt/lightpanda", "serve", "--host", "127.0.0.1", "--port", "43111", + "--http-cache-dir", str(_isolate / "http-cache"), ] kw = calls["kwargs"] assert kw["stdin"] is subprocess.DEVNULL @@ -129,6 +137,57 @@ class TestLaunch: assert record["owner_pid"] == os.getpid() assert record["start_time"] == 111 + def test_http_cache_dir_is_shared_across_sessions(self, monkeypatch, _isolate): + _, _, first = self._launch(monkeypatch) + with lp._servers_lock: + lp._servers.clear() + _, _, second = self._launch(monkeypatch) + cache = str(_isolate / "http-cache") + assert first["argv"][first["argv"].index("--http-cache-dir") + 1] == cache + assert second["argv"][second["argv"].index("--http-cache-dir") + 1] == cache + assert Path(cache).is_dir() + assert not list(Path(cache).glob("*.json")) # never confused with a session record + + def test_no_http_cache_flag_on_old_binary(self, monkeypatch, _isolate): + monkeypatch.setattr(lp, "_binary_supports_http_cache", lambda binary: False) + _, err, calls = self._launch(monkeypatch) + assert err is None + assert "--http-cache-dir" not in calls["argv"] + assert calls["argv"][-1] == "43111" + + def test_http_cache_probe_caches_and_detects_flag(self, monkeypatch, tmp_path): + monkeypatch.setattr(lp, "_binary_supports_http_cache", _real_probe) + _real_probe.cache_clear() + exe = _exe(tmp_path / "lightpanda") + runs = [] + + def fake_run(argv, **kwargs): + runs.append(argv) + return subprocess.CompletedProcess( + argv, returncode=0, + stdout="--http-cache-dir " if len(runs) == 1 else "", + ) + + monkeypatch.setattr(lp.subprocess, "run", fake_run) + assert lp._binary_supports_http_cache(str(exe)) is True + assert lp._binary_supports_http_cache(str(exe)) is True # cached, single probe + assert len(runs) == 1 + + lp._binary_supports_http_cache.cache_clear() + + def fake_run_old(argv, **kwargs): + return subprocess.CompletedProcess(argv, returncode=0, stdout="no such flag") + + monkeypatch.setattr(lp.subprocess, "run", fake_run_old) + assert lp._binary_supports_http_cache(str(exe)) is False + + def fake_run_hangs(argv, **kwargs): + raise subprocess.TimeoutExpired(cmd=argv, timeout=3.0) + + lp._binary_supports_http_cache.cache_clear() + monkeypatch.setattr(lp.subprocess, "run", fake_run_hangs) + assert lp._binary_supports_http_cache(str(exe)) is False + def test_block_private_networks_flag(self, monkeypatch): _, err, calls = self._launch(monkeypatch, block_private_networks=True) assert err is None diff --git a/tests/tools/test_code_execution.py b/tests/tools/test_code_execution.py index 2b32f2a026..25ccebd825 100644 --- a/tests/tools/test_code_execution.py +++ b/tests/tools/test_code_execution.py @@ -458,10 +458,11 @@ class TestStubSchemaDrift(unittest.TestCase): compile(src, "hermes_tools.py", "exec") # Verify specific parameter signatures are in the source - # search_files must accept context, offset, output_mode + # search_files must accept its pagination, output, and ordering controls self.assertIn("context", src) self.assertIn("offset", src) self.assertIn("output_mode", src) + self.assertIn("order", src) # patch must accept mode and patch params self.assertIn("mode", src) diff --git a/tests/tools/test_code_kernel.py b/tests/tools/test_code_kernel.py index 19f3179044..c3dbc29203 100644 --- a/tests/tools/test_code_kernel.py +++ b/tests/tools/test_code_kernel.py @@ -18,9 +18,15 @@ tests patch ``_load_config`` directly, mirroring test_code_execution_modes. import json import os +import shutil +import subprocess import sys +import tempfile +import textwrap +import time import unittest from contextlib import contextmanager +from pathlib import Path from unittest.mock import patch import pytest @@ -121,6 +127,51 @@ class TestSessionStatePersistence(unittest.TestCase): class TestKernelLifecycle(unittest.TestCase): + def test_kernel_exits_when_its_backend_parent_dies(self): + """A kernel must not outlive the host that spawned it, even when the + host dies without cleanup (SIGKILL/OOM/crash). Windows: inherited + SYNCHRONIZE handle; POSIX: inherited death pipe. Both are proven the + same way — kill the host mid-cell, the kernel is gone within seconds.""" + import psutil + + repo_root = str(Path(__file__).resolve().parents[2]) + host_src = textwrap.dedent(f""" + import json, os, sys, time + os.environ["HERMES_HOME"] = sys.argv[1] + sys.path.insert(0, {repo_root!r}) + from tools.code_kernel import SessionKernel, _spawn + k = SessionKernel(("parent-death",)) + _spawn(k, task_id="parent-death", child_python=sys.executable, + child_cwd="", sandbox_tools=frozenset(), max_tool_calls=1) + cell = json.dumps({{"id": "x", "code": "import os, time\\n" + "assert 'HERMES_KERNEL_PARENT_PROCESS_HANDLE' not in os.environ\\n" + "assert 'HERMES_KERNEL_PARENT_DEATH_FD' not in os.environ\\n" + "time.sleep(300)"}}) + "\\n" + k.proc.stdin.write(cell.encode()); k.proc.stdin.flush() + print(k.proc.pid, flush=True) + time.sleep(600) + """) + with tempfile.TemporaryDirectory() as home: + host = subprocess.Popen( + [sys.executable, "-c", host_src, home], + stdout=subprocess.PIPE, text=True, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + try: + kernel = psutil.Process(int(host.stdout.readline())) + time.sleep(0.5) + self.assertTrue(kernel.is_running(), "kernel never came up") + host.kill() + host.wait(timeout=10) + try: + kernel.wait(timeout=10) + except psutil.TimeoutExpired: + kernel.kill() + self.fail("session kernel survived its backend parent") + finally: + if host.poll() is None: + host.kill() + def test_timeout_kills_the_kernel_and_reports_state_loss(self): with _kernel_config(timeout=1): slow = _run("import time\ntime.sleep(30)") @@ -307,6 +358,31 @@ class TestKernelOwnershipAndLifecycle(unittest.TestCase): stale.proc.wait(timeout=10) self.assertFalse(stale.alive()) + def test_parallel_cells_share_one_kernel_process(self): + """Parallel cells for one owner race the first spawn. Each racer + used to see proc=None as 'dead', replace the registry entry, and + orphan the winner's process — 110 live kernels under a 4-capped + process (Sep 2026). Every kernel process must stay registry-owned.""" + import subprocess + import threading + + results = [] + with _kernel_config(): + def _cell(): + results.append(self._run_as("conv-a", "import time; time.sleep(0.3)", task_id="t")) + threads = [threading.Thread(target=_cell) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual([r["status"] for r in results], ["success"] * 6) + self.assertEqual(len(_KERNELS), 1) + live = subprocess.run( + ["pgrep", "-fc", "-P", str(os.getpid()), "hermes_kernel_runner"], + capture_output=True, text=True, + ).stdout.strip() + self.assertEqual(live, "1") + class TestPerCellRpcAuthority(unittest.TestCase): """Interpreter state persists across cells; RPC authority must not.""" diff --git a/tests/tools/test_code_kernel_remote.py b/tests/tools/test_code_kernel_remote.py index 9b2a1343d2..bf8470dcc2 100644 --- a/tests/tools/test_code_kernel_remote.py +++ b/tests/tools/test_code_kernel_remote.py @@ -200,6 +200,81 @@ class TestOwnershipIsolation(RemoteKernelBase): self.assertEqual(remaining_owner, "owner-b") +class TestIdleReapAndCapEviction(RemoteKernelBase): + """Unlike local session kernels, remote kernels had no idle-reap or + process-wide cap: _REMOTE_KERNELS grew one entry per distinct + (owner, env_type, task_env_id) that was never revisited, for the life + of the gateway process.""" + + def test_idle_expired_kernel_is_reaped_on_next_call(self): + env = ScriptedEnv(_spawn_ok_handlers([_cell(), _cell()])) + execute_in_remote_kernel( + "print(1)", env=env, env_type="ssh", task_env_id="stale", + sandbox_tools=frozenset(), timeout=10, max_tool_calls=5, + reset=False, idle_exit=1800, + ) + self.assertEqual(len(_REMOTE_KERNELS), 1) + # Backdate the kernel's last_used past the idle window — simulates + # a key that is never revisited again. + for kernel in _REMOTE_KERNELS.values(): + kernel.last_used -= 2000 + # A call for a DIFFERENT key must reap the stale entry on entry, + # without ever touching or reviving it. + execute_in_remote_kernel( + "print(1)", env=env, env_type="ssh", task_env_id="fresh", + sandbox_tools=frozenset(), timeout=10, max_tool_calls=5, + reset=False, idle_exit=1800, + ) + owners = {key[0] for key in _REMOTE_KERNELS} + self.assertNotIn("stale", owners) + self.assertIn("fresh", owners) + + def test_over_cap_evicts_least_recently_used(self): + with patch("tools.code_kernel._lifecycle_limits", return_value=(2, 1800)): + env = ScriptedEnv(_spawn_ok_handlers([_cell() for _ in range(10)])) + for i in range(3): + execute_in_remote_kernel( + "print(1)", env=env, env_type="ssh", task_env_id=f"owner-{i}", + sandbox_tools=frozenset(), timeout=10, max_tool_calls=5, + reset=False, idle_exit=1800, + ) + self.assertEqual(len(_REMOTE_KERNELS), 2) + owners = {key[0] for key in _REMOTE_KERNELS} + self.assertNotIn("owner-0", owners) + self.assertIn("owner-1", owners) + self.assertIn("owner-2", owners) + + def test_eviction_skips_kernels_with_a_running_cell(self): + """Cap eviction must never kill a kernel mid-cell (the local-kernel + race from hermes-agent#101861): a busy kernel stays put and a + settled one goes instead, even if the busy one is older.""" + import threading + + gate = threading.Event() + + def slow_cat(command): + gate.wait(10) + return {"output": json.dumps(_cell()), "returncode": 0} + + busy_env = ScriptedEnv([ + ("nohup", lambda c: {"output": "PID:4242\n", "returncode": 0}), + ("kill -0", lambda c: {"output": "ALIVE\n", "returncode": 0}), + ("cat ", slow_cat), + ]) + with patch("tools.code_kernel._lifecycle_limits", return_value=(1, 1800)): + worker = threading.Thread(target=_run, args=(busy_env,), kwargs={"task": "busy"}) + worker.start() + while not any(k.attached for k in _REMOTE_KERNELS.values()): + pass + env = ScriptedEnv(_spawn_ok_handlers([_cell()])) + _run(env, task="settled") + owners = {key[0] for key in _REMOTE_KERNELS} + self.assertIn("busy", owners) + gate.set() + worker.join(10) + self.assertFalse(any("kill 4242" in c for c in busy_env.commands)) + + class TestDispatchIntegration(unittest.TestCase): """_execute_remote prefers the kernel and falls open to per-call.""" diff --git a/tests/tools/test_credential_files.py b/tests/tools/test_credential_files.py index 11e04b8bcb..20bc21e68e 100644 --- a/tests/tools/test_credential_files.py +++ b/tests/tools/test_credential_files.py @@ -113,6 +113,34 @@ class TestSkillsDirectoryMount: # Symlink should NOT be present assert not (safe_path / "evil_link").exists() + def test_sanitized_copy_skips_bookkeeping_dirs(self, tmp_path): + """The symlink-safe copy is what gets mounted, so it must apply the + same EXCLUDED_SKILL_DIRS rule as the per-file sync path.""" + hermes_home = tmp_path / ".hermes" + skills_dir = hermes_home / "skills" + (skills_dir / "cat" / "myskill" / "references").mkdir(parents=True) + (skills_dir / "cat" / "myskill" / "SKILL.md").write_text("# skill") + (skills_dir / "cat" / "myskill" / "references" / "api.md").write_text("ref") + for excluded in (".hub", ".curator_backups", "node_modules"): + junk = skills_dir / excluded / "vendored" + junk.mkdir(parents=True) + (junk / "blob.bin").write_bytes(b"\0" * 64) + # Force the sanitizing copy path. + secret = tmp_path / "secret.txt" + secret.write_text("TOP SECRET") + (skills_dir / "evil_link").symlink_to(secret) + + with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}): + mounts = get_skills_directory_mount() + + safe_path = Path(mounts[0]["host_path"]) + assert safe_path != skills_dir + assert (safe_path / "cat" / "myskill" / "SKILL.md").exists() + assert (safe_path / "cat" / "myskill" / "references" / "api.md").exists() + assert not (safe_path / "evil_link").exists() + for excluded in (".hub", ".curator_backups", "node_modules"): + assert not (safe_path / excluded).exists(), excluded + def test_no_symlinks_returns_original_dir(self, tmp_path): """When no symlinks exist, the original dir is returned (no copy).""" hermes_home = tmp_path / ".hermes" @@ -149,6 +177,46 @@ class TestIterSkillsFiles: # Symlink should be excluded assert not any("evil" in f["container_path"] for f in files) + def test_skips_excluded_bookkeeping_dirs(self, tmp_path): + """Bookkeeping and dependency dirs must not be uploaded to a sandbox. + + The sync path used a bare rglob("*"), so the .hub download cache, + .archive, curator backups and any node_modules/.git under a skills + tree were packed up on every sync even though the sandbox never + reads them. Sync now honours EXCLUDED_SKILL_DIRS like discovery. + """ + hermes_home = tmp_path / ".hermes" + skills_dir = hermes_home / "skills" + (skills_dir / "cat" / "myskill").mkdir(parents=True) + (skills_dir / "cat" / "myskill" / "SKILL.md").write_text("# skill") + # Progressive-disclosure support files must still be synced. + (skills_dir / "cat" / "myskill" / "references").mkdir() + (skills_dir / "cat" / "myskill" / "references" / "api.md").write_text("ref") + + for excluded in (".hub", ".archive", ".curator_backups", "node_modules"): + junk = skills_dir / excluded / "vendored" + junk.mkdir(parents=True) + (junk / "SKILL.md").write_text("# stale copy") + # Also nested inside an otherwise-valid skill package. + cache = skills_dir / "cat" / "myskill" / "__pycache__" + cache.mkdir() + (cache / "helper.cpython-311.pyc").write_text("bytecode") + + with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}): + files = iter_skills_files() + + paths = {f["container_path"] for f in files} + assert "/root/.hermes/skills/cat/myskill/SKILL.md" in paths + assert "/root/.hermes/skills/cat/myskill/references/api.md" in paths + for excluded in ( + ".hub", + ".archive", + ".curator_backups", + "node_modules", + "__pycache__", + ): + assert not any(excluded in path for path in paths), excluded + def test_empty_when_no_skills_dir(self, tmp_path): hermes_home = tmp_path / ".hermes" hermes_home.mkdir() diff --git a/tests/tools/test_cronjob_run_background.py b/tests/tools/test_cronjob_run_background.py index a35d4c4a8f..c120273c03 100644 --- a/tests/tools/test_cronjob_run_background.py +++ b/tests/tools/test_cronjob_run_background.py @@ -237,6 +237,28 @@ class TestInFlightDedupe: assert seen_during_run["registered"] is True assert "job-bg-09" not in sched.get_running_job_ids() # released after + def test_run_claimed_job_reports_exact_unknown_execution_not_stale_success(self): + from tools.cronjob_tools import _run_claimed_job + + def probe_run(job, **_kwargs): + job["execution_id"] = "exec-unknown" + return True + + with patch("cron.scheduler.run_one_job", side_effect=probe_run), \ + patch("cron.executions.get_execution", return_value={ + "id": "exec-unknown", + "status": "unknown", + "error": "worker owner exited", + }), \ + patch("tools.cronjob_tools.get_job", return_value={ + "last_status": "ok", + "last_error": None, + }): + res = _run_claimed_job(_job("job-bg-unknown")) + + assert res["success"] is False + assert res["error"] == "worker owner exited" + def test_background_dispatch_reports_running_job_immediately(self): """The dispatch path pre-checks the running set so a mid-run job reports in the tool response, not as a delayed completion event.""" diff --git a/tests/tools/test_cronjob_run_delivery_notice.py b/tests/tools/test_cronjob_run_delivery_notice.py new file mode 100644 index 0000000000..e82b2ad090 --- /dev/null +++ b/tests/tools/test_cronjob_run_delivery_notice.py @@ -0,0 +1,274 @@ +"""Honesty of the manual-run delivery notice (issue #83993). + +A manual ``cronjob(action='run')`` finishes with a completion summary line + + Delivery target: (output was delivered there by the job itself) + +that was appended UNCONDITIONALLY for non-local targets — even when +``run_one_job`` had just written ``last_delivery_error`` onto the refreshed +job record because the post-run delivery (telegram/discord/…) failed. The +calling agent then relayed "all good" over a failed delivery. + +The note must follow the refreshed job record: a set ``last_delivery_error`` +means delivery FAILED with the error text surfaced; an empty/missing error +keeps the legacy wording byte-for-byte (zero regression), and local jobs +always say saved-locally. +""" + +import contextlib +import time +from unittest.mock import patch + +import pytest + +from tools.cronjob_tools import _manual_run_delivery_note + + +@pytest.fixture(autouse=True) +def _clean_state(): + """Reset the shared async-delegation world around each test. + + The dispatch tests below submit real workers onto the process-wide + daemon executor in ``tools.async_delegation``. A finished worker parks + idle holding an ``_idle_semaphore`` token, so the NEXT dispatch in this + process REUSES that thread instead of spawning a fresh one — and only + the fresh-spawn path keeps upstream's dispatch-and-return test winning + its patch-visibility race: ``Thread.start()`` blocks the dispatching + thread until the worker has bootstrapped, so the worker performs + ``_run_claimed_job``'s lazy ``from cron.scheduler import run_one_job`` + while the test's patches are still active. On the idle-reuse path + ``submit`` returns with the GIL still held, the patch block unwinds + first, and the worker binds the REAL ``run_one_job`` — which then runs + the fake job for real ("no model configured") and the mock never fires. + Without this reset, test_cronjob_run_background.py's + ``test_dispatches_and_returns_handle_immediately`` fails + deterministically whenever this file runs before it. Mirrors + ``tests/tools/test_async_delegation.py::_clean_state``. + """ + from tools import async_delegation as ad + from tools.process_registry import process_registry + + ad._reset_for_tests() + while not process_registry.completion_queue.empty(): + process_registry.completion_queue.get_nowait() + yield + # Give just-drained workers a beat to finalize BEFORE resetting, so + # their completion events land now instead of leaking into the next + # test's queue (mirrors test_async_delegation.py). + deadline = time.monotonic() + 2.0 + while ad.active_count() and time.monotonic() < deadline: + time.sleep(0.02) + ad._reset_for_tests() + while not process_registry.completion_queue.empty(): + process_registry.completion_queue.get_nowait() + + +def _job(job_id, deliver): + """Per-test job dict with a UNIQUE id. + + Background workers outlive their test (daemon executor) and hold the id + in the scheduler's shared running set until the run finishes; reusing an + id across tests trips the in-flight dedupe guard on a straggler. + """ + return { + "id": job_id, + "name": f"dn run {job_id}", + "prompt": "hi", + "schedule": {"kind": "cron", "expr": "0 9 * * *"}, + "deliver": deliver, + } + + +@contextlib.contextmanager +def _bound_session_key(key): + """Bind the approval session key contextvar (background dispatch gate).""" + from tools.approval import _approval_session_key + + token = _approval_session_key.set(key) + try: + yield + finally: + _approval_session_key.reset(token) + + +def _dispatch_diag(res) -> str: + """Failure renderer for the wiring tests' dispatch asserts: the result + dict plus the scheduler running set, so a broken assert names the return + path that was actually taken instead of a bare KeyError.""" + try: + from cron.scheduler import get_running_job_ids + + running = sorted(get_running_job_ids()) + except Exception as e: # pragma: no cover - diagnostic only + running = f"" + return f"dispatch result: {res!r}; running: {running}" + + +def _drain_completion_event(delegation_id): + """Wait (bounded) for this delegation's completion event; requeue others. + + The runner executes on a daemon thread, so this must be called while the + test's patches are still active. + """ + from tools.process_registry import process_registry + + for _ in range(100): + try: + evt = process_registry.completion_queue.get_nowait() + except Exception: + time.sleep(0.05) + continue + if evt.get("delegation_id") == delegation_id: + return evt + process_registry.completion_queue.put(evt) + time.sleep(0.05) + return None + + +class TestDeliveryNote: + """``_manual_run_delivery_note`` — the summary-line wording contract.""" + + def test_local_always_saved_locally_only(self): + expected = " (output saved locally only)" + assert _manual_run_delivery_note("local", {}) == expected + # Local jobs never deliver — a stale delivery error must not leak in. + assert ( + _manual_run_delivery_note("local", {"last_delivery_error": "telegram 400"}) + == expected + ) + + def test_remote_without_error_keeps_legacy_wording(self): + expected = " (output was delivered there by the job itself)" + assert _manual_run_delivery_note("telegram", {}) == expected + assert ( + _manual_run_delivery_note("telegram", {"last_delivery_error": None}) + == expected + ) + assert ( + _manual_run_delivery_note("discord:#ops", {"last_delivery_error": " "}) + == expected + ) + + def test_empty_or_missing_deliver_reads_saved_locally(self): + """Falsy deliver = no target, and the fire-time path treats it as + "local" (no delivery, no delivery error) — the note must not claim + "delivered there" for a target that doesn't exist (#83993 class).""" + expected = " (output saved locally only)" + assert _manual_run_delivery_note("", {}) == expected + assert _manual_run_delivery_note(None, {}) == expected + # Falsy deliver never attempts delivery — a stale error (e.g. from an + # earlier deliver config) must not flip the wording either. + assert _manual_run_delivery_note("", {"last_delivery_error": "old"}) == expected + + def test_whitespace_deliver_defers_to_error_record(self): + """Whitespace-only deliver is NOT folded into local: fire time lets it + through as a target that fails to resolve, so the recorded error must + stay visible rather than being masked by a saved-locally wording.""" + note = _manual_run_delivery_note(" ", {"last_delivery_error": "no target"}) + assert "delivery FAILED" in note + assert "no target" in note + + def test_remote_with_error_says_delivery_failed(self): + note = _manual_run_delivery_note( + "telegram", {"last_delivery_error": "send failed: 400 Bad Request"} + ) + assert "delivery FAILED" in note + assert "send failed: 400 Bad Request" in note + + def test_remote_error_text_truncated_to_200_chars(self): + note = _manual_run_delivery_note("telegram", {"last_delivery_error": "E" * 500}) + assert "E" * 200 in note + assert "E" * 201 not in note + + +class TestRunnerSummaryWiring: + """The completion event the calling agent actually sees must follow the + refreshed job record — both directions of issue #83993.""" + + def test_delivery_failure_surfaces_in_completion_summary(self): + from tools.cronjob_tools import _try_dispatch_background_run + + job = _job("job-dn-01", "telegram") + with _bound_session_key("agent:main:telegram:dm:83993"): + with ( + patch( + "tools.cronjob_tools.claim_job_for_fire", + return_value=job, # claimed snapshot (return_job=True API) + ), + patch("cron.scheduler.run_one_job", return_value=True), + patch( + "tools.cronjob_tools.get_job", + return_value={ + # Post-#83993 record shape: mark_job_run writes + # delivery_failed (not ok) when only delivery failed. + "last_status": "delivery_failed", + "last_error": None, + "last_delivery_error": "telegram send failed: 400", + }, + ), + ): + res = _try_dispatch_background_run(job) + assert res.get("dispatched") is True, _dispatch_diag(res) + evt = _drain_completion_event(res["delegation_id"]) + assert evt is not None, "completion event never reached the queue" + summary = evt.get("summary") or "" + assert "Delivery target: telegram" in summary + assert "delivery FAILED" in summary + assert "telegram send failed: 400" in summary + assert "delivered there by the job itself" not in summary + # The headline must not read "Result: ok" over an undelivered run. + assert "Result: FAILED" in summary + assert "Result: ok" not in summary + + def test_empty_deliver_summary_states_local_not_phantom_target(self): + """End-to-end: an empty stored deliver must render as the local target + it behaves as at fire time — never a bare "Delivery target: " followed + by a delivered-there claim.""" + from tools.cronjob_tools import _try_dispatch_background_run + + job = _job("job-dn-03", "") + with _bound_session_key("agent:main:telegram:dm:86622"): + with ( + patch( + "tools.cronjob_tools.claim_job_for_fire", + return_value=job, # claimed snapshot (return_job=True API) + ), + patch("cron.scheduler.run_one_job", return_value=True), + patch( + "tools.cronjob_tools.get_job", + return_value={"last_status": "ok", "last_error": None}, + ), + ): + res = _try_dispatch_background_run(job) + assert res.get("dispatched") is True, _dispatch_diag(res) + evt = _drain_completion_event(res["delegation_id"]) + assert evt is not None, "completion event never reached the queue" + summary = evt.get("summary") or "" + assert "Delivery target: local (output saved locally only)" in summary + assert "delivered there by the job itself" not in summary + + def test_delivery_success_wording_unchanged_in_completion_summary(self): + from tools.cronjob_tools import _try_dispatch_background_run + + job = _job("job-dn-02", "telegram") + with _bound_session_key("agent:main:telegram:dm:83994"): + with ( + patch( + "tools.cronjob_tools.claim_job_for_fire", + return_value=job, # claimed snapshot (return_job=True API) + ), + patch("cron.scheduler.run_one_job", return_value=True), + patch( + "tools.cronjob_tools.get_job", + return_value={"last_status": "ok", "last_error": None}, + ), + ): + res = _try_dispatch_background_run(job) + assert res.get("dispatched") is True, _dispatch_diag(res) + evt = _drain_completion_event(res["delegation_id"]) + assert evt is not None, "completion event never reached the queue" + summary = evt.get("summary") or "" + assert ( + "Delivery target: telegram (output was delivered there by the job itself)" + ) in summary + assert "delivery FAILED" not in summary diff --git a/tests/tools/test_cronjob_run_immediate.py b/tests/tools/test_cronjob_run_immediate.py index beb8c098c3..aa0eb8b97f 100644 --- a/tests/tools/test_cronjob_run_immediate.py +++ b/tests/tools/test_cronjob_run_immediate.py @@ -291,3 +291,37 @@ class TestCronjobRunExecutesImmediately: assert len(calls) >= 2, calls finally: set_activity_callback(None) + + +class TestManualRunReportsDeliveryFailure: + """#83993: a manual run whose agent succeeded but whose delivery failed + must not come back as success=True with no error — the calling agent + relays that result to the user.""" + + def test_delivery_failed_status_is_not_success_and_surfaces_reason(self): + refreshed = { + "id": "job-run-1", + "last_status": "delivery_failed", + "last_error": None, + "last_delivery_error": "live adapter send failed: 502 (target telegram:123)", + } + with patch("tools.cronjob_tools.claim_job_for_fire", + return_value={**_JOB, "fire_claim": {"by": "manual-owner"}}), \ + patch("cron.scheduler.run_one_job", return_value=True), \ + patch("tools.cronjob_tools.get_job", return_value=refreshed): + res = _execute_job_now(dict(_JOB)) + + assert res["claimed"] is True + assert res["success"] is False + assert "502" in res["error"] + + def test_plain_ok_is_still_success_with_no_error(self): + with patch("tools.cronjob_tools.claim_job_for_fire", + return_value={**_JOB, "fire_claim": {"by": "manual-owner"}}), \ + patch("cron.scheduler.run_one_job", return_value=True), \ + patch("tools.cronjob_tools.get_job", + return_value={"id": "job-run-1", "last_status": "ok", "last_error": None, + "last_delivery_error": None}): + res = _execute_job_now(dict(_JOB)) + assert res["success"] is True + assert res["error"] is None diff --git a/tests/tools/test_cronjob_tools.py b/tests/tools/test_cronjob_tools.py index 0fbf0364a9..12aa49c80a 100644 --- a/tests/tools/test_cronjob_tools.py +++ b/tests/tools/test_cronjob_tools.py @@ -465,7 +465,7 @@ class TestAgentCannotSetModelPin: updated = json.loads( registry.dispatch( - "cronjob", + "cronjob_manage", { "action": "update", "job_id": job_id, @@ -499,7 +499,7 @@ class TestRegisteredHandlerForwardsAttachToSession: created = json.loads( registry.dispatch( - "cronjob", + "cronjob_manage", { "action": "create", "name": "Continuable cron canary", @@ -516,7 +516,7 @@ class TestRegisteredHandlerForwardsAttachToSession: stored = get_job(created["job_id"]) assert stored is not None assert stored.get("attach_to_session") is True - listing = json.loads(registry.dispatch("cronjob", {"action": "list"})) + listing = json.loads(registry.dispatch("cronjob_manage", {"action": "list"})) listed = next(j for j in listing["jobs"] if j["job_id"] == created["job_id"]) assert listed.get("attach_to_session") is True @@ -526,7 +526,7 @@ class TestRegisteredHandlerForwardsAttachToSession: created = json.loads( registry.dispatch( - "cronjob", + "cronjob_manage", { "action": "create", "name": "plain", @@ -540,7 +540,7 @@ class TestRegisteredHandlerForwardsAttachToSession: updated = json.loads( registry.dispatch( - "cronjob", + "cronjob_manage", { "action": "update", "job_id": created["job_id"], @@ -556,7 +556,7 @@ class TestRegisteredHandlerForwardsAttachToSession: disabled = json.loads( registry.dispatch( - "cronjob", + "cronjob_manage", { "action": "update", "job_id": created["job_id"], @@ -569,7 +569,7 @@ class TestRegisteredHandlerForwardsAttachToSession: stored = get_job(created["job_id"]) assert stored is not None assert stored.get("attach_to_session") is False - listing = json.loads(registry.dispatch("cronjob", {"action": "list"})) + listing = json.loads(registry.dispatch("cronjob_manage", {"action": "list"})) listed = next(j for j in listing["jobs"] if j["job_id"] == created["job_id"]) assert listed.get("attach_to_session") is False @@ -579,7 +579,7 @@ class TestRegisteredHandlerForwardsAttachToSession: created = json.loads( registry.dispatch( - "cronjob", + "cronjob_manage", { "action": "create", "schedule": "1h", @@ -592,7 +592,7 @@ class TestRegisteredHandlerForwardsAttachToSession: assert stored is not None assert "attach_to_session" not in stored # And the formatted list output must not invent the field either. - listed = json.loads(registry.dispatch("cronjob", {"action": "list"})) + listed = json.loads(registry.dispatch("cronjob_manage", {"action": "list"})) formatted = next( j for j in listed["jobs"] if j["job_id"] == created["job_id"] ) diff --git a/tests/tools/test_daemon_pool.py b/tests/tools/test_daemon_pool.py index 250cc86e59..9370afb46f 100644 --- a/tests/tools/test_daemon_pool.py +++ b/tests/tools/test_daemon_pool.py @@ -69,6 +69,30 @@ def test_wedged_worker_does_not_block_interpreter_exit(): assert "main-done" in proc.stdout +def test_submit_propagates_caller_contextvars(): + """Pool workers inherit contextvars set in the submitting context. + + Stdlib ThreadPoolExecutor snapshots the caller's context with + ``copy_context()``; some bundled CPython runtime builds strip that, so + the daemon pool restores it explicitly. Without the fix this returns + the default because the worker runs in a bare context. + """ + from contextvars import ContextVar + + var = ContextVar("daemon_pool_test_var", default="unset") + + pool = DaemonThreadPoolExecutor(max_workers=1) + try: + token = var.set("hello") + try: + seen = pool.submit(var.get).result(timeout=10) + finally: + var.reset(token) + assert seen == "hello" + finally: + pool.shutdown(wait=True) + + def _repo_root(): import pathlib diff --git a/tests/tools/test_delegate_batch_tag.py b/tests/tools/test_delegate_batch_tag.py new file mode 100644 index 0000000000..168298840a --- /dev/null +++ b/tests/tools/test_delegate_batch_tag.py @@ -0,0 +1,119 @@ +"""Batch tag on delegation progress lines (#p1-campaign feedback, Sep 2026). + +When a parent fans out N subagents and a child fans out its own M, both +batches print ``[n/N]`` completion lines to the same console. Without a +batch tag ``✓ [3/3]`` and ``✓ [3/9]`` are indistinguishable. Every progress +surface carries a human-readable ``set N`` ordinal (not a raw id slice). +""" +import types + +import pytest + +import tools.delegate_tool as dt +from tools.delegate_tool import _batch_prefix, _build_child_progress_callback, format_batch_tag + + +@pytest.fixture(autouse=True) +def _fresh_ordinals(monkeypatch): + monkeypatch.setattr(dt, "_BATCH_ORDINALS", {}) + + +def test_format_batch_tag_assigns_stable_ordinals_per_batch(): + assert format_batch_tag("deleg_6a664903") == "set 1" + assert format_batch_tag("deleg_b2ac1234") == "set 2" + assert format_batch_tag("deleg_6a664903") == "set 1" # same batch, same label + assert format_batch_tag(None) == "" + assert format_batch_tag("") == "" + + +@pytest.mark.parametrize( + "deleg, idx, count, expected", + [ + ("deleg_6a664903", 2, 9, "[set 1 · 3/9] "), + (None, 2, 9, "[3/9] "), + ("deleg_6a664903", 0, 1, "[set 1] "), + (None, 0, 1, ""), + ], +) +def test_batch_prefix_shapes(deleg, idx, count, expected): + assert _batch_prefix(deleg, idx, count) == expected + + +class _Spinner: + def __init__(self): + self.lines = [] + + def print_above(self, line): + self.lines.append(line) + + def update_text(self, text): + self.lines.append(f"{text}") + + +def test_child_tree_lines_and_relayed_events_carry_batch_tag(): + relayed = [] + parent = types.SimpleNamespace( + _delegate_spinner=_Spinner(), + tool_progress_callback=lambda et, name=None, preview=None, args=None, **kw: relayed.append((et, kw)), + ) + ref = {} + cb = _build_child_progress_callback(2, "triage cluster", parent, 9, subagent_id="sa-2", session_ref=ref) + # Stamped by delegate_task AFTER the callback is built — must be picked up lazily. + ref["delegation_id"] = "deleg_6a664903" + ref["session_id"] = "child-sess" + + cb("subagent.start") + cb("tool.started", "terminal", "ls") + + tree = parent._delegate_spinner.lines + assert tree[0].startswith(" [set 1 · 3/9] ├─ 🔀 triage cluster") + assert tree[1].startswith(" [set 1 · 3/9] ├─ ") + assert all(kw.get("delegation_id") == "deleg_6a664903" for _, kw in relayed) + assert all(kw.get("child_session_id") == "child-sess" for _, kw in relayed) + + +def test_child_tree_prefix_without_batch_id_is_unchanged(): + parent = types.SimpleNamespace(_delegate_spinner=_Spinner(), tool_progress_callback=None) + cb = _build_child_progress_callback(0, "solo goal", parent, 3, session_ref={}) + cb("subagent.start") + assert parent._delegate_spinner.lines[0].startswith(" [1/3] ├─ 🔀 solo goal") + + +def test_batch_completion_lines_are_attributable_across_two_batches(monkeypatch, tmp_path): + """Two interleaved batches: every ✓ line names its own ``set N``.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + (tmp_path / ".hermes").mkdir() + lines = [] + parent = types.SimpleNamespace( + session_id="root", model="m", tool_progress_callback=None, _delegate_spinner=None, + _safe_print=lambda line: lines.append(line), + ) + monkeypatch.setattr( + dt, "_run_single_child", + lambda task_index, goal, child=None, parent_agent=None, **kw: { + "task_index": task_index, "status": "completed", "summary": "ok", + "error": None, "api_calls": 1, "duration_seconds": 1, + }, + ) + monkeypatch.setattr(dt, "_build_child_preserving_parent_tools", + lambda **kw: types.SimpleNamespace(tool_progress_callback=None)) + monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: { + "model": "m", "provider": "openrouter", "base_url": "https://x/v1", + "api_key": "k", "api_mode": "chat_completions"}) + + import re + + for n in (3, 9): + res = dt.delegate_task( + tasks=[{"goal": f"batch of {n}: worker task number {i}"} for i in range(n)], + parent_agent=parent, + ) + assert "error" not in str(res)[:20], res + headers = [re.match(r"\s*🔀 \[(set \d+)\] delegating (\d+) tasks", l) for l in lines] + headers = [m for m in headers if m] + assert [(m.group(1), int(m.group(2))) for m in headers] == [("set 1", 3), ("set 2", 9)] + + done = [l for l in lines if "✓ [" in l] + assert len(done) == 12 + assert sum(1 for l in done if "✓ [set 1 · " in l and "/3]" in l) == 3 + assert sum(1 for l in done if "✓ [set 2 · " in l and "/9]" in l) == 9 diff --git a/tests/tools/test_delegate_child_transcript_release.py b/tests/tools/test_delegate_child_transcript_release.py new file mode 100644 index 0000000000..de4e274b5a --- /dev/null +++ b/tests/tools/test_delegate_child_transcript_release.py @@ -0,0 +1,145 @@ +"""Finished delegate children must not pin their transcripts in the parent heap. + +Profiled parent (1,320 children over 13h) reached 2.6 GB RSS: every closed +child AIAgent stayed reachable, and each still owned a shallow copy of its +full message list. Two retainers were proven with ``gc.get_referrers``: + +1. ``AIAgent.close()`` cleared ``_session_messages`` but not the + ``_db_flush_scan_prefix`` snapshot (``messages[:]``) or the streamed-text + accumulator, so every message dict stayed alive through the agent. +2. ``bind_subagent_parent`` stored the agent (each child binds ITSELF for + its own turn) strongly in a ContextVar; asyncio Handles/Futures scheduled + during the turn snapshot that Context and live as long as the background + LSP / kernel loops do, so the child object itself was never collected. +""" + +from __future__ import annotations + +import gc +import json +import weakref +from unittest.mock import MagicMock + +from agent.subagent_lifecycle import ( + _ACTIVE_PARENT_AGENT, + bind_subagent_parent, + get_active_subagent_parent, +) +from run_agent import AIAgent + + +def _bare_agent() -> AIAgent: + agent = AIAgent.__new__(AIAgent) + agent._active_children = [] + import threading + + agent._active_children_lock = threading.Lock() + agent._session_db = None + agent.session_id = "child-x" + return agent + + +def test_close_releases_transcript_shadow_copies(): + agent = _bare_agent() + + class Payload(str): # weakref-able stand-in for a message content string + pass + + payload = Payload("x" * 50_000) + big = {"role": "tool", "content": payload} + agent._session_messages = [big] + agent._db_flush_scan_prefix = agent._session_messages[:] + agent._streamed_assistant_text_parts = ["y" * 10_000] + probe = weakref.ref(payload) + + agent.close() + + assert agent._session_messages == [] + assert agent._db_flush_scan_prefix is None + assert agent._streamed_assistant_text_parts == [] + del big, payload + gc.collect() + assert probe() is None, "closed agent still owns its message dicts" + + +def test_bind_subagent_parent_does_not_pin_agent(): + agent = _bare_agent() + probe = weakref.ref(agent) + snapshots = [] + with bind_subagent_parent(agent): + assert get_active_subagent_parent() is agent + import contextvars + + # An asyncio Handle scheduled inside the turn keeps this snapshot. + snapshots.append(contextvars.copy_context()) + assert get_active_subagent_parent() is None + assert snapshots[0][_ACTIVE_PARENT_AGENT] is not agent + del agent + gc.collect() + assert probe() is None, "Context snapshot still pins the agent" + + +def test_bind_subagent_parent_accepts_non_weakrefable_doubles(): + class Slots: + __slots__ = () + + double = Slots() + with bind_subagent_parent(double): + assert get_active_subagent_parent() is double + + +def _fake_child(messages): + child = MagicMock() + child._credential_pool = None + child._delegate_role = "leaf" + child.session_estimated_cost_usd = 0.0123 + child.session_cost_status = "estimated" + child.session_id = "child-sess" + child.run_conversation.return_value = { + "final_response": "the summary", + "completed": True, + "interrupted": False, + "api_calls": 3, + "messages": messages, + } + return child + + +def test_run_single_child_result_json_unchanged_by_transcript_release(): + """Pin: the parent-visible result entry is byte-identical whether or not + the child released its transcript at close() (the entry never carried + ``messages``; only summary/tool_trace/tokens/cost derive from them).""" + from tests.tools.test_delegate import _make_mock_parent + from tools.delegate_tool import _run_single_child + + messages = [ + {"role": "user", "content": "goal"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "z" * 5000}, + {"role": "assistant", "content": "the summary"}, + ] + results = [] + for _ in range(2): + child = _fake_child([dict(m) for m in messages]) + entry = _run_single_child( + task_index=0, goal="goal", child=child, parent_agent=_make_mock_parent() + ) + child.close.assert_called_once() + entry.pop("duration_seconds", None) + results.append(json.dumps(entry, sort_keys=True, default=str)) + assert results[0] == results[1] + parsed = json.loads(results[0]) + assert parsed["summary"] == "the summary" + assert parsed["tool_trace"][0]["tool"] == "read_file" + assert parsed["cost_usd"] == 0.0123 + assert "messages" not in parsed diff --git a/tests/tools/test_desktop_tools_diet.py b/tests/tools/test_desktop_tools_diet.py index 4329662b78..6ab8d20a88 100644 --- a/tests/tools/test_desktop_tools_diet.py +++ b/tests/tools/test_desktop_tools_diet.py @@ -26,14 +26,24 @@ class TestConsolidatedToolsets(unittest.TestCase): self.assertEqual(proj, ["desktop_project"]) def test_registry_serves_only_new_names(self): - from model_tools import get_tool_definitions + """Post-#97979 the GUI surface defers by default, so assemble with + the legacy everything-eager override (defer: []) — the contract + pinned here is the RENAME (new names only, dead names gone), not + the deferral policy.""" + from unittest.mock import patch as _patch - names = { - t["function"]["name"] - for t in get_tool_definitions( - quiet_mode=True, enabled_toolsets=["desktop_ui", "project"] - ) - } + from model_tools import get_tool_definitions + from tools.tool_search import ToolSearchConfig + + legacy = ToolSearchConfig.from_raw({"enabled": "on", "defer": []}) + with _patch("tools.tool_search.load_config_readonly", return_value=legacy), \ + _patch("tools.tool_search.load_config", return_value=legacy): + names = { + t["function"]["name"] + for t in get_tool_definitions( + quiet_mode=True, enabled_toolsets=["desktop_ui", "project"] + ) + } self.assertIn("desktop_preview", names) self.assertIn("desktop_project", names) for dead in ( diff --git a/tests/tools/test_docker_config_migrate.py b/tests/tools/test_docker_config_migrate.py index 5a8ec0cd8c..4472f92198 100644 --- a/tests/tools/test_docker_config_migrate.py +++ b/tests/tools/test_docker_config_migrate.py @@ -63,9 +63,10 @@ def test_docker_config_migrate_backs_up_and_migrates_legacy_config(tmp_path: Pat assert "Migrating config schema 12 ->" in proc.stdout raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) assert raw["_config_version"] == DEFAULT_CONFIG["_config_version"] - # v24→25 lowers the old default model_catalog TTL; v32→33 folds + # v24→25 lowers the old default model_catalog TTL to 1h, v39→40 drops + # that default so ttl_minutes (20) applies; v32→33 folds # max_async_children into max_concurrent_children. - assert raw["model_catalog"]["ttl_hours"] == 1 + assert "ttl_hours" not in raw["model_catalog"] assert raw["delegation"] == {"max_concurrent_children": 8} assert list(tmp_path.glob("config.yaml.bak-*")) assert list(tmp_path.glob(".env.bak-*")) diff --git a/tests/tools/test_file_operations.py b/tests/tools/test_file_operations.py index 1da5477199..8e60200308 100644 --- a/tests/tools/test_file_operations.py +++ b/tests/tools/test_file_operations.py @@ -1,12 +1,13 @@ """Tests for tools/file_operations.py — deny list, result dataclasses, helpers.""" import os -import re import pytest import subprocess from pathlib import Path from unittest.mock import MagicMock +from tests.tools.file_ops_fakes import READ_SENTINEL_RE, compound_read_output +from tools.environments.local import _find_bash, _msys_to_windows_path, LocalEnvironment from tools.file_operations import ( _is_write_denied, ReadResult, @@ -155,10 +156,16 @@ class TestSearchResult: assert d["matches"][0]["path"] == "a.py" - def test_truncated_flag(self): + def test_truncated_flag_marks_total_as_lower_bound(self): r = SearchResult(total_count=100, truncated=True) d = r.to_dict() assert d["truncated"] is True + assert d["total_count_is_lower_bound"] is True + + def test_untruncated_total_omits_lower_bound_flag(self): + r = SearchResult(total_count=100) + d = r.to_dict() + assert "total_count_is_lower_bound" not in d class TestSearchResultDensify: @@ -255,16 +262,29 @@ def make_real_subprocess_env(cwd: str, include_stderr: bool = False) -> MagicMoc env.cwd = cwd def execute(command, **kwargs): + stdin_data = kwargs.get("stdin_data") + is_windows = os.name == "nt" + if is_windows: + # Match LocalEnvironment: commands are POSIX scripts executed by + # Git Bash, and stdin bytes must bypass Windows newline rewriting. + command = [_find_bash(), "-c", command] completed = subprocess.run( command, - shell=True, - text=True, + shell=not is_windows, + text=not is_windows, capture_output=True, - input=kwargs.get("stdin_data"), + input=(stdin_data.encode("utf-8", "surrogateescape") + if is_windows and stdin_data is not None else stdin_data), + ) + output = ( + completed.stdout.decode("utf-8", "replace") + if is_windows else completed.stdout ) - output = completed.stdout if include_stderr: - output += completed.stderr + output += ( + completed.stderr.decode("utf-8", "replace") + if is_windows else completed.stderr + ) return { "output": output, "returncode": completed.returncode, @@ -303,19 +323,14 @@ class TestShellFileOpsHelpers: def side_effect(command, **kwargs): commands.append(command) - # The size probe gates `wc -c` behind `[ -f ]` so a FIFO or device - # cannot block the read; it still reports a plain byte count. - if command.startswith("if [ -f ") or command.startswith("wc -c"): - return {"output": "5\n", "returncode": 0} - if command.startswith("head -c") and "| base64" in command: - import base64 as b64 - return {"output": b64.b64encode(b"hello").decode(), "returncode": 0} - if command.startswith("head -c"): - return {"output": "hello", "returncode": 0} - if command.startswith("sed -n"): - return {"output": "hello\n", "returncode": 0} - if command.startswith("wc -l"): - return {"output": "1\n", "returncode": 0} + m = READ_SENTINEL_RE.search(command) + if m: + return { + "output": compound_read_output( + m.group(0), size=5, sample=b"hello", content="hello\n", total_lines=1 + ), + "returncode": 0, + } return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect @@ -323,16 +338,22 @@ class TestShellFileOpsHelpers: result = ops.read_file(r"C:\Users\alice\notes.txt") assert result.error is None - assert commands[0] == ( + # One compound probe carries every stage; each embeds the MSYS path. + # The size probe gates `wc -c` behind `[ -f ]` so a FIFO or device + # cannot block the read; it still reports a plain byte count. + assert len(commands) == 1 + probe = commands[0] + assert probe.startswith( "if [ -f '/c/Users/alice/notes.txt' ]; " "then wc -c < '/c/Users/alice/notes.txt' 2>/dev/null; " + ) + assert "head -c 1000 '/c/Users/alice/notes.txt' 2>/dev/null | base64" in probe + assert "sed -n '1,2000p' '/c/Users/alice/notes.txt' 2>/dev/null | cut -b1-8001" in probe + assert "wc -l < '/c/Users/alice/notes.txt'" in probe + assert ( "elif [ -e '/c/Users/alice/notes.txt' ]; " "then echo __hermes_not_regular__; " - "else exit 1; fi" - ) - assert commands[1] == "head -c 1000 '/c/Users/alice/notes.txt' 2>/dev/null | base64" - assert commands[2] == "sed -n '1,2000p' '/c/Users/alice/notes.txt' | cut -b1-8001" - assert commands[3] == "wc -l < '/c/Users/alice/notes.txt'" + ) in probe def test_is_likely_binary_by_extension(self, file_ops): assert file_ops._is_likely_binary("photo.png") is True @@ -355,14 +376,15 @@ class TestShellFileOpsHelpers: ) def side_effect(command, **kwargs): - if command.startswith("if [ -f ") or command.startswith("wc -c"): - return {"output": "12\n", "returncode": 0} - if command.startswith("head -c"): - return {"output": "print('ok')\n", "returncode": 0} - if command.startswith("sed -n"): - return {"output": leaked, "returncode": 0} - if command.startswith("wc -l"): - return {"output": "1\n", "returncode": 0} + m = READ_SENTINEL_RE.search(command) + if m: + return { + "output": compound_read_output( + m.group(0), size=12, sample=b"print('ok')\n", + content=leaked, total_lines=1, + ), + "returncode": 0, + } return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect @@ -437,7 +459,7 @@ class TestSearchPathValidation: class TestSearchFilesFallbackHiddenPaths: def _make_env(self): - return make_real_subprocess_env("/") + return LocalEnvironment("/") @pytest.mark.platforms("linux") def test_hidden_root_with_hidden_ancestor_includes_files(self, tmp_path, monkeypatch): @@ -779,17 +801,19 @@ class TestByteLayerBinaryDetection: # --- integration: read_file over the mocked terminal ------------------ def _dispatch(self, cjk_bytes): - import base64 as b64 - def side_effect(command, **kwargs): - if command.startswith("if [ -f ") or command.startswith("wc -c"): - return {"output": f"{len(cjk_bytes)}\n", "returncode": 0} - if command.startswith("head -c") and "| base64" in command: - return {"output": b64.b64encode(cjk_bytes[:1000]).decode(), "returncode": 0} - if command.startswith("sed -n"): - return {"output": cjk_bytes.decode("utf-8", errors="replace"), "returncode": 0} - if command.startswith("wc -l"): - return {"output": "1\n", "returncode": 0} + m = READ_SENTINEL_RE.search(command) + if m: + return { + "output": compound_read_output( + m.group(0), + size=len(cjk_bytes), + sample=cjk_bytes[:1000], + content=cjk_bytes.decode("utf-8", errors="replace"), + total_lines=1, + ), + "returncode": 0, + } return {"output": "", "returncode": 0} return side_effect diff --git a/tests/tools/test_file_operations_edge_cases.py b/tests/tools/test_file_operations_edge_cases.py index 0865801911..103c172884 100644 --- a/tests/tools/test_file_operations_edge_cases.py +++ b/tests/tools/test_file_operations_edge_cases.py @@ -8,6 +8,7 @@ Covers: import pytest from unittest.mock import MagicMock, patch +from tests.tools.file_ops_fakes import READ_SENTINEL_RE, compound_read_output from tools.file_operations import ShellFileOperations, _parse_search_context_line @@ -205,14 +206,15 @@ class TestPaginationBounds: def fake_exec(command, *args, **kwargs): commands.append(command) - if command.startswith("if [ -f ") or command.startswith("wc -c"): - return MagicMock(exit_code=0, stdout="12") - if command.startswith("head -c"): - return MagicMock(exit_code=0, stdout="line1\nline2\n") - if command.startswith("sed -n"): - return MagicMock(exit_code=0, stdout="line1\n") - if command.startswith("wc -l"): - return MagicMock(exit_code=0, stdout="2") + m = READ_SENTINEL_RE.search(command) + if m: + return MagicMock( + exit_code=0, + stdout=compound_read_output( + m.group(0), size=12, sample=b"line1\nline2\n", + content="line1\n", total_lines=2, + ), + ) return MagicMock(exit_code=0, stdout="") with patch.object(ops, "_exec", side_effect=fake_exec): @@ -220,8 +222,9 @@ class TestPaginationBounds: assert result.error is None assert "1|line1" in result.content - sed_commands = [cmd for cmd in commands if cmd.startswith("sed -n")] - assert sed_commands == ["sed -n '1,1p' 'notes.txt' | cut -b1-8001"] + # The clamped range rides the single compound probe. + assert len(commands) == 1 + assert "sed -n '1,1p' 'notes.txt' 2>/dev/null | cut -b1-8001" in commands[0] def test_search_clamps_offset_and_limit_before_building_head_pipeline(self): env = MagicMock() @@ -233,7 +236,7 @@ class TestPaginationBounds: commands.append(command) if command.startswith("test -e"): return MagicMock(exit_code=0, stdout="exists") - if command.startswith("rg --files"): + if "--files" in command: return MagicMock(exit_code=0, stdout="a.py\n") return MagicMock(exit_code=0, stdout="") @@ -242,9 +245,9 @@ class TestPaginationBounds: result = ops.search("*.py", target="files", path=".", offset=-4, limit=-2) assert result.files == ["a.py"] - rg_commands = [cmd for cmd in commands if cmd.startswith("rg --files")] + rg_commands = [cmd for cmd in commands if "--files" in cmd] assert rg_commands - assert "| head -n 1" in rg_commands[0] + assert "| head -n 2" in rg_commands[0] # ========================================================================= diff --git a/tests/tools/test_file_ops_single_roundtrip.py b/tests/tools/test_file_ops_single_roundtrip.py new file mode 100644 index 0000000000..b7d3e95835 --- /dev/null +++ b/tests/tools/test_file_ops_single_roundtrip.py @@ -0,0 +1,474 @@ +"""``read_file`` / ``write_file`` cost one shell round-trip, not four. + +Real ``LocalEnvironment`` against ``tmp_path`` (no mocks), with a spy on +``env.execute`` counting round-trips. The cases below are exactly the ones +that used to need their own probe (existence, size, binary sample, page, +line count, trailing newline), so each proves the compound reply carries +that answer. +""" + +import logging +import os +import sys +import threading +from unittest.mock import patch + +import pytest + +from tools.environments.local import LocalEnvironment +from tools.file_operations import ExecuteResult, ShellFileOperations + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell probes") + +READ_PROBE_MARK = "__HERMES_RF_" + + +@pytest.fixture(scope="module") +def _local_env(tmp_path_factory): + """One real LocalEnvironment per module; constructing one costs ~0.8 s.""" + return LocalEnvironment(cwd=str(tmp_path_factory.mktemp("file-ops"))) + + +@pytest.fixture +def _ops(_local_env, tmp_path): + """(ops, calls): file ops over the real local shell, every execute recorded.""" + env = _local_env + env.cwd = str(tmp_path) + calls = [] + real_execute = type(env).execute.__get__(env, type(env)) + + def spy(command, *args, **kwargs): + calls.append(command) + return real_execute(command, *args, **kwargs) + + env.execute = spy + try: + yield ShellFileOperations(env, cwd=str(tmp_path)), calls + finally: + env.__dict__.pop("execute", None) + + +@pytest.fixture +def shell(_ops, monkeypatch): + """Pin the shell path even where a native fast path exists.""" + monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0") + return _ops + + +@pytest.fixture +def native(_ops, monkeypatch): + """Same wiring with the native fast path on.""" + monkeypatch.delenv("HERMES_NATIVE_FILE_READ", raising=False) + return _ops + + +def _write(tmp_path, name, data: bytes): + p = tmp_path / name + p.write_bytes(data) + return str(p) + + +class TestReadFileOneRoundTrip: + def test_text_read_is_one_round_trip(self, shell, tmp_path): + ops, calls = shell + p = _write(tmp_path, "a.txt", b"one\ntwo\nthree\n") + r = ops.read_file(p) + assert len(calls) == 1 and READ_PROBE_MARK in calls[0] + assert r.error is None + # ``_add_line_numbers`` numbers the empty tail after the final + # newline: long-standing behaviour, preserved byte for byte. + assert r.content == "1|one\n2|two\n3|three\n4|" + assert (r.total_lines, r.file_size, r.truncated) == (3, 14, False) + + def test_no_trailing_newline_needs_no_extra_probe(self, shell, tmp_path): + ops, calls = shell + p = _write(tmp_path, "b.txt", b"a\nb") + r = ops.read_file(p) + assert len(calls) == 1 + # ``cut`` newline-terminates the last line; the artifact is stripped + # from the same reply that used to need a fifth ``tail -c 1`` call. + assert r.content == "1|a\n2|b" + assert r.total_lines == 1 # wc -l semantics, unchanged + + def test_pagination_window_and_hint(self, shell, tmp_path): + ops, calls = shell + p = _write(tmp_path, "c.txt", b"".join(b"l%d\n" % i for i in range(1, 11))) + r = ops.read_file(p, offset=3, limit=2) + assert len(calls) == 1 + assert r.content == "3|l3\n4|l4\n5|" + assert r.truncated is True and r.total_lines == 10 + assert "offset=5" in r.hint + + def test_offset_past_eof_note(self, shell, tmp_path): + ops, calls = shell + p = _write(tmp_path, "c.txt", b"".join(b"l%d\n" % i for i in range(1, 6))) + r = ops.read_file(p, offset=50) + assert len(calls) == 1 + assert r.content == "" and r.error is None + assert "beyond the end" in r.hint and "5" in r.hint + + def test_empty_file(self, shell, tmp_path): + ops, calls = shell + r = ops.read_file(_write(tmp_path, "e.txt", b"")) + assert len(calls) == 1 + assert r.error is None and r.content == "" and r.total_lines == 0 + assert "empty" in r.hint + + def test_bom_stripped_on_first_page(self, shell, tmp_path): + ops, calls = shell + r = ops.read_file(_write(tmp_path, "f.txt", "hello\n".encode("utf-8"))) + assert len(calls) == 1 + assert r.content == "1|hello\n2|" + + def test_crlf_bytes_survive(self, shell, tmp_path): + ops, calls = shell + r = ops.read_file(_write(tmp_path, "g.txt", b"x\r\ny\r\n")) + assert r.content == "1|x\r\n2|y\r\n3|" + + def test_long_line_clamped_and_marked(self, shell, tmp_path): + ops, calls = shell + r = ops.read_file(_write(tmp_path, "L.txt", b"a" * 9000 + b"\nshort\n")) + assert len(calls) == 1 + first, second, tail = r.content.split("\n") + assert first.endswith("... [truncated]") and len(first) < 9000 + assert second == "2|short" and tail == "3|" + + def test_relative_path_resolves_against_env_cwd(self, shell, tmp_path): + ops, calls = shell + _write(tmp_path, "rel.txt", b"here\n") + r = ops.read_file("rel.txt") + assert r.error is None and r.content == "1|here\n2|" + + def test_sentinel_lookalike_in_content_reads_intact(self, shell, tmp_path): + ops, calls = shell + lookalike = "__HERMES_RF_" + "ab" * 16 + "__" + p = _write(tmp_path, "s.txt", f"x\n{lookalike}\ny\n".encode("utf-8")) + r = ops.read_file(p) + assert r.error is None and r.total_lines == 3 + assert r.content == f"1|x\n2|{lookalike}\n3|y\n4|" + + +class TestReadFileNonTextPaths: + def test_missing_file_probes_once_then_suggests(self, shell, tmp_path): + ops, calls = shell + _write(tmp_path, "notes.txt", b"x\n") + r = ops.read_file(str(tmp_path / "note.txt")) + assert READ_PROBE_MARK in calls[0] + assert r.error and "File not found" in r.error + assert any(s.endswith("notes.txt") for s in r.similar_files) + + def test_unicode_variant_retry_still_works(self, shell, tmp_path): + ops, calls = shell + # A curly apostrophe vs the ASCII one: visually identical in a + # terminal, and — unlike NFC/NFD — never aliased by the filesystem + # (APFS resolves NFD lookups to NFC files directly, which would skip + # the retry path this test exists to exercise). + on_disk = "it\u2019s.txt" + typed = "it's.txt" + assert on_disk != typed + _write(tmp_path, on_disk, b"accent\n") + r = ops.read_file(str(tmp_path / typed)) + assert r.error is None and r.content == "1|accent\n2|" + assert r.hint is not None and "unicode-equivalent" in r.hint + + def test_directory_is_not_regular(self, shell, tmp_path): + ops, calls = shell + r = ops.read_file(str(tmp_path)) + assert len(calls) == 1 + assert r.error and "not a regular file" in r.error + + def test_binary_sample_detected_in_same_reply(self, shell, tmp_path): + ops, calls = shell + p = _write(tmp_path, "blob", b"\x00\x01\x02" + b"\x00" * 50) + r = ops.read_file(p) + assert READ_PROBE_MARK in calls[0] + assert r.is_binary is True and r.error + # Only the UTF-16 rescue may add round-trips, never a second sample. + assert not any("head -c 1000" in c for c in calls[1:]) + + def test_image_extension_stops_at_size_probe(self, shell, tmp_path): + ops, calls = shell + r = ops.read_file(_write(tmp_path, "p.png", b"\x89PNG\r\n")) + assert len(calls) == 1 and READ_PROBE_MARK not in calls[0] + assert r.is_image is True and r.file_size == 6 + + @pytest.mark.linux_only + def test_fifo_returns_not_regular_without_blocking(self, shell, tmp_path): + if not hasattr(os, "mkfifo"): + pytest.skip("no mkfifo") + ops, calls = shell + fifo = tmp_path / "pipe" + os.mkfifo(fifo) + box = {} + + def run(): + box["r"] = ops.read_file(str(fifo)) + + t = threading.Thread(target=run, daemon=True) + t.start() + t.join(20) + assert not t.is_alive(), "read_file blocked on a writer-less FIFO" + assert "not a regular file" in box["r"].error + assert len(calls) == 1 + + +class TestWriteFileRoundTrips: + """write_file: one probe, one atomic write, one hash check (three calls).""" + + @staticmethod + def _execs(calls): + return [c for c in calls] + + def test_new_text_file_is_three_round_trips(self, shell, tmp_path): + ops, calls = shell + p = str(tmp_path / "new.txt") + r = ops.write_file(p, "line one\nline two\n") + assert r.error is None and r.verified is True + assert len(calls) == 3 + assert "__HERMES_WF_" in calls[0] # probe + assert "mv -f" in calls[1] # atomic write + assert calls[2].startswith("sha256sum ") # verify + assert (tmp_path / "new.txt").read_bytes() == b"line one\nline two\n" + + def test_crlf_file_keeps_crlf_from_the_probe(self, shell, tmp_path): + ops, calls = shell + p = tmp_path / "crlf.txt" + p.write_bytes(b"a\r\nb\r\n") + r = ops.write_file(str(p), "x\ny\n") + assert r.error is None and len(calls) == 3 + assert p.read_bytes() == b"x\r\ny\r\n" + + def test_bom_is_read_from_disk_and_preserved(self, shell, tmp_path): + ops, calls = shell + p = tmp_path / "bom.txt" + p.write_bytes("old\n".encode("utf-8")) + r = ops.write_file(str(p), "new\n") + assert r.error is None and len(calls) == 3 + assert p.read_bytes() == "new\n".encode("utf-8") + + def test_pre_content_read_rides_the_same_probe(self, shell, tmp_path): + """A lintable extension wants the old text (lint delta); it comes + back in the probe reply instead of a separate ``cat``.""" + ops, calls = shell + p = tmp_path / "code.py" + p.write_bytes(b"x = 1\r\ny = 2\r\n") + r = ops.write_file(str(p), "x = 1\ny = 3\n") + assert r.error is None + probes = [c for c in calls if "__HERMES_WF_" in c] + assert len(probes) == 1 and "cat " in probes[0] + assert not any(c.startswith("cat ") for c in calls) + assert p.read_bytes() == b"x = 1\r\ny = 3\r\n" + + def test_missing_file_probe_does_not_block_the_write(self, shell, tmp_path): + ops, calls = shell + p = tmp_path / "deep" / "er" / "new.md" + r = ops.write_file(str(p), "hi\n") + assert r.error is None and r.dirs_created is True + assert len(calls) == 3 + assert p.read_bytes() == b"hi\n" + + def test_unparseable_probe_reply_falls_back_to_separate_probes(self, shell, tmp_path): + ops, calls = shell + p = tmp_path / "crlf.txt" + p.write_bytes(b"a\r\nb\r\n") + real_exec = ops._exec + + def garbled(command, *args, **kwargs): + if "__HERMES_WF_" in command: + return ExecuteResult(stdout="[Command timed out after 1s]\n", exit_code=124) + return real_exec(command, *args, **kwargs) + + with patch.object(ops, "_exec", side_effect=garbled): + r = ops.write_file(str(p), "x\ny\n") + assert r.error is None + assert p.read_bytes() == b"x\r\ny\r\n" + + +class TestNativeRead: + def test_native_read_makes_no_shell_call(self, native, tmp_path): + ops, calls = native + r = ops.read_file(_write(tmp_path, "a.txt", b"one\ntwo\n")) + assert calls == [] + assert r.error is None and r.content == "1|one\n2|two\n3|" + assert (r.total_lines, r.file_size) == (2, 8) + + def test_kill_switch_routes_to_the_shell(self, native, tmp_path, monkeypatch): + ops, calls = native + p = _write(tmp_path, "a.txt", b"one\n") + monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0") + ops.read_file(p) + assert len(calls) == 1 and READ_PROBE_MARK in calls[0] + + def test_non_local_environment_keeps_the_shell_path(self): + from unittest.mock import MagicMock + + env = MagicMock() + env.cwd = "/tmp" + assert ShellFileOperations(env)._native_read_enabled() is False + + def test_tilde_still_expands_through_the_shell(self, native): + ops, calls = native + r = ops.read_file("~/hermes-no-such-file-7f3a.txt") + assert calls[0] == "echo $HOME" + assert r.error and "File not found" in r.error + + def test_injection_lookalike_path_is_never_expanded(self, native, tmp_path): + ops, calls = native + marker = tmp_path / "pwned" + r = ops.read_file(f"~; echo PWNED > {marker}") + assert r.error and not marker.exists() + # The text reaches the shell only single-quoted, inside the missing- + # file recovery's directory listing; the tilde probe is a fixed + # ``echo $HOME`` that never embeds the path. Nothing else runs. + for c in calls: + assert c == "echo $HOME" or c.startswith("ls -1 '~; echo PWNED"), c + + @pytest.mark.linux_only + def test_fifo_refused_without_a_shell_and_without_blocking(self, native, tmp_path): + if not hasattr(os, "mkfifo"): + pytest.skip("no mkfifo") + ops, calls = native + fifo = tmp_path / "pipe" + os.mkfifo(fifo) + box = {} + + def run(): + box["r"] = ops.read_file(str(fifo)) + + t = threading.Thread(target=run, daemon=True) + t.start() + t.join(20) + assert not t.is_alive(), "native read_file blocked on a writer-less FIFO" + assert "not a regular file" in box["r"].error + assert calls == [] + + +# The native reader scans 1 MiB chunks and clamps each page line to +# ``4 * get_max_line_length() + 1`` bytes (8001 by default), exactly as +# ``sed | cut -b1-N`` does. These shapes put a newline, a line, the clamp +# point, a CRLF pair and EOF precisely on those chunk boundaries. +_CHUNK = 1 << 20 +_CLAMP = 8001 + +PARITY_CASES = [ + ("plain", b"one\ntwo\nthree\n", {}), + ("no_trailing_newline", b"a\nb", {}), + ("blank_tail", b"a\n\n", {}), + ("crlf", b"x\r\ny\r\n", {}), + ("lone_cr", b"a\rb\n", {}), + ("bom", "hello\n".encode("utf-8"), {}), + ("empty", b"", {}), + ("single_no_newline", b"solo", {}), + ("only_newline", b"\n", {}), + ("unicode", "héllo wörld\n汉字\n".encode("utf-8"), {}), + ("long_line", b"a" * 9000 + b"\nshort\n", {}), + ("multibyte_long_line", ("汉" * 4000 + "\nx\n").encode("utf-8"), {}), + ("multi_chunk_line", b"b" * 3_000_000 + b"\nz\n", {}), + ("newline_last_byte_of_chunk", b"a" * (_CHUNK - 1) + b"\nsecond\n", {}), + ("newline_first_byte_of_next_chunk", b"a" * _CHUNK + b"\nsecond\n", {}), + ("line_spans_three_boundaries", b"p\n" + b"b" * (3 * _CHUNK + 5) + b"\nz\n", {}), + ( + "clamp_fills_on_boundary", + b"x" * (_CHUNK - _CLAMP - 1) + b"\n" + b"c" * 20000 + b"\ntail\n", + {"offset": 2, "limit": 3}, + ), + ( + "clamp_fills_after_boundary", + b"x" * (_CHUNK - _CLAMP) + b"\n" + b"c" * 20000 + b"\ntail\n", + {"offset": 2, "limit": 3}, + ), + ("eof_midline_on_boundary", b"l1\n" + b"d" * (2 * _CHUNK - 3), {}), + ("blank_lines_on_boundary", b"e" * (_CHUNK - 2) + b"\n\n\n" + b"f\n", {}), + ("crlf_split_on_boundary", b"r" * (_CHUNK - 1) + b"\r\n" + b"s\r\n", {}), + ( + "page_starts_in_second_chunk", + b"q\n" * (_CHUNK // 2 + 3) + b"target1\ntarget2\n", + {"offset": _CHUNK // 2 + 3, "limit": 4}, + ), + ("window", b"".join(b"l%d\n" % i for i in range(1, 11)), {"offset": 3, "limit": 2}), + ("window_reaches_eof", b"".join(b"l%d\n" % i for i in range(1, 11)), {"offset": 9, "limit": 5}), + ("past_eof", b"".join(b"l%d\n" % i for i in range(1, 6)), {"offset": 50}), + ("nul_binary", b"\x00\x01\x02" * 20, {}), + ("latin1_tail", b"caf\xe9\n", {}), + ("sentinel_lookalike", b"x\n__HERMES_RF_" + b"ab" * 16 + b"__\ny\n", {}), +] + + +class TestNativeReadParity: + """The native path must be indistinguishable from the shell path.""" + + @pytest.mark.parametrize("name,data,kwargs", PARITY_CASES, ids=[c[0] for c in PARITY_CASES]) + def test_shell_and_native_agree(self, native, tmp_path, monkeypatch, name, data, kwargs): + ops, calls = native + p = _write(tmp_path, f"{name}.txt", data) + monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0") + via_shell = ops.read_file(p, **kwargs).to_dict() + assert calls and READ_PROBE_MARK in calls[0] + calls.clear() + monkeypatch.delenv("HERMES_NATIVE_FILE_READ") + via_native = ops.read_file(p, **kwargs).to_dict() + assert via_native == via_shell + # The native path touches the shell only for the UTF-16 rescue of binaries. + assert not any(READ_PROBE_MARK in c for c in calls) + + def test_special_paths_agree(self, native, tmp_path, monkeypatch): + ops, calls = native + _write(tmp_path, "real.txt", b"target\n") + os.symlink(tmp_path / "real.txt", tmp_path / "link.txt") + os.symlink(tmp_path / "gone", tmp_path / "dangling.txt") + (tmp_path / "sub").mkdir() + _write(tmp_path, "pic.png", b"\x89PNG\r\n") + _write(tmp_path, "notes.txt", b"n\n") + for p in ( + str(tmp_path / "link.txt"), + str(tmp_path / "dangling.txt"), + str(tmp_path / "sub"), + str(tmp_path / "pic.png"), + str(tmp_path / "note.txt"), # missing → similar-file suggestions + "real.txt", # relative to env.cwd + ): + monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0") + via_shell = ops.read_file(p).to_dict() + monkeypatch.delenv("HERMES_NATIVE_FILE_READ") + calls.clear() + via_native = ops.read_file(p).to_dict() + assert via_native == via_shell, p + assert not any(READ_PROBE_MARK in c for c in calls), p + + +class TestCompoundFallback: + def test_unparseable_reply_falls_back_to_sequential_probes(self, shell, tmp_path): + ops, calls = shell + p = _write(tmp_path, "a.txt", b"one\ntwo\n") + real_exec = ops._exec + + def garbled(command, *args, **kwargs): + if READ_PROBE_MARK in command: + return ExecuteResult(stdout="[Command timed out after 1s]\n", exit_code=124) + return real_exec(command, *args, **kwargs) + + with patch.object(ops, "_exec", side_effect=garbled): + r = ops.read_file(p) + assert r.error is None and r.content == "1|one\n2|two\n3|" + assert r.total_lines == 2 + + def test_fallback_is_logged_at_debug(self, shell, tmp_path, caplog): + """A backend that keeps falling back shows up in debug logs.""" + ops, calls = shell + p = _write(tmp_path, "a.txt", b"one\n") + real_exec = ops._exec + + def garbled(command, *args, **kwargs): + if READ_PROBE_MARK in command: + return ExecuteResult(stdout="garbage\n", exit_code=0) + return real_exec(command, *args, **kwargs) + + with caplog.at_level(logging.DEBUG, logger="tools.file_operations"), \ + patch.object(ops, "_exec", side_effect=garbled): + r = ops.read_file(p) + assert r.error is None and r.content == "1|one\n2|" + assert any( + "falling back to sequential probes" in rec.getMessage() + and str(p) in rec.getMessage() + for rec in caplog.records + ) diff --git a/tests/tools/test_file_read_guards.py b/tests/tools/test_file_read_guards.py index 492f22f36a..2076351510 100644 --- a/tests/tools/test_file_read_guards.py +++ b/tests/tools/test_file_read_guards.py @@ -398,7 +398,7 @@ class TestFileDedup(unittest.TestCase): _read_tracker.clear() self._tmpdir = _make_safe_tempdir("hermes-dedup-") self._tmpfile = os.path.join(self._tmpdir, "dedup_test.txt") - with open(self._tmpfile, "w") as f: + with open(self._tmpfile, "w", encoding="utf-8") as f: f.write("line one\nline two\n") def tearDown(self): @@ -470,7 +470,7 @@ class TestDedupStubLoopGuard(unittest.TestCase): _read_tracker.clear() self._tmpdir = tempfile.mkdtemp() self._tmpfile = os.path.join(self._tmpdir, "loop_test.txt") - with open(self._tmpfile, "w") as f: + with open(self._tmpfile, "w", encoding="utf-8") as f: f.write("line one\nline two\n") def tearDown(self): @@ -537,7 +537,7 @@ class TestDedupStubLoopGuard(unittest.TestCase): # File changes — mtime updates time.sleep(0.05) - with open(self._tmpfile, "w") as f: + with open(self._tmpfile, "w", encoding="utf-8") as f: f.write("brand new content\n") r4 = json.loads(read_file_tool(self._tmpfile, task_id="loop")) @@ -598,10 +598,16 @@ class TestDedupStubLoopGuard(unittest.TestCase): reset_file_dedup("loop") - # Fresh session — real read, no stub, no block + # Post-compression: block counters cleared and exact content is served + # once because the earlier payload may no longer be in context. r4 = json.loads(read_file_tool(self._tmpfile, task_id="loop")) self.assertNotIn("error", r4) self.assertNotIn("dedup", r4) + self.assertIn("content", r4) + + # The next unchanged read in this generation is lightweight again. + r5 = json.loads(read_file_tool(self._tmpfile, task_id="loop")) + self.assertTrue(r5.get("dedup")) # --------------------------------------------------------------------------- @@ -609,14 +615,13 @@ class TestDedupStubLoopGuard(unittest.TestCase): # --------------------------------------------------------------------------- class TestDedupResetOnCompression(unittest.TestCase): - """reset_file_dedup should clear the dedup cache so post-compression - reads return full content.""" + """Compaction starts a new full-content recovery generation.""" def setUp(self): _read_tracker.clear() self._tmpdir = tempfile.mkdtemp() self._tmpfile = os.path.join(self._tmpdir, "compress_test.txt") - with open(self._tmpfile, "w") as f: + with open(self._tmpfile, "w", encoding="utf-8") as f: f.write("original content\n") def tearDown(self): @@ -628,10 +633,10 @@ class TestDedupResetOnCompression(unittest.TestCase): pass @patch("tools.file_tools._get_file_ops") - def test_reset_clears_dedup(self, mock_ops): - """After reset_file_dedup, the same read returns full content.""" + def test_first_post_compaction_read_recovers_exact_content(self, mock_ops): + """First post-compaction read is full; later reads deduplicate.""" mock_ops.return_value = _make_fake_ops( - content="original content\n", file_size=18, + content="SECRET_EXACT_LINE=42\n", file_size=21, ) # First read — populates dedup cache read_file_tool(self._tmpfile, task_id="comp") @@ -643,10 +648,15 @@ class TestDedupResetOnCompression(unittest.TestCase): # Simulate compression reset_file_dedup("comp") - # Read again — should get full content + # Exact prior bytes may have been omitted from the summary, so the + # first read in the new generation must restore them. r_post = json.loads(read_file_tool(self._tmpfile, task_id="comp")) - self.assertNotEqual(r_post.get("dedup"), True, - "Post-compression read should return full content") + self.assertNotIn("dedup", r_post) + self.assertIn("SECRET_EXACT_LINE=42", r_post.get("content", "")) + + # The persisted mtime map still saves tokens after that recovery read. + r_again = json.loads(read_file_tool(self._tmpfile, task_id="comp")) + self.assertTrue(r_again.get("dedup")) @patch("tools.file_tools._get_file_ops") @@ -662,13 +672,12 @@ class TestDedupResetOnCompression(unittest.TestCase): reset_file_dedup("loop") - # 3rd read — counter should still be at 2 from before reset - # (dedup was hit for read 2, but consecutive counter was 1 for that) - # After reset, this read goes through full path, incrementing to 2 + # First read in the new generation returns full content, not a stale + # block or a stub that points to compacted-away bytes. r3 = json.loads(read_file_tool(self._tmpfile, task_id="loop")) - # Should NOT be blocked or warned — counter restarted since dedup - # intercepted reads before they reached the counter self.assertNotIn("error", r3) + self.assertNotIn("dedup", r3) + self.assertIn("content", r3) # --------------------------------------------------------------------------- @@ -767,7 +776,7 @@ class TestWriteInvalidatesDedup(unittest.TestCase): _read_tracker.clear() self._tmpdir = _make_safe_tempdir("hermes-write-dedup-") self._tmpfile = os.path.join(self._tmpdir, "write_dedup.txt") - with open(self._tmpfile, "w") as f: + with open(self._tmpfile, "w", encoding="utf-8") as f: f.write("original content\n") def tearDown(self): diff --git a/tests/tools/test_file_state_registry.py b/tests/tools/test_file_state_registry.py index 30ef964178..adc11dd67f 100644 --- a/tests/tools/test_file_state_registry.py +++ b/tests/tools/test_file_state_registry.py @@ -25,6 +25,7 @@ import unittest from tools import file_state from tools.file_tools import ( + clear_file_ops_cache, read_file_tool, write_file_tool, patch_tool, @@ -123,6 +124,54 @@ class FileStateRegistryUnitTests(unittest.TestCase): ta.join(timeout=3.0) tb.join(timeout=3.0) + def test_lock_path_state_is_released_after_last_waiter(self): + p = self._mk() + first_entered = threading.Event() + release_first = threading.Event() + second_entered = threading.Event() + + def first() -> None: + with file_state.lock_path(p): + first_entered.set() + release_first.wait(timeout=2.0) + + def second() -> None: + first_entered.wait(timeout=2.0) + with file_state.lock_path(p): + second_entered.set() + + ta = threading.Thread(target=first) + tb = threading.Thread(target=second) + ta.start() + tb.start() + self.assertTrue(first_entered.wait(timeout=2.0)) + time.sleep(0.02) + self.assertFalse(second_entered.is_set()) + release_first.set() + ta.join(timeout=3.0) + tb.join(timeout=3.0) + + registry = file_state.get_registry() + self.assertTrue(second_entered.is_set()) + self.assertNotIn(p, registry._path_locks) + self.assertNotIn(p, registry._path_lock_users) + + def test_clear_file_ops_cache_releases_task_state(self): + p = self._mk() + task_id = "finished-task" + file_state.record_read(task_id, p) + + from tools import file_tools + + file_tools._read_tracker[task_id] = {"dedup": {}} + file_tools._patch_failure_tracker[task_id] = {p: 2} + + clear_file_ops_cache(task_id) + + self.assertEqual(file_state.known_reads(task_id), []) + self.assertNotIn(task_id, file_tools._read_tracker) + self.assertNotIn(task_id, file_tools._patch_failure_tracker) + def test_kill_switch_env_var(self): p = self._mk() diff --git a/tests/tools/test_interrupt.py b/tests/tools/test_interrupt.py index 67c5fbf662..899a7f570d 100644 --- a/tests/tools/test_interrupt.py +++ b/tests/tools/test_interrupt.py @@ -62,6 +62,105 @@ class TestInterruptModule: assert other_tid in _interrupted_threads # other thread untouched _interrupted_threads.discard(other_tid) + def test_run_if_not_interrupted_skips_callback_when_already_interrupted(self): + from tools.interrupt import run_if_not_interrupted, set_interrupt + + callbacks = [] + set_interrupt(True) + try: + assert run_if_not_interrupted(lambda: callbacks.append("claimed")) is False + finally: + set_interrupt(False) + + assert callbacks == [] + + @pytest.mark.parametrize("callback_should_fail", [False, True]) + def test_run_if_not_interrupted_orders_callback_before_concurrent_interrupt( + self, callback_should_fail, monkeypatch + ): + import tools.interrupt as interrupt + + class CallbackFailure(Exception): + pass + + original_lock = interrupt._lock + attempting_interrupt_lock = threading.Event() + interrupt_published = threading.Event() + publisher_lock_contention = [] + callback_observations = [] + setters = [] + setter_tids = [] + + class ObservedLock: + def __enter__(self): + if ( + threading.current_thread() in setters + and not attempting_interrupt_lock.is_set() + ): + acquired = original_lock.acquire(blocking=False) + publisher_lock_contention.append(not acquired) + attempting_interrupt_lock.set() + if acquired: + return self + original_lock.acquire() + return self + + def __exit__(self, exc_type, exc_value, traceback): + original_lock.release() + + interrupt.set_interrupt(False) + with original_lock: + baseline = ( + set(interrupt._interrupted_threads), + dict(interrupt._interrupt_reasons), + ) + monkeypatch.setattr(interrupt, "_lock", ObservedLock()) + + def publish_interrupt(): + setter_tids.append(threading.get_ident()) + try: + interrupt.set_interrupt(True) + interrupt_published.set() + finally: + interrupt.set_interrupt(False) + + def callback(): + setter = threading.Thread(target=publish_interrupt) + setters.append(setter) + setter.start() + assert attempting_interrupt_lock.wait(5) + assert publisher_lock_contention == [True] + callback_observations.append(interrupt_published.is_set()) + if callback_should_fail: + raise CallbackFailure + + try: + if callback_should_fail: + with pytest.raises(CallbackFailure): + interrupt.run_if_not_interrupted(callback) + else: + assert interrupt.run_if_not_interrupted(callback) is True + assert interrupt_published.wait(5) + finally: + for setter in setters: + if setter.ident is not None: + setter.join(timeout=5) + interrupt.set_interrupt(False) + + assert setters + assert all(not setter.is_alive() for setter in setters) + assert setter_tids + assert callback_observations == [False] + assert interrupt_published.is_set() + with original_lock: + final_state = ( + set(interrupt._interrupted_threads), + dict(interrupt._interrupt_reasons), + ) + assert final_state == baseline + assert all(setter_tid not in final_state[0] for setter_tid in setter_tids) + assert all(setter_tid not in final_state[1] for setter_tid in setter_tids) + # --------------------------------------------------------------------------- # Unit tests: pre-tool interrupt check diff --git a/tests/tools/test_macos_protected_search.py b/tests/tools/test_macos_protected_search.py index c0e3ce44e4..035cefb9d4 100644 --- a/tests/tools/test_macos_protected_search.py +++ b/tests/tools/test_macos_protected_search.py @@ -1,5 +1,6 @@ """macOS TCC-safe behavior for broad file searches.""" +import re from pathlib import Path import tools.file_operations as file_operations @@ -32,6 +33,17 @@ PROTECTED_NAMES = { } +def _rg_files_commands(commands): + return [command for command in commands if "--files" in command] + + +def _find_commands(commands): + return [ + command for command in commands + if command.startswith("find ") or "; find " in command + ] + + def test_broad_home_search_excludes_macos_protected_folders(tmp_path): home = tmp_path / "Users" / "alice" @@ -72,7 +84,7 @@ def test_broad_file_search_passes_protected_globs_to_ripgrep(tmp_path, monkeypat result = ops.search("*.txt", path=str(home), target="files") - rg_command = next(command for command in env.commands if command.startswith("rg --files")) + rg_command = _rg_files_commands(env.commands)[0] for dirname in PROTECTED_NAMES: assert f"!{dirname}/**" in rg_command assert result.warning is not None @@ -94,7 +106,7 @@ def test_broad_content_search_passes_protected_globs_to_ripgrep(tmp_path, monkey assert f"!{dirname}/**" in rg_command -def test_legacy_ripgrep_file_fallback_keeps_protected_globs(tmp_path, monkeypatch): +def test_empty_ripgrep_file_search_is_one_scan_with_protected_globs(tmp_path, monkeypatch): home = tmp_path / "Users" / "alice" home.mkdir(parents=True) env = RecordingEnvironment(home) @@ -104,8 +116,8 @@ def test_legacy_ripgrep_file_fallback_keeps_protected_globs(tmp_path, monkeypatc ops.search("*.txt", path=str(home), target="files") - rg_commands = [command for command in env.commands if command.startswith("rg --files")] - assert len(rg_commands) == 2 + rg_commands = _rg_files_commands(env.commands) + assert len(rg_commands) == 1 for command in rg_commands: assert "!Downloads/**" in command @@ -169,7 +181,7 @@ def test_remote_backend_never_prunes(tmp_path, monkeypatch): result = ops.search("*.txt", path=str(home), target="files") - rg_command = next(command for command in env.commands if command.startswith("rg --files")) + rg_command = _rg_files_commands(env.commands)[0] assert "!Downloads/**" not in rg_command assert result.warning is None @@ -185,13 +197,173 @@ def test_find_fallback_prunes_protected_directories(tmp_path, monkeypatch): ops.search("*.txt", path=str(home), target="files") - find_commands = [command for command in env.commands if command.startswith("find ")] + find_commands = _find_commands(env.commands) assert find_commands for command in find_commands: assert _bash_safe_path(str(home / "Downloads")) in command assert "-prune" in command +def _multi_root_protected_search(tmp_path, monkeypatch, engine): + home = tmp_path / "Users" / "alice" + downloads = home / "Downloads" + downloads.mkdir(parents=True) + env = RecordingEnvironment(home) + ops = ShellFileOperations(env) + monkeypatch.setattr(file_operations, "_HOME", str(home)) + monkeypatch.setattr(file_operations.sys, "platform", "darwin") + monkeypatch.setattr(ops, "_has_command", lambda command: command == engine) + path_checks = 0 + + def execute(command, cwd=None, **kwargs): + nonlocal path_checks + env.commands.append(command) + if command.startswith("test -e"): + path_checks += 1 + output = "not_found\n" if path_checks == 1 else "exists\n" + return {"output": output, "returncode": 0} + if "--files" in command or command.startswith("set -o pipefail; find "): + return {"output": "", "returncode": 0} + return {"output": "yes\n", "returncode": 0} + + env.execute = execute + result = ops.search("*.txt", path=f"{home} {downloads}", target="files") + return ops, env, result, downloads + + +def test_rg_multi_root_keeps_explicit_protected_root_and_reports_actual_skips( + tmp_path, monkeypatch +): + ops, env, result, downloads = _multi_root_protected_search( + tmp_path, monkeypatch, "rg" + ) + + command = _rg_files_commands(env.commands)[0] + absolute_operand = downloads.as_posix() in command + anchored_operand = ( + f"cd {ops._escape_shell_arg(downloads.parent.as_posix())} &&" in command + and " -- '.' 'Downloads' 2>/dev/null" in command + ) + assert absolute_operand or anchored_operand + assert "!Downloads/**" not in command + assert "path contained 2 entries" in (result.warning or "") + assert "macOS protected folders" in (result.warning or "") + protected_warning = result.warning.split("macOS protected folders", 1)[1] + assert "Desktop" in protected_warning + assert "Downloads" not in protected_warning + + +def test_find_multi_root_keeps_explicit_protected_root_and_reports_actual_skips( + tmp_path, monkeypatch +): + ops, env, result, downloads = _multi_root_protected_search( + tmp_path, monkeypatch, "find" + ) + + command = _find_commands(env.commands)[0] + assert "Downloads" in command + assert re.search(r"(?/dev/null" in command + assert result.error is None + + def test_real_ripgrep_does_not_descend_into_protected_folder(tmp_path, monkeypatch): home = tmp_path / "Users" / "alice" safe = home / "safe" diff --git a/tests/tools/test_mcp_capability_gating.py b/tests/tools/test_mcp_capability_gating.py index 5facbd24e4..25dd7084aa 100644 --- a/tests/tools/test_mcp_capability_gating.py +++ b/tests/tools/test_mcp_capability_gating.py @@ -295,4 +295,54 @@ class TestKeepaliveProbeFallback: assert task._ping_unsupported is False + async def test_silent_ping_drop_falls_back_to_list_tools(self): + """Regression for #97245: a server that silently drops ping (no + response at all) produces a TimeoutError. If list_tools succeeds, + the transport is alive — latch _ping_unsupported and return + normally instead of reconnect-looping.""" + task = MCPServerTask("test") + task.initialize_result = _caps(tools=SimpleNamespace()) + task.session = SimpleNamespace( + send_ping=AsyncMock(side_effect=asyncio.TimeoutError()), + list_tools=AsyncMock(return_value=SimpleNamespace(tools=[])), + ) + + # Should NOT raise — the server is alive. + await task._keepalive_probe() + + task.session.send_ping.assert_awaited_once() + task.session.list_tools.assert_awaited_once() + assert task._ping_unsupported is True + + async def test_silent_ping_drop_both_fail_propagates(self): + """When both ping AND list_tools time out, it is a genuine liveness + failure — propagate so the caller reconnects.""" + task = MCPServerTask("test") + task.initialize_result = _caps(tools=SimpleNamespace()) + task.session = SimpleNamespace( + send_ping=AsyncMock(side_effect=asyncio.TimeoutError()), + list_tools=AsyncMock(side_effect=asyncio.TimeoutError()), + ) + + with pytest.raises((TimeoutError, asyncio.TimeoutError)): + await task._keepalive_probe() + + assert task._ping_unsupported is False + + async def test_silent_ping_drop_no_tools_propagates(self): + """A server that has no tools capability and times out on ping has no + fallback probe — the timeout must propagate immediately.""" + task = MCPServerTask("test") + task.initialize_result = _caps(prompts=SimpleNamespace()) # no tools + task.session = SimpleNamespace( + send_ping=AsyncMock(side_effect=asyncio.TimeoutError()), + list_tools=AsyncMock(), + ) + + with pytest.raises((TimeoutError, asyncio.TimeoutError)): + await task._keepalive_probe() + + # list_tools must not be called — no tools capability advertised. + task.session.list_tools.assert_not_called() + assert task._ping_unsupported is False diff --git a/tests/tools/test_mcp_death_supervisor.py b/tests/tools/test_mcp_death_supervisor.py new file mode 100644 index 0000000000..f89ff64b46 --- /dev/null +++ b/tests/tools/test_mcp_death_supervisor.py @@ -0,0 +1,820 @@ +"""Contract tests for the shared parent-death supervisor for stdio MCP servers. + +The end-to-end tests here spawn real processes and really SIGKILL a real parent, +because the whole point of this module is behaviour that only exists when a +process dies without running any Python cleanup. A mocked parent death proves +nothing about the guarantee. +""" + +import asyncio +import contextlib +import io +import os +import signal +import subprocess +import sys +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from tools import mcp_death_supervisor, mcp_tool + +pytestmark = pytest.mark.skipif( + os.name != "posix", reason="the supervisor is POSIX-only (process groups)" +) + +SUPERVISOR = os.path.join(os.path.dirname(mcp_tool.__file__), "mcp_death_supervisor.py") + +# Long enough that nothing here can pass because the victim exited on its own. +_VICTIM = [sys.executable, "-c", "import time; time.sleep(300)"] + + +def _alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except (ProcessLookupError, OSError): + return False + return True + + +def _wait_gone(pid: int, timeout: float = 15.0) -> bool: + """Wait for a process this test does NOT own to disappear.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not _alive(pid): + return True + time.sleep(0.05) + return False + + +def _wait_exited(proc: subprocess.Popen, timeout: float = 15.0) -> bool: + """Wait for a direct child of this test to exit. + + ``os.kill(pid, 0)`` cannot be used for our own children: a killed child + stays a zombie until someone reaps it, and signalling a zombie succeeds. + """ + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + return False + return True + + +def _kill(pid: int) -> None: + try: + os.kill(pid, signal.SIGKILL) + except (ProcessLookupError, OSError): + pass + + +# --------------------------------------------------------------------------- +# Target safety: this process signals whole process GROUPS, so a bad target is +# unusually expensive. killpg(0, ...) would signal the supervisor's own group. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("pgid", [0, 1, -1, -5]) +def test_refuses_process_groups_that_are_never_a_valid_target(pgid): + assert mcp_death_supervisor._is_safe_target( + pgid, own_pgid=4242, parent_pgid=777 + ) is False + + +def test_refuses_its_own_group_and_the_parents_group(): + assert mcp_death_supervisor._is_safe_target( + 4242, own_pgid=4242, parent_pgid=777 + ) is False + assert mcp_death_supervisor._is_safe_target( + 777, own_pgid=4242, parent_pgid=777 + ) is False + + +def test_accepts_an_unrelated_group(): + assert mcp_death_supervisor._is_safe_target( + 999, own_pgid=4242, parent_pgid=777 + ) is True + + +# --------------------------------------------------------------------------- +# Control protocol +# --------------------------------------------------------------------------- + + +def test_registrations_survive_to_eof_and_unregistrations_are_dropped(): + stream = io.StringIO("register 111\nregister 222\nunregister 111\n") + + still_registered = mcp_death_supervisor._serve( + stream, own_pgid=4242, parent_pgid=777 + ) + + assert still_registered == {222} + + +def test_garbage_lines_do_not_cost_us_the_other_registrations(): + # A corrupted byte on the control pipe must not take down reaping for every + # other server -- that would turn a cosmetic bug into leaked processes. + stream = io.StringIO( + "register 111\n" + "\n" + "register\n" + "register notanumber\n" + "register 222 333\n" + "explode 444\n" + "register 555\n" + ) + + still_registered = mcp_death_supervisor._serve( + stream, own_pgid=4242, parent_pgid=777 + ) + + assert still_registered == {111, 555} + + +def test_a_writer_that_never_sends_a_newline_cannot_grow_us_without_bound(): + """Found for real: iterating the stream let /dev/zero reach 15 GB. + + The supervisor is the last line of defense against leaked MCP servers, so + it must not be the process that dies under memory pressure -- and a reader + that buffers until a newline arrives is exactly that risk. + """ + huge = "register " + ("0" * 10_000_000) + "\nregister 222\n" + + still_registered = mcp_death_supervisor._serve( + io.StringIO(huge), own_pgid=4242, parent_pgid=777 + ) + + # The overlong line is skipped, and the stream resyncs on the next one. + assert still_registered == {222} + + +def test_a_line_truncated_by_the_cap_is_never_acted_on(): + # Truncation must not turn one pgid into a different, valid-looking one: + # "register 999999" clipped to "register 9" would reap the wrong group. + stream = io.StringIO("register " + "9" * (mcp_death_supervisor._MAX_LINE_CHARS)) + + assert mcp_death_supervisor._serve( + stream, own_pgid=4242, parent_pgid=777 + ) == set() + + +def test_unsafe_targets_are_rejected_at_registration_time(): + stream = io.StringIO("register 0\nregister 777\nregister 999\n") + + still_registered = mcp_death_supervisor._serve( + stream, own_pgid=4242, parent_pgid=777 + ) + + assert still_registered == {999} + + +def test_refuses_to_run_inside_the_parents_own_process_group(): + # Started without start_new_session, a killpg of the parent's group would + # take the supervisor out before it could reap. It must not pretend to work. + proc = subprocess.run( + [sys.executable, SUPERVISOR, "--parent-pgid", str(os.getpgid(0))], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + + assert proc.returncode == 2 + assert "process group" in proc.stderr + + +# --------------------------------------------------------------------------- +# End to end: real processes, real death +# --------------------------------------------------------------------------- + + +def test_reaps_a_registered_group_when_the_control_pipe_reaches_eof(): + victim = subprocess.Popen(_VICTIM, start_new_session=True) + supervisor = subprocess.Popen( + [sys.executable, SUPERVISOR, "--parent-pgid", str(os.getpgid(0))], + stdin=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + supervisor.stdin.write(f"register {os.getpgid(victim.pid)}\n") + supervisor.stdin.flush() + assert victim.poll() is None, "victim should outlive registration" + + # EOF is the death signal, whatever closed the pipe. + supervisor.stdin.close() + + assert _wait_exited(victim), "registered group survived parent death" + finally: + _kill(victim.pid) + _kill(supervisor.pid) + victim.wait(timeout=10) + supervisor.wait(timeout=10) + + +def test_leaves_an_unregistered_group_alone_at_eof(): + # The other failure direction, and the more damaging one: a clean Hermes + # shutdown unregisters as it tears each server down, so EOF must not become + # a kill-everything event for servers that were handed back. + survivor = subprocess.Popen(_VICTIM, start_new_session=True) + supervisor = subprocess.Popen( + [sys.executable, SUPERVISOR, "--parent-pgid", str(os.getpgid(0))], + stdin=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + pgid = os.getpgid(survivor.pid) + supervisor.stdin.write(f"register {pgid}\nunregister {pgid}\n") + supervisor.stdin.flush() + supervisor.stdin.close() + + supervisor.wait(timeout=15) + assert survivor.poll() is None, "a cleanly unregistered server was killed" + finally: + _kill(survivor.pid) + _kill(supervisor.pid) + survivor.wait(timeout=10) + supervisor.wait(timeout=10) + + +# A stand-in for Hermes: registers a real child, then blocks forever holding the +# only write end of the control pipe. SIGKILLing it is the scenario the whole +# module exists for -- no cleanup code of ours gets to run. +_FAKE_PARENT = """ +import os, subprocess, sys, time + +supervisor = sys.argv[1] +victim = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(300)"], start_new_session=True +) +sup = subprocess.Popen( + [sys.executable, supervisor, "--parent-pgid", str(os.getpgid(0))], + stdin=subprocess.PIPE, text=True, start_new_session=True, +) +sup.stdin.write("register %d\\n" % os.getpgid(victim.pid)) +sup.stdin.flush() +print("%d %d" % (victim.pid, sup.pid), flush=True) +time.sleep(300) +""" + + +# Reparented-to-init processes are by definition outside this test's subtree, +# so cleaning them up trips conftest's live-system kill guard. Real signal +# delivery to a real orphan is the entire point of these two tests. +@pytest.mark.live_system_guard_bypass +def test_reaps_the_server_when_the_registering_parent_is_sigkilled(tmp_path): + script = tmp_path / "fake_parent.py" + script.write_text(_FAKE_PARENT) + + parent = subprocess.Popen( + [sys.executable, str(script), SUPERVISOR], + stdout=subprocess.PIPE, + text=True, + ) + victim_pid = supervisor_pid = None + try: + victim_pid, supervisor_pid = ( + int(x) for x in parent.stdout.readline().split() + ) + assert _alive(victim_pid) + + # No graceful anything: the parent never runs another line of Python. + parent.kill() + parent.wait(timeout=10) + + assert _wait_gone(victim_pid), ( + "stdio MCP server survived kill -9 of its Hermes parent" + ) + finally: + for pid in (victim_pid, supervisor_pid): + if pid is not None: + _kill(pid) + _kill(parent.pid) + + +@pytest.mark.live_system_guard_bypass +def test_reaps_a_grandchild_left_in_the_registered_group(tmp_path): + # Real shape of the bug: mcp-remote exits but leaves the `node` it spawned + # behind. The grandchild reparents to init but keeps the pgid, so killpg + # still reaches it -- which is why we track groups and not pids. + script = tmp_path / "leaky_server.py" + script.write_text( + "import subprocess, sys\n" + "child = subprocess.Popen([sys.executable, '-c'," + " 'import time; time.sleep(300)'])\n" + "print(child.pid, flush=True)\n" + ) + + # start_new_session mirrors how the MCP SDK spawns stdio servers. + server = subprocess.Popen( + [sys.executable, str(script)], + stdout=subprocess.PIPE, + text=True, + start_new_session=True, + ) + grandchild_pid = int(server.stdout.readline()) + server.wait(timeout=10) # the direct child exits; the grandchild does not + + supervisor = subprocess.Popen( + [sys.executable, SUPERVISOR, "--parent-pgid", str(os.getpgid(0))], + stdin=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + assert _alive(grandchild_pid), "grandchild should outlive its parent" + # server.pid is its own pgid leader, captured at spawn time exactly as + # mcp_tool records it -- still usable after the leader itself exited. + supervisor.stdin.write(f"register {server.pid}\n") + supervisor.stdin.flush() + supervisor.stdin.close() + + assert _wait_gone(grandchild_pid), "orphaned grandchild was not reaped" + finally: + _kill(grandchild_pid) + _kill(supervisor.pid) + supervisor.wait(timeout=10) + + +# --------------------------------------------------------------------------- +# Client side: what mcp_tool tells the supervisor +# --------------------------------------------------------------------------- + + +class _FakeSupervisor: + """Stands in for the supervisor process, recording the control stream.""" + + def __init__(self, exited=False): + self.stdin = io.StringIO() + self.pid = 4242 + self._exited = exited + self._sent = "" + self.closed = False + _real_close = self.stdin.close + + def _close(): + # Mirror a real pipe: capture what was written before the write + # end goes away, so tests can still assert on the control stream. + self._sent = self.stdin.getvalue() + self.closed = True + _real_close() + + self.stdin.close = _close + + def poll(self): + return 1 if self._exited else None + + def wait(self, timeout=None): + self.waited = True + return 0 + + def lines(self): + if self.closed: + return self._sent.splitlines() + return self.stdin.getvalue().splitlines() + + +@pytest.fixture(autouse=True) +def _reset_client_state(): + yield + mcp_tool._death_supervisor = None + mcp_tool._supervised_pgids.clear() + + +@pytest.fixture +def all_groups_alive(monkeypatch): + """Answer every liveness probe with "this group exists". + + The protocol tests below register synthetic pgids that were never real + process groups. Without this, the liveness prune correctly discards them + before the control stream can be asserted on -- so state the precondition + rather than letting these tests depend on pid-space luck. + """ + monkeypatch.setattr(mcp_tool.os, "killpg", lambda pgid, sig: None) + + +def test_register_starts_the_supervisor_once_and_reuses_it(monkeypatch, all_groups_alive): + spawned = [] + + def _spawn(): + fake = _FakeSupervisor() + spawned.append(fake) + return fake + + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", _spawn) + + mcp_tool._update_death_supervisor("register", [111]) + mcp_tool._update_death_supervisor("register", [222]) + + assert len(spawned) == 1, "each register spawned its own supervisor" + assert spawned[0].lines() == ["register 111", "register 222"] + + +def test_unregister_is_forwarded(monkeypatch, all_groups_alive): + fake = _FakeSupervisor() + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", lambda: fake) + + mcp_tool._update_death_supervisor("register", [111]) + mcp_tool._update_death_supervisor("unregister", [111]) + + assert fake.lines() == ["register 111", "unregister 111"] + assert mcp_tool._supervised_pgids == set() + + +def test_supervisor_is_released_once_nothing_is_left_to_reap(monkeypatch, all_groups_alive): + """An empty registration set must not keep a supervisor resident. + + A gateway that once connected a stdio server would otherwise carry a + ~15 MB process and a live pipe for the rest of its life. Closing our + write end is the same EOF the supervisor treats as parent death; with + nothing registered it exits without reaping. The next register starts a + fresh one, exactly like the dead-supervisor replay path. + """ + spawned = [] + + def _spawn(): + fake = _FakeSupervisor() + spawned.append(fake) + return fake + + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", _spawn) + + mcp_tool._update_death_supervisor("register", [111, 222]) + mcp_tool._update_death_supervisor("unregister", [111]) + assert not spawned[0].closed, "released the supervisor while a group was still registered" + + mcp_tool._update_death_supervisor("unregister", [222]) + assert spawned[0].closed, "supervisor kept resident with nothing left to reap" + assert getattr(spawned[0], "waited", False), "released supervisor was never wait()ed -> zombie until the next Popen" + assert spawned[0].lines()[-1] == "unregister 222", "release happened before the last unregister was sent" + assert mcp_tool._death_supervisor is None + + mcp_tool._update_death_supervisor("register", [333]) + assert len(spawned) == 2 and spawned[1].lines() == ["register 333"] + + +def test_supervisor_survives_the_real_eof_release(): + """End to end: closing the control pipe with nothing registered exits cleanly.""" + if os.name != "posix": + pytest.skip("POSIX-only supervisor") + child = subprocess.Popen(_VICTIM, start_new_session=True) + try: + mcp_tool._update_death_supervisor("register", [os.getpgid(child.pid)]) + proc = mcp_tool._death_supervisor + assert proc is not None and proc.poll() is None + mcp_tool._update_death_supervisor("unregister", [os.getpgid(child.pid)]) + assert mcp_tool._death_supervisor is None + assert proc.wait(timeout=10) == 0, "supervisor did not exit on the release EOF" + assert child.poll() is None, "release reaped a group that had been unregistered" + finally: + _kill(child.pid) + child.wait(timeout=10) + + +def test_unregister_alone_does_not_start_a_supervisor(monkeypatch): + spawned = [] + monkeypatch.setattr( + mcp_tool, + "_spawn_death_supervisor", + lambda: spawned.append(1) or _FakeSupervisor(), + ) + + mcp_tool._update_death_supervisor("unregister", [111]) + + assert spawned == [] + + +def test_a_dead_supervisor_is_replaced_and_live_coverage_replayed(monkeypatch, all_groups_alive): + dead = _FakeSupervisor(exited=True) + replacement = _FakeSupervisor() + queue = [dead, replacement] + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", lambda: queue.pop(0)) + + mcp_tool._update_death_supervisor("register", [111]) + mcp_tool._update_death_supervisor("register", [222]) + + # Losing the supervisor must not silently drop the server registered with + # it -- the replacement has to be told about 111 as well as 222. + assert set(replacement.lines()) == {"register 111", "register 222"} + + +def test_replay_does_not_resurrect_an_unregistered_group(monkeypatch, all_groups_alive): + dead = _FakeSupervisor(exited=True) + replacement = _FakeSupervisor() + queue = [dead, replacement] + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", lambda: queue.pop(0)) + + mcp_tool._update_death_supervisor("register", [111]) + mcp_tool._update_death_supervisor("register", [222]) + mcp_tool._update_death_supervisor("unregister", [111]) + + assert mcp_tool._supervised_pgids == {222} + # 111 was legitimately replayed to the replacement (it was live when the + # dead supervisor was swapped out), then unregistered. What must never + # happen is a replay AFTER the unregister bringing it back. + lines = replacement.lines() + assert lines.index("unregister 111") > lines.index("register 111") + assert "register 111" not in lines[lines.index("unregister 111") :] + mcp_tool._update_death_supervisor("register", [333]) # any later replay/append + assert "register 111" not in replacement.lines()[len(lines) :] + + +def test_a_broken_pipe_never_propagates_into_a_live_mcp_session(monkeypatch, all_groups_alive): + class _BrokenPipe(_FakeSupervisor): + def __init__(self): + super().__init__() + + class _Stdin: + def write(self, _payload): + raise BrokenPipeError("supervisor exited after poll()") + + def flush(self): + pass + + self.stdin = _Stdin() + + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", _BrokenPipe) + + mcp_tool._update_death_supervisor("register", [111]) # must not raise + + # Dropped, so the next registration respawns instead of writing into a + # pipe that is known to be dead. + assert mcp_tool._death_supervisor is None + + +def test_unregister_after_a_broken_pipe_rebuilds_coverage_for_survivors(monkeypatch, all_groups_alive): + """A lost supervisor must be replaced by the NEXT lifecycle event, whatever its verb. + + Sequence from the #93517 review: two groups live, the control pipe dies + (write fails, supervisor dropped, set retained), then a clean teardown + unregisters one of them. Keying the no-spawn fast path on the verb left + the survivor recorded but unsupervised; it must be keyed on the set. + """ + spawned = [] + + def _spawn(): + fake = _FakeSupervisor() + spawned.append(fake) + return fake + + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", _spawn) + mcp_tool._update_death_supervisor("register", [111, 222]) + + class _DeadStdin: + def write(self, _payload): + raise BrokenPipeError("supervisor died") + + def flush(self): + pass + + spawned[0].stdin = _DeadStdin() + mcp_tool._update_death_supervisor("register", [333]) # the write fails; supervisor dropped + assert mcp_tool._death_supervisor is None + assert mcp_tool._supervised_pgids == {111, 222, 333} + + mcp_tool._update_death_supervisor("unregister", [222]) + + assert len(spawned) == 2, "unregister after a lost supervisor did not respawn one" + assert sorted(spawned[1].lines()) == ["register 111", "register 333"], ( + "the replacement did not receive the surviving groups" + ) + assert mcp_tool._death_supervisor is spawned[1] + + +def test_a_supervisor_that_cannot_start_is_not_fatal(monkeypatch, all_groups_alive): + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", lambda: None) + + mcp_tool._update_death_supervisor("register", [111]) # must not raise + + assert mcp_tool._death_supervisor is None + + +@contextlib.contextmanager +def _stdio_connection(child_pid, fake_supervisor): + """Drive the real MCPServerTask._run_stdio with a known spawned child. + + Only the MCP transport itself is mocked. Everything the supervisor wiring + depends on -- child discovery, _filter_mcp_children, the real os.getpgid + lookup -- runs for real against ``child_pid``, so the pgid asserted on is + the pgid of an actual process rather than a fixture value. + """ + session = MagicMock() + session.initialize = AsyncMock() + session.list_tools = AsyncMock(return_value=SimpleNamespace(tools=[])) + + stdio_cm = MagicMock() + stdio_cm.__aenter__ = AsyncMock(return_value=(object(), object())) + stdio_cm.__aexit__ = AsyncMock(return_value=False) + session_cm = MagicMock() + session_cm.__aenter__ = AsyncMock(return_value=session) + session_cm.__aexit__ = AsyncMock(return_value=False) + + with ( + patch("tools.mcp_tool.stdio_client", return_value=stdio_cm), + patch("tools.mcp_tool.ClientSession", return_value=session_cm), + # First call is the pids_before baseline; the second reports our child + # as the newly spawned server. + patch( + "tools.mcp_tool._snapshot_child_pids", + side_effect=[set(), {child_pid}], + ), + patch("tools.mcp_tool._write_stderr_log_header"), + patch("tools.mcp_tool._get_mcp_stderr_log", return_value=None), + patch( + "tools.mcp_tool._spawn_death_supervisor", + return_value=fake_supervisor, + ), + ): + yield mcp_tool.MCPServerTask("supervisor-wiring") + + +@pytest.mark.skipif(not mcp_tool._MCP_AVAILABLE, reason="MCP SDK not installed") +def test_connecting_a_stdio_server_registers_its_real_process_group(): + fake = _FakeSupervisor() + child = subprocess.Popen(_VICTIM, start_new_session=True) + try: + with _stdio_connection(child.pid, fake) as server: + asyncio.run(server.start({"command": "echo", "args": ["hi"]})) + + assert f"register {os.getpgid(child.pid)}" in fake.lines(), ( + "connecting a stdio server did not hand its process group to the " + f"supervisor; control stream was {fake.lines()}" + ) + finally: + _kill(child.pid) + child.wait(timeout=10) + + +@pytest.mark.skipif(not mcp_tool._MCP_AVAILABLE, reason="MCP SDK not installed") +def test_a_server_that_exited_is_released_on_teardown(): + fake = _FakeSupervisor() + child = subprocess.Popen(_VICTIM, start_new_session=True) + pgid = None + try: + with _stdio_connection(child.pid, fake) as server: + + async def _connect_then_lose_the_child(): + await server.start({"command": "echo", "args": ["hi"]}) + nonlocal pgid + pgid = os.getpgid(child.pid) + # The server exits while connected. Reap it here so the + # teardown path sees a genuinely dead pid, not a zombie. + child.kill() + child.wait(timeout=10) + await server.shutdown() + + asyncio.run(_connect_then_lose_the_child()) + + assert f"register {pgid}" in fake.lines() + assert f"unregister {pgid}" in fake.lines(), ( + "a stdio server with nothing left alive stayed registered, so the " + f"supervisor would keep a stale group; stream was {fake.lines()}" + ) + finally: + _kill(child.pid) + + +@pytest.mark.skipif(not mcp_tool._MCP_AVAILABLE, reason="MCP SDK not installed") +def test_a_server_that_survived_teardown_stays_registered(): + # The case the whole module exists for: teardown did not manage to kill it. + # Releasing it here would hand the orphan back to nobody. + fake = _FakeSupervisor() + child = subprocess.Popen(_VICTIM, start_new_session=True) + try: + with _stdio_connection(child.pid, fake) as server: + + async def _connect_then_shutdown(): + await server.start({"command": "echo", "args": ["hi"]}) + await server.shutdown() + + asyncio.run(_connect_then_shutdown()) + + pgid = os.getpgid(child.pid) + assert f"register {pgid}" in fake.lines() + assert f"unregister {pgid}" not in fake.lines(), ( + "a server that outlived teardown was released from the supervisor, " + "so an ungraceful exit would leave it running forever" + ) + finally: + _kill(child.pid) + child.wait(timeout=10) + + +@pytest.mark.live_system_guard_bypass +def test_scoped_teardown_of_one_owner_keeps_the_other_owner_supervised(monkeypatch): + """Two owners (profiles / agents) each hold a stdio group; tearing one down + must release only that owner's group and leave the other covered, and the + per-process supervisor must then still know about the survivor. + + Exercises the real registry + ``_kill_orphaned_mcp_children`` scoping + rather than the control protocol alone (review request on #93517). + """ + fake = _FakeSupervisor() + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", lambda: fake) + monkeypatch.setattr(mcp_tool.time, "sleep", lambda _s: None) # skip the SIGTERM grace wait + a = subprocess.Popen(_VICTIM, start_new_session=True) + b = subprocess.Popen(_VICTIM, start_new_session=True) + try: + pg_a, pg_b = os.getpgid(a.pid), os.getpgid(b.pid) + with mcp_tool._lock: + mcp_tool._stdio_pids[a.pid] = "profile-a" + mcp_tool._stdio_pids[b.pid] = "profile-b" + mcp_tool._stdio_pgids[a.pid] = pg_a + mcp_tool._stdio_pgids[b.pid] = pg_b + mcp_tool._update_death_supervisor("register", [pg_a, pg_b]) + + mcp_tool._kill_orphaned_mcp_children(include_active=True, server_name="profile-a") + a.wait(timeout=10) + + assert b.poll() is None, "scoped teardown of profile-a killed profile-b's server" + assert f"unregister {pg_a}" in fake.lines() + assert f"unregister {pg_b}" not in fake.lines(), ( + "scoped teardown released the OTHER owner's group from the supervisor" + ) + assert mcp_tool._supervised_pgids == {pg_b} + assert b.pid in mcp_tool._stdio_pids and b.pid in mcp_tool._stdio_pgids + finally: + for p in (a, b): + _kill(p.pid) + try: + p.wait(timeout=10) + except Exception: # noqa: BLE001 - best-effort cleanup + pass + with mcp_tool._lock: + for p in (a, b): + mcp_tool._stdio_pids.pop(p.pid, None) + mcp_tool._stdio_pgids.pop(p.pid, None) + + +@pytest.mark.live_system_guard_bypass +def test_a_group_with_nothing_left_alive_is_forgotten_and_unregistered(monkeypatch): + """A dead group must not stay registered: its pgid can be recycled. + + Uses a real process so the liveness probe is answered by the kernel rather + than a fixture -- the whole point is that we notice actual death. + """ + fake = _FakeSupervisor() + monkeypatch.setattr(mcp_tool, "_spawn_death_supervisor", lambda: fake) + + doomed = subprocess.Popen(_VICTIM, start_new_session=True) + doomed_pgid = os.getpgid(doomed.pid) + survivor = subprocess.Popen(_VICTIM, start_new_session=True) + survivor_pgid = os.getpgid(survivor.pid) + try: + mcp_tool._update_death_supervisor("register", [doomed_pgid, survivor_pgid]) + assert mcp_tool._supervised_pgids == {doomed_pgid, survivor_pgid} + + # Reap it fully so the group is genuinely empty, not a zombie. + doomed.kill() + doomed.wait(timeout=10) + + # Any later registration change is when we notice. + mcp_tool._update_death_supervisor("register", [survivor_pgid]) + + assert doomed_pgid not in mcp_tool._supervised_pgids, ( + "a group with no members left stayed registered, so a recycled " + "pgid could later be reaped as if it were an MCP server" + ) + assert survivor_pgid in mcp_tool._supervised_pgids, ( + "pruning dropped a group that is still alive" + ) + assert f"unregister {doomed_pgid}" in fake.lines(), ( + "the supervisor was never told to forget the dead group" + ) + finally: + _kill(survivor.pid) + survivor.wait(timeout=10) + _kill(doomed.pid) + + +def test_pruning_keeps_groups_it_cannot_prove_are_gone(monkeypatch): + # An ambiguous probe (EPERM: exists but not ours) must not drop coverage -- + # losing a real registration is worse than keeping a doubtful one. + monkeypatch.setattr(mcp_tool, "_supervised_pgids", {111, 222}, raising=False) + + def _probe(pgid, sig): + if pgid == 111: + raise PermissionError("exists, not ours") + raise ProcessLookupError("gone") + + monkeypatch.setattr(mcp_tool.os, "killpg", _probe) + + stale = mcp_tool._prune_dead_supervised_pgids() + + assert stale == {222} + assert mcp_tool._supervised_pgids == {111} + + +def test_no_pgids_is_a_no_op(monkeypatch): + spawned = [] + monkeypatch.setattr( + mcp_tool, + "_spawn_death_supervisor", + lambda: spawned.append(1) or _FakeSupervisor(), + ) + + mcp_tool._update_death_supervisor("register", []) + + assert spawned == [] diff --git a/tests/tools/test_mcp_lazy_start.py b/tests/tools/test_mcp_lazy_start.py index 85312c0fc2..dedb3de115 100644 --- a/tests/tools/test_mcp_lazy_start.py +++ b/tests/tools/test_mcp_lazy_start.py @@ -285,7 +285,7 @@ class TestLazyFirstUseConnect: patch.object(registry, "deregister") as mock_dereg: assert mcp._ensure_lazy_server_connected("playwright") is True - mock_dereg.assert_called_once_with("mcp_playwright_tool_x") + mock_dereg.assert_called_once_with("mcp_playwright_tool_x", scope=None) def test_lazy_connect_failure_records_cooldown(self): mcp._lazy_server_configs["playwright"] = {"command": "npx", "lazy": True} diff --git a/tests/tools/test_mcp_npx_cached_bin.py b/tests/tools/test_mcp_npx_cached_bin.py new file mode 100644 index 0000000000..40a16640d8 --- /dev/null +++ b/tests/tools/test_mcp_npx_cached_bin.py @@ -0,0 +1,208 @@ +"""``npx -y `` should spawn the cached binary, not a resident `npm exec`. + +`npx` resolves the package and then FORKS, staying alive as the real server's +parent for the whole process lifetime while doing no work. Measured on a +4-agent host that is ~48 MB of private memory per MCP server — and it buys +nothing, because Hermes already wraps the child in its own parent-death +watchdog, so npx's supervision is a second parent nobody reads. + +Removing it must stay conservative: a cache miss, a version-pinned spec, or an +ambiguous ``bin`` map all fall back to plain `npx` so a cold machine still +installs normally. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from tools.mcp_tool import _npx_cached_bin + + +def _cache(tmp_path, *, package, deps=None, bin_field, make_bin=True, entry="abc123"): + """Build a fake npx cache entry the way npm lays one out.""" + root = tmp_path / ".npm" / "_npx" / entry + (root / "node_modules" / package).mkdir(parents=True) + (root / "package.json").write_text( + json.dumps({"dependencies": deps if deps is not None else {package: "^1.0.0"}}), + encoding="utf-8", + ) + (root / "node_modules" / package / "package.json").write_text( + json.dumps({"name": package, "bin": bin_field}), encoding="utf-8" + ) + bindir = root / "node_modules" / ".bin" + bindir.mkdir(parents=True, exist_ok=True) + name = bin_field if isinstance(bin_field, str) else list(bin_field)[0] + target = bindir / (os.path.basename(package) if isinstance(bin_field, str) else name) + if make_bin: + target.write_text("#!/usr/bin/env node\n", encoding="utf-8") + target.chmod(0o755) + return target + + +@pytest.fixture(autouse=True) +def _isolate_cache(tmp_path, monkeypatch): + monkeypatch.setenv("npm_config_cache", str(tmp_path / ".npm")) + yield + + +def test_cached_package_resolves_to_its_binary(tmp_path): + target = _cache(tmp_path, package="mcp-linear", bin_field={"mcp-linear": "dist/index.js"}) + + got = _npx_cached_bin(["-y", "mcp-linear"]) + + assert got == (str(target), []) + + +def test_scoped_package_and_trailing_args_survive(tmp_path): + target = _cache( + tmp_path, + package="@tacticlaunch/mcp-linear", + bin_field={"mcp-linear": "dist/index.js"}, + ) + + got = _npx_cached_bin(["-y", "@tacticlaunch/mcp-linear", "--port", "7"]) + + assert got == (str(target), ["--port", "7"]) + + +def test_uncached_package_falls_back_to_npx(tmp_path): + _cache(tmp_path, package="something-else", bin_field={"something-else": "i.js"}) + + assert _npx_cached_bin(["-y", "mcp-linear"]) is None + + +def test_version_pinned_spec_is_left_to_npx(tmp_path): + _cache(tmp_path, package="mcp-linear", bin_field={"mcp-linear": "dist/index.js"}) + + # The user pinned a build; npx owns that resolution and the cache key for + # a different version would not match this entry. + assert _npx_cached_bin(["-y", "mcp-linear@1.2.3"]) is None + + +def test_ambiguous_bin_map_is_left_to_npx(tmp_path): + _cache( + tmp_path, + package="multi", + bin_field={"one": "a.js", "two": "b.js"}, + ) + + # Which bin npx would choose is not ours to guess. + assert _npx_cached_bin(["-y", "multi"]) is None + + +def test_missing_or_non_executable_binary_falls_back(tmp_path): + _cache( + tmp_path, + package="mcp-linear", + bin_field={"mcp-linear": "dist/index.js"}, + make_bin=False, + ) + + assert _npx_cached_bin(["-y", "mcp-linear"]) is None + + +def test_no_cache_directory_at_all(tmp_path, monkeypatch): + monkeypatch.setenv("npm_config_cache", str(tmp_path / "nope")) + + assert _npx_cached_bin(["-y", "mcp-linear"]) is None + + +def test_corrupt_cache_manifest_is_skipped(tmp_path): + root = tmp_path / ".npm" / "_npx" / "broken" + root.mkdir(parents=True) + (root / "package.json").write_text("{ not json", encoding="utf-8") + + assert _npx_cached_bin(["-y", "mcp-linear"]) is None + + +@pytest.mark.parametrize("args", [[], ["-y"], ["--yes"], ["-p", "x"], None, "notalist"]) +def test_unusable_args_are_ignored(args): + assert _npx_cached_bin(args) is None + + +def test_osv_preflight_runs_before_the_swap(): + """The malware gate must still see `npx` + the package name. + + `_infer_ecosystem` keys off the command basename, so a command already + rewritten to `.../node_modules/.bin/mcp-linear` yields no ecosystem and + `check_package_for_malware` returns None — the gate silently becomes a + no-op. This pins the ordering: OSV inspects the original invocation. + """ + from tools.osv_check import _infer_ecosystem, _parse_package_from_args + + # What the preflight sees today, before any swap. + assert _infer_ecosystem("npx") == "npm" + assert _parse_package_from_args(["-y", "@tacticlaunch/mcp-linear"], "npm")[0] == ( + "@tacticlaunch/mcp-linear" + ) + + # What it would see if the swap happened first — nothing. + assert _infer_ecosystem("/home/u/.npm/_npx/abc/node_modules/.bin/mcp-linear") is None + + +def test_swap_happens_after_the_osv_call_in_source(): + """Structural guard for the ordering above. + + The swap and the preflight live in one async function; a future edit that + moves the swap earlier would disable the malware gate silently, and no + unit test of either piece alone would notice. + """ + from pathlib import Path as _P + + src = _P(__file__).resolve().parents[2] / "tools" / "mcp_tool.py" + text = src.read_text(encoding="utf-8") + osv_needle = "check_package_for_malware, command, args" + swap_needle = "cached = _npx_cached_bin(args)" + # Report a rename explicitly: a bare .index() ValueError here reads like a + # broken test rather than "someone renamed the thing this guards". + assert osv_needle in text, ( + f"cannot find the OSV preflight call ({osv_needle!r}) — it was renamed; " + "update this guard and re-verify the swap still happens after it" + ) + assert swap_needle in text, ( + f"cannot find the npx swap ({swap_needle!r}) — it was renamed; update " + "this guard and re-verify it still happens after the OSV preflight" + ) + + assert text.index(osv_needle) < text.index(swap_needle), ( + "the npx swap now precedes the OSV malware preflight, which silently " + "disables it: _infer_ecosystem keys off the command basename being " + "npx/uvx/pipx, so a rewritten command yields no ecosystem and " + "check_package_for_malware returns None" + ) + + +def test_windows_selects_launchers_never_the_sh_script(): + """On Windows the extensionless sh script must never be chosen. + + npm lays down three siblings per bin — ``, `.cmd`, + `.ps1` — and spawning the sh one from a Windows process fails, while + `os.access(X_OK)` there is effectively an existence check and cannot tell + them apart. Tested through the injectable helper rather than by patching + `os.name`, which breaks path handling process-wide (it took pytest's own + traceback formatting down when I tried). + """ + from tools.mcp_tool import _npx_bin_candidates + + win = _npx_bin_candidates("/c/bin", "mcp-linear", windows=True) + assert win == ["/c/bin/mcp-linear.cmd", "/c/bin/mcp-linear.exe"] + assert not any(c.endswith("mcp-linear") for c in win), "sh script must not be a candidate" + + assert _npx_bin_candidates("/bin", "mcp-linear", windows=False) == ["/bin/mcp-linear"] + + +def test_posix_resolution_uses_the_helper(tmp_path): + """The resolver honours the helper's ordering (POSIX path end-to-end).""" + target = _cache(tmp_path, package="mcp-linear", bin_field={"mcp-linear": "i.js"}) + + assert _npx_cached_bin(["-y", "mcp-linear"]) == (str(target), []) + + +def test_flag_after_the_spec_is_left_to_npx(tmp_path): + """`npx pkg -y` would forward -y to the server; that shape stays with npx.""" + _cache(tmp_path, package="mcp-linear", bin_field={"mcp-linear": "i.js"}) + + assert _npx_cached_bin(["mcp-linear", "-y"]) is None diff --git a/tests/tools/test_mcp_stability.py b/tests/tools/test_mcp_stability.py index f3a5591ebb..e1261bdda7 100644 --- a/tests/tools/test_mcp_stability.py +++ b/tests/tools/test_mcp_stability.py @@ -66,6 +66,36 @@ class TestStdioPidTracking: for pid in result: assert isinstance(pid, int) + def test_snapshot_sees_child_spawned_from_another_thread(self): + """/proc//task//children is per-thread; the MCP subprocess + is spawned from the background loop thread, so a main-thread-only + read misses it and every dead-child fast-fail / respawn / killpg + path silently no-ops.""" + import subprocess + import sys as _sys + import threading + + from tools.mcp_tool import _snapshot_child_pids + + procs = [] + started = threading.Event() + release = threading.Event() + + def _spawn(): + procs.append(subprocess.Popen([_sys.executable, "-c", "import time; time.sleep(30)"])) + started.set() + release.wait(10) # keep the spawning thread alive while we snapshot + + t = threading.Thread(target=_spawn, daemon=True) + t.start() + assert started.wait(10) + try: + assert procs[0].pid in _snapshot_child_pids() + finally: + release.set() + procs[0].kill() + procs[0].wait(5) + def test_kill_orphaned_handles_dead_pids(self): """_kill_orphaned_mcp_children gracefully handles already-dead PIDs.""" diff --git a/tests/tools/test_mcp_stdio_children_dead.py b/tests/tools/test_mcp_stdio_children_dead.py index 09398d5958..23ec4a8077 100644 --- a/tests/tools/test_mcp_stdio_children_dead.py +++ b/tests/tools/test_mcp_stdio_children_dead.py @@ -92,3 +92,36 @@ def test_watcher_resolves_when_all_children_are_dead(): ) asyncio.run(_run()) + + +def test_watch_ok_probe_does_not_create_unawaited_coroutine(): + """The fast-fail gate must inspect the watcher, not call it (#96044). + + The old probe — inspect.isawaitable(_watch_children()) — created a + fresh coroutine per stdio tool call and never awaited it, emitting + 'coroutine ... was never awaited' RuntimeWarnings under -W error and + churning the GC. Pin that the shipped source no longer calls the + watcher during the probe. + """ + import inspect as _inspect + + import tools.mcp_tool as mcp_mod + + src = _inspect.getsource(mcp_mod) + assert "isawaitable(_watch_children())" not in src + assert "iscoroutinefunction(_watch_children)" in src + + +def test_watch_ok_semantics_mock_vs_real(): + """MagicMock watchers stay on the plain-await path; real async defs + (and AsyncMock) qualify for the fast-fail race — same split the old + isawaitable(call) probe produced, without the coroutine leak.""" + import inspect as _inspect + from unittest.mock import AsyncMock, MagicMock + + async def _real_watcher(): # what the real method looks like + pass + + assert _inspect.iscoroutinefunction(_real_watcher) is True + assert _inspect.iscoroutinefunction(AsyncMock()) is True + assert _inspect.iscoroutinefunction(MagicMock()) is False diff --git a/tests/tools/test_mcp_stdio_fastfail_reconnect.py b/tests/tools/test_mcp_stdio_fastfail_reconnect.py index 80ae511981..7b9a934a52 100644 --- a/tests/tools/test_mcp_stdio_fastfail_reconnect.py +++ b/tests/tools/test_mcp_stdio_fastfail_reconnect.py @@ -1,16 +1,23 @@ -"""Regression tests for stdio fast-fail reconnect signaling (#95626 salvage). +"""Regression tests for dead stdio subprocess recovery (#95626 salvage). The #81995 fast-fail gate detects a dead stdio subprocess but the transport failure never cleared ``server.session``, so the transport-down reconnect path (which only fires when the session is gone/not-ready) never ran. The call failed fast — correctly — but nothing asked the server task to respawn the -subprocess, so every subsequent call kept failing until the idle keepalive -probe eventually noticed. Both fast-fail sites must signal a reconnect: +subprocess (#95626 added the reconnect signal). -- pre-call gate (children already dead when the call arrives): return a clean - "reconnecting" tool error and set ``_reconnect_event``; -- mid-call watcher race (children die while the RPC is in flight): raise the - fast-fail TimeoutError and set ``_reconnect_event``. +Signalling alone still lost the call: a gateway restart kills every MCP stdio +child, and the first call from a surviving agent session (or a cron run +spanning the restart) failed in 0.00s while the subprocess was respawned +seconds later. Both fast-fail sites now respawn AND retry once: + +- pre-call gate (children already dead when the call arrives); +- mid-call watcher race (children die while the RPC is in flight). + +Both must recover transparently, and both must stop after ONE retry so a +server that keeps dying parks via run()'s rapid-drop budget instead of +hot-cycling respawns forever. The error text must never claim a timeout — +that wording is what misdirected the original investigation. """ import asyncio @@ -23,10 +30,26 @@ import pytest pytest.importorskip("mcp") +def _success_result(): + result = MagicMock() + result.is_error = False + block = MagicMock() + block.text = "ok" + result.content = [block] + result.structured_content = None + result.meta = None + return result + + def _install_stub_server(mcp_tool_module, name: str, call_tool_impl, - *, children_dead): + *, children_dead, on_reconnect=None): """Fake MCP server with real-bool stdio liveness and a countable - reconnect event (mirrors tests/tools/test_mcp_circuit_breaker.py).""" + reconnect event (mirrors tests/tools/test_mcp_circuit_breaker.py). + + ``on_reconnect`` runs on the MCP loop thread when the reconnect event is + set — the hook tests use to simulate the server task respawning the + subprocess and publishing a fresh session. + """ server = MagicMock() server.name = name session = MagicMock() @@ -42,6 +65,8 @@ def _install_stub_server(mcp_tool_module, name: str, call_tool_impl, def set(self): self.set_calls += 1 + if on_reconnect is not None: + on_reconnect(server) server._reconnect_event = _ReconnectAdapter() server._ready = ready_flag @@ -64,63 +89,171 @@ def _cleanup(mcp_tool_module, name: str) -> None: mcp_tool_module._server_breaker_opened_at.pop(name, None) -def test_precall_dead_children_signal_reconnect(monkeypatch, tmp_path): - """Dead-at-call-time subprocess → clean reconnecting error + reconnect - signal, instead of a bare fast-fail that leaves the server dead.""" +def test_precall_dead_children_respawn_and_retry(monkeypatch, tmp_path): + """Dead-at-call-time subprocess (the gateway-restart case): respawn, + retry once, and hand the model a normal result — no error at all.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) from tools import mcp_tool from tools.mcp_tool import _make_tool_handler called = {"n": 0} + alive = {"v": False} async def _call_tool(*a, **kw): called["n"] += 1 - return MagicMock(is_error=False, content=[]) + return _success_result() + + def _respawn(server): + # What the server task does after a gateway restart: fresh child, + # fresh session object, _ready re-armed. + alive["v"] = True + new_session = MagicMock() + new_session.call_tool = _call_tool + server.session = new_session + server._ready.set() server = _install_stub_server( - mcp_tool, "srv-dead", _call_tool, children_dead=lambda: True + mcp_tool, "srv-dead", _call_tool, + children_dead=lambda: not alive["v"], + on_reconnect=_respawn, ) mcp_tool._ensure_mcp_loop() try: handler = _make_tool_handler("srv-dead", "tool1", 10.0) - result = handler({}) - parsed = json.loads(result) - assert "error" in parsed, parsed - assert "reconnect" in parsed["error"].lower(), parsed + parsed = json.loads(handler({})) + assert "error" not in parsed, parsed + assert parsed["result"] == "ok", parsed assert server._reconnect_event.set_calls == 1 - assert called["n"] == 0, "RPC must not be attempted on a dead transport" - # The error payload flows through the handler's JSON parse, which - # bumps the breaker exactly once (no double-bump at the gate). - assert mcp_tool._server_error_counts.get("srv-dead", 0) == 1 + assert called["n"] == 1, "exactly one RPC — the retry after respawn" + assert mcp_tool._server_error_counts.get("srv-dead", 0) == 0 finally: _cleanup(mcp_tool, "srv-dead") -def test_midcall_child_exit_signals_reconnect(monkeypatch, tmp_path): - """Subprocess dies while the RPC is in flight → fast-fail error AND a - reconnect signal so the next call lands on a respawned transport.""" +def test_midcall_child_exit_respawn_and_retry(monkeypatch, tmp_path): + """Subprocess dies while the RPC is in flight → respawn and retry once, + so the caller still gets its result.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) from tools import mcp_tool from tools.mcp_tool import _make_tool_handler + alive = {"v": True} + async def _hanging_call(*a, **kw): await asyncio.sleep(30) - server = _install_stub_server( - mcp_tool, "srv-midcall", _hanging_call, children_dead=lambda: False - ) + async def _good_call(*a, **kw): + return _success_result() async def _watch_children(): - return # children die immediately → watcher resolves first + # Resolves immediately while the child is dead; never while alive. + while alive["v"]: + await asyncio.sleep(0.05) + def _respawn(server): + alive["v"] = True + new_session = MagicMock() + new_session.call_tool = _good_call + server.session = new_session + server._ready.set() + + server = _install_stub_server( + mcp_tool, "srv-midcall", _hanging_call, + children_dead=lambda: not alive["v"], + on_reconnect=_respawn, + ) server._watch_stdio_children = _watch_children mcp_tool._ensure_mcp_loop() try: handler = _make_tool_handler("srv-midcall", "tool1", 10.0) - result = handler({}) - parsed = json.loads(result) - assert "error" in parsed, parsed - assert "exited mid-call" in parsed["error"], parsed + # The child dies once the RPC is in flight. + alive["v"] = False + parsed = json.loads(handler({})) + assert "error" not in parsed, parsed + assert parsed["result"] == "ok", parsed assert server._reconnect_event.set_calls == 1 finally: _cleanup(mcp_tool, "srv-midcall") + + +def test_dead_child_never_returning_is_not_reported_as_a_timeout( + monkeypatch, tmp_path, +): + """No fresh session inside the respawn window → a clean error that says + the subprocess exited, never that something timed out (the + old "failing the call fast instead of waiting 300s" wording sent the + investigation into a healthy remote backend).""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools import mcp_tool + from tools.mcp_tool import _make_tool_handler + + monkeypatch.setattr(mcp_tool, "_STDIO_RESPAWN_WAIT_SEC", 1.0) + called = {"n": 0} + + async def _call_tool(*a, **kw): + called["n"] += 1 + return _success_result() + + server = _install_stub_server( + mcp_tool, "srv-gone", _call_tool, children_dead=lambda: True, + ) + mcp_tool._ensure_mcp_loop() + try: + handler = _make_tool_handler("srv-gone", "tool1", 300.0) + parsed = json.loads(handler({})) + assert "error" in parsed, parsed + message = parsed["error"] + assert "exited" in message, message + for forbidden in ("TimeoutError", "300s", "timed out"): + assert forbidden not in message, message + assert server._reconnect_event.set_calls == 1 + assert called["n"] == 0, "RPC must not be attempted on a dead transport" + assert mcp_tool._server_error_counts.get("srv-gone", 0) == 1 + finally: + _cleanup(mcp_tool, "srv-gone") + + +def test_child_dying_again_after_respawn_does_not_hot_cycle( + monkeypatch, tmp_path, +): + """A server whose child dies immediately after every respawn gets ONE + retry per call, not an endless respawn loop — run()'s rapid-drop budget + is what parks it, and this path must not fight that.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools import mcp_tool + from tools.mcp_tool import _make_tool_handler + + monkeypatch.setattr(mcp_tool, "_STDIO_RESPAWN_WAIT_SEC", 1.0) + called = {"n": 0} + + async def _call_tool(*a, **kw): + called["n"] += 1 + return _success_result() + + def _respawn_then_die(server): + # Fresh session object (so the readiness wait succeeds) whose child + # is already dead again by the time the retry dispatches. + new_session = MagicMock() + new_session.call_tool = _call_tool + server.session = new_session + server._ready.set() + + server = _install_stub_server( + mcp_tool, "srv-flap", _call_tool, + children_dead=lambda: True, + on_reconnect=_respawn_then_die, + ) + mcp_tool._ensure_mcp_loop() + try: + handler = _make_tool_handler("srv-flap", "tool1", 10.0) + parsed = json.loads(handler({})) + assert "error" in parsed, parsed + assert "exited again" in parsed["error"], parsed + assert "do NOT retry" in parsed["error"], parsed + assert server._reconnect_event.set_calls == 1, ( + "one respawn request per tool call — never a retry loop" + ) + assert called["n"] == 0 + assert mcp_tool._server_error_counts.get("srv-flap", 0) == 1 + finally: + _cleanup(mcp_tool, "srv-flap") diff --git a/tests/tools/test_mcp_stdio_watchdog.py b/tests/tools/test_mcp_stdio_watchdog.py deleted file mode 100644 index 411695eea5..0000000000 --- a/tests/tools/test_mcp_stdio_watchdog.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Contract tests for the direct POSIX stdio MCP child watchdog.""" - -import os -import sys - -import pytest - -from tools import mcp_stdio_watchdog, mcp_tool - - -def test_is_orphaned_is_false_while_direct_parent_is_unchanged(): - original_ppid = 1234 - - assert mcp_stdio_watchdog._is_orphaned( - original_ppid, - getppid=lambda: original_ppid, - ) is False - - -@pytest.mark.skipif(os.name != "posix", reason="watchdog wrapping is POSIX-only") -def test_wrap_command_uses_stable_parent_pid_and_preserves_command_tail(): - parent_pid = os.getpid() - command = "/opt/hermes/bin/mcp-server" - command_args = ["--label", "value with spaces", "--", "literal-tail"] - - wrapped_command, wrapped_args = mcp_tool._wrap_command_with_watchdog( - command, - command_args, - ) - - assert wrapped_command == sys.executable - assert wrapped_args == [ - os.path.join(os.path.dirname(mcp_tool.__file__), "mcp_stdio_watchdog.py"), - "--ppid", - str(parent_pid), - "--", - command, - *command_args, - ] - assert "--create-time" not in wrapped_args diff --git a/tests/tools/test_mcp_structured_content.py b/tests/tools/test_mcp_structured_content.py index 7ce59324de..b8d401cfa4 100644 --- a/tests/tools/test_mcp_structured_content.py +++ b/tests/tools/test_mcp_structured_content.py @@ -163,6 +163,7 @@ class TestMetaPassthrough: assert data == {"result": "done"} def test_meta_with_structured_content(self, _patch_mcp_server): + """With usable text, structuredContent is suppressed but _meta rides.""" session = _patch_mcp_server session.call_tool = AsyncMock( return_value=_FakeCallToolResult( @@ -175,7 +176,6 @@ class TestMetaPassthrough: data = json.loads(handler({})) assert data == { "result": "txt", - "structuredContent": {"ok": True}, "_meta": {"com.example/k": "v"}, } @@ -215,3 +215,114 @@ class TestReservedMetaKeyPredicate: assert not mcp_tool._is_reserved_mcp_meta_key("com.example/x") assert not mcp_tool._is_reserved_mcp_meta_key("plain-key") assert not mcp_tool._is_reserved_mcp_meta_key("/leading-slash") + + +class TestContentStructuredArbitration: + """content and structuredContent are alternatives — never both. + + Ported from MoonshotAI/kimi-code#3234: spec-following servers render + their data into content (verbatim dual-emit or a faithful human + reorganisation), so forwarding both sent the same information twice. + """ + + def test_dual_emit_suppresses_structured(self, _patch_mcp_server): + """Verbatim dual-emit servers: model receives content only.""" + session = _patch_mcp_server + payload = {"items": [1, 2, 3]} + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult( + content=[_FakeContentBlock(json.dumps(payload))], + structuredContent=payload, + ) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + data = json.loads(handler({})) + assert data == {"result": json.dumps(payload)} + + def test_prose_summary_suppresses_structured(self, _patch_mcp_server): + """Lossy prose summaries also win — no heuristic is attempted.""" + session = _patch_mcp_server + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult( + content=[_FakeContentBlock("3 item(s) found")], + structuredContent={"items": [1, 2, 3]}, + ) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + data = json.loads(handler({})) + assert data == {"result": "3 item(s) found"} + + def test_whitespace_only_content_falls_back(self, _patch_mcp_server): + """Whitespace-only text is not usable content — fallback fires.""" + session = _patch_mcp_server + payload = {"status": "ok"} + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult( + content=[_FakeContentBlock(" \n")], + structuredContent=payload, + ) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + data = json.loads(handler({})) + assert data["structuredContent"] == payload + + def test_structured_only_still_surfaced(self, _patch_mcp_server): + """structuredContent-only servers keep working (#2596 fix preserved).""" + session = _patch_mcp_server + payload = {"only": "structured"} + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult( + content=[], + structuredContent=payload, + ) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + data = json.loads(handler({})) + assert data["result"] == payload + + +class TestDroppedBlockNotice: + """Unsupported content blocks surface a drop notice to the model. + + Ported from MoonshotAI/kimi-code#3227. + """ + + def test_unsupported_block_renders_notice(self, _patch_mcp_server): + session = _patch_mcp_server + # NOTE: no `uri` — a uri'd block without .resource is rendered as a + # resource link by _render_mcp_resource_block, not dropped. + weird = SimpleNamespace( + type="hologram", + mimeType="application/x-hologram", + size=1234, + ) + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult(content=[weird]) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + data = json.loads(handler({})) + assert "[MCP content dropped: unsupported block" in data["result"] + assert "type=hologram" in data["result"] + assert "mimeType=application/x-hologram" in data["result"] + assert "size=1234" in data["result"] + + def test_drop_notice_does_not_suppress_structured(self, _patch_mcp_server): + """A drop notice is not usable content — structured fallback fires.""" + session = _patch_mcp_server + weird = SimpleNamespace(type="hologram") + payload = {"real": "data"} + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult( + content=[weird], structuredContent=payload, + ) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + data = json.loads(handler({})) + assert data["structuredContent"] == payload + assert "[MCP content dropped" in data["result"] + + def test_notice_helper_minimal_block(self): + notice = mcp_tool._render_mcp_dropped_block_notice( + SimpleNamespace(), "mystery" + ) + assert notice == "[MCP content dropped: unsupported block (type=mystery)]" diff --git a/tests/tools/test_modal_sandbox_fixes.py b/tests/tools/test_modal_sandbox_fixes.py index c0148923f5..6e254fdd07 100644 --- a/tests/tools/test_modal_sandbox_fixes.py +++ b/tests/tools/test_modal_sandbox_fixes.py @@ -36,13 +36,22 @@ class TestToolResolution: def test_terminal_and_file_toolsets_resolve_all_tools(self): """enabled_toolsets=['terminal', 'file'] should produce 6 tools.""" + from unittest.mock import patch as _patch + from model_tools import get_tool_definitions - tools = get_tool_definitions( - enabled_toolsets=["terminal", "file"], - quiet_mode=True, - ) + from tools.tool_search import ToolSearchConfig + + # Pin the RESOLUTION contract independent of deferral policy — + # #97979 defers process_manage by default (legacy defer: [] override). + _legacy = ToolSearchConfig.from_raw({"enabled": "on", "defer": []}) + with _patch("tools.tool_search.load_config", return_value=_legacy), \ + _patch("tools.tool_search.load_config_readonly", return_value=_legacy): + tools = get_tool_definitions( + enabled_toolsets=["terminal", "file"], + quiet_mode=True, + ) names = {t["function"]["name"] for t in tools} - expected = {"terminal", "process", "read_file", "write_file", "search_files", "patch"} + expected = {"terminal", "process_manage", "read_file", "write_file", "search_files", "patch"} assert expected == names, f"Expected {expected}, got {names}" def test_terminal_tool_present(self): diff --git a/tests/tools/test_osv_check.py b/tests/tools/test_osv_check.py index 72e81058e6..d136602f02 100644 --- a/tests/tools/test_osv_check.py +++ b/tests/tools/test_osv_check.py @@ -1,6 +1,9 @@ """Tests for OSV malware check on MCP extension packages.""" import json +import time +from pathlib import Path + import pytest from unittest.mock import patch, MagicMock @@ -66,14 +69,18 @@ class TestParsePackageFromArgs: class TestCheckPackageForMalware: @pytest.fixture(autouse=True) - def _fresh_cache(self): + def _fresh_cache(self, tmp_path, monkeypatch): from tools import osv_check + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) with osv_check._cache_lock: osv_check._cache.clear() + osv_check._disk_cache_loaded = False + (tmp_path / "cache" / "osv_check.json").unlink(missing_ok=True) yield with osv_check._cache_lock: osv_check._cache.clear() - + osv_check._disk_cache_loaded = False + (tmp_path / "cache" / "osv_check.json").unlink(missing_ok=True) def test_clean_package(self): """Clean package returns None (allow).""" mock_response = MagicMock() @@ -189,6 +196,94 @@ class TestCheckPackageForMalware: check_package_for_malware("uvx", ["mcp-server-fetch"]) assert mock_url.call_count == 2 + def test_disk_cache_persists_and_reloads(self, tmp_path, monkeypatch): + """A warm disk cache is reused by a fresh in-process cache.""" + from tools import osv_check + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({"vulns": []}).encode() + mock_response.__enter__ = lambda s: s + mock_response.__exit__ = MagicMock(return_value=False) + + with patch("tools.osv_check.urllib.request.urlopen", return_value=mock_response) as mock_url: + check_package_for_malware("uvx", ["mcp-server-persist"]) + + cache_file = tmp_path / "cache" / "osv_check.json" + assert cache_file.exists(), "disk cache should be written after a warm result" + + with osv_check._cache_lock: + osv_check._cache.clear() + osv_check._disk_cache_loaded = False + + with patch("tools.osv_check.urllib.request.urlopen", return_value=mock_response) as mock_url2: + check_package_for_malware("uvx", ["mcp-server-persist"]) + + assert mock_url2.call_count == 0, "disk cache must satisfy the second call" + + def test_disk_cache_format_versioned(self, tmp_path, monkeypatch): + """Disk cache JSON has a version field and recoverable entries.""" + from tools import osv_check + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + mock_response = MagicMock() + mock_response.read.return_value = json.dumps({"vulns": []}).encode() + mock_response.__enter__ = lambda s: s + mock_response.__exit__ = MagicMock(return_value=False) + + with patch("tools.osv_check.urllib.request.urlopen", return_value=mock_response): + check_package_for_malware("uvx", ["mcp-server-format"]) + + cache_file = tmp_path / "cache" / "osv_check.json" + with open(cache_file, "r", encoding="utf-8") as f: + data = json.load(f) + assert data["version"] == osv_check._DISK_CACHE_VERSION + assert "entries" in data + key = "PyPI|mcp-server-format|" + assert key in data["entries"] + assert "expiry" in data["entries"][key] + assert data["entries"][key]["result"] is None + + def test_disk_cache_retries_after_transient_oserror(self, tmp_path, monkeypatch): + """A busy/unreadable cache file must not disable disk loads for the process.""" + from tools import osv_check + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + cache_file = tmp_path / "cache" / "osv_check.json" + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text( + json.dumps({ + "version": osv_check._DISK_CACHE_VERSION, + "entries": { + "PyPI|mcp-server-retry|": { + "expiry": time.time() + 3600, + "result": None, + } + }, + }), + encoding="utf-8", + ) + + real_open = open + calls = {"n": 0} + + def flaky_open(path, *args, **kwargs): + if Path(path) == cache_file: + calls["n"] += 1 + if calls["n"] == 1: + raise OSError("resource temporarily unavailable") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", flaky_open) + with osv_check._cache_lock: + osv_check._load_disk_cache() + assert osv_check._disk_cache_loaded is False + osv_check._load_disk_cache() + assert osv_check._disk_cache_loaded is True + assert ("PyPI", "mcp-server-retry", None) in osv_check._cache + class TestLiveOsvQuery: """Live integration test against the real OSV API. Skipped if offline.""" diff --git a/tests/tools/test_process_registry.py b/tests/tools/test_process_registry.py index 0bb4fdada0..ffaf349a51 100644 --- a/tests/tools/test_process_registry.py +++ b/tests/tools/test_process_registry.py @@ -2,6 +2,8 @@ import json import os +import shlex +import shutil import signal import subprocess import sys @@ -783,7 +785,7 @@ class TestSpawnEnvSanitization: def __init__(self): self.commands = [] self._responses = iter([ - {"output": "hello\n"}, + {"output": "6 0\nhello\n"}, {"output": "1\n"}, {"output": "0\n"}, ]) @@ -804,11 +806,180 @@ class TestSpawnEnvSanitization: "/path with spaces/hermes_bg.exit", ) - assert env.commands[0][0] == "cat '/path with spaces/hermes_bg.log' 2>/dev/null" + assert "'/path with spaces/hermes_bg.log'" in env.commands[0][0] + assert "cat '/path with spaces/hermes_bg.log'" not in env.commands[0][0] assert env.commands[1][0] == "kill -0 \"$(cat '/path with spaces/hermes_bg.pid' 2>/dev/null)\" 2>/dev/null; echo $?" assert env.commands[2][0] == "cat '/path with spaces/hermes_bg.exit' 2>/dev/null" +class TestEnvPollerIncrementalRead: + """The sandbox log poller must read only new bytes, not the whole file. + + Reading the whole file every poll made one poll cost grow with the total + output so far, so a long noisy job re-sent all of its output over the + docker or SSH channel every two seconds. + """ + + @staticmethod + def _run_poller(registry, session, responses): + """Drive one poll cycle and hand back the commands the env saw.""" + + class FakeEnv: + def __init__(self): + self.commands = [] + self._responses = iter(responses) + + def execute(self, command, **kwargs): + self.commands.append(command) + return next(self._responses) + + env = FakeEnv() + with patch("tools.process_registry.time.sleep", return_value=None), \ + patch.object(registry, "_move_to_finished"): + registry._env_poller_loop( + session, env, "/tmp/bg.log", "/tmp/bg.pid", "/tmp/bg.exit" + ) + return env.commands + + def test_read_command_asks_only_for_new_bytes(self): + cmd = ProcessRegistry._log_delta_command("'/tmp/bg.log'", 4096) + # The offset is carried into the command, and the file is opened with + # tail rather than cat. + assert "O=4096" in cmd + assert "tail -c +$((O+1)) '/tmp/bg.log'" in cmd + assert "cat '/tmp/bg.log'" not in cmd + + def test_read_command_starts_from_zero_on_first_poll(self): + cmd = ProcessRegistry._log_delta_command("'/tmp/bg.log'", 0) + assert "O=0" in cmd + + @pytest.mark.skipif(not shutil.which("sh"), reason="needs a POSIX sh") + def test_read_command_holds_back_a_split_utf8_sequence(self, tmp_path): + """A multibyte character straddling two polls must not be split. + + The backend decodes each execute() result on its own, so returning + the first byte of an 'é' in one poll and the rest in the next would + yield replacement characters in the transcript (and break watch + patterns at the seam). Every prefix of a mixed ASCII/2/3/4-byte + string must come back decodable, with at most 3 bytes held back and + nothing held back once the trailing character is complete. + """ + full = "hé😀中a\n€bz🚀".encode() + log = tmp_path / "bg.log" + quoted = shlex.quote(str(log)) + for n in range(1, len(full) + 1): + log.write_bytes(full[:n]) + out = subprocess.run( + ["sh", "-c", ProcessRegistry._log_delta_command(quoted, 0)], + capture_output=True, timeout=30, + ).stdout + header, _, delta = out.partition(b"\n") + size, _offset = map(int, header.split()) + delta.decode("utf-8") # must not raise + assert delta == full[:size] + complete = full[:n].decode("utf-8", "ignore").encode() == full[:n] + assert (n - size) == 0 if complete else 0 < (n - size) <= 3 + + def test_first_poll_reads_from_the_start(self, registry): + session = _make_session(sid="proc_delta") + session.exited = False + commands = self._run_poller( + registry, + session, + [ + {"output": "11 0\nfirst chunk"}, + {"output": "1\n"}, + {"output": "0\n"}, + ], + ) + assert "O=0" in commands[0] + assert session.output_buffer == "first chunk" + + def test_delta_is_appended_not_replaced(self, registry): + session = _make_session(sid="proc_append", output="already here ") + session.exited = False + self._run_poller( + registry, + session, + [ + {"output": "8 0\nand new"}, + {"output": "1\n"}, + {"output": "0\n"}, + ], + ) + assert session.output_buffer == "already here and new" + + def test_second_poll_asks_from_where_the_first_one_stopped(self, registry): + session = _make_session(sid="proc_two_polls") + session.exited = False + commands = self._run_poller( + registry, + session, + [ + {"output": "11 0\nfirst chunk"}, + {"output": "0\n"}, # still running, poll again + {"output": "17 11\n and more"}, + {"output": "1\n"}, # gone now + {"output": "0\n"}, + ], + ) + assert "O=0" in commands[0] + # The second read starts at byte 11, so the first chunk is not sent + # a second time. + assert "O=11" in commands[2] + assert session.output_buffer == "first chunk and more" + + def test_truncated_log_drops_the_stale_buffer(self, registry): + session = _make_session(sid="proc_rotate") + session.exited = False + # The second read reports offset 0 even though the first one left off + # at byte 11. The file no longer reaches that byte, so it was rotated + # or truncated and the buffer we hold no longer matches it. + self._run_poller( + registry, + session, + [ + {"output": "11 0\nfirst chunk"}, + {"output": "0\n"}, # still running, poll again + {"output": "5 0\nfresh"}, + {"output": "1\n"}, + {"output": "0\n"}, + ], + ) + assert session.output_buffer == "fresh" + + def test_unreadable_header_leaves_the_buffer_alone(self, registry): + session = _make_session(sid="proc_bad", output="keep me") + session.exited = False + # No header at all, for example when the shell is missing one of the + # tools the command needs. + self._run_poller( + registry, + session, + [ + {"output": ""}, + {"output": "1\n"}, + {"output": "0\n"}, + ], + ) + assert session.output_buffer == "keep me" + + def test_buffer_stays_within_the_cap(self, registry): + session = _make_session(sid="proc_cap") + session.exited = False + session.max_output_chars = 10 + self._run_poller( + registry, + session, + [ + {"output": "20 0\n" + "x" * 20}, + {"output": "1\n"}, + {"output": "0\n"}, + ], + ) + assert session.output_buffer == "x" * 10 + + # ========================================================================= # Popen leak prevention # ========================================================================= diff --git a/tests/tools/test_refresh_agent_mcp_tools.py b/tests/tools/test_refresh_agent_mcp_tools.py index b1aa95e12f..ee88a58ff9 100644 --- a/tests/tools/test_refresh_agent_mcp_tools.py +++ b/tests/tools/test_refresh_agent_mcp_tools.py @@ -224,3 +224,116 @@ def test_wait_returns_instantly_when_no_discovery_thread(monkeypatch): t0 = time.time() mcp_startup.wait_for_mcp_discovery() assert time.time() - t0 < 0.2 # never blocks on the bound when nothing's pending + + +# --------------------------------------------------------------------------- +# preserve_prefix: the tool array is a cached request prefix (#100336) +# --------------------------------------------------------------------------- + + +def _registered(monkeypatch, names): + """Make the registry report exactly *names* as still registered.""" + from tools import registry as registry_mod + + entries = [types.SimpleNamespace(name=n) for n in names] + monkeypatch.setattr( + registry_mod.registry, "get_all_entries", lambda: entries, raising=False + ) + + +def _serve(monkeypatch, defs): + import model_tools + + monkeypatch.setattr(model_tools, "get_tool_definitions", lambda **kw: list(defs)) + + +def test_preserve_prefix_carries_a_flapping_tool_forward(monkeypatch): + """A check_fn flip must not shrink a live session's tool prefix. + + ``browser_navigate``'s availability probe fails this turn (headless box, + expired credential, docker blip) so ``get_tool_definitions`` omits it. The + tool is still *registered* — only its probe flapped — so the snapshot must + keep it, byte-for-byte, instead of forking the cached prefix. + """ + agent = _agent(["read_file", "browser_navigate", "terminal"]) + before = list(agent.tools) + + _serve(monkeypatch, [_tool("read_file"), _tool("terminal")]) + _registered(monkeypatch, ["read_file", "browser_navigate", "terminal"]) + + added = mcp_tool.refresh_agent_mcp_tools(agent, preserve_prefix=True) + + assert added == set() + assert agent.tools == before + assert "browser_navigate" in agent.valid_tool_names + + +def test_preserve_prefix_appends_late_arrivals_at_the_tail(monkeypatch): + """``get_definitions`` sorts by name, so a late tool can splice in at 0. + + Under ``preserve_prefix`` the live order is authoritative and the new tool + extends the array, leaving every earlier byte where the provider cached it. + """ + agent = _agent(["read_file", "terminal"]) + + # Sorted order would put the new tool first. + _serve(monkeypatch, [_tool("aaa_mcp_late"), _tool("read_file"), _tool("terminal")]) + _registered(monkeypatch, ["aaa_mcp_late", "read_file", "terminal"]) + + added = mcp_tool.refresh_agent_mcp_tools(agent, preserve_prefix=True) + + assert added == {"aaa_mcp_late"} + assert [t["function"]["name"] for t in agent.tools] == [ + "read_file", "terminal", "aaa_mcp_late", + ] + + +# --------------------------------------------------------------------------- +# tools[] freeze: eviction rebuild + the /reload-mcp re-probe hatch +# --------------------------------------------------------------------------- + + +def test_eviction_rebuild_restores_the_sessions_saved_tool_order(monkeypatch): + """A fresh AIAgent for an EXISTING session must keep the saved tools[] pin. + + Gateway agent-cache eviction rebuilds the agent; ``agent_init`` re-probes + every ``check_fn`` and ``browser_navigate``'s flips false. The persisted + name list stands in for the missing predecessor: the tool is carried + forward from the registry schema, byte-for-byte in its old slot. + """ + from tools import registry as registry_mod + + saved = ["read_file", "browser_navigate", "terminal"] + entries = {n: types.SimpleNamespace(name=n, schema=_tool(n)["function"]) for n in saved} + monkeypatch.setattr(registry_mod.registry, "get_all_entries", lambda: list(entries.values()), raising=False) + monkeypatch.setattr(registry_mod.registry, "get_entry", lambda name, **kw: entries.get(name), raising=False) + + rebuilt = _agent(["read_file", "terminal"]) # probe flipped: browser_navigate gone + changed = mcp_tool.restore_agent_tool_prefix(rebuilt, saved) + + assert changed is True + assert [t["function"]["name"] for t in rebuilt.tools] == saved + assert rebuilt.valid_tool_names == set(saved) + + +def test_reprobe_tool_availability_drops_cached_check_fn_verdicts(monkeypatch): + """/reload-mcp is the explicit hatch: a cached False must be re-probed.""" + from tools import registry as registry_mod + import model_tools + + verdict = {"ok": False} + + def probe(): + return verdict["ok"] + + monkeypatch.setattr(registry_mod, "check_fn_cache_scope", lambda: "test-scope") + assert registry_mod._check_fn_cached(probe) is False + verdict["ok"] = True + assert registry_mod._check_fn_cached(probe) is False # TTL cache replays stale verdict + with model_tools._tool_defs_cache_lock: + model_tools._tool_defs_cache[("sentinel",)] = [] + + mcp_tool.reprobe_tool_availability() + + assert registry_mod._check_fn_cached(probe) is True + assert ("sentinel",) not in model_tools._tool_defs_cache diff --git a/tests/tools/test_search_budget_truncation.py b/tests/tools/test_search_budget_truncation.py index 432327bd61..9e58c48ba1 100644 --- a/tests/tools/test_search_budget_truncation.py +++ b/tests/tools/test_search_budget_truncation.py @@ -1,7 +1,10 @@ +import shlex from unittest.mock import MagicMock import pytest +import tools.file_operations as file_operations +from tools.environments.local import LocalEnvironment from tools.file_operations import ExecuteResult, ShellFileOperations, _search_stdout_and_limit @@ -71,3 +74,247 @@ def test_real_rg_error_still_hard_fails(ops, monkeypatch): assert result.error == "Search failed: rg: regex parse error:" assert result.limit_reason is None + + +class FindRecordingEnvironment: + is_local = False + cwd = "/narrow" + + def __init__(self, output="", code=0): + self.output = output + self.code = code + self.commands = [] + + def execute(self, command, **kwargs): + self.commands.append((command, kwargs)) + if command.startswith("command -v find"): + return {"output": "yes\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "", "returncode": 1} + if "find " in command: + return {"output": self.output, "returncode": self.code} + return {"output": "", "returncode": 1} + + @property + def find_commands(self): + return [ + item for item in self.commands + if item[0].startswith("find ") or "; find " in item[0] + ] + + +class MultiRootFindEnvironment(FindRecordingEnvironment): + def execute(self, command, **kwargs): + self.commands.append((command, kwargs)) + if command.startswith("test -e "): + output = "not_found\n" if "'/one/.hidden /two/.cache'" in command else "exists\n" + return {"output": output, "returncode": 0} + if command.startswith("command -v find"): + return {"output": "yes\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "", "returncode": 1} + if "find " in command: + return {"output": self.output, "returncode": self.code} + return {"output": "", "returncode": 1} + + +def test_find_discovery_is_one_unsorted_pruned_bounded_scan(): + env = FindRecordingEnvironment("/narrow/a.py\n/narrow/b.py\n/narrow/c.py\n/narrow/d.py\n") + result = ShellFileOperations(env)._search_files( + "*.py", "/narrow", limit=2, offset=1, order="discovery" + ) + assert result.files == ["/narrow/b.py", "/narrow/c.py"] + assert result.truncated is True + assert len(env.find_commands) == 1 + command, kwargs = env.find_commands[0] + assert "-printf" not in command + assert "sort " not in command + assert "-prune" in command + assert "head -n 4" in command + assert kwargs["timeout"] <= 60 + + +def test_find_modified_is_one_exact_scan_without_bsd_retry(): + env = FindRecordingEnvironment("30 /narrow/new.py\n20 /narrow/mid.py\n10 /narrow/old.py\n") + result = ShellFileOperations(env)._search_files( + "*.py", "/narrow", limit=1, offset=1, order="modified" + ) + assert result.files == ["/narrow/mid.py"] + assert result.truncated is True + assert len(env.find_commands) == 1 + command, _ = env.find_commands[0] + assert "-printf '%T@ %p\\n'" in command + assert "sort -rn" in command + assert "head -n 3" in command + + +def test_no_rg_multi_root_modified_is_one_globally_sorted_scan(): + env = MultiRootFindEnvironment( + "30 /two/.cache/new.py\n10 /one/.hidden/old.py\n" + ) + + result = ShellFileOperations(env).search( + "*.py", + path="/one/.hidden /two/.cache", + target="files", + order="modified", + limit=1, + ) + + assert result.error is None + assert result.files == ["/two/.cache/new.py"] + assert result.truncated is True + assert len(env.find_commands) == 1 + command, kwargs = env.find_commands[0] + assert "find '/one/.hidden' '/two/.cache'" in command + assert "sort -rn" in command + assert "head -n 2" in command + assert "! -path '/one/.hidden'" in command + assert "! -path '/two/.cache'" in command + assert kwargs["timeout"] <= 60 + + +def test_find_dash_prefixed_relative_root_is_an_explicit_operand( + tmp_path, monkeypatch +): + dash_root = tmp_path / "--version" + ordinary_root = tmp_path / "ordinary" + dash_root.mkdir() + ordinary_root.mkdir() + (dash_root / "dash.py").write_text("", encoding="utf-8") + (ordinary_root / "plain.py").write_text("", encoding="utf-8") + + ops = ShellFileOperations(LocalEnvironment(str(tmp_path))) + monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") + executed = [] + real_exec = ops._exec + + def recording_exec(command, **kwargs): + if command.startswith("set -o pipefail; find "): + executed.append(command) + return real_exec(command, **kwargs) + + monkeypatch.setattr(ops, "_exec", recording_exec) + result = ops._search_files( + "*.py", ["--version", "ordinary"], limit=10, offset=0 + ) + + assert result.error is None + assert sorted(result.files) == ["./--version/dash.py", "ordinary/plain.py"] + assert len(executed) == 1 + command_tokens = shlex.split(executed[0].removeprefix("set -o pipefail; ")) + assert "./--version" in command_tokens + assert "--version" not in command_tokens + assert all("find (GNU findutils)" not in path for path in result.files) + + +def test_find_modified_capability_failure_is_actionable_without_retry(): + env = FindRecordingEnvironment("", code=1) + result = ShellFileOperations(env)._search_files( + "*.py", "/narrow", limit=2, offset=0, order="modified" + ) + assert "modification-time" in (result.error or "") + assert len(env.find_commands) == 1 + + +@pytest.mark.parametrize( + ("order", "output"), + [ + ( + "discovery", + "/narrow/a.py\n/narrow/b.py\n/narrow/c.py\n/narrow/d.py\n", + ), + ( + "modified", + "40 /narrow/a.py\n30 /narrow/b.py\n20 /narrow/c.py\n10 /narrow/d.py\n", + ), + ], +) +def test_find_sigpipe_is_benign_only_after_fetch_limit_rows(order, output): + result = ShellFileOperations(FindRecordingEnvironment(output, code=141))._search_files( + "*.py", "/narrow", limit=2, offset=1, order=order + ) + + assert result.error is None + assert result.files == ["/narrow/b.py", "/narrow/c.py"] + assert result.truncated is True + + +@pytest.mark.parametrize( + ("order", "output", "error_fragment"), + [ + ("discovery", "/narrow/partial.py\n", "bounded find traversal"), + ("modified", "10 /narrow/partial.py\n", "modification-time"), + ], +) +def test_find_sigpipe_with_fewer_than_fetch_limit_rows_fails_closed( + order, output, error_fragment +): + result = ShellFileOperations(FindRecordingEnvironment(output, code=141))._search_files( + "*.py", "/narrow", limit=2, offset=1, order=order + ) + + assert error_fragment in (result.error or "") + assert result.files == [] + assert result.total_count == 0 + + +@pytest.mark.parametrize( + ("order", "output", "error_fragment"), + [ + ("discovery", "/narrow/partial.py\n", "bounded find traversal"), + ("modified", "/narrow/not-a-timestamp.py\n", "modification-time"), + ], +) +def test_find_hard_error_discards_partial_output(order, output, error_fragment): + env = FindRecordingEnvironment(output, code=2) + + result = ShellFileOperations(env)._search_files( + "*.py", "/narrow", limit=2, offset=0, order=order + ) + + assert error_fragment in (result.error or "") + assert result.files == [] + assert result.total_count == 0 + + +def test_find_timeout_preserves_partial_results_and_limit_reason(): + env = FindRecordingEnvironment(timeout_output("/narrow/partial.py"), code=124) + + result = ShellFileOperations(env)._search_files( + "*.py", "/narrow", limit=2, offset=0, order="discovery" + ) + + assert result.files == ["/narrow/partial.py"] + assert_timed_out(result) + + +def test_find_zero_match_exit_zero_is_success(): + result = ShellFileOperations(FindRecordingEnvironment("", code=0))._search_files( + "*.missing", "/narrow", limit=2, offset=0, order="discovery" + ) + + assert result.error is None + assert result.files == [] + + +def test_local_broad_no_rg_refuses_before_find(monkeypatch, tmp_path): + home = tmp_path / "home" + home.mkdir() + ops = ShellFileOperations(LocalEnvironment(str(home))) + monkeypatch.setattr(file_operations, "_HOME", str(home)) + monkeypatch.setattr(file_operations.os.path, "isfile", lambda path: False) + commands = [] + + def fake_exec(command, **kwargs): + commands.append((command, kwargs)) + if command.startswith("command -v rg"): + return ExecuteResult("", 1) + if command.startswith("command -v find"): + return ExecuteResult("yes\n", 0) + raise AssertionError(f"broad fallback must not execute: {command}") + + monkeypatch.setattr(ops, "_exec", fake_exec) + result = ops._search_files("*.py", str(home), 10, 0, "discovery") + assert "ripgrep" in (result.error or "").lower() + assert not any(command.startswith("find ") for command, _ in commands) diff --git a/tests/tools/test_search_files_cpu_windows.py b/tests/tools/test_search_files_cpu_windows.py new file mode 100644 index 0000000000..652f7255c9 --- /dev/null +++ b/tests/tools/test_search_files_cpu_windows.py @@ -0,0 +1,300 @@ +"""Concurrency admission tests for expensive filename walks.""" + +from concurrent.futures import ThreadPoolExecutor +import threading +import types + +import pytest + +from tools.environments.local import LocalEnvironment +from tools.file_operations import ( + _ACTIVE_FILENAME_SEARCH_ROOTS, + _FILENAME_SEARCH_ADMISSION, + _normalized_filename_search_root, + SearchResult, + ShellFileOperations, +) +from tools.interrupt import set_interrupt + + +class RemoteEnvironment: + is_local = False + cwd = "/workspace" + + def execute(self, command, **kwargs): + raise AssertionError(f"unexpected backend command: {command}") + + +def _operations(env, scan): + operations = ShellFileOperations(env) + operations._resolve_command = lambda command: "/usr/bin/rg" if command == "rg" else None + operations._search_files_rg = types.MethodType(scan, operations) + return operations + + +def test_same_backend_class_and_root_serialize_five_filename_walks(): + entered = threading.Event() + release = threading.Event() + counter_lock = threading.Lock() + active = 0 + maximum_active = 0 + completed = 0 + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + nonlocal active, maximum_active, completed + with counter_lock: + active += 1 + maximum_active = max(maximum_active, active) + entered.set() + assert release.wait(5) + with counter_lock: + active -= 1 + completed += 1 + return SearchResult(files=[str(path)], total_count=1) + + operations = [_operations(RemoteEnvironment(), scan) for _ in range(5)] + with ThreadPoolExecutor(max_workers=5) as pool: + futures = [ + pool.submit(operation._search_files, "*.py", "/repo", 50, 0) + for operation in operations + ] + assert entered.wait(5) + release.set() + results = [future.result(timeout=5) for future in futures] + + assert all(result.error is None for result in results) + assert completed == 5 + assert maximum_active == 1 + + +def test_different_roots_can_enter_filename_walks_together(): + both_entered = threading.Barrier(2) + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + both_entered.wait(5) + return SearchResult(files=[str(path)], total_count=1) + + first = _operations(RemoteEnvironment(), scan) + second = _operations(RemoteEnvironment(), scan) + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [ + pool.submit(first._search_files, "*.py", "/one", 50, 0), + pool.submit(second._search_files, "*.py", "/two", 50, 0), + ] + assert [future.result(timeout=5).error for future in futures] == [None, None] + + +def test_different_backend_classes_can_walk_the_same_root_together(): + class OtherRemoteEnvironment(RemoteEnvironment): + pass + + both_entered = threading.Barrier(2) + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + both_entered.wait(5) + return SearchResult(files=[str(path)], total_count=1) + + first = _operations(RemoteEnvironment(), scan) + second = _operations(OtherRemoteEnvironment(), scan) + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [ + pool.submit(first._search_files, "*.py", "/same", 50, 0), + pool.submit(second._search_files, "*.py", "/same", 50, 0), + ] + assert [future.result(timeout=5).error for future in futures] == [None, None] + + +def test_overlapping_multi_root_sets_are_claimed_atomically(monkeypatch): + first_entered = threading.Event() + release_first = threading.Event() + second_waiting = threading.Event() + lock = threading.Lock() + active = 0 + maximum_active = 0 + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + nonlocal active, maximum_active + with lock: + active += 1 + maximum_active = max(maximum_active, active) + if path == ["/a", "/b"]: + first_entered.set() + if path == ["/a", "/b"]: + assert release_first.wait(5) + with lock: + active -= 1 + return SearchResult(files=[str(path)], total_count=1) + + first = _operations(RemoteEnvironment(), scan) + second = _operations(RemoteEnvironment(), scan) + original_wait = _FILENAME_SEARCH_ADMISSION.wait + + def observed_wait(timeout=None): + second_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(_FILENAME_SEARCH_ADMISSION, "wait", observed_wait) + with ThreadPoolExecutor(max_workers=2) as pool: + first_future = pool.submit(first._search_files, "*.py", ["/a", "/b"], 50, 0) + assert first_entered.wait(5) + second_future = pool.submit(second._search_files, "*.py", ["/b", "/c"], 50, 0) + assert second_waiting.wait(5) + release_first.set() + assert first_future.result(timeout=5).error is None + assert second_future.result(timeout=5).error is None + + assert maximum_active == 1 + + +def test_interrupted_waiter_returns_without_dispatch_or_late_dispatch(monkeypatch): + holder_entered = threading.Event() + release_holder = threading.Event() + waiter_waiting = threading.Event() + waiter_tid = [] + dispatches = [] + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + dispatches.append(threading.get_ident()) + holder_entered.set() + assert release_holder.wait(5) + return SearchResult(files=[str(path)], total_count=1) + + holder = _operations(RemoteEnvironment(), scan) + waiter = _operations(RemoteEnvironment(), scan) + + original_wait = _FILENAME_SEARCH_ADMISSION.wait + + def observed_wait(timeout=None): + waiter_waiting.set() + return original_wait(timeout) + + monkeypatch.setattr(_FILENAME_SEARCH_ADMISSION, "wait", observed_wait) + + def run_waiter(): + waiter_tid.append(threading.get_ident()) + return waiter._search_files("*.py", "/repo", 50, 0) + + with ThreadPoolExecutor(max_workers=2) as pool: + holder_future = pool.submit(holder._search_files, "*.py", "/repo", 50, 0) + assert holder_entered.wait(5) + waiter_future = pool.submit(run_waiter) + assert waiter_waiting.wait(5) + set_interrupt(True, waiter_tid[0]) + try: + interrupted = waiter_future.result(timeout=5) + assert "interrupted" in (interrupted.error or "").lower() + assert len(dispatches) == 1 + release_holder.set() + assert holder_future.result(timeout=5).error is None + assert len(dispatches) == 1 + finally: + set_interrupt(False, waiter_tid[0]) + release_holder.set() + + +def test_interrupt_published_after_final_sample_prevents_filename_dispatch(monkeypatch): + sampled_clear = threading.Event() + resume_acquire = threading.Event() + worker_tid = [] + dispatches = [] + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + dispatches.append(threading.get_ident()) + return SearchResult(files=[str(path)], total_count=1) + + operations = _operations(RemoteEnvironment(), scan) + original_is_interrupted = __import__( + "tools.interrupt", fromlist=["is_interrupted"] + ).is_interrupted + + def pause_after_clear_sample(): + interrupted = original_is_interrupted() + if not interrupted and threading.get_ident() == worker_tid[0]: + sampled_clear.set() + assert resume_acquire.wait(5) + return interrupted + + monkeypatch.setattr( + "tools.file_operations.tool_interrupt.is_interrupted", + pause_after_clear_sample, + ) + + def run_search(): + worker_tid.append(threading.get_ident()) + return operations._search_files("*.py", "/repo", 50, 0) + + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(run_search) + assert sampled_clear.wait(5) + set_interrupt(True, worker_tid[0]) + resume_acquire.set() + try: + result = future.result(timeout=5) + finally: + set_interrupt(False, worker_tid[0]) + resume_acquire.set() + + assert "interrupted" in (result.error or "").lower() + assert dispatches == [] + assert _ACTIVE_FILENAME_SEARCH_ROOTS == set() + + +def test_empty_filename_roots_are_rejected_before_engine_resolution(): + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + raise AssertionError("filename engine dispatched") + + operations = _operations(RemoteEnvironment(), scan) + operations._resolve_command = lambda command: (_ for _ in ()).throw( + AssertionError(f"engine resolution attempted: {command}") + ) + + result = operations._search_files("*.py", [], 50, 0) + + assert "at least one search root" in (result.error or "").lower() + assert _ACTIVE_FILENAME_SEARCH_ROOTS == set() + + +@pytest.mark.parametrize("raised", [Exception, KeyboardInterrupt, SystemExit, BaseException]) +def test_admission_releases_after_every_base_exception_path(raised): + attempts = 0 + + def scan(self, pattern, path, limit, offset, order, rg_executable=None): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise raised("engine failed") + return SearchResult(files=[str(path)], total_count=1) + + operations = _operations(RemoteEnvironment(), scan) + with pytest.raises(raised, match="engine failed"): + operations._search_files("*.py", "/repo", 50, 0) + + result = operations._search_files("*.py", "/repo", 50, 0) + assert result.error is None + assert attempts == 2 + assert _ACTIVE_FILENAME_SEARCH_ROOTS == set() + + +def test_remote_roots_are_normalized_lexically_against_backend_cwd(monkeypatch): + env = RemoteEnvironment() + monkeypatch.setattr( + "tools.file_operations.os.path.abspath", + lambda path: (_ for _ in ()).throw(AssertionError("controller resolution used")), + ) + + relative = _normalized_filename_search_root(env, "repo/../repo", "/controller") + absolute = _normalized_filename_search_root(env, "/workspace/repo", "/controller") + + assert relative == "/workspace/repo" + assert absolute == relative + + +@pytest.mark.windows_only +def test_windows_local_root_spellings_share_one_normalized_key(): + env = LocalEnvironment.__new__(LocalEnvironment) + env.cwd = "C:/Repo" + + native = _normalized_filename_search_root(env, r"C:\Repo\src\..", "C:/ignored") + msys = _normalized_filename_search_root(env, "/c/Repo", "C:/ignored") + + assert native == msys diff --git a/tests/tools/test_search_files_engine_selection.py b/tests/tools/test_search_files_engine_selection.py new file mode 100644 index 0000000000..72e9100f84 --- /dev/null +++ b/tests/tools/test_search_files_engine_selection.py @@ -0,0 +1,539 @@ +"""Behavior tests for file-search ordering and ripgrep selection.""" + +import json +import re + +import pytest + +from tools.environments.local import LocalEnvironment +from tools.file_operations import SearchResult, ShellFileOperations +from tools.file_tools import SEARCH_FILES_SCHEMA, _handle_search_files, search_tool + + +class RecordingEnvironment: + is_local = False + cwd = "/repo" + + def __init__(self, *, rg_output="/repo/one.py\n/repo/two.py\n", rg_code=0): + self.commands = [] + self.rg_output = rg_output + self.rg_code = rg_code + + def execute(self, command, **kwargs): + self.commands.append(command) + if command.startswith("test -e "): + return {"output": "exists\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/opt/Rip Grep/rg\n", "returncode": 0} + if "--version" in command: + return {"output": "ripgrep 14.1.1\n", "returncode": 0} + if "--files" in command: + return {"output": self.rg_output, "returncode": self.rg_code} + return {"output": "", "returncode": 1} + + @property + def rg_commands(self): + return [command for command in self.commands if "--files" in command] + + +def test_schema_exposes_fast_discovery_default_and_exact_modified_opt_in(): + order = SEARCH_FILES_SCHEMA["parameters"]["properties"]["order"] + + assert order["enum"] == ["discovery", "modified"] + assert order["default"] == "discovery" + assert "fast bounded traversal order" in order["description"] + assert "exact global newest-first" in order["description"] + assert "ignored for content" in order["description"] + + +def test_default_file_search_runs_one_bounded_unsorted_rg_command(): + env = RecordingEnvironment() + ops = ShellFileOperations(env) + + result = ops.search("*.py", path="/repo", target="files", limit=1, offset=1) + + assert result.files == ["/repo/two.py"] + assert len(env.rg_commands) == 1 + assert "--sortr" not in env.rg_commands[0] + assert "head -n 3" in env.rg_commands[0] + + +@pytest.mark.parametrize("engine", ["rg", "find"]) +def test_bounded_filename_total_is_serialized_as_a_lower_bound(engine, monkeypatch): + conceptual_files = [f"/repo/file-{index:03}.py" for index in range(200)] + env = RecordingEnvironment() + + def execute(command, **kwargs): + env.commands.append(command) + if command.startswith("test -e "): + return {"output": "exists\n", "returncode": 0} + if command.startswith("command -v rg"): + return { + "output": "/usr/bin/rg\n" if engine == "rg" else "", + "returncode": 0 if engine == "rg" else 1, + } + if "--files" in command or command.startswith("set -o pipefail; find "): + fetch_limit = int(re.search(r"head -n (\d+)", command).group(1)) + return { + "output": "\n".join(conceptual_files[:fetch_limit]) + "\n", + "returncode": 0, + } + return {"output": "", "returncode": 1} + + env.execute = execute + ops = ShellFileOperations(env) + if engine == "find": + monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") + + result = ops.search("*.py", path="/repo", target="files", limit=50) + serialized = result.to_dict() + + assert result.total_count == 51 + assert len(result.files) == 50 + assert serialized["truncated"] is True + assert serialized["total_count_is_lower_bound"] is True + + +def test_modified_file_search_runs_one_exact_order_rg_command(): + env = RecordingEnvironment() + ops = ShellFileOperations(env) + + result = ops.search("*.py", path="/repo", target="files", order="modified") + + assert result.error is None + assert len(env.rg_commands) == 1 + assert "--sortr=modified" in env.rg_commands[0] + + +def test_modified_zero_match_exit_one_is_valid_without_capability_error(): + env = RecordingEnvironment(rg_output="", rg_code=1) + result = ShellFileOperations(env).search( + "*.missing", path="/repo", target="files", order="modified" + ) + assert result.error is None + assert result.files == [] + assert len(env.rg_commands) == 1 + + +@pytest.mark.parametrize( + "version", + [ + "ripgrep 13.0.0\n", + "ripgrep unknown\n", + "ripgrep 14 garbage\n", + "ripgrep 14\n", + "ripgrep 14.1\n", + "ripgrep 14.1.1-\n", + "ripgrep 14.1.1+\n", + "ripgrep 14.1.1-alpha..1\n", + "ripgrep 14.1.1+build..2\n", + "ripgrep 14.1.1-01\n", + "ripgrep 014.1.1\n", + "ripgrep 14.01.1\n", + "ripgrep 14.1.01\n", + ], +) +def test_modified_requires_parseable_ripgrep_14_before_search(version): + env = RecordingEnvironment() + + def execute(command, **kwargs): + env.commands.append(command) + if command.startswith("test -e "): + return {"output": "exists\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/opt/Rip Grep/rg\n", "returncode": 0} + if "--version" in command: + return {"output": version, "returncode": 0} + raise AssertionError(f"search must not run: {command}") + + env.execute = execute + ops = ShellFileOperations(env) + first = ops.search("*.py", path="/repo", target="files", order="modified") + second = ops.search("*.py", path="/repo", target="files", order="modified") + assert "ripgrep 14" in (first.error or "").lower() + assert second.error == first.error + assert env.rg_commands == [] + assert len([c for c in env.commands if "--version" in c]) == 1 + + +def test_modified_accepts_complete_ripgrep_semver_with_revision_text(): + env = RecordingEnvironment() + original_execute = env.execute + + def execute(command, **kwargs): + if "--version" in command: + env.commands.append(command) + return {"output": "ripgrep 14.1.1 (rev abc123)\n", "returncode": 0} + return original_execute(command, **kwargs) + + env.execute = execute + result = ShellFileOperations(env).search( + "*.py", path="/repo", target="files", order="modified" + ) + + assert result.error is None + assert len(env.rg_commands) == 1 + + +def test_modified_accepts_ripgrep_semver_with_prerelease_and_build_metadata(): + env = RecordingEnvironment() + original_execute = env.execute + + def execute(command, **kwargs): + if "--version" in command: + env.commands.append(command) + return { + "output": "ripgrep 14.1.1-alpha.1+build.2\n", + "returncode": 0, + } + return original_execute(command, **kwargs) + + env.execute = execute + result = ShellFileOperations(env).search( + "*.py", path="/repo", target="files", order="modified" + ) + + assert result.error is None + assert len(env.rg_commands) == 1 + + +def test_empty_discovery_output_is_zero_matches_without_retry(): + env = RecordingEnvironment(rg_output="", rg_code=0) + ops = ShellFileOperations(env) + + result = ops.search("*.missing", path="/repo", target="files") + + assert result.error is None + assert result.files == [] + assert result.total_count == 0 + assert len(env.rg_commands) == 1 + + +def test_modified_capability_failure_is_actionable_and_not_downgraded(): + env = RecordingEnvironment(rg_output="", rg_code=2) + ops = ShellFileOperations(env) + + result = ops.search("*.py", path="/repo", target="files", order="modified") + + assert len(env.rg_commands) == 1 + assert result.error is not None + assert "exact modification-time order" in result.error.lower() + assert "ripgrep" in result.error + + +@pytest.mark.parametrize("order", ["discovery", "modified"]) +def test_rg_partial_output_with_error_exit_fails_closed(order): + env = RecordingEnvironment(rg_output="/repo/partial.py\n", rg_code=2) + + result = ShellFileOperations(env).search( + "*.py", path="/repo", target="files", order=order + ) + + assert result.error is not None + assert result.files == [] + assert len(env.rg_commands) == 1 + + +@pytest.mark.parametrize("order", ["discovery", "modified"]) +def test_rg_sigpipe_is_benign_only_after_fetch_limit_paths(order): + output = "".join(f"/repo/{name}.py\n" for name in ("a", "b", "c", "d")) + env = RecordingEnvironment(rg_output=output, rg_code=141) + + result = ShellFileOperations(env).search( + "*.py", path="/repo", target="files", limit=2, offset=1, order=order + ) + + assert result.error is None + assert result.files == ["/repo/b.py", "/repo/c.py"] + assert result.truncated is True + + +@pytest.mark.parametrize("order", ["discovery", "modified"]) +def test_rg_sigpipe_with_fewer_than_fetch_limit_paths_fails_closed(order): + env = RecordingEnvironment(rg_output="/repo/partial.py\n", rg_code=141) + + result = ShellFileOperations(env).search( + "*.py", path="/repo", target="files", limit=2, offset=1, order=order + ) + + assert result.error is not None + assert result.files == [] + assert result.total_count == 0 + + +def test_invalid_direct_file_order_returns_structured_error(): + env = RecordingEnvironment() + ops = ShellFileOperations(env) + + result = ops.search("*.py", path="/repo", target="files", order="random") + + assert isinstance(result, SearchResult) + assert result.error == "Invalid file search order 'random'; expected 'discovery' or 'modified'." + assert env.rg_commands == [] + + +def test_handler_forwards_modified_order(monkeypatch): + captured = {} + + def fake_search_tool(**kwargs): + captured.update(kwargs) + return "{}" + + monkeypatch.setattr("tools.file_tools.search_tool", fake_search_tool) + + _handle_search_files({"pattern": "*.py", "target": "files", "order": "modified"}) + + assert captured["order"] == "modified" + + +def test_repeated_search_key_distinguishes_order(monkeypatch): + class StubOperations: + def search(self, **kwargs): + return SearchResult() + + monkeypatch.setattr("tools.file_tools._get_file_ops", lambda task_id: StubOperations()) + task_id = "engine-order-key" + for _ in range(3): + assert "BLOCKED" not in json.loads( + search_tool("*.py", target="files", order="discovery", task_id=task_id) + ).get("error", "") + + changed = json.loads( + search_tool("*.py", target="files", order="modified", task_id=task_id) + ) + + assert "BLOCKED" not in changed.get("error", "") + + +class RipgrepInvocationEnvironment(RecordingEnvironment): + def execute(self, command, **kwargs): + self.commands.append(command) + if command.startswith("test -e "): + return {"output": "exists\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/opt/Rip Grep/rg\n", "returncode": 0} + if "--version" in command: + return {"output": "ripgrep 14.1.1\n", "returncode": 0} + if "--files" in command: + return {"output": "/repo/a.py\n", "returncode": 0} + if "--line-number" in command: + return {"output": "/repo/a.py:1:needle\n", "returncode": 0} + if "--count-matches" in command: + return {"output": "", "returncode": 1} + return {"output": "", "returncode": 1} + + +def test_resolved_executable_with_spaces_is_used_by_every_rg_invocation(): + env = RipgrepInvocationEnvironment() + ops = ShellFileOperations(env) + + assert ops.search("*.py", path="/repo", target="files").files + assert ops.search("needle", path="/repo", target="content").matches + assert ops._zero_match_probe("absent", "/repo", None) is None + + invocations = [ + command for command in env.commands + if any(flag in command for flag in ("--files", "--line-number", "--count-matches")) + ] + assert invocations + assert all("'/opt/Rip Grep/rg'" in command for command in invocations) + assert all(not re.search(r"(?:^|[; ])rg\s", command) for command in invocations) + assert len([c for c in env.commands if c.startswith("command -v rg")]) == 1 + + +def test_non_rg_command_cache_keeps_cached_misses_and_bool_values(): + env = RecordingEnvironment() + ops = ShellFileOperations(env) + assert ops._has_command("find") is False + assert ops._has_command("find") is False + assert ops._command_cache == {"find": False} + assert len([c for c in env.commands if c.startswith("command -v find")]) == 1 + + +@pytest.mark.windows_only +def test_off_path_windows_rg_miss_is_reprobed_then_success_is_cached( + tmp_path, monkeypatch +): + local_app_data = tmp_path / "Local Data" + candidate = local_app_data / "Microsoft" / "WinGet" / "Links" / "rg.exe" + monkeypatch.setenv("LOCALAPPDATA", str(local_app_data)) + monkeypatch.setenv("USERPROFILE", str(tmp_path / "User Profile")) + monkeypatch.delenv("SCOOP", raising=False) + ops = ShellFileOperations(LocalEnvironment(str(tmp_path))) + probes = [] + + def command_v_miss(command, **kwargs): + probes.append(command) + from tools.file_operations import ExecuteResult + return ExecuteResult(stdout="", exit_code=1) + + monkeypatch.setattr(ops, "_exec", command_v_miss) + + assert ops._resolve_command("rg") is None + candidate.parent.mkdir(parents=True) + candidate.write_text("") + expected = str(candidate).replace("\\", "/") + assert ops._resolve_command("rg") == expected + assert ops._resolve_command("rg") == expected + assert probes == ["command -v rg 2>/dev/null", "command -v rg 2>/dev/null"] + + +def test_remote_resolution_never_probes_controller_host_paths(tmp_path, monkeypatch): + monkeypatch.setenv("LOCALAPPDATA", str(tmp_path / "controller-local")) + monkeypatch.setenv("USERPROFILE", str(tmp_path / "controller-user")) + env = RecordingEnvironment() + + def miss(command, **kwargs): + env.commands.append(command) + return {"output": "", "returncode": 1} + + env.execute = miss + ops = ShellFileOperations(env) + + assert ops._resolve_command("rg") is None + assert env.commands == ["command -v rg 2>/dev/null"] + assert str(tmp_path) not in env.commands[0] + + +@pytest.mark.windows_only +def test_remote_msys_shaped_executable_is_not_rewritten_as_controller_path(): + env = RecordingEnvironment() + + def execute(command, **kwargs): + env.commands.append(command) + if command.startswith("test -e "): + return {"output": "exists\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/c/remote-tools/rg\n", "returncode": 0} + if "--files" in command: + return {"output": "/repo/a.py\n", "returncode": 0} + return {"output": "", "returncode": 1} + + env.execute = execute + + result = ShellFileOperations(env).search("*.py", path="/repo", target="files") + + assert result.files + assert "'/c/remote-tools/rg' --files" in env.rg_commands[0] + assert "C:/remote-tools/rg" not in env.rg_commands[0] + + +@pytest.mark.windows_only +def test_every_windows_drive_root_is_broad_even_when_home_is_on_another_drive( + tmp_path, monkeypatch +): + import tools.file_operations as file_operations + + monkeypatch.setattr(file_operations, "_HOME", "C:/Users/alice") + ops = ShellFileOperations(LocalEnvironment(str(tmp_path))) + + assert ops._is_broad_local_search_root("D:/") is True + assert ops._is_broad_local_search_root("D:/repo") is False + + +def test_modified_multi_path_search_preserves_exact_order_request(): + env = RecordingEnvironment() + + def execute(command, **kwargs): + env.commands.append(command) + if command.startswith("test -e "): + if "'/one /two'" in command: + return {"output": "not_found\n", "returncode": 0} + return {"output": "exists\n", "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/usr/bin/rg\n", "returncode": 0} + if "--version" in command: + return {"output": "ripgrep 14.1.1\n", "returncode": 0} + if "--files" in command: + return {"output": "/two/new.py\n/one/old.py\n", "returncode": 0} + return {"output": "", "returncode": 1} + + env.execute = execute + result = ShellFileOperations(env).search( + "*.py", path="/one /two", target="files", order="modified" + ) + + assert result.files == ["/two/new.py", "/one/old.py"] + assert len(env.rg_commands) == 1 + assert "--sortr=modified" in env.rg_commands[0] + assert "'/one'" in env.rg_commands[0] + assert "'/two'" in env.rg_commands[0] + + +def test_comma_delimited_file_roots_preserve_internal_spaces_in_one_search(): + env = RecordingEnvironment(rg_output="C:/root one/a.py\nC:/root two/b.py\n") + combined = "C:/root one, C:/root two" + + path_checks = 0 + + def execute(command, **kwargs): + nonlocal path_checks + env.commands.append(command) + if command.startswith("test -e "): + path_checks += 1 + output = "not_found\n" if path_checks == 1 else "exists\n" + return {"output": output, "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/usr/bin/rg\n", "returncode": 0} + if "--files" in command: + return {"output": env.rg_output, "returncode": 0} + return {"output": "", "returncode": 1} + + env.execute = execute + result = ShellFileOperations(env).search( + "*.py", path=combined, target="files" + ) + + assert result.error is None + assert result.files == ["C:/root one/a.py", "C:/root two/b.py"] + assert len(env.rg_commands) == 1 + assert "'C:/root one' 'C:/root two'" in env.rg_commands[0] + assert "path contained 2 entries" in (result.warning or "") + + +def test_multi_path_modified_capability_error_propagates(): + env = RecordingEnvironment() + + def execute(command, **kwargs): + env.commands.append(command) + if command.startswith("test -e "): + output = "not_found\n" if "'/one /two'" in command else "exists\n" + return {"output": output, "returncode": 0} + if command.startswith("command -v rg"): + return {"output": "/usr/bin/rg\n", "returncode": 0} + if "--version" in command: + return {"output": "ripgrep 13.0.0\n", "returncode": 0} + raise AssertionError(command) + + env.execute = execute + result = ShellFileOperations(env).search( + "*.py", path="/one /two", target="files", order="modified" + ) + assert "ripgrep 14" in (result.error or "").lower() + assert env.rg_commands == [] + + +def test_modified_timeout_preserves_partial_results_and_limit_reason(): + env = RecordingEnvironment( + rg_output="/repo/partial.py\n[Command timed out after 60s]\n", + rg_code=124, + ) + + result = ShellFileOperations(env).search( + "*.py", path="/repo", target="files", order="modified" + ) + + assert result.files == ["/repo/partial.py"] + assert result.truncated is True + assert result.limit_reason == "search_timeout" + + +def test_order_is_ignored_for_content_search(): + env = RipgrepInvocationEnvironment() + + result = ShellFileOperations(env).search( + "needle", path="/repo", target="content", order="not-a-file-order" + ) + + assert result.error is None + assert result.matches diff --git a/tests/tools/test_search_giant_line_containment.py b/tests/tools/test_search_giant_line_containment.py new file mode 100644 index 0000000000..a31c6509fa --- /dev/null +++ b/tests/tools/test_search_giant_line_containment.py @@ -0,0 +1,100 @@ +"""Giant single-line file containment in content search (cline/cline#13525 port). + +A match inside a serialized dump (multi-MB single-line JSON, minified +bundle) used to make rg/grep emit the ENTIRE matched line into stdout: +``head -n`` counts lines, so a 40MB match line crossed the transport +untruncated and was buffered whole into Python before the per-match +[:500] clamp ran (measured 42MB transport / ~180MB peak alloc for one +match). The fix bounds lines at the search-engine layer: rg gets +``--max-columns 2000 --max-columns-preview``; the grep fallbacks pipe +through ``cut -c1-2000``. + +These tests run the REAL pipelines via bash (no mocked stdout) so the +flag/pipe behavior of the installed rg/grep is what's exercised. +""" + +import os +import shutil +import subprocess + +import pytest + +from tools.file_operations import ShellFileOperations + +# Big enough to prove containment, small enough to keep the test fast. +GIANT = 5 * 1024 * 1024 # 5MB single line +# Generous ceiling: pre-fix stdout for one giant match is >= GIANT bytes. +STDOUT_CEILING = 1 * 1024 * 1024 + + +class RecordingEnv: + """Local bash executor that records the largest stdout it returned.""" + + def __init__(self, cwd): + self.cwd = cwd + self.max_stdout = 0 + + def execute(self, command, timeout=60, **kwargs): + proc = subprocess.run( + ["bash", "-c", command], + capture_output=True, text=True, errors="replace", + timeout=timeout + 30, + ) + out = proc.stdout + (proc.stderr or "") + self.max_stdout = max(self.max_stdout, len(out)) + return {"output": out, "returncode": proc.returncode} + + +@pytest.fixture() +def giant_dir(tmp_path): + (tmp_path / "trace.json").write_text( + '{"needle": "' + "x" * GIANT + '"}', encoding="utf-8" + ) + (tmp_path / "small.py").write_text("needle = 1\n", encoding="utf-8") + return tmp_path + + +def _ops(giant_dir, engine): + env = RecordingEnv(str(giant_dir)) + ops = ShellFileOperations(env) + ops._has_command = lambda cmd: cmd == engine + return ops, env + + +@pytest.mark.parametrize("engine", ["rg", "grep"]) +def test_giant_single_line_match_is_bounded(giant_dir, engine): + if shutil.which(engine) is None: + pytest.skip(f"{engine} not installed") + ops, env = _ops(giant_dir, engine) + + result = ops.search("needle", path=str(giant_dir), target="content") + + assert result.error is None + paths = {os.path.basename(m.path) for m in result.matches} + # The giant-file match must still be REPORTED (preview, not omission)... + assert paths == {"trace.json", "small.py"} + assert all(len(m.content) <= 500 for m in result.matches) + # ...but its full line must never have crossed the transport. + assert env.max_stdout < STDOUT_CEILING, ( + f"{engine} pipeline returned {env.max_stdout} bytes of stdout — " + "giant matched line was not truncated at the engine layer" + ) + + +@pytest.mark.parametrize("engine", ["rg", "grep"]) +@pytest.mark.parametrize("output_mode", ["files_only", "count"]) +def test_line_cap_skipped_for_path_and_count_modes(giant_dir, engine, output_mode): + """files_only/count lines are paths/counts — never giant, never cut.""" + if shutil.which(engine) is None: + pytest.skip(f"{engine} not installed") + ops, env = _ops(giant_dir, engine) + + result = ops.search("needle", path=str(giant_dir), target="content", + output_mode=output_mode) + + assert result.error is None + if output_mode == "files_only": + assert {os.path.basename(f) for f in result.files} == {"trace.json", "small.py"} + else: + assert {os.path.basename(k) for k in result.counts} == {"trace.json", "small.py"} + assert env.max_stdout < STDOUT_CEILING diff --git a/tests/tools/test_search_zero_match_and_multipath.py b/tests/tools/test_search_zero_match_and_multipath.py index a001447051..f3d9fbbed8 100644 --- a/tests/tools/test_search_zero_match_and_multipath.py +++ b/tests/tools/test_search_zero_match_and_multipath.py @@ -58,6 +58,86 @@ class TestZeroMatchProbe: # Same class as the casing probe: the path must be in the hint. assert "conf.cfg" in r.get("warning", "") + def test_hidden_probe_prunes_dependency_trees_and_keeps_local_ignored(self, proj, monkeypatch): + d = proj / "proj" + dependency = d / "node_modules" / "package" / ".hidden" + dependency.mkdir(parents=True) + dependency_file = dependency / "dependency.js" + dependency_file.write_text("BOUNDED_HIDDEN_TOKEN = true\n") + local = d / ".project-local" + local.mkdir() + local_file = local / "settings.cfg" + local_file.write_text("BOUNDED_HIDDEN_TOKEN = true\n") + (d / ".gitignore").write_text("node_modules/\n.project-local/\n") + + # Drive the public search seam while recording the commands that the + # zero-match probe actually executes. The real rg calls still run. + from tools.file_tools import _get_file_ops + + task_id = "t-zm-pruned-hidden" + ops = _get_file_ops(task_id=task_id) + commands = [] + real_exec = ops._exec + + def recording_exec(command, *args, **kwargs): + commands.append(command) + return real_exec(command, *args, **kwargs) + + monkeypatch.setattr(ops, "_exec", recording_exec) + r = json.loads(search_tool("BOUNDED_HIDDEN_TOKEN", path=str(d), task_id=task_id)) + warning = r.get("warning", "") + + assert r["total_count"] == 0 + assert "hidden or gitignored" in warning + assert local_file.name in warning + assert dependency_file.name not in warning + + hidden_probe_commands = [ + command for command in commands + if "--hidden" in command and "--no-ignore" in command + ] + assert len(hidden_probe_commands) == 1 + hidden_probe = hidden_probe_commands[0] + assert "--glob" in hidden_probe + assert "'!node_modules/**'" in hidden_probe + assert "'!**/node_modules/**'" in hidden_probe + + def test_hidden_probe_prunes_explicit_dependency_root(self, proj, monkeypatch): + d = proj / "proj" + dependency = d / "node_modules" / "package" / ".hidden" + dependency.mkdir(parents=True) + (dependency / "dependency.js").write_text("EXPLICIT_ROOT_TOKEN = true\n") + (d / ".gitignore").write_text("node_modules/\n") + + from tools.file_tools import _get_file_ops + + task_id = "t-zm-explicit-pruned-root" + ops = _get_file_ops(task_id=task_id) + commands = [] + real_exec = ops._exec + + def recording_exec(command, *args, **kwargs): + commands.append(command) + return real_exec(command, *args, **kwargs) + + monkeypatch.setattr(ops, "_exec", recording_exec) + r = json.loads(search_tool( + "EXPLICIT_ROOT_TOKEN", + path=str(d / "node_modules"), + task_id=task_id, + )) + + assert r["total_count"] == 0 + assert "warning" not in r + hidden_probe_commands = [ + command for command in commands + if "--hidden" in command and "--no-ignore" in command + ] + assert len(hidden_probe_commands) == 1 + hidden_probe = hidden_probe_commands[0] + assert "'!node_modules/**'" in hidden_probe + assert "'!**/node_modules/**'" in hidden_probe + def test_probe_path_list_is_capped(self, proj): d = proj / "proj" for i in range(8): diff --git a/tests/tools/test_session_search.py b/tests/tools/test_session_search.py index 635bcb91e4..1d428364f2 100644 --- a/tests/tools/test_session_search.py +++ b/tests/tools/test_session_search.py @@ -114,11 +114,41 @@ class TestFormatTimestamp: # ========================================================================= class TestBrowseShape: + def test_browse_uses_bounded_recent_path(self): + class _DB: + rich_called = False + bounded_kwargs = None + + def list_recent_sessions_bounded(self, **kwargs): + self.bounded_kwargs = kwargs + return [] + + def list_sessions_rich(self, **_kwargs): + self.rich_called = True + raise AssertionError("unbounded rich listing must not be used") + + db = _DB() + result = json.loads(session_search(db=db)) + + assert result["success"] is True + assert db.rich_called is False + assert db.bounded_kwargs["timeout_seconds"] == 3.0 + + def test_browse_fails_closed_without_bounded_database_capability(self): + class _LegacyDB: + def list_sessions_rich(self, **_kwargs): + raise AssertionError("known-unbounded fallback must not be called") + + result = json.loads(session_search(db=_LegacyDB())) + + assert result["success"] is False + assert "does not support bounded recent-session browse" in result["error"] + def test_lazy_database_is_released_after_search(self, monkeypatch): class _DB: released = 0 - def list_sessions_rich(self, **_kwargs): + def list_recent_sessions_bounded(self, **_kwargs): return [] db = _DB() @@ -140,7 +170,7 @@ class TestBrowseShape: def __init__(self): self.closed = 0 - def list_sessions_rich(self, **_kwargs): + def list_recent_sessions_bounded(self, **_kwargs): return [] def close(self): diff --git a/tests/tools/test_terminal_compound_background.py b/tests/tools/test_terminal_compound_background.py index d0f1762fe9..beaa1d95f4 100644 --- a/tests/tools/test_terminal_compound_background.py +++ b/tests/tools/test_terminal_compound_background.py @@ -12,6 +12,10 @@ The rewriter fixes this by wrapping the tail in a brace group — the current shell. No subshell fork, no wait. """ +import shutil +import subprocess + +import pytest from tools.terminal_tool import _rewrite_compound_background as rewrite @@ -100,3 +104,105 @@ class TestEdgeCases: def test_tabs_between_tokens(self): assert rewrite("A\t&&\tB\t&") == "A\t&&\t{ B\t& }" + + +class TestTrailingStatementSeparator: + """A statement after the backgrounded compound on the SAME line. + + In ``A && B & C`` the trailing ``&`` is both the background operator and + the separator between the compound and ``C``. The rewrite consumes that + ``&`` into the brace group; without restoring a separator the result is + ``A && { B & } C`` — a bash syntax error (a brace group must be terminated + by ``;``, ``&``, ``|``, a newline, or ``)``/``}`` before the next command). + That mangles a valid command into one that fails entirely. + """ + + def test_trailing_command_gets_separator(self): + assert rewrite("echo hi && sleep 5 & echo done") == ( + "echo hi && { sleep 5 & } ; echo done" + ) + + def test_trailing_chain_gets_separator(self): + assert rewrite("a && b & c && d") == "a && { b & } ; c && d" + + def test_redirect_then_trailing_command(self): + assert rewrite("echo hi && sleep 5 &>/dev/null & echo done") == ( + "echo hi && { sleep 5 &>/dev/null & } ; echo done" + ) + + def test_existing_semicolon_separator_untouched(self): + # An explicit `;` already separates the group; don't add a second one. + assert rewrite("a && b &; c") == "a && { b & }; c" + + def test_newline_separator_untouched(self): + # A newline already terminates the brace group — no `;` needed. + assert rewrite("a && b &\necho next") == "a && { b & }\necho next" + + def test_pipe_after_group_untouched(self): + # `{ ...; } | cmd` is valid; the pipe is its own terminator. + assert rewrite("a && b & | cat") == "a && { b & } | cat" + + def test_redirect_prefix_on_trailing_command_gets_separator(self): + # `&>` after the group is a redirect for the NEXT command, not a + # terminator: `{ b & } &>/dev/null c` is a syntax error. + assert rewrite("a && b & &>/dev/null c") == "a && { b & } ; &>/dev/null c" + + def test_case_arm_terminator_untouched(self): + # `;;` already terminates the arm; adding `;` would leave an empty + # command between `;` and `;;`, which bash rejects. + assert rewrite("case $x in p) b && c & ;; esac") == "case $x in p) b && { c & } ;; esac" + + def test_separator_is_idempotent(self): + once = rewrite("echo hi && sleep 5 & echo done") + assert rewrite(once) == once + + def test_second_background_then_trailing(self): + assert rewrite("echo a && sleep 5 & echo b & echo c") == ( + "echo a && { sleep 5 & } ; echo b & echo c" + ) + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash not available") +class TestRewriteIsValidBash: + """The rewrite must always produce syntactically valid bash. + + This is the crux of the trailing-statement bug: a mangled command fails + with a confusing syntax error and neither half runs. ``bash -n`` parses + without executing, so it catches the corruption directly. + """ + + @pytest.mark.parametrize( + "command", + [ + "echo hi && sleep 5 & echo done", + "a && b & c && d", + "echo hi && sleep 5 &>/dev/null & echo done", + "echo a && sleep 5 & echo b & echo c", + "A && B &", + "A && B &; C", + "A && B &\nC", + "cd /tmp && python3 -m http.server 0 &>/dev/null & curl localhost", + "a && b & &>/dev/null c", + "case $x in p) b && c & ;; esac", + "A && B & echo x\nC && D & echo y && E & echo z", + ], + ) + def test_rewrite_parses(self, command): + rewritten = rewrite(command) + result = subprocess.run( + ["bash", "-n", "-c", rewritten], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"rewrite produced invalid bash: {rewritten!r}\n{result.stderr}" + ) + + def test_trailing_statement_actually_runs(self): + # End-to-end: the command after the backgrounded compound must run. + rewritten = rewrite("echo first && true & echo SECOND_RAN") + result = subprocess.run( + ["bash", "-c", rewritten], capture_output=True, text=True + ) + assert result.returncode == 0 + assert "SECOND_RAN" in result.stdout diff --git a/tests/tools/test_terminal_scope_multiplex.py b/tests/tools/test_terminal_scope_multiplex.py new file mode 100644 index 0000000000..fb48010b63 --- /dev/null +++ b/tests/tools/test_terminal_scope_multiplex.py @@ -0,0 +1,184 @@ +"""Per-turn terminal scope isolation under profile multiplexing (#68559 class). + +One multiplexed process serves several profiles, but terminal.* used to +resolve through the process-global ``TERMINAL_*`` env vars bridged once at +startup — so every routed profile inherited the launch profile's backend, +cwd, docker mounts and shared-container key (#68559, #94200, #101132, +#95470). ``tools.terminal_scope`` installs the routed profile's COMPLETE +terminal policy as a ContextVar at each profile boundary; readers resolve +ONLY from it (omitted key → defined default, never ``os.environ``) and an +unresolvable policy fails closed. +""" + +import json +import os + +import pytest + +from tools.terminal_scope import ( + TerminalPolicyRefusal, + TerminalPolicyUnavailable, + get_terminal_scope, + install_profile_terminal_scope, + reset_terminal_scope, + set_terminal_scope, + terminal_env, +) + +_LAUNCH_CWD = "/home/launch-user/private" +_LAUNCH_VOLUMES = '["/host/secret:/data:rw"]' + + +@pytest.fixture(autouse=True) +def _polluted_launch_env(monkeypatch, tmp_path): + """Launch profile A bridged a docker backend with sensitive policy into + the process env; every test proves a routed profile observes none of it.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + monkeypatch.setenv("TERMINAL_ENV", "docker") + monkeypatch.setenv("TERMINAL_CWD", _LAUNCH_CWD) + monkeypatch.setenv("TERMINAL_DOCKER_VOLUMES", _LAUNCH_VOLUMES) + monkeypatch.setenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "alpha-shared") + monkeypatch.setenv("TERMINAL_SSH_HOST", "10.10.0.103") + monkeypatch.setattr("agent.secret_scope.build_profile_secret_scope", lambda _h: {}) + monkeypatch.setattr("hermes_cli.env_loader.hydrate_profile_secret_sources", lambda _h: None) + import tools.terminal_tool as tt + + monkeypatch.setattr(tt, "_terminal_config_bridge_attempted", True) + yield + + +def _profile(tmp_path, name, config_yaml="", dotenv=""): + home = tmp_path / "profiles" / name + home.mkdir(parents=True) + if config_yaml: + (home / "config.yaml").write_text(config_yaml, encoding="utf-8") + if dotenv: + (home / ".env").write_text(dotenv, encoding="utf-8") + return home + + +def test_no_scope_keeps_process_env_behavior(): + """Single-process CLI/TUI (no scope bound) is byte-identical to before.""" + assert terminal_env("TERMINAL_ENV") == "docker" + assert terminal_env("TERMINAL_SSH_HOST") == "10.10.0.103" + + +def test_scoped_read_never_falls_through_to_process_env(): + """Omitted key under a scope → defined default, NOT the ambient value.""" + token = set_terminal_scope({"TERMINAL_ENV": "local"}) + try: + assert terminal_env("TERMINAL_ENV") == "local" + assert terminal_env("TERMINAL_SSH_HOST") == "" + assert terminal_env("TERMINAL_DOCKER_VOLUMES", "[]") == "[]" + assert os.environ["TERMINAL_ENV"] == "docker" # never mutated + finally: + reset_terminal_scope(token) + + +@pytest.mark.parametrize( + "config_yaml,dotenv", + [ + pytest.param("terminal:\n backend: local\n cwd: {cwd}\n", "", id="config-yaml"), + pytest.param("", "TERMINAL_ENV=local\nTERMINAL_CWD={cwd}\n", id="dotenv-only"), + ], +) +def test_routed_turn_reads_every_terminal_consumer_from_profile( + tmp_path, config_yaml, dotenv +): + """Leak matrix through the REAL gateway boundary: a routed local profile + with its own cwd must be seen as such by every terminal.* consumer — + terminal_tool config, container key resolution, docker media translation, + file_tools/runtime_cwd cwd anchors, and the browser/env_probe backend + checks — with none of launch profile A's docker policy showing through.""" + import gateway.run as gw + import tools.terminal_tool as tt + from agent import runtime_cwd + from gateway.platforms import base as gbase + from tools import browser_tool, env_probe, file_tools + + b_cwd = tmp_path / "b-work" + b_cwd.mkdir() + home = _profile( + tmp_path, "bee", + config_yaml.format(cwd=b_cwd), dotenv.format(cwd=b_cwd), + ) + + with gw._profile_runtime_scope(home): + cfg = tt._get_env_config() + assert cfg["env_type"] == "local" + assert cfg["cwd"] == str(b_cwd) + assert cfg["docker_volumes"] == [] + assert cfg["docker_shared_container_key"] == "" + assert tt._resolve_container_task_id(None) == "default" + assert gbase._parse_docker_volume_mounts() == [] + assert not any( + "alpha-shared" in c for c in gbase._docker_sandbox_dir_candidates("agent:bee:x") + ) + assert file_tools._configured_terminal_cwd() == str(b_cwd) + assert runtime_cwd.resolve_agent_cwd() == b_cwd + assert browser_tool._is_local_backend() is True + # env_probe bails out with "" for remote backends; a local profile + # must not be treated as remote just because the launch env is docker. + assert env_probe._resolve_terminal_backend() == "local" + assert get_terminal_scope() is None + # Process env untouched — the launch profile's own turns are unchanged. + assert os.environ["TERMINAL_DOCKER_VOLUMES"] == _LAUNCH_VOLUMES + + +def test_profile_omitting_keys_gets_defaults_not_launch_values(tmp_path): + """#101132/#95470: a docker profile that does NOT set docker_volumes or + docker_shared_container_key must not inherit the launch profile's.""" + import gateway.run as gw + import tools.terminal_tool as tt + + home = _profile(tmp_path, "bee", "terminal:\n backend: docker\n") + with gw._profile_runtime_scope(home): + cfg = tt._get_env_config() + assert cfg["env_type"] == "docker" + assert cfg["docker_volumes"] == [] + assert cfg["docker_shared_container_key"] == "" + assert cfg["ssh_host"] == "" + assert cfg["cwd"] != _LAUNCH_CWD + assert json.loads(os.environ["TERMINAL_DOCKER_VOLUMES"]) # A unchanged + + +def test_malformed_profile_config_refuses_execution(tmp_path): + """Unresolvable policy → refusal scope; terminal_tool refuses instead of + running under the launch process's ambient policy (fail closed).""" + from tools.terminal_tool import terminal_tool + + home = _profile(tmp_path, "broken", "terminal: [unclosed\n") + token = install_profile_terminal_scope(home) + try: + assert isinstance(get_terminal_scope(), TerminalPolicyRefusal) + with pytest.raises(TerminalPolicyUnavailable): + terminal_env("TERMINAL_ENV") + result = terminal_tool(command="whoami") + assert "terminal policy unavailable" in result + finally: + reset_terminal_scope(token) + + +def test_gateway_runtime_scope_resets_on_error(tmp_path): + import gateway.run as gw + + home = _profile(tmp_path, "qa", "terminal:\n backend: local\n") + with pytest.raises(RuntimeError): + with gw._profile_runtime_scope(home): + assert terminal_env("TERMINAL_ENV") == "local" + raise RuntimeError("turn blew up") + assert get_terminal_scope() is None + + +def test_tui_and_cron_boundaries_bind_and_reset(tmp_path): + import tui_gateway.server as server + from tools.terminal_scope import install_and_reset_profile_terminal_scope + + home = _profile(tmp_path, "dash", "terminal:\n backend: local\n") + with server._session_profile_runtime_scope({"profile_home": str(home)}): + assert terminal_env("TERMINAL_ENV") == "local" + assert terminal_env("TERMINAL_SSH_HOST") == "" + assert get_terminal_scope() is None + with install_and_reset_profile_terminal_scope(home): # cron fire helper + assert terminal_env("TERMINAL_ENV") == "local" + assert get_terminal_scope() is None diff --git a/tests/tools/test_terminal_tool_requirements.py b/tests/tools/test_terminal_tool_requirements.py index b87bca9da4..bdf85c9d04 100644 --- a/tests/tools/test_terminal_tool_requirements.py +++ b/tests/tools/test_terminal_tool_requirements.py @@ -360,3 +360,34 @@ class TestCheckFnTransientFailureSuppression: assert "terminal" not in names assert "execute_code" not in names + + +class TestUnscopedSecretReadLogging: + """#100697: with multiplexing on, boot-time check_fns run before any + profile secret scope exists, so get_secret fails closed with + UnscopedSecretError. That expected signal must not be logged like a + crashed check_fn (WARNING + traceback); an unscoped read reported while + the scope was *resolved* is a genuinely lost scope and stays loud.""" + + def test_expected_fail_closed_probe_is_quiet_but_lost_scope_stays_loud(self, caplog): + import logging + + import tools.registry as reg + from agent.secret_scope import get_secret, set_multiplex_active + + def probe(): + return bool(get_secret("REGISTRY_LOG_PROBE_TOKEN", "")) + + set_multiplex_active(True) + try: + with caplog.at_level(logging.DEBUG, logger="tools.registry"): + assert reg._run_check_fn_uncached(probe, unresolved_scope=True) is False + boot = [r for r in caplog.records if r.name == "tools.registry"] + caplog.clear() + assert reg._run_check_fn_uncached(probe, unresolved_scope=False) is False + lost = [r for r in caplog.records if r.name == "tools.registry"] + finally: + set_multiplex_active(False) + + assert boot and all(r.levelno == logging.DEBUG and r.exc_info is None for r in boot) + assert any(r.levelno >= logging.WARNING and r.exc_info for r in lost) diff --git a/tests/tools/test_tip_tool.py b/tests/tools/test_tip_tool.py index 7134b2799a..d8c9098a67 100644 --- a/tests/tools/test_tip_tool.py +++ b/tests/tools/test_tip_tool.py @@ -24,7 +24,7 @@ def emitted(monkeypatch): def test_lives_in_the_gui_surface_toolset(monkeypatch): """Scoped by toolset, not by the backend's env — see AGENTS.md.""" monkeypatch.delenv("HERMES_DESKTOP", raising=False) - entry = registry.get_entry("tip") + entry = registry.get_entry("show_tip") assert entry is not None assert entry.toolset == "desktop_ui" @@ -33,7 +33,7 @@ def test_lives_in_the_gui_surface_toolset(monkeypatch): def test_answers_to_the_appearance_switch(): """Tips off has to mean the model never sees the tool. See tests/tools/test_display_toggles.py for the config end of it.""" - entry = registry.get_entry("tip") + entry = registry.get_entry("show_tip") assert entry is not None assert entry.check_fn is tt.check_tips_enabled diff --git a/tests/tools/test_tool_search.py b/tests/tools/test_tool_search.py index 3e2c80e8ad..4a6b654d82 100644 --- a/tests/tools/test_tool_search.py +++ b/tests/tools/test_tool_search.py @@ -96,7 +96,30 @@ class TestClassification: assert not is_deferrable_tool_name(name), name assert name not in _HERMES_CORE_TOOLS - def test_gui_surface_alone_does_not_activate_the_bridge(self): + def test_gui_surface_defers_by_default(self): + """2026-08 core-deferral reversal: the curated defer set (GUI surface + included) hides behind the bridge BY DEFAULT. project tools not in + the defer set stay direct.""" + from tools.registry import discover_builtin_tools + from tools.tool_search import ToolSearchConfig, assemble_tool_defs + + discover_builtin_tools() + assembled = assemble_tool_defs( + [_td(name, f"GUI {name}") for name in + {"read_window_below", "apply_layout", "project_list"}], + context_length=200_000, + config=ToolSearchConfig.from_raw({"enabled": "on"}), + ) + assert assembled.activated + names = {td["function"]["name"] for td in assembled.tool_defs} + assert "read_window_below" not in names + assert "apply_layout" not in names + # project_list is NOT in the curated defer set → stays direct. + assert "project_list" in names + + def test_defer_override_restores_legacy_direct_gui(self): + """tools.tool_search.defer: [] restores the everything-eager legacy: + GUI tools alone no longer activate the bridge.""" from tools.registry import discover_builtin_tools from tools.tool_search import ToolSearchConfig, assemble_tool_defs @@ -105,14 +128,15 @@ class TestClassification: assembled = assemble_tool_defs( [_td(name, f"GUI {name}") for name in names], context_length=200_000, - config=ToolSearchConfig.from_raw({"enabled": "on"}), + config=ToolSearchConfig.from_raw({"enabled": "on", "defer": []}), ) assert not assembled.activated assert {td["function"]["name"] for td in assembled.tool_defs} == names - def test_gui_surface_stays_direct_when_mcp_activates_the_bridge(self): - """MCP/plugin tools turn Tool Search on; the session's GUI tools stay - in the model-facing array so HUD can still name read_window_below.""" + def test_core_working_set_never_defers_even_with_mcp_active(self): + """The bridge activates for MCP, but working-set core tools (terminal, + files, memory...) stay direct — the deferral set is the CURATED list, + not all of core.""" from tools.registry import discover_builtin_tools, registry from tools.tool_search import ( BRIDGE_TOOL_NAMES, @@ -131,8 +155,8 @@ class TestClassification: assembled = assemble_tool_defs( [ - _td("read_window_below", "Identify the window below"), - _td("apply_layout", "Apply a layout preset"), + _td("terminal", "Run a command"), + _td("memory", "Persistent memory"), _td("computer_use", "Drive the OS"), _td(mcp_name, "Deferred MCP capability"), ], @@ -144,7 +168,38 @@ class TestClassification: assert assembled.activated assert mcp_name not in names assert BRIDGE_TOOL_NAMES <= names - assert {"read_window_below", "apply_layout", "computer_use"} <= names + assert {"terminal", "memory"} <= names + # computer_use IS in the curated defer set → behind the bridge. + assert "computer_use" not in names + + def test_clarify_stays_eager_by_default(self): + """PR #97979 A/B verdict (288 runs, 3 model tiers): clarify deferred + collapsed structured ask-the-user usage 18/18 → 7/18 (gpt-terra 0/6); + models fell back to plain-text questions. The ask-the-user affordance + must stay ambient — clarify is NOT in the curated default defer set, + and assembles as a direct tool even when the bridge is active.""" + from tools.registry import discover_builtin_tools + from tools.tool_search import ( + _DEFAULT_DEFERRED_TOOLS, + ToolSearchConfig, + assemble_tool_defs, + ) + + assert "clarify" not in _DEFAULT_DEFERRED_TOOLS + + discover_builtin_tools() + assembled = assemble_tool_defs( + [ + _td("clarify", "Ask the user clarifying questions"), + _td("computer_use", "Drive the OS"), + ], + context_length=200_000, + config=ToolSearchConfig.from_raw({"enabled": "on"}), + ) + assert assembled.activated # computer_use still activates the bridge + names = {td["function"]["name"] for td in assembled.tool_defs} + assert "clarify" in names + assert "computer_use" not in names def test_unknown_tool_not_deferrable(self): """Defensive: a tool name we cannot resolve to a registry entry must @@ -658,9 +713,24 @@ class TestDeferredCallSchemaProbe: registry.register( name=name, handler=_handler, - schema={"type": "function", - "function": {"name": name, "description": f"desc {name}", - "parameters": params}}, + schema={"name": name, "description": f"desc {name}", + "parameters": params}, + toolset=toolset, + ) + + @staticmethod + def _register_schema(name, toolset, params, calls): + from tools.registry import registry + + def _handler(args, task_id=None, **kw): + calls.append(args) + return json.dumps({"ok": True, "args": args}) + + registry.register( + name=name, + handler=_handler, + schema={"name": name, "description": f"desc {name}", + "parameters": params}, toolset=toolset, ) @@ -696,3 +766,166 @@ class TestDeferredCallSchemaProbe: )) assert result.get("ok") is True assert result.get("doc") == "abc" + + def test_invalid_enum_is_blocked_before_dispatch(self): + import model_tools + + calls = [] + name = "mcp_probe_enum_validation" + toolset = "mcp-probe-enum-validation" + self._register_schema(name, toolset, { + "type": "object", + "properties": { + "priority": {"type": "string", "enum": ["low", "high"]}, + }, + "required": ["priority"], + }, calls) + + result = json.loads(model_tools.handle_function_call( + function_name="tool_call", + function_args={"name": name, "arguments": {"priority": "urgent"}}, + enabled_toolsets=[toolset], + )) + + assert calls == [] + assert result["path"] == "arguments.priority" + assert result["constraint"] == "enum" + assert "NOT invoked" in result["error"] + + @pytest.mark.parametrize( + ("suffix", "arguments", "expected_path", "expected_constraint"), + [ + ( + "nested_type", + {"options": {"count": "not-an-integer"}}, + "arguments.options.count", + "type", + ), + ( + "nested_required", + {"options": {}}, + "arguments.options", + "required", + ), + ( + "nested_extra", + {"options": {"count": 1, "extra": True}}, + "arguments.options", + "additionalProperties", + ), + ], + ) + def test_validator_reports_nested_constraint_path( + self, suffix, arguments, expected_path, expected_constraint, + ): + from tools.tool_search import validate_deferred_call_args + + calls = [] + name = f"mcp_probe_{suffix}" + self._register_schema(name, "mcp-probe-nested", { + "type": "object", + "properties": { + "options": { + "type": "object", + "properties": {"count": {"type": "integer"}}, + "required": ["count"], + "additionalProperties": False, + }, + }, + "required": ["options"], + }, calls) + + result = json.loads(validate_deferred_call_args(name, arguments)) + + assert result["path"] == expected_path + assert result["constraint"] == expected_constraint + + def test_coercible_arguments_validate_then_dispatch_repaired(self): + import model_tools + + calls = [] + name = "mcp_probe_coercion_validation" + toolset = "mcp-probe-coercion-validation" + self._register_schema(name, toolset, { + "type": "object", + "properties": {"count": {"type": "integer"}}, + "required": ["count"], + }, calls) + + result = json.loads(model_tools.handle_function_call( + function_name="tool_call", + function_args={"name": name, "arguments": {"count": "42"}}, + enabled_toolsets=[toolset], + )) + + assert result["ok"] is True + assert calls == [{"count": 42}] + + def test_nullable_extension_remains_accepted(self): + import model_tools + + calls = [] + name = "mcp_probe_nullable_validation" + toolset = "mcp-probe-nullable-validation" + self._register_schema(name, toolset, { + "type": "object", + "properties": {"value": {"type": "string", "nullable": True}}, + "required": ["value"], + }, calls) + + result = json.loads(model_tools.handle_function_call( + function_name="tool_call", + function_args={"name": name, "arguments": {"value": None}}, + enabled_toolsets=[toolset], + )) + + assert result["ok"] is True + assert calls == [{"value": None}] + + def test_schema_normalization_preserves_literal_enum_objects(self): + from tools.tool_search import validate_deferred_call_args + + calls = [] + name = "mcp_probe_literal_enum_validation" + enum_value = {"nullable": True, "$ref": "literal-not-a-schema"} + self._register_schema(name, "mcp-probe-literal-enum", { + "type": "object", + "properties": {"value": {"enum": [enum_value]}}, + "required": ["value"], + }, calls) + + assert validate_deferred_call_args(name, {"value": enum_value}) is None + + def test_malformed_schema_fails_open(self): + import model_tools + + calls = [] + name = "mcp_probe_malformed_validation" + toolset = "mcp-probe-malformed-validation" + self._register_schema(name, toolset, { + "type": "object", + "properties": {"value": {"type": "not-a-json-schema-type"}}, + }, calls) + + result = json.loads(model_tools.handle_function_call( + function_name="tool_call", + function_args={"name": name, "arguments": {"value": "kept"}}, + enabled_toolsets=[toolset], + )) + + assert result["ok"] is True + assert calls == [{"value": "kept"}] + + def test_external_ref_fails_open_without_resolution(self): + from tools.tool_search import validate_deferred_call_args + + calls = [] + name = "mcp_probe_external_ref_validation" + self._register_schema(name, "mcp-probe-external-ref", { + "type": "object", + "properties": { + "payload": {"$ref": "https://example.invalid/schema.json"}, + }, + }, calls) + + assert validate_deferred_call_args(name, {"payload": {"anything": True}}) is None diff --git a/tests/tools/test_tour_tool.py b/tests/tools/test_tour_tool.py index 1ab40e21ac..1cd2196cc3 100644 --- a/tests/tools/test_tour_tool.py +++ b/tests/tools/test_tour_tool.py @@ -14,7 +14,7 @@ def _run(**kwargs): def test_lives_in_the_gui_surface_toolset(monkeypatch): """Scoped by toolset, not by the backend's env — see AGENTS.md.""" monkeypatch.delenv("HERMES_DESKTOP", raising=False) - entry = registry.get_entry("tour") + entry = registry.get_entry("gui_tour") assert entry is not None assert entry.toolset == "desktop_ui" @@ -23,7 +23,7 @@ def test_lives_in_the_gui_surface_toolset(monkeypatch): def test_answers_to_the_appearance_switch(): """Tours off has to mean the model never sees the tool. See tests/tools/test_display_toggles.py for the config end of it.""" - entry = registry.get_entry("tour") + entry = registry.get_entry("gui_tour") assert entry is not None assert entry.check_fn is tt.check_tours_enabled diff --git a/tests/tools/test_tts_lifecycle_leases.py b/tests/tools/test_tts_lifecycle_leases.py new file mode 100644 index 0000000000..7558848f8e --- /dev/null +++ b/tests/tools/test_tts_lifecycle_leases.py @@ -0,0 +1,305 @@ +"""TTS engine lifecycle driven by speech-output toggles (issue #100881). + +Local engines load lazily on first synthesis, so the first spoken reply after +"read replies aloud" / voice conversation turns on pays the model load as dead +air. The toggles now hold *leases*: acquiring warms the configured provider +into the SAME cache slot synthesis reads; releasing the last lease unloads +resident local models. +""" + +from __future__ import annotations + +import threading + +import pytest + +from tools import tts_tool + + +@pytest.fixture(autouse=True) +def _clean_lifecycle(monkeypatch): + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + yield + tts_tool._reset_tts_leases_for_tests() + for cache in tts_tool._LOCAL_TTS_MODEL_CACHES.values(): + cache.clear() + + +class _FakePiperVoice: + loads = 0 + synthesized: list = [] + + @classmethod + def load(cls, model_path, use_cuda=False): + cls.loads += 1 + inst = cls() + inst.model_path = model_path + return inst + + def synthesize_wav(self, text, wav_file, syn_config=None): + type(self).synthesized.append(text) + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 160) + + +@pytest.fixture +def fake_piper(monkeypatch, tmp_path): + _FakePiperVoice.loads = 0 + _FakePiperVoice.synthesized = [] + monkeypatch.setattr(tts_tool, "_import_piper", lambda: _FakePiperVoice) + # Pretend the voice is already on disk so no download subprocess runs. + voices_dir = tmp_path / "voices" + voices_dir.mkdir() + (voices_dir / "en_US-test-medium.onnx").write_bytes(b"onnx") + (voices_dir / "en_US-test-medium.onnx.json").write_text("{}") + cfg = {"provider": "piper", "piper": {"voice": "en_US-test-medium", "voices_dir": str(voices_dir)}} + monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: cfg) + return cfg + + +# -------------------------------------------------------------------------- +# warm_tts_provider: warm-up populates the exact slot synthesis reads +# -------------------------------------------------------------------------- + + +def test_warm_loads_piper_into_synthesis_cache(fake_piper, tmp_path): + result = tts_tool.warm_tts_provider(fake_piper) + + assert result["warmed"] is True + assert result["action"] == "loaded" + assert result["provider"] == "piper" + assert _FakePiperVoice.loads == 1 + assert len(tts_tool._piper_voice_cache) == 1 + + # The load that would have happened on the first reply is already done: + # synthesis reuses the warmed instance without loading again. + out = tts_tool._generate_piper_tts("hello", str(tmp_path / "out.wav"), fake_piper) + assert out.endswith(".wav") + assert _FakePiperVoice.loads == 1 + assert _FakePiperVoice.synthesized == ["hello"] + + +def test_warm_twice_is_a_cache_hit(fake_piper): + tts_tool.warm_tts_provider(fake_piper) + second = tts_tool.warm_tts_provider(fake_piper) + + assert second["action"] == "cached" + assert _FakePiperVoice.loads == 1 + + +def test_warm_reads_configured_provider_when_none_given(fake_piper): + result = tts_tool.warm_tts_provider() + assert result["provider"] == "piper" + assert result["action"] == "loaded" + + +def test_warm_never_raises_on_engine_failure(monkeypatch): + def _boom(): + raise ImportError("No module named 'piper'") + + monkeypatch.setattr(tts_tool, "_import_piper", _boom) + result = tts_tool.warm_tts_provider({"provider": "piper"}) + + assert result["warmed"] is False + assert result["action"] == "error" + assert "piper" in result["error"] + assert tts_tool._piper_voice_cache == {} + + +def test_warm_is_noop_for_cloud_provider_without_lazy_sdk(monkeypatch): + result = tts_tool.warm_tts_provider({"provider": "openai"}) + assert result == {"provider": "openai", "warmed": False, "action": "noop"} + + +def test_warm_lazy_sdk_provider_reports_cached_when_installed(monkeypatch): + import types + + fake = types.SimpleNamespace( + is_available=lambda feature: feature == "tts.edge", + ensure=lambda *a, **k: pytest.fail("ensure must not run when the SDK is present"), + ) + monkeypatch.setitem(__import__("sys").modules, "tools.lazy_deps", fake) + result = tts_tool.warm_tts_provider({"provider": "edge"}) + assert result["warmed"] is True + assert result["action"] == "cached" + + +def test_warm_lazy_sdk_provider_installs_when_missing(monkeypatch): + import types + + calls = [] + fake = types.SimpleNamespace( + is_available=lambda feature: False, + ensure=lambda feature, prompt: calls.append((feature, prompt)), + ) + monkeypatch.setitem(__import__("sys").modules, "tools.lazy_deps", fake) + result = tts_tool.warm_tts_provider({"provider": "edge"}) + assert result["action"] == "installed" + assert calls == [("tts.edge", False)] + + +# -------------------------------------------------------------------------- +# release_tts_provider +# -------------------------------------------------------------------------- + + +def test_release_drops_every_local_cache(fake_piper): + tts_tool.warm_tts_provider(fake_piper) + tts_tool._kittentts_model_cache["m"] = object() + + assert tts_tool.release_tts_provider() == {"released": 2} + assert tts_tool._piper_voice_cache == {} + assert tts_tool._kittentts_model_cache == {} + + +def test_release_scoped_to_one_provider(fake_piper): + tts_tool.warm_tts_provider(fake_piper) + tts_tool._kittentts_model_cache["m"] = object() + + assert tts_tool.release_tts_provider("kittentts") == {"released": 1} + assert len(tts_tool._piper_voice_cache) == 1 + + +def test_release_with_nothing_resident_is_zero(): + assert tts_tool.release_tts_provider() == {"released": 0} + + +# -------------------------------------------------------------------------- +# Leases: warm on acquire, unload only when the LAST holder releases +# -------------------------------------------------------------------------- + + +def test_acquire_warms_and_counts(fake_piper): + result = tts_tool.acquire_tts_lease("desktop:read-aloud") + assert result["leases"] == 1 + assert result["action"] == "loaded" + assert tts_tool.tts_lease_holders() == ["desktop:read-aloud"] + + +def test_last_release_unloads_but_earlier_release_does_not(fake_piper): + tts_tool.acquire_tts_lease("desktop:read-aloud") + tts_tool.acquire_tts_lease("tui:voice-tts") + assert len(tts_tool._piper_voice_cache) == 1 + + # One surface turning speech off must not pull the model from under the + # other surface that still speaks through this process. + first = tts_tool.release_tts_lease("desktop:read-aloud") + assert first == {"leases": 1, "released": 0} + assert len(tts_tool._piper_voice_cache) == 1 + + last = tts_tool.release_tts_lease("tui:voice-tts") + assert last == {"leases": 0, "released": 1} + assert tts_tool._piper_voice_cache == {} + + +def test_reacquire_is_idempotent_and_reheals_cache(fake_piper): + tts_tool.acquire_tts_lease("cli:voice-tts") + tts_tool.release_tts_provider() # something else dropped the model + result = tts_tool.acquire_tts_lease("cli:voice-tts") + + assert result["leases"] == 1 + assert result["action"] == "loaded" + assert _FakePiperVoice.loads == 2 + + +def test_release_unknown_lease_is_noop(fake_piper): + tts_tool.acquire_tts_lease("a") + assert tts_tool.release_tts_lease("never-acquired") == {"leases": 1, "released": 0} + assert len(tts_tool._piper_voice_cache) == 1 + + +def test_acquire_failure_still_registers_lease(monkeypatch): + def _boom(): + raise RuntimeError("engine missing") + + monkeypatch.setattr(tts_tool, "_import_piper", _boom) + result = tts_tool.acquire_tts_lease("desktop:conversation", {"provider": "piper"}) + assert result["action"] == "error" + assert result["leases"] == 1 + assert tts_tool.tts_lease_holders() == ["desktop:conversation"] + + +# -------------------------------------------------------------------------- +# Registry invariant: every local engine cache is release-able +# -------------------------------------------------------------------------- + + +def test_every_local_warmer_has_a_registered_cache(): + warmers = tts_tool._local_tts_warmers() + assert set(warmers) == set(tts_tool._LOCAL_TTS_MODEL_CACHES) + assert tts_tool._LOCAL_TTS_MODEL_CACHES["piper"] is tts_tool._piper_voice_cache + assert tts_tool._LOCAL_TTS_MODEL_CACHES["kittentts"] is tts_tool._kittentts_model_cache + + +# -------------------------------------------------------------------------- +# User-declared providers get the same signal (plugin warm()/release(), +# command warm_command/release_command) so a local TTS server can preload +# and unload on the speech toggles. +# -------------------------------------------------------------------------- + + +def test_plugin_provider_warm_and_release_follow_the_lease(monkeypatch): + from agent import tts_provider, tts_registry + + calls: list = [] + + class _ServerBacked(tts_provider.TTSProvider): + @property + def name(self): + return "my-server" + + def synthesize(self, text, output_path, **kw): + return output_path + + def warm(self): + calls.append("warm") + + def release(self): + calls.append("release") + + tts_registry._reset_for_tests() + tts_registry.register_provider(_ServerBacked()) + cfg = {"provider": "my-server"} + monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: cfg) + monkeypatch.setattr("hermes_cli.plugins._ensure_plugins_discovered", lambda force=False: None) + try: + assert tts_tool.acquire_tts_lease("desktop:read-aloud", cfg)["action"] == "warmed" + tts_tool.acquire_tts_lease("tui:voice-tts", cfg) + tts_tool.release_tts_lease("desktop:read-aloud") + assert calls == ["warm", "warm"] # still one holder — no release yet + tts_tool.release_tts_lease("tui:voice-tts") + assert calls == ["warm", "warm", "release"] + finally: + tts_registry._reset_for_tests() + + +def test_command_provider_runs_warm_and_release_commands(monkeypatch): + ran: list = [] + done = threading.Event() + + def _fake_run(command, timeout, env_passthrough=None): + ran.append(command) + done.set() + + monkeypatch.setattr(tts_tool, "_run_command_tts", _fake_run) + cfg = { + "provider": "srv", + "providers": {"srv": { + "command": "srv say {input_path} {output_path}", + "warm_command": "curl -s localhost:5002/load?model={model}", + "release_command": "curl -s localhost:5002/unload", + "model": "kokoro v1", + }}, + } + monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: cfg) + + assert tts_tool.acquire_tts_lease("desktop:read-aloud", cfg)["action"] == "warmed" + assert done.wait(5) + done.clear() + tts_tool.release_tts_lease("desktop:read-aloud") + assert done.wait(5) + assert ran == ["curl -s localhost:5002/load?model='kokoro v1'", "curl -s localhost:5002/unload"] diff --git a/tests/tools/test_tts_output_dir_profile_scope.py b/tests/tools/test_tts_output_dir_profile_scope.py new file mode 100644 index 0000000000..264718e20d --- /dev/null +++ b/tests/tools/test_tts_output_dir_profile_scope.py @@ -0,0 +1,60 @@ +"""Regression tests for profile-scoped TTS default output dir (#98749). + +``DEFAULT_OUTPUT_DIR`` was resolved once at import time, so long-lived +multi-profile runtimes (dashboard console, TUI/Desktop backend, cron, kanban +workers) kept writing synthesized audio into the launch profile's +``cache/audio`` even while the request was scoped to a different profile via +``HERMES_HOME`` or ``set_hermes_home_override()``. The call-time accessor +``_default_output_dir()`` re-resolves from the live profile-scoped home; +these pins keep the synthesis paths from re-freezing the launch profile. +""" + +import importlib +from pathlib import Path + + +def _reload_tts_tool(import_home: Path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(import_home)) + import tools.tts_tool as tts_tool + + return importlib.reload(tts_tool) + + +def test_default_output_dir_follows_contextvar_profile_override(tmp_path, monkeypatch): + """The web server scopes profiles via set_hermes_home_override() rather + than mutating the process env; the accessor must follow that override.""" + default_home = tmp_path / "default-home" + profile_home = tmp_path / "profiles" / "ramona" + default_home.mkdir(parents=True) + profile_home.mkdir(parents=True) + + tts_tool = _reload_tts_tool(default_home, monkeypatch) + + from hermes_constants import ( + reset_hermes_home_override, + set_hermes_home_override, + ) + + token = set_hermes_home_override(str(profile_home)) + try: + assert tts_tool._default_output_dir() == str( + profile_home / "cache" / "audio" + ) + finally: + reset_hermes_home_override(token) + + # Outside the override scope the launch home applies again. + assert tts_tool._default_output_dir() == str(default_home / "cache" / "audio") + + +def test_explicit_default_output_dir_monkeypatch_still_wins(tmp_path, monkeypatch): + """Existing tests and external patchers can still override + tools.tts_tool.DEFAULT_OUTPUT_DIR directly.""" + default_home = tmp_path / "default-home" + default_home.mkdir(parents=True) + + tts_tool = _reload_tts_tool(default_home, monkeypatch) + + monkeypatch.setattr(tts_tool, "DEFAULT_OUTPUT_DIR", "/custom/audio") + + assert tts_tool._default_output_dir() == "/custom/audio" diff --git a/tests/tools/test_vision_native_fast_path.py b/tests/tools/test_vision_native_fast_path.py index 5237ae665f..82b7cb57ce 100644 --- a/tests/tools/test_vision_native_fast_path.py +++ b/tests/tools/test_vision_native_fast_path.py @@ -90,6 +90,30 @@ class TestSupportsMediaInToolResults: assert _supports_media_in_tool_results("", "anything") is False assert _supports_media_in_tool_results(None, "anything") is False # type: ignore[arg-type] + def test_profile_tool_message_veto_overrides_supports_vision(self): + """supports_vision_tool_messages=False is a hard veto even when the + profile declares supports_vision=True (xiaomi/MiMo 400s on list-type + tool-result content, #89981).""" + assert _supports_media_in_tool_results("xiaomi", "mimo-v2.5") is False + + def test_profile_veto_applies_even_when_vision_capable_lookup_agrees(self): + """A capability source marking the model vision-capable must not + re-open the native fast path for a provider that rejects it.""" + from tools.vision_tools import _should_use_native_vision_fast_path + from agent.auxiliary_client import set_runtime_main, clear_runtime_main + from agent import image_routing + + set_runtime_main("xiaomi", "mimo-v2.5") + try: + with patch.object( + image_routing, "decide_image_input_mode", return_value="native" + ), patch.object( + image_routing, "_lookup_supports_vision", return_value=True + ): + assert _should_use_native_vision_fast_path() is False + finally: + clear_runtime_main() + # ─── _build_native_vision_tool_result ──────────────────────────────────────── diff --git a/tests/tui_gateway/test_bot_relay_methods.py b/tests/tui_gateway/test_bot_relay_methods.py index ceca846637..9b1c9ae31d 100644 --- a/tests/tui_gateway/test_bot_relay_methods.py +++ b/tests/tui_gateway/test_bot_relay_methods.py @@ -106,6 +106,52 @@ def test_deliver_requires_params(home): assert "error" in err +def test_deliver_lands_in_live_bot_chat_instead_of_subprocess(home, monkeypatch): + """#100523: a Desktop-owned Bot Chat receives the DM as a normal user turn. + + With the target's Bot Chat live in this gateway, the subprocess transport + would be fenced out by the single-owner lease and drop the payload. The + handler must route through prompt.submit (the composer's choke point) and + never spawn the CLI. + """ + spawned = [] + submitted = [] + + class _Proc: + returncode, stdout, stderr = 0, "pong", "" + + def _fake_run(argv, *a, **k): + # The server module's import-time update prefetch runs `git ...` on a + # daemon thread; only the relay's `hermes` CLI spawn is under test. + if argv and argv[0] != "git": + spawned.append(argv) + return _Proc() + + monkeypatch.setattr("subprocess.run", _fake_run) + monkeypatch.setitem( + srv._methods, "prompt.submit", lambda rid, p: submitted.append(p) or srv._ok(rid, {"status": "streaming"}) + ) + monkeypatch.setattr(srv, "_profile_home", lambda name: home / "profiles" / name) + monkeypatch.setitem( + srv._sessions, + "live-ops", + {"profile_home": str(home / "profiles" / "ops"), "pending_title": "Bot Chat", "history": []}, + ) + out = _result(srv._methods["bot_relay.deliver"](1, {"profile": "ops", "message": "ping"})) + # queued=True is the invariant: a DM never interrupts a turn in flight. + assert submitted == [{"session_id": "live-ops", "text": "ping", "queued": True}] + assert not spawned + assert "reply" in out + + # A live session titled anything else for the same profile does not qualify: + # the subprocess path runs exactly as before. + srv._sessions["live-ops"]["pending_title"] = "Scratch" + submitted.clear() + + out = _result(srv._methods["bot_relay.deliver"](2, {"profile": "ops", "message": "ping"})) + assert out["reply"] == "pong" and spawned and not submitted + + def test_reply_roundtrip_and_id_validation(home): envelope_id = "c" * 32 _result(srv._methods["bot_relay.reply"](1, {"id": envelope_id, "reply": "hi"})) diff --git a/tests/tui_gateway/test_change_watcher.py b/tests/tui_gateway/test_change_watcher.py index 9c612c555f..1e0d297072 100644 --- a/tests/tui_gateway/test_change_watcher.py +++ b/tests/tui_gateway/test_change_watcher.py @@ -64,6 +64,23 @@ def test_state_db_move_broadcasts_sessions_changed(watcher_home): assert ("sessions.changed", {}) in events +def test_served_profile_store_move_broadcasts_sessions_changed(watcher_home, monkeypatch): + """A backend serving a sibling profile must see that profile's state.db + move too — otherwise a routed profile's Bot Chat never refreshes (#99333).""" + home, events = watcher_home + bot_home = home / "profiles" / "bot" + bot_home.mkdir(parents=True) + monkeypatch.setattr(server, "_served_profile_homes", set()) + monkeypatch.setattr("hermes_cli.profiles.get_profile_dir", lambda name: home / "profiles" / name) + assert server._profile_home("bot") == bot_home + server._broadcast_watched_changes(now=0.0) + + (bot_home / "state.db").write_text("x") + server._broadcast_watched_changes(now=10.0) + + assert ("sessions.changed", {}) in events + + def test_gateway_state_move_broadcasts_platforms_changed(watcher_home): home, events = watcher_home server._broadcast_watched_changes(now=0.0) diff --git a/tests/tui_gateway/test_compute_host_late_compress_ack.py b/tests/tui_gateway/test_compute_host_late_compress_ack.py new file mode 100644 index 0000000000..3458ddbd10 --- /dev/null +++ b/tests/tui_gateway/test_compute_host_late_compress_ack.py @@ -0,0 +1,235 @@ +"""Regression tests for #97948 symptom A (salvaged from #99630). + +A manual /compress on a compute-host session used to block its RPC waiter for +a hard-coded 120s, return a 5019 timeout error, and then DROP the host's late +``control.ack`` — so the rotated session_key / history_version / session_info +never reached the gateway session and the desktop never refreshed. +""" + +import queue +import sys +import threading +import time +import types + +import pytest + +from tui_gateway import server +from tui_gateway.host_supervisor import HostSupervisor + + +def _supervisor() -> tuple[HostSupervisor, list]: + sup = HostSupervisor(argv=[sys.executable, "-c", ""], autostart=False) + sent: list = [] + sup._send_frame = lambda frame: sent.append(frame) + sup.start = lambda: None # never spawn a child + return sup, sent + + +def _session(**extra) -> dict: + return { + "agent": types.SimpleNamespace(), + "session_key": "old-session-key", + "history": [], + "history_lock": threading.Lock(), + "history_version": 3, + "running": False, + "attached_images": [], + "image_counter": 0, + "cols": 80, + "slash_worker": None, + "show_reasoning": False, + "tool_progress_mode": "all", + "_compute_host_active": True, + **extra, + } + + +# ── HostSupervisor: late-ack registration ─────────────────────────────────── + + +def test_control_timeout_registers_one_shot_late_ack_handler(): + sup, sent = _supervisor() + fired: list = [] + + with pytest.raises(queue.Empty): + sup.control("sid", route_name="session.compress", payload={"command": "/compress"}, + wait=True, timeout=0.05, on_late_ack=fired.append) + + request_id = sent[0]["request_id"] + assert request_id not in sup._pending_controls + assert request_id in sup._late_control_handlers + + late = {"type": "control.ack", "request_id": request_id, "result": {"status": "compressed"}} + sup._handle_host_frame(late) + assert fired == [late] + # One-shot: a duplicate ack for the same request is ignored. + sup._handle_host_frame(late) + assert fired == [late] + assert request_id not in sup._late_control_handlers + + +def test_control_timeout_without_handler_still_drops_late_ack(): + sup, sent = _supervisor() + with pytest.raises(queue.Empty): + sup.control("sid", route_name="session.compress", wait=True, timeout=0.05) + assert sup._late_control_handlers == {} + sup._handle_host_frame({"type": "control.ack", "request_id": sent[0]["request_id"]}) + + +def test_late_control_error_and_bare_error_frames_fire_handler(): + sup, sent = _supervisor() + fired: list = [] + for _ in range(2): + with pytest.raises(queue.Empty): + sup.control("sid", route_name="session.compress", wait=True, timeout=0.01, + on_late_ack=fired.append) + rid_a, rid_b = sent[0]["request_id"], sent[1]["request_id"] + sup._handle_host_frame({"type": "control.error", "request_id": rid_a, "message": "boom"}) + sup._handle_host_frame({"type": "error", "request_id": rid_b, "message": "bad frame"}) + assert [f["request_id"] for f in fired] == [rid_a, rid_b] + + +def test_late_ack_handlers_are_bounded_by_ttl_and_cap(monkeypatch): + from tui_gateway import host_supervisor as hs + + monkeypatch.setattr(hs, "_LATE_CONTROL_MAX", 3) + sup, _sent = _supervisor() + for i in range(5): + sup._register_late_control_handler(f"r{i}", lambda _f: None) + assert len(sup._late_control_handlers) == 3 + assert set(sup._late_control_handlers) == {"r2", "r3", "r4"} + + # TTL: an old registration is dropped on the next registration. + monkeypatch.setattr(hs, "_LATE_CONTROL_TTL_SECS", 0.0) + time.sleep(0.01) + sup._register_late_control_handler("fresh", lambda _f: None) + assert set(sup._late_control_handlers) == {"fresh"} + + +def test_host_crash_fails_outstanding_late_ack_handlers(): + sup, sent = _supervisor() + fired: list = [] + with pytest.raises(queue.Empty): + sup.control("sid", route_name="session.compress", wait=True, timeout=0.01, + on_late_ack=fired.append) + sup._fail_pending_turns(reason="crash", message="compute host exited with code 1") + assert len(fired) == 1 + assert fired[0]["type"] == "control.error" + assert fired[0]["request_id"] == sent[0]["request_id"] + assert sup._late_control_handlers == {} + + +# ── session.compress RPC: pending answer + late adoption ──────────────────── + + +@pytest.fixture +def compute_host_gateway(monkeypatch): + sup, sent = _supervisor() + emitted: list = [] + monkeypatch.setattr(server, "_compute_host_supervisor", sup) + monkeypatch.setattr(server, "_emit", lambda event, sid, payload=None: emitted.append((event, sid, payload))) + monkeypatch.setattr(server, "_session_uses_compute_host", lambda _s, cfg=None: True) + monkeypatch.setattr(server, "_compute_host_compress_wait_seconds", lambda cfg=None: 0.05) + monkeypatch.setattr(server, "_session_info", lambda _agent, _session=None: {"model": "mirrored"}) + session = _session() + server._sessions["sid"] = session + try: + yield sup, sent, emitted, session + finally: + server._sessions.pop("sid", None) + + +def _late_ack(request_id: str) -> dict: + return { + "type": "control.ack", + "sid": "sid", + "request_id": request_id, + "route_name": "session.compress", + "result": {"status": "compressed", "removed": 12, "summary": {"headline": "Compressed 14 → 2"}}, + "session_key": "rotated-session-key", + "history_version": 9, + "message_count": 2, + "session_info": {"model": "host-model", "usage": {"total": 111}}, + } + + +def test_session_compress_reports_pending_and_adopts_late_ack(compute_host_gateway): + sup, sent, emitted, session = compute_host_gateway + + resp = server.handle_request({"id": "1", "method": "session.compress", "params": {"session_id": "sid"}}) + + assert "error" not in resp, resp + assert resp["result"]["status"] == "pending" + assert resp["result"]["turn_isolation"] is True + assert "background" in resp["result"]["message"] + assert sent[0]["route_name"] == "session.compress" + # Nothing adopted yet, the host is still working. + assert session["session_key"] == "old-session-key" + assert emitted == [] + + sup._handle_host_frame(_late_ack(sent[0]["request_id"])) + + assert session["session_key"] == "rotated-session-key" + assert session["history_version"] == 9 + assert session["_metadata_message_count"] == 2 + assert session["_metadata_mirror"]["model"] == "host-model" + events = [(event, payload) for event, _sid, payload in emitted] + assert ("session.info", {"model": "mirrored"}) in events + assert ("status.update", {"kind": "compacted", "text": "✓ Context compression complete"}) in events + + +def test_session_compress_late_control_error_surfaces_as_error_event(compute_host_gateway): + sup, sent, emitted, session = compute_host_gateway + + resp = server.handle_request({"id": "1", "method": "session.compress", "params": {"session_id": "sid"}}) + assert resp["result"]["status"] == "pending" + + sup._handle_host_frame({"type": "control.error", "request_id": sent[0]["request_id"], "message": "provider down"}) + + assert session["session_key"] == "old-session-key" + assert ("error", "sid", {"message": "compression failed: provider down"}) in emitted + + +def test_session_compress_late_ack_ignored_after_session_closed(compute_host_gateway): + sup, sent, emitted, session = compute_host_gateway + server.handle_request({"id": "1", "method": "session.compress", "params": {"session_id": "sid"}}) + server._sessions.pop("sid") + + sup._handle_host_frame(_late_ack(sent[0]["request_id"])) + + assert session["session_key"] == "old-session-key" + assert emitted == [] + + +def test_slash_compress_route_reports_pending_and_adopts_late_ack(compute_host_gateway): + sup, sent, emitted, session = compute_host_gateway + + resp = server.handle_request( + {"id": "1", "method": "slash.exec", "params": {"session_id": "sid", "command": "/compress"}} + ) + + assert "error" not in resp, resp + assert "compression still running in the background" in resp["result"]["output"] + assert sent[0]["route_name"] == "slash.compress" + + sup._handle_host_frame({**_late_ack(sent[0]["request_id"]), "route_name": "slash.compress"}) + assert session["session_key"] == "rotated-session-key" + assert any(event == "session.info" for event, _sid, _p in emitted) + + +# ── wait budget follows compression.context_total_ceiling_seconds ─────────── + + +def test_compress_wait_budget_follows_config_ceiling(): + assert server._compute_host_compress_wait_seconds({"compression": {}}) == 630.0 + assert server._compute_host_compress_wait_seconds( + {"compression": {"context_total_ceiling_seconds": 200}} + ) == 230.0 + # Never below the historical 120s floor, never above the RPC-safe cap. + assert server._compute_host_compress_wait_seconds( + {"compression": {"context_total_ceiling_seconds": 10, "context_timeout_seconds": 0}} + ) == 120.0 + assert server._compute_host_compress_wait_seconds( + {"compression": {"context_total_ceiling_seconds": 99999}} + ) == server._COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS diff --git a/tests/tui_gateway/test_cross_process_orphan_ownership.py b/tests/tui_gateway/test_cross_process_orphan_ownership.py index 481b837c07..1a9de04ff9 100644 --- a/tests/tui_gateway/test_cross_process_orphan_ownership.py +++ b/tests/tui_gateway/test_cross_process_orphan_ownership.py @@ -270,6 +270,84 @@ def test_automatic_cleanup_preserves_corrupt_registry_without_overwrite( assert state_path.read_text(encoding="utf-8") == corrupt +def test_own_live_lease_ids_reports_live_owners_and_skips_the_excluded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _Lease: + def __init__(self, lease_id: str) -> None: + self.lease_id = lease_id + + first = _Lease("first") + second = _Lease("second") + monkeypatch.setattr( + server, + "_sessions", + { + "one": {"active_session_lease": first}, + "two": {"active_session_lease": second}, + "three": {"active_session_lease": None}, + }, + ) + + assert server._own_live_lease_ids() == {"first", "second"} + assert server._own_live_lease_ids(exclude=first) == {"second"} + + +def test_automatic_cleanup_reclaims_own_orphan_lease_not_treated_as_sibling( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + profile_home = tmp_path / "profile-home" + session_id = "own-orphan-session" + owner_lease, message = server._claim_active_session_slot( + session_id, + live_session_id="vanished-runtime", + surface="desktop", + profile_home=profile_home, + ) + assert owner_lease is not None and message is None + # The owner vanished minutes ago; a lease written seconds ago is still + # inside the self-orphan grace window and must be left alone. + monkeypatch.setattr( + "hermes_cli.active_sessions._SELF_ORPHAN_GRACE_SECONDS", 0.0 + ) + ended: list[tuple[str, str]] = [] + + class _FakeDB: + def get_session(self, target: str) -> dict[str, str]: + return {"id": target, "source": "desktop"} + + def end_session(self, target: str, reason: str) -> None: + ended.append((target, reason)) + + @contextlib.contextmanager + def _profile_db(_session: dict): + yield _FakeDB() + + monkeypatch.setattr(server, "_sessions", {}) + monkeypatch.setattr(server, "_session_db", _profile_db) + monkeypatch.setattr( + server, "_notify_session_boundary", lambda *args, **kwargs: None + ) + monkeypatch.setattr( + "tools.async_delegation.interrupt_for_session", lambda *args, **kwargs: None + ) + session = { + "active_session_lease": None, + "agent": None, + "history": [], + "history_lock": threading.Lock(), + "profile_home": str(profile_home), + "session_key": session_id, + "slash_worker": None, + "source": "desktop", + } + + server._finalize_session(session, end_reason="ws_orphan_reap") + + assert ended == [(session_id, "ws_orphan_reap")] + assert active_session_registry_snapshot(registry_home=profile_home) == [] + + def test_liveness_guard_serializes_cross_process_acquire(tmp_path: Path) -> None: home = tmp_path / "guard-home" waiting_file = tmp_path / "child-waiting" diff --git a/tests/tui_gateway/test_gui_surface_toolsets.py b/tests/tui_gateway/test_gui_surface_toolsets.py index b22f04feb0..92463fe453 100644 --- a/tests/tui_gateway/test_gui_surface_toolsets.py +++ b/tests/tui_gateway/test_gui_surface_toolsets.py @@ -27,8 +27,8 @@ GUI_TOOLS = { "read_window_below", "react_to_message", "setup_mcp", - "tip", - "tour", + "show_tip", + "gui_tour", } @@ -43,7 +43,13 @@ def no_desktop_env(monkeypatch): class TestDesktopUiToolset: def test_holds_exactly_the_gui_affordances(self): - assert set(resolve_toolset("desktop_ui")) == GUI_TOOLS + # apply_layout registers into desktop_ui via the registry (not the + # static toolsets.py list), so force discovery first — otherwise the + # result depends on which earlier test imported tool modules + # (pre-existing ordering flake, surfaced by the #97979 test sweep). + from tools.registry import discover_builtin_tools + discover_builtin_tools() + assert set(resolve_toolset("desktop_ui")) == GUI_TOOLS | {"apply_layout"} def test_stays_off_the_core_tool_list(self): """Core ships on every API call — a GUI-only tool must not be there.""" diff --git a/tests/tui_gateway/test_hud_surface_note.py b/tests/tui_gateway/test_hud_surface_note.py index cfb201e2d1..029664b351 100644 --- a/tests/tui_gateway/test_hud_surface_note.py +++ b/tests/tui_gateway/test_hud_surface_note.py @@ -117,7 +117,12 @@ class TestTurnRouting: assert assembled.activated assert mcp_name not in names - assert server._hud_surface_note(_session(tools=names, client_surface="hud")) == ( + # Production computes the note from agent.valid_tool_names — the + # GRANTED set — not from the visible post-assembly schemas. Under + # #97979 the HUD kit (read_window_below, computer_use) is deferred + # behind the bridge yet still granted/callable, so the note must + # survive assembly unchanged. + assert server._hud_surface_note(_session(tools=FULL_KIT, client_surface="hud")) == ( hud_surface_note(FULL_KIT) ) diff --git a/tests/tui_gateway/test_projects_rpc.py b/tests/tui_gateway/test_projects_rpc.py index 1cd5a320c2..f2faa74262 100644 --- a/tests/tui_gateway/test_projects_rpc.py +++ b/tests/tui_gateway/test_projects_rpc.py @@ -763,11 +763,13 @@ def test_projects_reads_are_scoped_to_the_requested_profile(monkeypatch, tmp_pat assert coder_tree["projects"][0]["sessionCount"] == 1 assert launch_tree["scoped_session_ids"] == ["launch-session"] assert coder_tree["scoped_session_ids"] == ["coder-session"] + assert [s["profile"] for s in coder_tree["projects"][0]["previewSessions"]] == ["coder"] assert coder_sessions["project"]["id"] == coder_project["id"] assert coder_sessions["project"]["sessionCount"] == 1 lane = coder_sessions["project"]["repos"][0]["groups"][0] assert [s["id"] for s in lane["sessions"]] == ["coder-session"] + assert [s["profile"] for s in lane["sessions"]] == ["coder"] def test_projects_tree_is_scoped_to_the_requested_profile(monkeypatch, tmp_path): diff --git a/tests/tui_gateway/test_protocol.py b/tests/tui_gateway/test_protocol.py index 59d44c18f3..1f5040f261 100644 --- a/tests/tui_gateway/test_protocol.py +++ b/tests/tui_gateway/test_protocol.py @@ -806,6 +806,122 @@ def test_session_resume_rejects_runaway_transcript_before_history_load( assert "safe resume limit is 20000" in response["error"]["message"] +def test_session_resume_deferred_and_omitted_paths_guard_the_tip_only(server, monkeypatch): + """A deep compression lineage behind a small tip must open on Desktop. + + Desktop's cold resume sends ``defer_history`` + ``omit_messages`` and pages + the transcript over REST, so the process only ever holds the tip segment. + Counting the whole lineage there returned 4130 for the healthiest sessions + (85 compaction segments / ~29k rows / ~700-row tip: Bot Chat stuck on + "Waking up…"). The guard must count what each path loads. + """ + calls = [] + + class _DB: + def get_session(self, sid): + return {"id": sid, "message_count": 28_730} + + def get_session_by_title(self, _title): + return None + + def resolve_resume_session_id(self, sid): + return sid + + def assert_resume_safe(self, sid, max_messages=None, *, tip_only=False): + calls.append(tip_only) + if not tip_only: + from hermes_state import SessionResumeTooLargeError + + raise SessionResumeTooLargeError(20_001, 20_000) + return 666 + + def reopen_session(self, _sid): + raise RuntimeError("stop before history load") + + monkeypatch.setattr(server, "_get_db", lambda: _DB()) + monkeypatch.setattr(server, "_enable_gateway_prompts", lambda: None) + + for params in ( + {"defer_history": True, "omit_messages": True, "source": "desktop"}, + {"omit_messages": True}, + {"lazy": True}, + ): + calls.clear() + response = server.handle_request( + { + "id": "r-tip", + "method": "session.resume", + "params": {"session_id": "deep-lineage", **params}, + } + ) + err = response.get("error") or {} + assert err.get("code") != 4130, params + assert calls == [True], params + + # The non-deferred, non-omitted resume materializes the full lineage in + # memory, so it keeps the lineage-wide bound. + calls.clear() + response = server.handle_request( + {"id": "r-full", "method": "session.resume", "params": {"session_id": "deep-lineage"}} + ) + assert response["error"]["code"] == 4130 + assert calls == [False] + + +def test_deferred_hydration_falls_back_to_tip_when_lineage_exceeds_limit(server, monkeypatch): + """The hydration worker never loads a lineage the guard would refuse.""" + import threading + + from hermes_state import SessionResumeTooLargeError + + tip = [{"role": "user", "content": "tip"}] + reads = [] + + class _DB: + def reopen_session(self, _sid): + return True + + def assert_resume_safe(self, sid, max_messages=None, *, tip_only=False): + if not tip_only: + raise SessionResumeTooLargeError(20_001, 20_000) + return 1 + + def get_resume_conversations(self, _sid): + reads.append("lineage") + raise AssertionError("must not materialize the runaway lineage") + + def get_ancestor_display_prefix(self, _sid): + reads.append("prefix") + raise AssertionError("must not materialize the runaway lineage") + + def get_messages_as_conversation(self, sid, **kwargs): + reads.append(("tip", kwargs.get("repair_alternation"))) + return list(tip) + + built = threading.Event() + monkeypatch.setattr(server, "_start_agent_build", lambda _sid, _session: built.set()) + monkeypatch.setattr(server, "_maybe_schedule_auto_continue", lambda *_a, **_k: None) + + session = server._deferred_session_record( + "deep-lineage", cols=80, cwd="/tmp", history=[], lease=None + ) + session["resume_history_ready"] = threading.Event() + session["resume_hydrating"] = True + session["resume_message_count"] = 28_730 + server._sessions["hyd"] = session + try: + server._schedule_resume_hydration("hyd", "deep-lineage", _DB()) + assert session["resume_history_ready"].wait(timeout=5) + assert built.wait(timeout=5) + assert session.get("resume_history_error") is None + assert session["history"] == tip + assert session["display_history_prefix"] == [] + assert session["resume_message_count"] == 1 + assert reads == [("tip", True)] + finally: + server._sessions.pop("hyd", None) + + def test_session_resume_guard_failure_fails_open(server, monkeypatch): """A transient guard error must not block resume (fail open, log only).""" reopened = [] diff --git a/tests/tui_gateway/test_resume_live_profile_scope.py b/tests/tui_gateway/test_resume_live_profile_scope.py new file mode 100644 index 0000000000..f6fd13d84c --- /dev/null +++ b/tests/tui_gateway/test_resume_live_profile_scope.py @@ -0,0 +1,108 @@ +"""``session.resume`` reuses a live session only within the requested profile. + +The live registry is keyed by bare stored session id, and stored ids are +timestamp-based, so the same id can legitimately be live under profile A while +profile B's store also holds it. The resume fast path (and the post-build +re-check / ``_claim_or_reuse_live``) used to hand profile B's resume profile +A's runtime — the turn then ran with A's persona and wrote A's memory +(#100029). Pinned here: + +* resume with profile B never reuses profile A's live session of the same id; +* the launch profile (no ``profile``) still matches live records that carry + no ``profile_home`` — the pre-existing single-profile contract. +""" + +from __future__ import annotations + +import pytest + +from tui_gateway import server + + +class _DB: + """Minimal ``SessionDB`` stand-in: every profile store knows ``s1``.""" + + def __init__(self, db_path=None, **_kwargs): + self.db_path = db_path + + def close(self): + pass + + def get_session(self, target): + return {"id": "s1", "cwd": ""} if target == "s1" else None + + def get_session_by_title(self, _target): + return None + + def resolve_resume_session_id(self, target): + return target + + def reopen_session(self, _target): + pass + + def get_resume_conversations(self, _target): + return ([], []) + + def get_ancestor_display_prefix(self, _target): + return [] + + def get_messages_as_conversation(self, _target, **_kwargs): + return [] + + +@pytest.fixture() +def homes(monkeypatch, tmp_path): + homes = {name: tmp_path / name for name in ("a", "b")} + for home in homes.values(): + home.mkdir() + monkeypatch.setattr("hermes_state.get_shared_session_db", _DB) + monkeypatch.setattr(server, "_get_db", lambda: _DB()) + monkeypatch.setattr(server, "_profile_home", lambda p: homes.get(p) if p else None) + monkeypatch.setattr(server, "_profile_configured_cwd", lambda _home: str(tmp_path)) + monkeypatch.setattr(server, "_enable_gateway_prompts", lambda: None) + monkeypatch.setattr(server, "_schedule_agent_build", lambda *a, **k: None) + monkeypatch.setattr(server, "_schedule_session_cap_enforcement", lambda *a, **k: None) + monkeypatch.setattr(server, "_maybe_schedule_auto_continue", lambda *a, **k: None) + monkeypatch.setattr(server, "_default_session_cwd", lambda *a, **k: str(tmp_path)) + monkeypatch.setattr(server, "_child_run_active", lambda _key: False) + monkeypatch.setattr( + server, "_live_session_payload", lambda sid, session, **_k: {"session_id": sid} + ) + known = set(server._sessions) + yield homes + with server._sessions_lock: + for sid in [s for s in server._sessions if s not in known]: + server._sessions.pop(sid, None) + + +def _resume(**params): + return server.handle_request({"id": "1", "method": "session.resume", "params": params}) + + +def _register_live(sid: str, profile_home) -> dict: + record = {"session_key": "s1", "history": [], "last_active": 0.0} + if profile_home is not None: + record["profile_home"] = str(profile_home) + with server._sessions_lock: + server._sessions[sid] = record + return record + + +def test_resume_with_other_profile_never_reuses_live_session(homes): + _register_live("live-a", homes["a"]) + + same = _resume(session_id="s1", profile="a", source="desktop") + assert same["result"]["session_id"] == "live-a" + + other = _resume(session_id="s1", profile="b", source="desktop") + new_sid = other["result"]["session_id"] + assert new_sid != "live-a" + assert server._sessions[new_sid]["profile_home"] == str(homes["b"]) + + +def test_launch_profile_still_matches_records_without_profile_home(homes): + _register_live("live-launch", None) + _register_live("live-a", homes["a"]) + + assert _resume(session_id="s1", source="desktop")["result"]["session_id"] == "live-launch" + assert _resume(session_id="s1", profile="a", source="desktop")["result"]["session_id"] == "live-a" diff --git a/tests/tui_gateway/test_session_resume_db_ownership.py b/tests/tui_gateway/test_session_resume_db_ownership.py index 29f050ec30..3e4aa9cbfb 100644 --- a/tests/tui_gateway/test_session_resume_db_ownership.py +++ b/tests/tui_gateway/test_session_resume_db_ownership.py @@ -98,7 +98,7 @@ def profile_dbs(monkeypatch, tmp_path): # The handler builds nothing on the paths under test; keep it hermetic and # off the real agent/secret/HERMES_HOME machinery. monkeypatch.setattr(server, "_enable_gateway_prompts", lambda: None) - monkeypatch.setattr(server, "_find_live_session_by_key", lambda _key: None) + monkeypatch.setattr(server, "_find_live_session_by_key", lambda _key, *_a: None) monkeypatch.setattr(server, "_schedule_agent_build", lambda *a, **k: None) monkeypatch.setattr(server, "_schedule_session_cap_enforcement", lambda *a, **k: None) monkeypatch.setattr(server, "_maybe_schedule_auto_continue", lambda *a, **k: None) @@ -196,7 +196,7 @@ def test_resume_closes_profile_db_on_live_session_fast_path(profile_dbs, monkeyp monkeypatch.setattr( server, "_find_live_session_by_key", - lambda _key: ("live-sid", live_session), + lambda _key, *_a: ("live-sid", live_session), ) monkeypatch.setattr( server, diff --git a/tests/tui_gateway/test_turn_finished_failure_cause.py b/tests/tui_gateway/test_turn_finished_failure_cause.py new file mode 100644 index 0000000000..307b89a020 --- /dev/null +++ b/tests/tui_gateway/test_turn_finished_failure_cause.py @@ -0,0 +1,385 @@ +"""A failed TUI turn must say why in its own record (#89117). + +#89117 is a report made entirely of two log lines:: + + tui_turn finished: ui_session=0dfcee58 status=error error_retained=True duration=0.9s + tui_turn finished: ui_session=093285e9 status=error error_retained=True duration=0.9s + +That is the whole evidence, and it is not enough to act on: a provider 4xx, a +budget wall, a billing block and a crashed finalizer all produce exactly those +characters. The bookend was added by #86865 to trace compression rotations, so +it carries identities and a coarse status by design — but it is also the *only* +record the returned-error path writes. A sub-second failure almost always takes +that path (the provider rejected the request before any work happened), so the +quietest failures are precisely the ones with nothing to read. The exception +path at least prints ``[gateway-turn] : `` to stderr. + +These tests pin the cause into the record on both failure paths, and pin the +content discipline #86865 established while doing it: prompts are never logged, +and the provider's message is redacted and length-capped, because a 4xx body +can quote the request that produced it. +""" + +from __future__ import annotations + +import logging +import threading +import types + +import pytest + +from tui_gateway import server + + +class _InlineThread: + """Run the turn synchronously so tests observe its final state.""" + + def __init__(self, target=None, daemon=None, args=(), kwargs=None): + self._target = target + self._args = args + self._kwargs = kwargs or {} + + def start(self): + if self._target is not None: + self._target(*self._args, **self._kwargs) + + def is_alive(self): + return False + + def join(self, timeout=None): + return None + + +def _session(agent=None, **extra): + return { + "agent": agent if agent is not None else types.SimpleNamespace(), + "session_key": "gw-session-key", + "history": [], + "history_lock": threading.Lock(), + "history_version": 0, + "running": True, + "attached_images": [], + "image_counter": 0, + "cols": 80, + "slash_worker": None, + "show_reasoning": False, + "tool_progress_mode": "all", + "inflight_turn": None, + **extra, + } + + +@pytest.fixture() +def turn_env(monkeypatch, tmp_path): + """Neutralize the turn pipeline's environment-heavy side paths.""" + monkeypatch.setattr(server.threading, "Thread", _InlineThread) + monkeypatch.setattr(server, "_emit", lambda *a, **k: None) + monkeypatch.setattr(server, "_wire_callbacks", lambda sid: None) + monkeypatch.setattr(server, "_sync_agent_model_with_config", lambda sid, session: None) + monkeypatch.setattr(server, "_session_cwd", lambda session: str(tmp_path)) + monkeypatch.setattr(server, "_register_session_cwd", lambda session: None) + monkeypatch.setattr(server, "_tts_stream_begin", lambda: None) + monkeypatch.setattr(server, "_sync_session_key_after_compress", lambda *a, **k: None) + monkeypatch.setattr(server, "_get_usage", lambda agent: {}) + + +def _finished(caplog): + records = [r for r in caplog.records if "tui turn finished" in r.getMessage()] + assert len(records) == 1, f"expected exactly one bookend, got {len(records)}" + return records[0].getMessage() + + +def _run(session, prompt="go"): + server._run_prompt_submit("rid", "ui-sid", session, prompt) + + +def _agent_returning(result): + return types.SimpleNamespace( + session_id="agent-sid-1", + run_conversation=lambda *a, **k: result, + clear_interrupt=lambda: None, + ) + + +class TestTheReportedRecordNowNamesItsCause: + + def test_returned_error_carries_the_provider_message(self, turn_env, caplog): + """The reporter's exact line shape, with the missing half filled in.""" + session = _session(agent=_agent_returning({ + "final_response": "", + "error": "Error code: 402 - {'error': {'message': 'insufficient credits'}}", + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + msg = _finished(caplog) + assert "status=error" in msg + assert "error_retained=True" in msg + assert "insufficient credits" in msg, ( + "a record that says only status=error is what #89117 is about" + ) + + def test_structured_failure_reason_is_logged_when_present(self, turn_env, caplog): + """The billing wall already ships a machine-readable reason; use it. + + ``failure_reason`` is the field the client renders a billing-specific + recovery surface from, so it is the one field guaranteed to be stable + enough to grep a log for across releases. + """ + session = _session(agent=_agent_returning({ + "final_response": "", + "error": "payment required", + "failure_reason": "billing_wall", + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + assert "failure_reason=billing_wall" in _finished(caplog) + + def test_exception_path_carries_the_exception(self, turn_env, caplog): + """The other failure path, so one grep covers both.""" + def _boom(*a, **k): + raise RuntimeError("connection reset mid-stream") + + session = _session(agent=types.SimpleNamespace( + session_id="agent-sid-1", + run_conversation=_boom, + clear_interrupt=lambda: None, + )) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + msg = _finished(caplog) + assert "status=error" in msg + assert "failure_reason=RuntimeError" in msg + assert "connection reset mid-stream" in msg + + def test_successful_turn_stays_exactly_as_it_was(self, turn_env, caplog): + """No cost to the common case: a clean turn gains no new fields.""" + session = _session(agent=_agent_returning({"final_response": "done"})) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + msg = _finished(caplog) + assert "status=complete" in msg + assert "cause=" not in msg + assert "failure_reason=" not in msg + + +class TestContentDiscipline: + """#86865's rule — the record logs identities, never content.""" + + SECRETISH_PROMPT = "please rotate QDRANT_API_KEY=hunter2-super-secret now" + + def test_prompt_is_never_logged_even_when_the_turn_fails(self, turn_env, caplog): + session = _session(agent=_agent_returning({ + "final_response": "", + "error": "provider rejected the request", + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session, self.SECRETISH_PROMPT) + + msg = _finished(caplog) + assert "hunter2" not in msg + assert "QDRANT_API_KEY" not in msg + + def test_secrets_echoed_back_by_the_provider_are_redacted(self, turn_env, caplog): + """The load-bearing safety test. + + A 4xx body frequently quotes the request. Without redaction, adding the + cause to a log record would take a header the user never chose to log + and write it to disk — turning a diagnostics improvement into a secret + leak. This is why the cause goes through ``redact_sensitive_text`` and + not ``str()``. + """ + session = _session(agent=_agent_returning({ + "final_response": "", + "error": ( + "400 from provider; request headers were " + "Authorization: Bearer sk-proj-abcdefghijklmnopqrstuvwxyz0123456789" + ), + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + msg = _finished(caplog) + assert "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789" not in msg + # The diagnostic value survives the redaction — this is the point. + assert "400 from provider" in msg + + SENTINEL = "the marmalade inventory for Q3 was discontinued in March" + + def test_a_prompt_the_provider_quotes_back_does_not_reach_the_record( + self, turn_env, caplog + ): + """Secret redaction is not prompt omission, and this is the difference. + + A provider that rejects a request routinely quotes it back. The quoted + material is the user's own prose: it matches no credential pattern, so + ``redact_sensitive_text`` passes it through untouched, and adding the + cause to this record would newly persist user content that #86865 + deliberately kept out of it. The sentinel here is deliberately benign + for that reason: nothing about it looks like a secret. + """ + session = _session(agent=_agent_returning({ + "final_response": "", + "error": ( + "400 Bad Request from provider: messages[0].content was " + "rejected: '" + self.SENTINEL + "'" + ), + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session, "Summarise this: " + self.SENTINEL) + + msg = _finished(caplog) + assert self.SENTINEL not in msg + assert "marmalade" not in msg + assert "" in msg, "the removal should be visible, not silent" + # The whole point of the cause survives the removal. + assert "400 Bad Request from provider" in msg + + def test_a_provider_message_that_shares_nothing_is_untouched( + self, turn_env, caplog + ): + """The echo strip must not eat diagnostics that merely sit near a prompt.""" + session = _session(agent=_agent_returning({ + "final_response": "", + "error": "429 rate limited; retry after 30s", + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session, "Summarise this: " + self.SENTINEL) + + msg = _finished(caplog) + assert "429 rate limited; retry after 30s" in msg + assert "" not in msg + + def test_a_huge_provider_body_cannot_flood_the_log(self, turn_env, caplog): + """An HTML error page or a full request echo is a log-volume problem.""" + session = _session(agent=_agent_returning({ + "final_response": "", + "error": "upstream said: " + ("x" * 9000), + "failed": True, + })) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + msg = _finished(caplog) + assert len(msg) < 700 + assert "upstream said" in msg + assert "…" in msg, "truncation should be visible, not silent" + + def test_a_multiline_traceback_stays_one_record(self, turn_env, caplog): + """One accepted prompt, one finished record — including its cause. + + A cause spanning lines would break every log pipeline that treats the + bookend as a single greppable line, which is the only reason it is + useful for an intermittent bug like this one. + """ + def _boom(*a, **k): + raise RuntimeError("first line\nsecond line\n\tthird") + + session = _session(agent=types.SimpleNamespace( + session_id="agent-sid-1", + run_conversation=_boom, + clear_interrupt=lambda: None, + )) + + with caplog.at_level(logging.INFO, logger="tui_gateway.server"): + _run(session) + + msg = _finished(caplog) + assert "\n" not in msg + assert "first line second line third" in msg + + +class TestDetailHelperDirectly: + """``_turn_failure_detail`` in isolation — the branches the paths can't reach.""" + + def test_nothing_to_say_produces_nothing(self): + assert server._turn_failure_detail("", None) == "" + assert server._turn_failure_detail(None) == "" + + def test_fragment_carries_its_own_leading_space(self): + """The bookend appends it unconditionally, so it must self-format.""" + out = server._turn_failure_detail("boom") + assert out.startswith(" ") + + def test_an_exception_with_no_message_still_names_its_type(self): + assert "KeyError" in server._turn_failure_detail(KeyError()) + + def test_the_prompt_argument_is_optional(self): + """Callers without a prompt in scope still get the secret contract.""" + out = server._turn_failure_detail("Bearer sk-proj-supersecretvalue1234") + assert "supersecretvalue1234" not in out + + def test_a_broken_redactor_fails_closed(self, monkeypatch): + """If redaction cannot run, the raw message must not reach the log. + + Failing open here would be worse than logging nothing: the whole reason + the cause is safe to log is that it went through the redactor. + """ + import agent.redact + + def _explode(*a, **k): + raise RuntimeError("redactor unavailable") + + monkeypatch.setattr(agent.redact, "redact_sensitive_text", _explode) + + out = server._turn_failure_detail("Bearer sk-proj-supersecretvalue") + assert "supersecretvalue" not in out + assert "unredactable" in out + + +class TestPromptEchoStripping: + """``_strip_prompt_echo`` in isolation: the boundaries of the guarantee.""" + + def test_an_overlap_below_the_window_is_not_an_echo(self): + """Short shared phrases are coincidence, and eating them costs detail.""" + out = server._strip_prompt_echo("400: invalid model", "invalid model") + assert out == "400: invalid model" + + def test_a_json_escaped_echo_is_stripped_too(self): + """A provider handing back its own request body often hands it escaped.""" + prompt = "please summarise the Q3 marmalade inventory memo for me" + message = 'upstream body: {"messages": [{"content": "' + prompt + '"}]}' + out = server._strip_prompt_echo(message, prompt) + assert "marmalade" not in out + assert "" in out + + def test_an_echo_is_removed_before_the_length_cap_applies(self): + """A quote must not survive by starting inside the kept prefix.""" + prompt = "the confidential merger memorandum for the northern division" + error = ("x" * 200) + " echoed request: " + prompt + out = server._turn_failure_detail(error, None, prompt) + assert "merger memorandum" not in out + assert "confidential" not in out + + def test_a_prompt_shorter_than_the_window_cannot_blank_the_message(self): + """A one-word prompt must not turn every message into .""" + out = server._strip_prompt_echo("provider said no", "hi") + assert out == "provider said no" + + def test_whitespace_shape_does_not_hide_an_echo(self): + """Both sides are collapsed, so a re-wrapped quote still matches.""" + prompt = "the marmalade inventory for Q3 was discontinued in March" + error = ( + "rejected: the marmalade inventory\n" + " for Q3 was discontinued in March" + ) + out = server._turn_failure_detail(error, None, prompt) + assert "marmalade" not in out diff --git a/tests/tui_gateway/test_ws_surrogate_send.py b/tests/tui_gateway/test_ws_surrogate_send.py new file mode 100644 index 0000000000..14006f38b8 --- /dev/null +++ b/tests/tui_gateway/test_ws_surrogate_send.py @@ -0,0 +1,63 @@ +"""Lone UTF-16 surrogates must not tear down the Desktop WebSocket (#97288).""" + +from __future__ import annotations + +import asyncio + +from tui_gateway.ws import WSTransport, _sanitize_ws_text + + +LONE_SURROGATE = "\ud83d" + + +def test_sanitize_ws_text_makes_utf8_encodable() -> None: + dirty = f"gateway.ready {LONE_SURROGATE} payload" + out = _sanitize_ws_text(dirty) + out.encode("utf-8") + assert LONE_SURROGATE not in out + + +def test_sanitize_ws_text_leaves_valid_text_unchanged() -> None: + clean = '{"type":"gateway.ready","ok":true}' + assert _sanitize_ws_text(clean) is clean or _sanitize_ws_text(clean) == clean + + +class _FakeWS: + def __init__(self) -> None: + self.sent: list[str] = [] + self.raise_on: str | None = None + + async def send_text(self, line: str) -> None: + line.encode("utf-8") + if self.raise_on is not None and self.raise_on in line: + raise UnicodeEncodeError("utf-8", line, 0, 1, "surrogates not allowed") + self.sent.append(line) + + +def test_safe_send_sanitizes_surrogate_and_keeps_connection() -> None: + async def _run() -> None: + loop = asyncio.get_running_loop() + ws = _FakeWS() + transport = WSTransport(ws, loop, peer="127.0.0.1:1") + dirty = f'{{"type":"gateway.ready","x":"{LONE_SURROGATE}"}}' + await transport._safe_send_many(["first", dirty, "third"]) + assert transport.closed is False + assert ws.sent[0] == "first" + assert ws.sent[-1] == "third" + assert LONE_SURROGATE not in "".join(ws.sent) + assert len(ws.sent) == 3 + + asyncio.run(_run()) + + +def test_unicode_encode_error_does_not_close_socket() -> None: + async def _run() -> None: + loop = asyncio.get_running_loop() + ws = _FakeWS() + ws.raise_on = "BOOM" + transport = WSTransport(ws, loop, peer="127.0.0.1:1") + await transport._safe_send_many(["ok-a", "BOOM-frame", "ok-b"]) + assert transport.closed is False + assert ws.sent == ["ok-a", "ok-b"] + + asyncio.run(_run()) diff --git a/tools/annotate_preview_tool.py b/tools/annotate_preview_tool.py index eebe33ed5f..1b709d0134 100644 --- a/tools/annotate_preview_tool.py +++ b/tools/annotate_preview_tool.py @@ -89,7 +89,7 @@ def annotate_preview_tool( ANNOTATE_PREVIEW_SCHEMA = { "name": "annotate_preview", "description": ( - "Leave a LASTING mark on the preview-pane page (drive_preview's own " + "Highlight elements on the preview-pane page, lastingly (drive_preview's own " "marks fade; annotations stay until removed) — point at findings, " "flag what you're about to change, keep your place. Use the refs " "from drive_preview action='elements'. add: outline one element " diff --git a/tools/bot_mode_dm.py b/tools/bot_mode_dm.py index 95c8caa316..0da909519b 100644 --- a/tools/bot_mode_dm.py +++ b/tools/bot_mode_dm.py @@ -609,6 +609,17 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: ) # Re-emit the transport's streams: stdout is the reply text the # completion notification carries back to the sending agent. + if proc.returncode != 0 and "already has a live owner" in (proc.stderr or ""): + # #100523: the target's Bot Chat is held live by another + # surface (Desktop). The turn never ran, so tell the sender + # plainly instead of leaking a raw lease error + exit code. + who = argv[argv.index("-p") + 1] if "-p" in argv[:-1] else "the teammate" + print(json.dumps({ + "error": f"Delivery failed: @{who}'s Bot Chat is open on another " + "surface right now, so your message was NOT delivered. Try again later.", + "reason": "target_busy", + })) + return 1 if proc.stdout: sys.stdout.write(proc.stdout) sys.stdout.flush() diff --git a/tools/bot_relay.py b/tools/bot_relay.py index 8ac61e4b8b..60d9c40ea3 100644 --- a/tools/bot_relay.py +++ b/tools/bot_relay.py @@ -524,7 +524,10 @@ def waiter_command(root: Path | str, envelope: dict) -> str: " print('Reply from ' + label + ':')\n" " print(d.get('reply') or '(empty reply)')\n" " sys.exit(0)\n" - " time.sleep(2)\n" + # 250ms cadence: the reply file is written once by the target + # gateway's deliver path; a 2s sleep here added up to 2s of dead + # air to every cross-machine reply for no benefit (stat is cheap). + " time.sleep(0.25)\n" f"print('No reply from ' + label + ' within {REPLY_WAIT_SECONDS}s. The message may " "still be delivered when the Desktop reconnects; do not resend blindly.')\n" "sys.exit(1)\n" diff --git a/tools/browser_lightpanda.py b/tools/browser_lightpanda.py index bb64d2c82a..800c05aa05 100644 --- a/tools/browser_lightpanda.py +++ b/tools/browser_lightpanda.py @@ -12,6 +12,7 @@ reaper and the atexit sweep; it calls :func:`launch_lightpanda` / left behind by a crashed Hermes. """ +import functools import json import logging import os @@ -115,6 +116,43 @@ def _state_dir() -> Path: return path +def _http_cache_dir() -> Path: + """Filesystem HTTP cache shared by every Lightpanda this Hermes spawns. + + Shared rather than per-session so a cached asset survives session churn. + Lightpanda holds it in sqlite (WAL) with a best-effort write path, and + ``--http-cache-entry-limit`` (upstream default 1000, not passed here) + bounds it without Hermes managing eviction. + """ + path = _state_dir() / "http-cache" + path.mkdir(parents=True, exist_ok=True) + return path + + +_HTTP_CACHE_FLAG = "--http-cache-dir" + + +@functools.lru_cache(maxsize=1) +def _binary_supports_http_cache(binary: str) -> bool: + """True if ``lightpanda serve`` accepts ``--http-cache-dir``. + + The flag landed upstream in 0.3.x; older binaries fatally reject it + ("unknown argument"), which would break every launch. Probing ``help`` + output keeps working across future flag additions without parsing + versions, and the lru_cache keeps it once per binary per process. + """ + try: + proc = subprocess.run( + [binary, "help"], + capture_output=True, text=True, timeout=3.0, + stdin=subprocess.DEVNULL, + ) + return _HTTP_CACHE_FLAG in ((proc.stdout or "") + (proc.stderr or "")) + except Exception as e: + logger.debug("lightpanda http-cache probe failed (%s); assuming no", e) + return False + + def _record_path(session_name: str) -> Path: return _state_dir() / f"{session_name}.json" @@ -217,6 +255,8 @@ def launch_lightpanda( port = _pick_free_loopback_port() argv = [binary, "serve", "--host", "127.0.0.1", "--port", str(port)] + if _binary_supports_http_cache(binary): + argv += [_HTTP_CACHE_FLAG, str(_http_cache_dir())] if block_private_networks: argv.append("--block-private-networks") log_path = str(_state_dir() / f"{session_name}.log") diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 27fb443aab..c9b79e6be8 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -50,6 +50,7 @@ Usage: """ import atexit +import contextlib import functools import json import logging @@ -72,6 +73,8 @@ from hermes_constants import ( get_hermes_home_override, hermes_home_key, node_tool_runnable, + reset_hermes_home_override, + set_hermes_home_override, ) from utils import env_int, is_truthy_value from hermes_cli.config import DEFAULT_CONFIG, cfg_get @@ -1013,7 +1016,11 @@ def _is_local_backend() -> bool: return False # When terminal runs in a container, browser on host can access # internal networks the terminal can't → treat as non-local. - terminal_backend = os.getenv("TERMINAL_ENV", "local").strip().lower() + # Scope-aware: under gateway multiplexing the routed profile's backend + # lives in the per-turn terminal scope, not the process env (#68559). + from tools.terminal_scope import terminal_env + + terminal_backend = terminal_env("TERMINAL_ENV", "local").strip().lower() return terminal_backend in ("local", "") @@ -2172,6 +2179,16 @@ BROWSER_ORPHAN_GRACE_SECONDS = max(3600, BROWSER_SESSION_INACTIVITY_TIMEOUT * 20 # Track last activity time per session _session_last_activity: Dict[str, float] = {} +# Owner Hermes home per session (#86402). The inactivity janitor is one +# process-global thread started by whichever profile first opens a browser, +# so it has no profile scope of its own; under multiplexing every cleanup +# must re-enter the *owning* profile's scope (copy_context at spawn would pin +# the first profile's secrets onto every other profile's teardown). +_session_owner_homes: Dict[str, str] = {} +# Consecutive janitor cleanup failures per session; after +# MAX_INACTIVITY_CLEANUP_FAILURES the session is force-reaped (#100738). +_cleanup_failures: Dict[str, int] = {} +MAX_INACTIVITY_CLEANUP_FAILURES = 3 # Session keys flagged suspect after a command timeout (#72205 / #85125 3b). # Written by _BrowserSessionBackend.mark_suspect (cheap, lock-free — a single @@ -2315,6 +2332,8 @@ def _emergency_cleanup_all_sessions(): with _cleanup_lock: _active_sessions.clear() _session_last_activity.clear() + _session_owner_homes.clear() + _cleanup_failures.clear() _recording_sessions.clear() # Lightpanda servers (Browser Use mode) are processes we spawned; the @@ -2349,6 +2368,41 @@ atexit.register(_emergency_cleanup_all_sessions) # Inactivity Cleanup Functions # ============================================================================= +@contextlib.contextmanager +def _session_owner_scope(task_id: str): + """Run under the Hermes home + secret scope that owns ``task_id``'s + browser session (recorded by ``_update_session_activity``). + + No-op when no owner was recorded. Mirrors + ``gateway.run._profile_runtime_scope`` — the janitor thread is + process-global, so each session's teardown must re-enter its OWN + profile's scope rather than inherit whichever profile spawned the + thread; never falls through to ``os.environ``. + """ + owner_home = _session_owner_homes.get(task_id) + if owner_home is None: + yield + return + + from agent.secret_scope import ( + build_profile_secret_scope, + reset_secret_scope, + set_secret_scope, + ) + from hermes_cli.env_loader import hydrate_profile_secret_sources + + home_token = set_hermes_home_override(owner_home) + try: + hydrate_profile_secret_sources(Path(owner_home)) + secret_token = set_secret_scope(build_profile_secret_scope(Path(owner_home))) + try: + yield + finally: + reset_secret_scope(secret_token) + finally: + reset_hermes_home_override(home_token) + + def _cleanup_inactive_browser_sessions(): """ Clean up browser sessions that have been inactive for longer than the timeout. @@ -2356,6 +2410,11 @@ def _cleanup_inactive_browser_sessions(): This function is called periodically by the background cleanup thread to automatically close sessions that haven't been used recently, preventing orphaned sessions (local or Browserbase) from accumulating. + + Each session is torn down under its owner profile's scope (#86402). A + session whose cleanup keeps failing is force-reaped after + ``MAX_INACTIVITY_CLEANUP_FAILURES`` attempts instead of retrying forever + (#100738); only a successful cleanup clears its failure count. """ current_time = time.time() sessions_to_cleanup = [] @@ -2366,15 +2425,33 @@ def _cleanup_inactive_browser_sessions(): sessions_to_cleanup.append(task_id) for task_id in sessions_to_cleanup: + elapsed = int(current_time - _session_last_activity.get(task_id, current_time)) + logger.info("Cleaning up inactive session for task: %s (inactive for %ss)", task_id, elapsed) try: - elapsed = int(current_time - _session_last_activity.get(task_id, current_time)) - logger.info("Cleaning up inactive session for task: %s (inactive for %ss)", task_id, elapsed) - cleanup_browser(task_id) + with _session_owner_scope(task_id): + cleanup_browser(task_id) with _cleanup_lock: - if task_id in _session_last_activity: - del _session_last_activity[task_id] + _session_last_activity.pop(task_id, None) + _session_owner_homes.pop(task_id, None) + _cleanup_failures.pop(task_id, None) except Exception as e: - logger.warning("Error cleaning up inactive session %s: %s", task_id, e) + with _cleanup_lock: + failures = _cleanup_failures[task_id] = _cleanup_failures.get(task_id, 0) + 1 + if failures < MAX_INACTIVITY_CLEANUP_FAILURES: + logger.warning("Error cleaning up inactive session %s (attempt %d/%d): %s", + task_id, failures, MAX_INACTIVITY_CLEANUP_FAILURES, e) + continue + logger.error("Browser cleanup failed %d times for inactive session %s; " + "force-reaping: %s", failures, task_id, e) + try: + with _session_owner_scope(task_id): + _force_reap_browser_session(task_id) + except Exception as reap_exc: + logger.error("Force-reap of browser session %s failed: %s", task_id, reap_exc) + finally: + with _cleanup_lock: + _session_owner_homes.pop(task_id, None) + _cleanup_failures.pop(task_id, None) def _write_owner_pid(socket_dir: str, session_name: str) -> None: @@ -2755,9 +2832,16 @@ def _stop_browser_cleanup_thread(): def _update_session_activity(task_id: str): - """Update the last activity timestamp for a session.""" + """Update the last activity timestamp for a session. + + Also records the owning Hermes home on first sight so the process-global + janitor can tear the session down under its owner's scope (#86402). An + activity touch deliberately does NOT reset ``_cleanup_failures`` — only a + successful cleanup does. + """ with _cleanup_lock: _session_last_activity[task_id] = time.time() + _session_owner_homes.setdefault(task_id, str(get_hermes_home())) # Register cleanup thread stop on exit @@ -3342,9 +3426,9 @@ def _kill_process_tree(proc: "subprocess.Popen") -> None: descendants on both platforms. Fires SIGTERM then SIGKILL back-to-back with no grace period between - them (unlike ``tools/mcp_stdio_watchdog.py``'s ``_terminate_process_group``, - which waits between signals because it's reacting to a live daemon being - orphaned). By the time this is called, the caller has already burned its + them (unlike ``tools/mcp_death_supervisor.py``'s ``_reap``, which waits + between signals because it's reacting to a live daemon being orphaned). + By the time this is called, the caller has already burned its full timeout budget waiting for a graceful exit — there's nothing to gain from waiting again here, only more delay on an already-timed-out call. @@ -3383,7 +3467,7 @@ def _legacy_kill_process_tree(proc: "subprocess.Popen") -> None: # on Windows), but resolve them defensively via getattr anyway so an # accidental future refactor that drops that guard degrades to a plain # kill() instead of AttributeError — same discipline as - # tools/mcp_stdio_watchdog.py's _terminate_process_group. + # tools/mcp_death_supervisor.py's _reap. killpg = getattr(os, "killpg", None) if killpg is None: # windows-footgun: ok - non-POSIX fallback try: @@ -5819,6 +5903,91 @@ def cleanup_browser(task_id: Optional[str] = None) -> None: _last_active_session_key.pop(bare_task_id, None) +def _release_session_resources(task_id: str, session_info: Dict[str, Any]) -> None: + """Untrack ``task_id``, close its cloud provider session, kill its daemon. + + The unconditional tail of ``_cleanup_single_browser_session``; also the + whole of the janitor's force-reap path (#100738), which skips the polite + agent-browser/Camofox ``close`` that kept failing but must still release + the cloud session and the local Chromium. + """ + bb_session_id = session_info.get("bb_session_id", "unknown") + # Now remove from tracking under lock + with _cleanup_lock: + _active_sessions.pop(task_id, None) + _session_last_activity.pop(task_id, None) + _session_owner_homes.pop(task_id, None) + _cleanup_failures.pop(task_id, None) + + # Cloud mode: close the cloud browser session via provider API. + # Local sidecars have bb_session_id=None so this no-ops for them. + if bb_session_id: + provider = _get_cloud_provider() + if provider is not None: + try: + provider.close_session(bb_session_id) + except Exception as e: + logger.warning("Could not close cloud browser session: %s", e) + + # Kill the daemon process and clean up socket directory + session_name = session_info.get("session_name", "") + if session_name: + socket_dir = os.path.join(_socket_safe_tmpdir(), f"agent-browser-{session_name}") + if os.path.exists(socket_dir): + # agent-browser writes {session}.pid in the socket dir + pid_file = os.path.join(socket_dir, f"{session_name}.pid") + if os.path.isfile(pid_file): + try: + from tools.process_registry import ProcessRegistry + daemon_pid = int(Path(pid_file).read_text(encoding="utf-8").strip()) + # The .pid file lives in a world-writable temp dir and + # PIDs recycle: verify this really is our daemon for + # this session before tree-killing, and pin the + # identity with a start-time fingerprint so the kill + # refuses if the PID is swapped between check and kill. + if _verify_reapable_browser_daemon( + daemon_pid, socket_dir, session_name): + from gateway.status import get_process_start_time + daemon_start = get_process_start_time(daemon_pid) + if daemon_start is not None: + ProcessRegistry._terminate_host_pid( + daemon_pid, daemon_start) + logger.debug("Killed daemon pid %s for %s", daemon_pid, session_name) + else: + logger.debug( + "Skipped daemon kill for %s: no start-time " + "fingerprint for pid %s", session_name, daemon_pid) + else: + logger.debug( + "Skipped daemon kill for %s: pid %s failed identity " + "verification", session_name, daemon_pid) + except (ProcessLookupError, ValueError, PermissionError, OSError): + logger.debug("Could not kill daemon pid for %s (already dead or inaccessible)", session_name) + shutil.rmtree(socket_dir, ignore_errors=True) + + +def _force_reap_browser_session(task_id: str) -> None: + """Janitor last resort after repeated cleanup failures (#100738). + + Skips the ``close`` round-trips that keep failing and goes straight to + ``_release_session_resources`` (cloud close + daemon kill + untrack). + """ + _stop_cdp_supervisor(task_id) + with _cleanup_lock: + session_info = _active_sessions.get(task_id) + _session_last_activity.pop(task_id, None) + _recording_sessions.discard(task_id) + if session_info: + _release_session_resources(task_id, session_info) + # Same ownership-binding drop as cleanup_browser(). + if _is_local_sidecar_key(task_id): + bare_task_id = task_id[: -len(_LOCAL_SUFFIX)] + if _last_active_session_key.get(bare_task_id) == task_id: + _last_active_session_key.pop(bare_task_id, None) + else: + _last_active_session_key.pop(task_id, None) + + def _cleanup_single_browser_session(task_id: str) -> None: """Internal: reap a single browser session by its exact session key.""" # Stop the CDP supervisor for this task FIRST so we close our WebSocket @@ -5879,56 +6048,7 @@ def _cleanup_single_browser_session(task_id: str) -> None: except Exception as e: logger.warning("agent-browser close failed for task %s: %s", task_id, e) - # Now remove from tracking under lock - with _cleanup_lock: - _active_sessions.pop(task_id, None) - _session_last_activity.pop(task_id, None) - - # Cloud mode: close the cloud browser session via provider API. - # Local sidecars have bb_session_id=None so this no-ops for them. - if bb_session_id: - provider = _get_cloud_provider() - if provider is not None: - try: - provider.close_session(bb_session_id) - except Exception as e: - logger.warning("Could not close cloud browser session: %s", e) - - # Kill the daemon process and clean up socket directory - session_name = session_info.get("session_name", "") - if session_name: - socket_dir = os.path.join(_socket_safe_tmpdir(), f"agent-browser-{session_name}") - if os.path.exists(socket_dir): - # agent-browser writes {session}.pid in the socket dir - pid_file = os.path.join(socket_dir, f"{session_name}.pid") - if os.path.isfile(pid_file): - try: - from tools.process_registry import ProcessRegistry - daemon_pid = int(Path(pid_file).read_text(encoding="utf-8-sig").strip()) - # The .pid file lives in a world-writable temp dir and - # PIDs recycle: verify this really is our daemon for - # this session before tree-killing, and pin the - # identity with a start-time fingerprint so the kill - # refuses if the PID is swapped between check and kill. - if _verify_reapable_browser_daemon( - daemon_pid, socket_dir, session_name): - from gateway.status import get_process_start_time - daemon_start = get_process_start_time(daemon_pid) - if daemon_start is not None: - ProcessRegistry._terminate_host_pid( - daemon_pid, daemon_start) - logger.debug("Killed daemon pid %s for %s", daemon_pid, session_name) - else: - logger.debug( - "Skipped daemon kill for %s: no start-time " - "fingerprint for pid %s", session_name, daemon_pid) - else: - logger.debug( - "Skipped daemon kill for %s: pid %s failed identity " - "verification", session_name, daemon_pid) - except (ProcessLookupError, ValueError, PermissionError, OSError): - logger.debug("Could not kill daemon pid for %s (already dead or inaccessible)", session_name) - shutil.rmtree(socket_dir, ignore_errors=True) + _release_session_resources(task_id, session_info) logger.debug("Removed task %s from active sessions", task_id) else: diff --git a/tools/close_terminal_tool.py b/tools/close_terminal_tool.py index 24277f44da..4ae8088fbd 100644 --- a/tools/close_terminal_tool.py +++ b/tools/close_terminal_tool.py @@ -30,7 +30,7 @@ def close_terminal_tool(process_id: str) -> str: CLOSE_TERMINAL_SCHEMA = { "name": "close_terminal", "description": ( - "Close the read-only terminal tab for one of your background processes in " + "Hide a background process's terminal tab (process keeps running) in " "the Hermes desktop GUI (the tabs mirroring terminal(background=true) runs). " "This does NOT kill the process — it only drops the tab/view; the output " "keeps buffering and the user can reopen it from the status stack. Use it " diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index 36f4b939d6..630ff8d8d8 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -408,9 +408,9 @@ _TOOL_STUBS = { ), "search_files": ( "search_files", - 'pattern: str, target: str = "content", path: str = ".", file_glob: str = None, limit: int = 50, offset: int = 0, output_mode: str = "content", context: int = 0', + 'pattern: str, target: str = "content", path: str = ".", file_glob: str = None, limit: int = 50, offset: int = 0, output_mode: str = "content", context: int = 0, order: str = "discovery"', '"""Search file contents (target="content") or find files by name (target="files"). Returns dict with "matches"."""', - '{"pattern": pattern, "target": target, "path": path, "file_glob": file_glob, "limit": limit, "offset": offset, "output_mode": output_mode, "context": context}', + '{"pattern": pattern, "target": target, "path": path, "file_glob": file_glob, "limit": limit, "offset": offset, "output_mode": output_mode, "context": context, "order": order}', ), "patch": ( "patch", @@ -1553,6 +1553,21 @@ def execute_code( "Use normal tool calls (terminal, read_file, write_file, ...) instead." ) + # Fail closed under a terminal-policy refusal scope (#68559): the routed + # profile's terminal policy could not be resolved and execute_code runs on + # the configured terminal backend — refuse rather than inheriting the + # launch process's ambient policy. + try: + from tools.terminal_scope import enforce_no_refusal + + enforce_no_refusal() + except Exception as refusal: + return tool_error( + f"execute_code refused: {refusal} " + "(profile terminal policy unresolved; fix the profile's " + "config.yaml / .env and retry)" + ) + if not code or not code.strip(): return tool_error( "No code provided. execute_code requires a non-empty 'code' " @@ -2282,7 +2297,9 @@ def _resolve_child_cwd(mode: str, staging_dir: str, task_id: str = "") -> str: session_cwd = None if session_cwd and os.path.isdir(session_cwd): return session_cwd - raw = os.environ.get("TERMINAL_CWD", "").strip() + from agent.runtime_cwd import scope_terminal_cwd + + raw = scope_terminal_cwd().strip() if raw: expanded = os.path.expanduser(raw) if os.path.isdir(expanded): @@ -2314,7 +2331,7 @@ _TOOL_DOC_LINES = [ " write_file(path: str, content: str) -> dict\n" " Always overwrites the entire file."), ("search_files", - " search_files(pattern: str, target=\"content\", path=\".\", file_glob=None, limit=50) -> dict\n" + " search_files(pattern: str, target=\"content\", path=\".\", file_glob=None, limit=50, order=\"discovery\") -> dict\n" " target: \"content\" (search inside files) or \"files\" (find files by name). Returns {\"matches\": [...]}"), ("patch", " patch(path: str, old_string: str, new_string: str, replace_all: bool = False) -> dict\n" diff --git a/tools/code_kernel.py b/tools/code_kernel.py index 6c60aa7dc9..e832a4033b 100644 --- a/tools/code_kernel.py +++ b/tools/code_kernel.py @@ -78,12 +78,101 @@ import io import json import os import sys +import threading import traceback _SENTINEL = os.environ["HERMES_KERNEL_SENTINEL"] _CAPTURE_LIMIT = {capture_limit} _SPILL_DIR = os.environ.get("HERMES_KERNEL_SPILL_DIR", "") _SPILL_CAP = {spill_cap} +_PARENT_PROCESS_HANDLE = os.environ.pop("HERMES_KERNEL_PARENT_PROCESS_HANDLE", "") +_PARENT_DEATH_FD = os.environ.pop("HERMES_KERNEL_PARENT_DEATH_FD", "") + + +def _start_parent_death_pipe_watchdog(): + """POSIX twin of the Windows handle watchdog: exit when the parent dies. + + The host holds the only write end of an inherited pipe; a blocking read + returns EOF the instant the host exits by ANY means (SIGKILL, OOM, crash), + exactly like the MCP death supervisor. Stdin EOF alone is not enough: the + main loop only sees it between cells, so a kernel SIGKILLed mid-cell + outlived its host. Not PR_SET_PDEATHSIG — that is bound to the spawning + THREAD, and kernels are spawned from per-cell threads that exit. + """ + global _PARENT_DEATH_FD + raw_fd = _PARENT_DEATH_FD + _PARENT_DEATH_FD = "" + if sys.platform == "win32" or not raw_fd: + return + try: + fd = int(raw_fd) + os.set_inheritable(fd, False) + except (OSError, ValueError): + return + + def _wait(): + try: + while os.read(fd, 1): + pass + except OSError: + pass + os._exit(0) + + threading.Thread(target=_wait, name="hermes-parent-watchdog", daemon=True).start() + + +def _start_parent_process_watchdog(): + """Exit when the exact Windows parent process object is signaled. + + The inherited SYNCHRONIZE handle names a process object, not a reusable + PID. Missing or invalid handles fail open so watchdog setup can never kill + an otherwise healthy kernel. + """ + global _PARENT_PROCESS_HANDLE + raw_handle = _PARENT_PROCESS_HANDLE + _PARENT_PROCESS_HANDLE = "" + if sys.platform != "win32" or not raw_handle: + return + try: + import ctypes + from ctypes import wintypes + + handle = int(raw_handle) + if handle <= 0: + return + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] + kernel32.WaitForSingleObject.restype = wintypes.DWORD + kernel32.SetHandleInformation.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + wintypes.DWORD, + ] + kernel32.SetHandleInformation.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + # This process needs the handle, but user code spawned by a cell must + # not pass it any further. If Windows refuses to clear inheritance, + # disable the watchdog rather than leak the handle into cell children. + if not kernel32.SetHandleInformation(handle, 0x00000001, 0): + kernel32.CloseHandle(handle) + return + except (ImportError, OSError, TypeError, ValueError): + return + + def _wait(): + try: + result = kernel32.WaitForSingleObject(handle, 0xFFFFFFFF) + finally: + kernel32.CloseHandle(handle) + if result == 0x00000000: # WAIT_OBJECT_0: the parent exited + os._exit(0) + + threading.Thread(target=_wait, name="hermes-parent-watchdog", daemon=True).start() + + +_start_parent_process_watchdog() +_start_parent_death_pipe_watchdog() # The persistent cell namespace. `__name__` is `__main__` so scripts behave # like the per-call path; builtins resolve normally through exec. @@ -259,8 +348,15 @@ class SessionKernel: self.stop_event = threading.Event() self.rpc_token: str = "" self.sentinel: str = "" + self.death_pipe_w: Optional[int] = None self.tool_call_log: List = [] self.tool_call_counter: List[int] = [0] + # Cells currently attached to this kernel (bumped under _KERNELS_LOCK + # when a caller selects it, dropped when its cell settles). Reaping + # and cap-eviction skip kernels with attached cells: tearing one down + # mid-spawn rmtree'd the staging dir under the spawner + # (FileNotFoundError) and killed live cells (Sep 2026). + self.attached: int = 0 self.response_q: "queue.Queue[dict]" = queue.Queue() self.raw_chunks: List[bytes] = [] self.raw_bytes = [0] @@ -273,6 +369,17 @@ class SessionKernel: def alive(self) -> bool: return self.proc is not None and self.proc.poll() is None + def dead(self) -> bool: + """True only once a spawned process has exited. + + A kernel whose ``proc`` is still ``None`` is mid-spawn, not dead: + parallel cells for one owner race the first cell's ``_spawn``, and + treating the pending kernel as dead made every racer replace it, + orphaning the winner's process outside the registry (110 live + kernels under one 4-capped process, Sep 2026). + """ + return self.proc is not None and self.proc.poll() is not None + _KERNELS: Dict[Tuple, SessionKernel] = {} _KERNELS_LOCK = threading.Lock() @@ -380,18 +487,18 @@ def _reap_unlocked() -> List[SessionKernel]: doomed = [ key for key, kernel in _KERNELS.items() - if now - kernel.last_used > idle_timeout + if kernel.attached == 0 and now - kernel.last_used > idle_timeout ] return [_KERNELS.pop(key) for key in doomed] def _evict_over_cap_unlocked(keep: Tuple) -> List[SessionKernel]: - """Pop least-recently-used kernels beyond the process-wide cap.""" + """Pop least-recently-used idle kernels beyond the process-wide cap.""" cap, _ = _lifecycle_limits() if len(_KERNELS) <= cap: return [] by_age = sorted( - (key for key in _KERNELS if key != keep), + (key for key in _KERNELS if key != keep and _KERNELS[key].attached == 0), key=lambda key: _KERNELS[key].last_used, ) doomed = by_age[: len(_KERNELS) - cap] @@ -403,6 +510,12 @@ atexit.register(shutdown_all_kernels) def _teardown(kernel: SessionKernel) -> None: kernel.stop_event.set() + if kernel.death_pipe_w is not None: + try: + os.close(kernel.death_pipe_w) + except OSError: + pass + kernel.death_pipe_w = None if kernel.proc is not None and kernel.proc.poll() is None: from tools.code_execution_tool import _kill_process_group @@ -593,18 +706,75 @@ def _spawn(kernel: SessionKernel, *, task_id: str, child_python: str, # timeout — a kernel outlives the 300s window between cells. child_env["HERMES_RPC_PERSISTENT"] = "1" - kernel.proc = subprocess.Popen( - [child_python, runner_path], - # Strict mode resolves an empty cwd: the kernel's own staging dir - # then plays the per-call tmpdir's role. - cwd=child_cwd or kernel.tmpdir, - env=child_env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - start_new_session=True, - creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, - ) + parent_process_handle = None + close_parent_process_handle = None + startupinfo = None + if _IS_WINDOWS: + try: + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.GetCurrentProcessId.argtypes = [] + kernel32.GetCurrentProcessId.restype = wintypes.DWORD + kernel32.OpenProcess.argtypes = [ + wintypes.DWORD, + wintypes.BOOL, + wintypes.DWORD, + ] + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + close_parent_process_handle = kernel32.CloseHandle + parent_process_handle = kernel32.OpenProcess( + 0x00100000, # SYNCHRONIZE + True, # inherited only by the explicitly allow-listed child + kernel32.GetCurrentProcessId(), + ) + if parent_process_handle: + child_env["HERMES_KERNEL_PARENT_PROCESS_HANDLE"] = str( + int(parent_process_handle) + ) + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = { + "handle_list": [int(parent_process_handle)] + } + except (AttributeError, ImportError, OSError, TypeError, ValueError): + if parent_process_handle and close_parent_process_handle is not None: + close_parent_process_handle(parent_process_handle) + child_env.pop("HERMES_KERNEL_PARENT_PROCESS_HANDLE", None) + parent_process_handle = None + close_parent_process_handle = None + startupinfo = None + + death_r: Optional[int] = None + pass_fds: Tuple[int, ...] = () + if not _IS_WINDOWS: + death_r, kernel.death_pipe_w = os.pipe() + child_env["HERMES_KERNEL_PARENT_DEATH_FD"] = str(death_r) + pass_fds = (death_r,) + + try: + kernel.proc = subprocess.Popen( + [child_python, runner_path], + # Strict mode resolves an empty cwd: the kernel's own staging dir + # then plays the per-call tmpdir's role. + cwd=child_cwd or kernel.tmpdir, + env=child_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + start_new_session=True, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, + close_fds=True, + pass_fds=pass_fds, + startupinfo=startupinfo, + ) + finally: + if parent_process_handle and close_parent_process_handle is not None: + close_parent_process_handle(parent_process_handle) + if death_r is not None: + os.close(death_r) # Deliberately NOT propagate_context_to_thread: that would freeze the # spawning cell's context/callbacks into the server thread for the @@ -664,18 +834,60 @@ def execute_in_session_kernel( with _KERNELS_LOCK: expired = _reap_unlocked() kernel = _KERNELS.get(key) - if kernel is not None and (reset or not kernel.alive()): + if kernel is not None and (reset or kernel.dead()): _KERNELS.pop(key, None) - expired.append(kernel) + if kernel.attached == 0: + expired.append(kernel) kernel = None state_reset = True if kernel is None: kernel = SessionKernel(key) _KERNELS[key] = kernel kernel.last_used = time.monotonic() + kernel.attached += 1 expired.extend(_evict_over_cap_unlocked(keep=key)) for doomed in expired: _teardown(doomed) + try: + return _run_cell( + kernel, key, code, task_id=task_id, child_python=child_python, + child_cwd=child_cwd, sandbox_tools=sandbox_tools, timeout=timeout, + max_tool_calls=max_tool_calls, is_interrupted=is_interrupted, + exec_start=exec_start, state_reset=state_reset, + ) + finally: + with _KERNELS_LOCK: + kernel.attached -= 1 + kernel.last_used = time.monotonic() + # Dropped from the registry (reset/dead/reaped) while cells were + # still attached: the last one out owns the teardown. + orphaned = kernel.attached == 0 and _KERNELS.get(key) is not kernel + if orphaned: + _teardown(kernel) + + +def _run_cell( + kernel: SessionKernel, + key: Tuple, + code: str, + *, + task_id: str, + child_python: str, + child_cwd: str, + sandbox_tools: frozenset, + timeout: int, + max_tool_calls: int, + is_interrupted, + exec_start: float, + state_reset: bool, +) -> str: + from tools.code_execution_tool import ( + _sandbox_failure_hint, + _truncate_stdout_text, + ) + from agent.redact import redact_sensitive_text + from tools.ansi_strip import strip_ansi + reused = kernel.proc is not None # Captured on the calling thread BEFORE the cell runs — the same @@ -731,7 +943,8 @@ def execute_in_session_kernel( # No safe way to interrupt one cell in place: kill the kernel, # report the state loss, let the next call respawn. with _KERNELS_LOCK: - _KERNELS.pop(key, None) + if _KERNELS.get(key) is kernel: + _KERNELS.pop(key, None) _teardown(kernel) duration = round(time.monotonic() - exec_start, 2) @@ -810,7 +1023,8 @@ def execute_in_session_kernel( elif cell_status == "exit": # The cell called sys.exit(): honor it as end-of-kernel. with _KERNELS_LOCK: - _KERNELS.pop(key, None) + if _KERNELS.get(key) is kernel: + _KERNELS.pop(key, None) _teardown(kernel) result["kernel"]["ended"] = True if cell_stderr: @@ -822,7 +1036,8 @@ def execute_in_session_kernel( + (": " + stderr_raw.strip() if stderr_raw.strip() else ".") ) with _KERNELS_LOCK: - _KERNELS.pop(key, None) + if _KERNELS.get(key) is kernel: + _KERNELS.pop(key, None) _teardown(kernel) elif cell_stderr: result["output"] = stdout_text + "\n--- stderr ---\n" + cell_stderr @@ -831,7 +1046,8 @@ def execute_in_session_kernel( except Exception as exc: # pragma: no cover - defensive parity with per-call logger.error("session kernel failed: %s: %s", type(exc).__name__, exc, exc_info=True) with _KERNELS_LOCK: - _KERNELS.pop(key, None) + if _KERNELS.get(key) is kernel: + _KERNELS.pop(key, None) _teardown(kernel) return json.dumps({ "status": "error", diff --git a/tools/code_kernel_remote.py b/tools/code_kernel_remote.py index 3b3e9a2293..9412eebb54 100644 --- a/tools/code_kernel_remote.py +++ b/tools/code_kernel_remote.py @@ -156,6 +156,10 @@ class RemoteKernel: last_used: float = field(default_factory=time.monotonic) execution_count: int = 0 cell_seq: int = 0 + # Cells currently running on this kernel. Reap/evict skip attached + # kernels: killing one mid-cell tears the runner out from under a live + # poll loop (same guard as tools.code_kernel, hermes-agent#101861). + attached: int = 0 def _kernel_key(owner: str, env_type: str, task_env_id: str) -> Tuple: @@ -219,6 +223,44 @@ def shutdown_remote_kernels_for_owner(owner: str) -> None: _kill(kernel) +def _reap_unlocked(idle_timeout: int) -> List["RemoteKernel"]: + """Pop idle-expired remote kernels; caller tears them down outside the lock. + + Mirrors tools.code_kernel._reap_unlocked. The remote runner itself + self-exits after the same idle window (REMOTE_KERNEL_RUNNER_SOURCE's + IDLE_EXIT_SECONDS), so this only needs to clear the HOST-side + bookkeeping entry — without it, _REMOTE_KERNELS grows one entry per + distinct (owner, env_type, task_env_id) that is never revisited, for + the life of the gateway process. + """ + now = time.monotonic() + doomed = [ + key + for key, kernel in _REMOTE_KERNELS.items() + if kernel.attached == 0 and now - kernel.last_used > idle_timeout + ] + return [_REMOTE_KERNELS.pop(key) for key in doomed] + + +def _evict_over_cap_unlocked(keep: Tuple) -> List["RemoteKernel"]: + """Pop least-recently-used remote kernels beyond the process-wide cap. + + Mirrors tools.code_kernel._evict_over_cap_unlocked, reusing the same + max_session_kernels config as an independent bound on _REMOTE_KERNELS. + """ + from tools.code_kernel import _lifecycle_limits + + cap, _ = _lifecycle_limits() + if len(_REMOTE_KERNELS) <= cap: + return [] + by_age = sorted( + (key for key in _REMOTE_KERNELS if key != keep and _REMOTE_KERNELS[key].attached == 0), + key=lambda key: _REMOTE_KERNELS[key].last_used, + ) + doomed = by_age[: len(_REMOTE_KERNELS) - cap] + return [_REMOTE_KERNELS.pop(key) for key in doomed] + + atexit.register(shutdown_all_remote_kernels) @@ -317,11 +359,6 @@ def execute_in_remote_kernel( the ``kernel`` sub-dict, matching the local kernel's result shape. """ from tools.code_kernel import _resolve_owner - from tools.code_execution_tool import ( - _rpc_poll_loop, - _ship_file_to_remote, - ) - from tools.thread_context import propagate_context_to_thread owner = _resolve_owner(task_env_id) key = _kernel_key(owner, env_type, task_env_id) @@ -329,7 +366,10 @@ def execute_in_remote_kernel( state_reset = False with _REMOTE_KERNELS_LOCK: + expired = _reap_unlocked(idle_exit) kernel = _REMOTE_KERNELS.get(key) + for doomed in expired: + _kill(doomed) if kernel is not None and reset: with _REMOTE_KERNELS_LOCK: @@ -359,6 +399,44 @@ def execute_in_remote_kernel( _REMOTE_KERNELS[key] = kernel kernel.last_used = time.monotonic() + with _REMOTE_KERNELS_LOCK: + kernel.attached += 1 + evicted = _evict_over_cap_unlocked(keep=key) + for doomed in evicted: + _kill(doomed) + try: + return _run_remote_cell( + kernel, key, code, env=env, task_env_id=task_env_id, + sandbox_tools=sandbox_tools, timeout=timeout, + max_tool_calls=max_tool_calls, reused=reused, + state_reset=state_reset, state_lost=state_lost, + ) + finally: + with _REMOTE_KERNELS_LOCK: + kernel.attached -= 1 + kernel.last_used = time.monotonic() + + +def _run_remote_cell( + kernel: RemoteKernel, + key: Tuple, + code: str, + *, + env, + task_env_id: str, + sandbox_tools: frozenset, + timeout: int, + max_tool_calls: int, + reused: bool, + state_reset: bool, + state_lost: bool, +) -> Dict[str, Any]: + from tools.code_execution_tool import ( + _rpc_poll_loop, + _ship_file_to_remote, + ) + from tools.thread_context import propagate_context_to_thread + kernel.cell_seq += 1 seq = f"{kernel.cell_seq:06d}" q_cells = shlex.quote(f"{kernel.kernel_dir}/cells") diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index e512980bd6..227340d7b0 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -218,6 +218,7 @@ _NON_APP_WINDOW_TITLE_PREFIXES = ( # Setting it to "0" disables telemetry; absence => the binary's own default # (telemetry ON upstream). _CUA_TELEMETRY_ENV_VAR = "CUA_DRIVER_RS_TELEMETRY_ENABLED" +_CUA_NATIVE_WAYLAND_ENV_VAR = "CUA_DRIVER_RS_ENABLE_WAYLAND" def _computer_use_cfg() -> Dict[str, Any]: @@ -356,15 +357,20 @@ def _computer_use_max_image_dimension() -> Optional[int]: def cua_driver_child_env(base_env: Optional[Dict[str, str]] = None) -> Dict[str, str]: """Return the environment dict for spawning cua-driver. - Starts from ``base_env`` (defaults to ``os.environ``) and, when telemetry - is disabled (the default), injects ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0``. - When the user has opted in, the var is left untouched so cua-driver uses - its own default. Used by every cua-driver spawn site (MCP backend, status, - doctor, install) so the policy is applied consistently. + Starts from ``base_env`` (defaults to ``os.environ``), applies the Hermes + telemetry policy, and bridges an explicit native-Wayland config opt-in only + when the child has a Wayland display. Used by every cua-driver spawn site + so CLI and gateway runtimes share one policy. """ env = dict(base_env if base_env is not None else os.environ) if _cua_telemetry_disabled(): env[_CUA_TELEMETRY_ENV_VAR] = "0" + if ( + sys.platform == "linux" + and env.get("WAYLAND_DISPLAY") + and bool(_computer_use_cfg().get("native_wayland", False)) + ): + env[_CUA_NATIVE_WAYLAND_ENV_VAR] = "1" return env diff --git a/tools/computer_use/doctor.py b/tools/computer_use/doctor.py index 1379c8d95c..bd59f7e1fd 100644 --- a/tools/computer_use/doctor.py +++ b/tools/computer_use/doctor.py @@ -742,11 +742,18 @@ def _apply_display_count_guard(report: Dict[str, Any]) -> Dict[str, Any]: return report +def _wayland_environment_context(report: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if report.get("platform") != "linux" or not os.environ.get("WAYLAND_DISPLAY"): + return None + return {"scope": "cli_process", "gateway_environment_checked": False} + + def _print_text_report( report: Dict[str, Any], color: bool, *, identity: Optional[Dict[str, Any]] = None, + environment: Optional[Dict[str, Any]] = None, ) -> None: """Render the report in the same style as `cua-driver call health_report` would (one line per check + a summary footer). @@ -797,6 +804,12 @@ def _print_text_report( elif cli_v and not mismatch: # Still show the resolved path; version already matches header. pass + if environment: + print(f" {col_dim}environment: current CLI process{col_reset}") + print( + f" {col_dim}gateway environment was not checked; active gateway " + f"computer_use sessions use that process environment{col_reset}" + ) if mismatch: warn = col_yellow if color else "" print( @@ -881,6 +894,7 @@ def run_doctor( return 2 identity = _build_identity(binary, report) + environment = _wayland_environment_context(report) if json_output: # Additive envelope: preserve the upstream health_report keys and @@ -888,12 +902,19 @@ def run_doctor( # that only read overall/checks keep working. payload = dict(report) payload["hermes_identity"] = identity + if environment: + payload["hermes_environment"] = environment json.dump(payload, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") else: if color is None: color = sys.stdout.isatty() - _print_text_report(report, color=bool(color), identity=identity) + _print_text_report( + report, + color=bool(color), + identity=identity, + environment=environment, + ) overall = report.get("overall") if overall in ("degraded", "failed"): diff --git a/tools/credential_files.py b/tools/credential_files.py index e7cc8f028e..631eb03710 100644 --- a/tools/credential_files.py +++ b/tools/credential_files.py @@ -28,6 +28,8 @@ from pathlib import Path from typing import Dict, List, Optional from hermes_cli.config import cfg_get +from agent.skill_utils import EXCLUDED_SKILL_DIRS + try: # pragma: no cover - exercised via the fail-closed test below from agent.file_safety import get_read_block_error except ImportError: # noqa: F401 - sentinel consumed in register_credential_file @@ -324,16 +326,19 @@ def _safe_skills_path(skills_dir: Path) -> str: safe_dir = Path(tempfile.mkdtemp(prefix="hermes-skills-safe-")) _safe_skills_tempdir = safe_dir - for item in skills_dir.rglob("*"): - if item.is_symlink(): - continue - rel = item.relative_to(skills_dir) - target = safe_dir / rel - if item.is_dir(): - target.mkdir(parents=True, exist_ok=True) - elif item.is_file(): - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(str(item), str(target)) + # Same exclusion rule as the per-file sync path (_iter_syncable_files): + # the sanitized copy is what gets mounted, so it must not carry the + # bookkeeping trees either. Prune before descending so a multi-GB + # .curator_backups is never even walked. + for dirpath, dirnames, filenames in os.walk(skills_dir): + dirnames[:] = sorted(d for d in dirnames if d not in EXCLUDED_SKILL_DIRS) + base = Path(dirpath) + (safe_dir / base.relative_to(skills_dir)).mkdir(parents=True, exist_ok=True) + for name in filenames: + item = base / name + if item.is_symlink() or not item.is_file(): + continue + shutil.copy2(str(item), str(safe_dir / item.relative_to(skills_dir))) def _cleanup(): if safe_dir.is_dir(): @@ -344,15 +349,40 @@ def _safe_skills_path(skills_dir: Path) -> str: return str(safe_dir) +def _iter_syncable_files(root: Path): + """Yield ``(path, rel)`` for every regular, non-symlink file under *root* + that a sandbox should receive. + + Prunes ``agent.skill_utils.EXCLUDED_SKILL_DIRS`` *before* descending, so + the walk never enters local bookkeeping and dependency trees (``.hub`` + download cache, ``.archive``, ``.curator_backups``, ``node_modules``, + ``__pycache__``, ``.git``, ...) that the remote agent never reads — the + sync path agrees with discovery on what counts as skill content. + + This deliberately does not use ``is_excluded_skill_path()``, which also + prunes ``references/``, ``templates/``, ``assets/`` and ``scripts/``. + Those hold progressive-disclosure support files and bundled scripts the + sandbox does execute, so they must keep syncing. + """ + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = sorted(d for d in dirnames if d not in EXCLUDED_SKILL_DIRS) + base = Path(dirpath) + for name in filenames: + item = base / name + if item.is_symlink() or not item.is_file(): + continue + yield item, item.relative_to(root) + + def iter_skills_files( container_base: str = "/root/.hermes", ) -> List[Dict[str, str]]: """Yield individual (host_path, container_path) entries for skills files. Includes both the local skills dir and any external dirs configured via - skills.external_dirs. Skips symlinks entirely. Preferred for backends - that upload files individually (Daytona, Modal) rather than mounting a - directory. + skills.external_dirs. Skips symlinks and anything under + EXCLUDED_SKILL_DIRS entirely. Preferred for backends that upload files + individually (Daytona, Modal) rather than mounting a directory. """ result: List[Dict[str, str]] = [] @@ -360,10 +390,7 @@ def iter_skills_files( skills_dir = hermes_home / "skills" if skills_dir.is_dir(): container_root = f"{container_base.rstrip('/')}/skills" - for item in skills_dir.rglob("*"): - if item.is_symlink() or not item.is_file(): - continue - rel = item.relative_to(skills_dir) + for item, rel in _iter_syncable_files(skills_dir): result.append({ "host_path": str(item), "container_path": f"{container_root}/{rel}", @@ -376,10 +403,7 @@ def iter_skills_files( if not ext_dir.is_dir(): continue container_root = f"{container_base.rstrip('/')}/external_skills/{idx}" - for item in ext_dir.rglob("*"): - if item.is_symlink() or not item.is_file(): - continue - rel = item.relative_to(ext_dir) + for item, rel in _iter_syncable_files(ext_dir): result.append({ "host_path": str(item), "container_path": f"{container_root}/{rel}", @@ -388,10 +412,7 @@ def iter_skills_files( if not proj_dir.is_dir(): continue container_root = f"{container_base.rstrip('/')}/project_skills/{idx}" - for item in proj_dir.rglob("*"): - if item.is_symlink() or not item.is_file(): - continue - rel = item.relative_to(proj_dir) + for item, rel in _iter_syncable_files(proj_dir): result.append({ "host_path": str(item), "container_path": f"{container_root}/{rel}", diff --git a/tools/cronjob_tools.py b/tools/cronjob_tools.py index 3bc08f4a19..1e34bbb787 100644 --- a/tools/cronjob_tools.py +++ b/tools/cronjob_tools.py @@ -770,6 +770,7 @@ def _format_job(job: Dict[str, Any]) -> Dict[str, Any]: "last_run_at": job.get("last_run_at"), "last_status": job.get("last_status"), "last_delivery_error": job.get("last_delivery_error"), + "last_delivery_unverified": job.get("last_delivery_unverified"), "last_fire_error": job.get("last_fire_error"), "enabled": job.get("enabled", True), # Derive from enabled so half-paused records never render as paused. @@ -920,6 +921,30 @@ def _forward_relay_fronted_run( ) +def _manual_run_delivery_note(deliver: str, refreshed: Dict[str, Any]) -> str: + """Parenthetical delivery note for a manual run's completion summary. + + Follows the refreshed job record (#83993): ``run_one_job`` writes + ``last_delivery_error`` via ``mark_job_run`` when the post-run delivery + (telegram/discord/…) failed, and the summary must not claim success over + that record — the calling agent relays this line to the user. Local jobs + never deliver; an empty/missing error keeps the legacy wording + byte-for-byte. + """ + # Falsy deliver ("", stored JSON null) means no delivery target — the + # fire-time path normalizes it to "local" (no delivery, output persisted + # in last_output, no delivery error), so it must read as saved-locally, + # not as a delivered remote target. Whitespace-only values are NOT folded + # in here: they keep falling through to the error check, where the + # fire-time "no delivery target resolved" error gets surfaced. + if not deliver or deliver == "local": + return " (output saved locally only)" + err = str(refreshed.get("last_delivery_error") or "").strip() + if not err: + return " (output was delivered there by the job itself)" + return f" (⚠ delivery FAILED: {err[:200]})" + + def _execute_job_now( job: Dict[str, Any], extra_prompt: Optional[str] = None ) -> Dict[str, Any]: @@ -1100,11 +1125,33 @@ def _run_claimed_job( _registered = False release_running_job(job_id) refreshed = get_job(job_id) or {} - ok = refreshed.get("last_status") == "ok" + execution = None + execution_id = job.get("execution_id") + if execution_id: + from cron.executions import get_execution + + execution = get_execution(str(execution_id)) + last_status = refreshed.get("last_status") + # "delivery_failed" (#83993): the agent run itself succeeded but the + # output never reached the user. That is NOT a success for the caller + # — the calling agent relays this result — so report it as failed + # and surface the delivery error, which lives in last_delivery_error + # (last_error is None for these runs, and a bare success=False with + # error=None reads as an unexplained failure). + ok = last_status == "ok" + run_error = refreshed.get("last_error") + if last_status == "delivery_failed" and not run_error: + run_error = refreshed.get("last_delivery_error") + if execution is not None and execution.get("status") != "completed": + ok = False + run_error = ( + execution.get("error") + or f"execution ended in {execution.get('status') or 'unknown'} state" + ) return { "claimed": True, "success": bool(processed and ok), - "error": refreshed.get("last_error"), + "error": run_error, } except Exception as e: @@ -1334,7 +1381,14 @@ def _try_dispatch_background_run( max_async = 3 started_at = time.time() - deliver = job.get("deliver", "local") + # Canonicalize with the scheduler's own normalizer so the summary states + # the same target fire time will use: falsy ("", stored JSON null) reads + # "local", legacy list-form deliver flattens to its comma string. Read + # from the claimed snapshot — the owner-bearing record the run actually + # executes — not the pre-claim `job` the tool loaded. + from cron.scheduler import _normalize_deliver_value + + deliver = _normalize_deliver_value(claimed_job.get("deliver", "local")) def _runner() -> Dict[str, Any]: res = _run_claimed_job(claimed_job, extra_prompt=extra_prompt) @@ -1345,11 +1399,7 @@ def _try_dispatch_background_run( f"Result: {'ok' if res.get('success') else 'FAILED'}" + (f" — {res.get('error')}" if res.get("error") else ""), f"Delivery target: {deliver}" - + ( - " (output was delivered there by the job itself)" - if deliver != "local" - else " (output saved locally only)" - ), + + _manual_run_delivery_note(deliver, refreshed), ] if refreshed.get("next_run_at"): lines.append(f"Next scheduled run: {refreshed['next_run_at']}") @@ -1482,6 +1532,7 @@ def cronjob( monitor_script: Optional[str] = None, monitor_url: Optional[str] = None, reasoning_effort: Optional[str] = None, + failure_deliver: Optional[Union[str, List[str]]] = None, task_id: str = None, session_id: Optional[str] = None, ) -> str: @@ -1540,6 +1591,12 @@ def cronjob( # bot-chat deliver targets are machine-local: named profiles must # exist here, and a bad name should fail the CREATE, not the run. bot_chat_error = _validate_bot_chat_deliver(_normalize_deliver_param(deliver)) + if bot_chat_error: + return tool_error(bot_chat_error, success=False) + # failure_deliver shares deliver's grammar and validators (NS-788). + bot_chat_error = _validate_bot_chat_deliver( + _normalize_deliver_param(failure_deliver) + ) if bot_chat_error: return tool_error(bot_chat_error, success=False) @@ -1598,6 +1655,9 @@ def cronjob( # dispatch below: models do not make model-config # decisions (standing policy). reasoning_effort=reasoning_effort, + failure_deliver=_resolve_cron_context_deliver( + _normalize_deliver_param(failure_deliver) + ), ) except CronSchedulerRegistrationError as exc: _partial = exc.to_dict() @@ -1800,6 +1860,19 @@ def cronjob( updates["deliver"] = _resolve_cron_context_deliver( _normalize_deliver_param(deliver) ) + if failure_deliver is not None: + # '' clears the override (job falls back to deliver on + # failures); non-empty values share deliver's validation + # AND its cron-context origin resolution (a job created + # from inside a cron run must never store literal + # 'origin' — same rule as deliver). + _norm_fd = _normalize_deliver_param(failure_deliver) + if _norm_fd: + bot_chat_error = _validate_bot_chat_deliver(_norm_fd) + if bot_chat_error: + return tool_error(bot_chat_error, success=False) + _norm_fd = _resolve_cron_context_deliver(_norm_fd) + updates["failure_deliver"] = _norm_fd if skills is not None or skill is not None: canonical_skills = _canonical_skills(skill, skills) updates["skills"] = canonical_skills @@ -1953,7 +2026,7 @@ def cronjob( CRONJOB_SCHEMA = { - "name": "cronjob", + "name": "cronjob_manage", "description": """Manage scheduled cron jobs: action='create' schedules a job from a prompt and/or skills; 'list' inspects jobs; 'update'/'pause'/'resume'/'remove' manage one by job_id (always list first — never guess job IDs); 'run' fires a job immediately in the BACKGROUND (returns a handle at once, outcome re-enters the conversation when done — do not wait or poll; optional 'prompt' adds transient context for that fire only). Jobs run in a fresh session with no current-chat context, so prompts must be self-contained, and the agent's FINAL RESPONSE is what gets delivered — cron runs are autonomous and cannot ask questions. Prefer updating an existing job over creating near-duplicates.""", @@ -1989,6 +2062,10 @@ Jobs run in a fresh session with no current-chat context, so prompts must be sel "type": "string", "description": "Where the job's output is POSTED as a one-way message (the job itself always runs in a fresh session with no chat context). Omit to address the chat/topic this job was created from. Otherwise: 'local' (save only, no delivery), 'all' (every connected home channel, resolved at fire time), 'bot-chat' or 'bot-chat:' (inject into a Bot Chat as a real message), or platform:chat_id:thread_id (e.g. 'telegram:-1001234567890:17585'). Comma-combine like 'origin,all'." }, + "failure_deliver": { + "type": "string", + "description": "Optional override target for FAILURE notices only (same grammar as deliver). When set, engine failure/interruption notices go here instead of the deliver target; 'local' suppresses them entirely (state still recorded in cron list/run history). Use for jobs delivering into shared channels where failure noise is unwanted. Omit = failures follow deliver (default). On update, '' clears." + }, "skills": { "type": "array", "items": {"type": "string"}, @@ -2079,6 +2156,7 @@ def _cronjob_handler(args, **kw): name=args.get("name"), repeat=args.get("repeat"), deliver=args.get("deliver"), + failure_deliver=args.get("failure_deliver"), include_disabled=args.get("include_disabled", True), skill=args.get("skill"), skills=args.get("skills"), @@ -2103,7 +2181,7 @@ def _cronjob_handler(args, **kw): registry.register( - name="cronjob", + name="cronjob_manage", toolset="cronjob", schema=CRONJOB_SCHEMA, handler=_cronjob_handler, diff --git a/tools/daemon_pool.py b/tools/daemon_pool.py index 2fb5a61d0a..33e99c5143 100644 --- a/tools/daemon_pool.py +++ b/tools/daemon_pool.py @@ -16,8 +16,16 @@ exit hook insists on joining. - the interpreter's non-daemon thread join at shutdown skips them. Semantics are otherwise identical (initializer/initargs, work queue, -idle-thread reuse). Use it for any pool whose work is best-effort or -independently interruptible and must never hold the process open: +idle-thread reuse), plus context propagation: ``submit`` snapshots the +submitting context with ``copy_context()`` and runs each work item inside +it. Stdlib ``ThreadPoolExecutor`` only does this from Python 3.14; on the +3.11-3.13 runtimes Hermes ships, a bare pool worker starts with an EMPTY +Context and silently drops contextvar-based state (profile secret scope, +HERMES_HOME override) — under the multiplexed gateway a credential read in +such a worker fails closed with ``UnscopedSecretError``. Propagating by +default makes every consumer safe even when it forgets +``propagate_context_to_thread``. Use it for any pool whose work is +best-effort or independently interruptible and must never hold the process open: concurrent tool execution, background memory sync, catalog fan-out, subagent timeout wrappers. Do NOT use it for work that must complete before exit (durable writes) — those belong on foreground threads with @@ -30,6 +38,7 @@ import threading import weakref from concurrent.futures import ThreadPoolExecutor from concurrent.futures.thread import _worker +from contextvars import copy_context __all__ = ["DaemonThreadPoolExecutor"] @@ -37,6 +46,25 @@ __all__ = ["DaemonThreadPoolExecutor"] class DaemonThreadPoolExecutor(ThreadPoolExecutor): """ThreadPoolExecutor variant whose workers do not block process exit.""" + def submit(self, fn, /, *args, **kwargs): + """Submit a callable, propagating the caller's contextvars. + + Python 3.14's ``ThreadPoolExecutor`` snapshots the submitting + context with ``copy_context()`` and runs each work item inside it; + 3.11-3.13 (the runtimes Hermes ships) do not, so a pool worker + starts with an empty Context and loses the multiplexed profile + secret scope / HERMES_HOME override. Do it here unconditionally so + the daemon pool behaves identically on every runtime; on 3.14+ the + inner ``ctx.run`` re-applies the same immutable context and is a + no-op. + """ + ctx = copy_context() + + def _run_with_context(*call_args, **call_kwargs): + return ctx.run(fn, *call_args, **call_kwargs) + + return super().submit(_run_with_context, *args, **kwargs) + def _adjust_thread_count(self) -> None: # Mirrors CPython's implementation (3.8–3.13) with two changes: # daemon=True and no _threads_queues registration. diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index d295068843..02a999cab7 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -53,7 +53,7 @@ DELEGATE_BLOCKED_TOOLS = frozenset( "clarify", # no user interaction "memory", # no writes to shared MEMORY.md "send_message", # no cross-platform side effects - "cronjob", # no scheduling more work in the parent's name + "cronjob_manage", # no scheduling more work in the parent's name ] ) @@ -1402,6 +1402,41 @@ def _blocked_toolsets_for_role(role: str) -> List[str]: ) +_BATCH_ORDINALS: Dict[str, int] = {} +_BATCH_ORDINALS_LOCK = threading.Lock() + + +def format_batch_tag(delegation_id: Optional[str]) -> str: + """Short human tag identifying which delegation batch a line belongs to. + + ``deleg_6a664903`` → ``set 1`` (first batch seen in this process), + the next distinct id → ``set 2``, and so on. Several batches (a parent's + fan-out plus a child's nested fan-out, or two concurrent tools) print + interleaved ``[n/N]`` progress lines to the same console; without a batch + tag a ``✓ [3/3]`` and a ``✓ [3/9]`` are indistinguishable, and a raw hex + slice (``[b2ac 3/9]``) is attributable but unreadable. Empty string when + no id is known so callers can concatenate unconditionally. + """ + if not isinstance(delegation_id, str) or not delegation_id: + return "" + with _BATCH_ORDINALS_LOCK: + n = _BATCH_ORDINALS.get(delegation_id) + if n is None: + n = len(_BATCH_ORDINALS) + 1 + _BATCH_ORDINALS[delegation_id] = n + return f"set {n}" + + +def _batch_prefix(delegation_id: Optional[str], task_index: int, task_count: int) -> str: + """``[set 2 · 3/9] `` for batch children, ``[set 2] `` for a lone child, + ``[3/9] `` / ``""`` when the batch id is unknown.""" + tag = format_batch_tag(delegation_id) + if task_count > 1: + inner = f"{tag} · {task_index + 1}/{task_count}" if tag else f"{task_index + 1}/{task_count}" + return f"[{inner}] " + return f"[{tag}] " if tag else "" + + def _emit_parent_console(parent_agent, line: str) -> None: """Emit a human-readable progress line to the parent's console. @@ -1454,8 +1489,14 @@ def _build_child_progress_callback( if not spinner and not parent_cb: return None # No display → no callback → zero behavior change - # Show 1-indexed prefix only in batch mode (multiple tasks) - prefix = f"[{task_index + 1}] " if task_count > 1 else "" + # Show 1-indexed prefix only in batch mode (multiple tasks). The batch tag + # (short delegation id) is resolved lazily from session_ref because the + # callback is built before delegate_task stamps ``_delegation_id`` on the + # child; delegate_task drops the id into the same shared ref. + def _prefix() -> str: + deleg = session_ref.get("delegation_id") if session_ref else None + return _batch_prefix(deleg, task_index, task_count) + goal_label = (goal or "").strip() # Gateway: batch tool names, flush periodically @@ -1484,6 +1525,8 @@ def _build_child_progress_callback( # event lets UIs open/inspect the subagent's session directly. if session_ref and session_ref.get("session_id"): kw["child_session_id"] = str(session_ref["session_id"]) + if session_ref and session_ref.get("delegation_id"): + kw["delegation_id"] = str(session_ref["delegation_id"]) kw["tool_count"] = _tool_count[0] return kw @@ -1510,7 +1553,7 @@ def _build_child_progress_callback( (goal_label[:55] + "...") if len(goal_label) > 55 else goal_label ) try: - spinner.print_above(f" {prefix}├─ 🔀 {short}") + spinner.print_above(f" {_prefix()}├─ 🔀 {short}") except Exception as e: logger.debug("Spinner print_above failed: %s", e) _relay("subagent.start", preview=preview or goal_label or "", **kwargs) @@ -1529,7 +1572,7 @@ def _build_child_progress_callback( duration_seconds=kwargs.get("duration_seconds"), ) try: - spinner.print_above(f" {prefix}├─ {_fail_line}") + spinner.print_above(f" {_prefix()}├─ {_fail_line}") except Exception as e: logger.debug("Spinner print_above failed: %s", e) _relay("subagent.complete", preview=preview, **kwargs) @@ -1563,7 +1606,7 @@ def _build_child_progress_callback( if spinner: short = (text[:55] + "...") if len(text) > 55 else text try: - spinner.print_above(f' {prefix}├─ 💭 "{short}"') + spinner.print_above(f' {_prefix()}├─ 💭 "{short}"') except Exception as e: logger.debug("Spinner print_above failed: %s", e) _relay("subagent.thinking", preview=text) @@ -1583,12 +1626,12 @@ def _build_child_progress_callback( summary_text = tool_name or preview or "" if spinner and summary_text: try: - spinner.print_above(f" {prefix}├─ 🔀 {summary_text}") + spinner.print_above(f" {_prefix()}├─ 🔀 {summary_text}") except Exception as e: logger.debug("Spinner print_above failed: %s", e) if parent_cb: try: - parent_cb("subagent_progress", f"{prefix}{summary_text}") + parent_cb("subagent_progress", f"{_prefix()}{summary_text}") except Exception as e: logger.debug("Parent callback relay failed: %s", e) return @@ -1610,7 +1653,7 @@ def _build_child_progress_callback( from agent.display import get_tool_emoji emoji = get_tool_emoji(tool_name or "") - line = f" {prefix}├─ {emoji} {tool_name}" + line = f" {_prefix()}├─ {emoji} {tool_name}" if short: line += f' "{short}"' try: @@ -1623,14 +1666,14 @@ def _build_child_progress_callback( _batch.append(tool_name or "") if len(_batch) >= _BATCH_SIZE: summary = ", ".join(_batch) - _relay("subagent.progress", preview=f"🔀 {prefix}{summary}") + _relay("subagent.progress", preview=f"🔀 {_prefix()}{summary}") _batch.clear() def _flush(): """Flush remaining batched tool names to gateway on completion.""" if parent_cb and _batch: summary = ", ".join(_batch) - _relay("subagent.progress", preview=f"🔀 {prefix}{summary}") + _relay("subagent.progress", preview=f"🔀 {_prefix()}{summary}") _batch.clear() _callback._flush = _flush @@ -2138,6 +2181,9 @@ def _build_child_agent( # Now the child exists, its session id can ride on every relayed event # (including the spawn_requested below — first emit happens after this). child_session_ref["session_id"] = getattr(child, "session_id", "") or "" + # Same shared ref receives the batch id once delegate_task stamps it, so + # the display prefix and relayed events can tag which batch this is. + child._progress_identity_ref = child_session_ref # Set delegation depth so children can't spawn grandchildren child._delegate_depth = child_depth # Stash the post-degrade role for introspection (leaf if the @@ -2637,7 +2683,8 @@ def _run_single_child( # gateway inactivity timeout doesn't fire while the subagent is working. # Without this, the parent's _last_activity_ts freezes when delegate_task # starts and the gateway eventually kills the agent for "no activity". - _heartbeat_stop = threading.Event() + # Runs on the shared periodic scheduler thread (agent/periodic_scheduler) + # rather than one daemon thread per child; returning False stops it. # Stale detection: track the child's (tool, iteration, activity_ts) across # heartbeat cycles. If none advances, count the cycle as stale. # Different thresholds for idle vs in-tool (see _HEARTBEAT_STALE_CYCLES_*). @@ -2647,87 +2694,85 @@ def _run_single_child( _last_seen_tool = [None] # type: list _last_seen_activity_ts = [None] # type: list _stale_count = [0] + _heartbeat_handle = [None] # type: list - def _heartbeat_loop(): - while not _heartbeat_stop.wait(_HEARTBEAT_INTERVAL): - if parent_agent is None: - continue - touch = getattr(parent_agent, "_touch_activity", None) - if not touch: - continue - # Pull detail from the child's own activity tracker - desc = f"delegate_task: subagent {task_index} working" - try: - child_summary = child.get_activity_summary() - child_tool = child_summary.get("current_tool") - child_iter = child_summary.get("api_call_count", 0) - child_max = child_summary.get("max_iterations", 0) - child_activity_ts = child_summary.get("last_activity_ts") + def _heartbeat_tick(): + if parent_agent is None: + return + touch = getattr(parent_agent, "_touch_activity", None) + if not touch: + return + # Pull detail from the child's own activity tracker + desc = f"delegate_task: subagent {task_index} working" + try: + child_summary = child.get_activity_summary() + child_tool = child_summary.get("current_tool") + child_iter = child_summary.get("api_call_count", 0) + child_max = child_summary.get("max_iterations", 0) + child_activity_ts = child_summary.get("last_activity_ts") - # Stale detection: count cycles where iteration, current_tool, - # AND last_activity_ts are all frozen. A child running a - # legitimately long-running tool keeps current_tool set; a - # child waiting on a slow model refreshes last_activity_ts - # via direct_api_call's activity heartbeat — neither should - # look stale at the idle threshold. - iter_advanced = child_iter > _last_seen_iter[0] - tool_changed = child_tool != _last_seen_tool[0] - activity_advanced = ( - child_activity_ts is not None - and ( - _last_seen_activity_ts[0] is None - or child_activity_ts > _last_seen_activity_ts[0] - ) + # Stale detection: count cycles where iteration, current_tool, + # AND last_activity_ts are all frozen. A child running a + # legitimately long-running tool keeps current_tool set; a + # child waiting on a slow model refreshes last_activity_ts + # via direct_api_call's activity heartbeat — neither should + # look stale at the idle threshold. + iter_advanced = child_iter > _last_seen_iter[0] + tool_changed = child_tool != _last_seen_tool[0] + activity_advanced = ( + child_activity_ts is not None + and ( + _last_seen_activity_ts[0] is None + or child_activity_ts > _last_seen_activity_ts[0] ) - if iter_advanced or tool_changed or activity_advanced: - _last_seen_iter[0] = child_iter - _last_seen_tool[0] = child_tool - if child_activity_ts is not None: - _last_seen_activity_ts[0] = child_activity_ts - _stale_count[0] = 0 - else: - _stale_count[0] += 1 + ) + if iter_advanced or tool_changed or activity_advanced: + _last_seen_iter[0] = child_iter + _last_seen_tool[0] = child_tool + if child_activity_ts is not None: + _last_seen_activity_ts[0] = child_activity_ts + _stale_count[0] = 0 + else: + _stale_count[0] += 1 - # Pick threshold based on whether the child is currently - # inside a tool call. In-tool threshold is high enough to - # cover legitimately slow tools; idle threshold stays - # tight so the gateway timeout can fire on a truly wedged - # child. - stale_limit = ( - _HEARTBEAT_STALE_CYCLES_IN_TOOL - if child_tool - else _HEARTBEAT_STALE_CYCLES_IDLE + # Pick threshold based on whether the child is currently + # inside a tool call. In-tool threshold is high enough to + # cover legitimately slow tools; idle threshold stays + # tight so the gateway timeout can fire on a truly wedged + # child. + stale_limit = ( + _HEARTBEAT_STALE_CYCLES_IN_TOOL + if child_tool + else _HEARTBEAT_STALE_CYCLES_IDLE + ) + if _stale_count[0] >= stale_limit: + logger.warning( + "Subagent %d appears stale (no progress for %d " + "heartbeat cycles, tool=%s) — stopping heartbeat", + task_index, + _stale_count[0], + child_tool or "", ) - if _stale_count[0] >= stale_limit: - logger.warning( - "Subagent %d appears stale (no progress for %d " - "heartbeat cycles, tool=%s) — stopping heartbeat", - task_index, - _stale_count[0], - child_tool or "", - ) - break # stop touching parent, let gateway timeout fire + return False # stop touching parent, let gateway timeout fire - if child_tool: + if child_tool: + desc = ( + f"delegate_task: subagent running {child_tool} " + f"(iteration {child_iter}/{child_max})" + ) + else: + child_desc = child_summary.get("last_activity_desc", "") + if child_desc: desc = ( - f"delegate_task: subagent running {child_tool} " + f"delegate_task: subagent {child_desc} " f"(iteration {child_iter}/{child_max})" ) - else: - child_desc = child_summary.get("last_activity_desc", "") - if child_desc: - desc = ( - f"delegate_task: subagent {child_desc} " - f"(iteration {child_iter}/{child_max})" - ) - except Exception: - pass - try: - touch(desc) - except Exception: - pass - - _heartbeat_thread = threading.Thread(target=_heartbeat_loop, daemon=True) + except Exception: + pass + try: + touch(desc) + except Exception: + pass # Register the live agent in the module-level registry so the TUI can # target it by subagent_id (kill, pause, status queries). Unregistered @@ -2838,7 +2883,9 @@ def _run_single_child( } try: - _heartbeat_thread.start() + from agent.periodic_scheduler import schedule as _schedule_periodic + + _heartbeat_handle[0] = _schedule_periodic(_heartbeat_tick, _HEARTBEAT_INTERVAL) if child_progress_cb: try: child_progress_cb("subagent.start", preview=goal) @@ -3572,14 +3619,12 @@ def _run_single_child( return _error_entry finally: - # Stop the heartbeat thread so it doesn't keep touching parent activity - # after the child has finished (or failed). Guard the join: .start() - # now lives inside the try block, so if it raised (OS thread - # exhaustion) the thread was never started and Thread.join() would - # raise RuntimeError. ident is None until start() succeeds. - _heartbeat_stop.set() - if _heartbeat_thread.ident is not None: - _heartbeat_thread.join(timeout=5) + # Stop the heartbeat so it doesn't keep touching parent activity + # after the child has finished (or failed). The handle is None if + # scheduling itself raised (OS thread exhaustion on first use). + # wait=5 mirrors the old thread join: an in-flight tick finishes. + if _heartbeat_handle[0] is not None: + _heartbeat_handle[0].cancel(wait=5) # Drop the TUI-facing registry entry. Safe to call even if the # child was never registered (e.g. ID missing on test doubles). @@ -4091,6 +4136,18 @@ def delegate_task( live_deleg_id, live_writers, live_paths = create_live_transcripts( task_list, context, model=creds.get("model"), provider=creds.get("provider") ) + # Announce the batch tag once so the later ``[tag n/N]`` completion lines + # (and any nested batch's lines interleaving with them) are attributable. + if n_tasks > 1 and live_deleg_id: + _hdr = f"🔀 [{format_batch_tag(live_deleg_id)}] delegating {n_tasks} tasks" + _hdr_spinner = getattr(parent_agent, "_delegate_spinner", None) + if _hdr_spinner: + try: + _hdr_spinner.print_above(f" {_hdr}") + except Exception: + _emit_parent_console(parent_agent, f" {_hdr}") + else: + _emit_parent_console(parent_agent, f" {_hdr}") # Capture the ORIGINATING session's wake target BEFORE any child agent is # constructed: _build_child_agent() -> AIAgent() -> agent_init calls @@ -4179,6 +4236,9 @@ def delegate_task( # attribution (child-started background processes report under it). if live_deleg_id: setattr(child, "_delegation_id", live_deleg_id) + _ident_ref = getattr(child, "_progress_identity_ref", None) + if isinstance(_ident_ref, dict): + _ident_ref["delegation_id"] = live_deleg_id children.append((i, t, child)) def _execute_and_aggregate(*, honor_parent_interrupt: bool = True) -> dict: @@ -4314,7 +4374,9 @@ def delegate_task( status = entry.get("status", "?") icon = "✓" if status == "completed" else "✗" remaining = n_tasks - completed_count - completion_line = f"{icon} [{idx+1}/{n_tasks}] {label} ({dur}s)" + _tag = format_batch_tag(live_deleg_id) + _slot = f"{_tag} · {idx+1}/{n_tasks}" if _tag else f"{idx+1}/{n_tasks}" + completion_line = f"{icon} [{_slot}] {label} ({dur}s)" # Failed/errored/timed-out children: say WHY on the # same line, cleaned to one short human-readable # fragment — a bare ✗ reads as "silently dropped". @@ -4336,7 +4398,7 @@ def delegate_task( if spinner_ref and remaining > 0: try: spinner_ref.update_text( - f"🔀 {remaining} task{'s' if remaining != 1 else ''} remaining" + f"🔀 {'[' + _tag + '] ' if _tag else ''}{remaining} task{'s' if remaining != 1 else ''} remaining" ) except Exception as e: logger.debug("Spinner update_text failed: %s", e) diff --git a/tools/env_probe.py b/tools/env_probe.py index 32b8f6e6b6..d1eaf08c12 100644 --- a/tools/env_probe.py +++ b/tools/env_probe.py @@ -198,18 +198,23 @@ def _pip_python_version() -> Optional[str]: return None +def _resolve_terminal_backend() -> str: + """Scope-aware terminal backend name (``local`` when unresolvable).""" + try: + from tools.terminal_scope import terminal_env + + return (terminal_env("TERMINAL_ENV") or "local").strip().lower() + except Exception: # never let policy resolution break prompt building + logger.debug("terminal backend resolution failed", exc_info=True) + return "local" + + def _build_probe_line() -> str: """Build the one-liner. Returns "" when nothing notable is detected. Emit only when SOMETHING is off — the goal is to save the model from hitting an avoidable wall, not to narrate a healthy environment. """ - # Bail out if a remote terminal backend is configured; the host's - # Python state isn't where the agent's tools run. - backend = (os.getenv("TERMINAL_ENV") or "local").strip().lower() - if backend in _REMOTE_BACKENDS or _plugin_backend_is_remote(backend): - return "" - py3_ver = _python_version_of("python3") py_ver = _python_version_of("python") # for systems with a `python` alias py3_has_pip = _has_pip_module("python3") if py3_ver else False @@ -305,6 +310,15 @@ def get_environment_probe_line(*, force_refresh: bool = False) -> str: _PROBE_GEN += 1 _WAIT_ALREADY_TIMED_OUT = False + # Resolve the backend HERE, in the caller's context: under gateway + # multiplexing the routed profile's backend lives in the per-turn terminal + # scope, which the bare probe worker thread does not inherit (#68559). A + # remote backend answers "" without consulting the cache — the cached line + # describes the HOST toolchain, not where that profile's tools run. + backend = _resolve_terminal_backend() + if backend in _REMOTE_BACKENDS or _plugin_backend_is_remote(backend): + return "" + if _PROBE_DONE.is_set(): return _CACHED_LINE or "" diff --git a/tools/file_operations.py b/tools/file_operations.py index cbef8fc1e2..2644be38a0 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -28,11 +28,15 @@ Usage: import base64 import binascii import os +import posixpath import re +import secrets import sys import difflib import hashlib import json +import logging +import threading import unicodedata from abc import ABC, abstractmethod from dataclasses import dataclass, field @@ -46,6 +50,10 @@ from agent.file_safety import ( get_write_denied_error, is_write_denied as _shared_is_write_denied, ) +from agent.search_policy import SEARCH_PRUNE_DIR_NAMES +from tools import interrupt as tool_interrupt + +logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -65,6 +73,71 @@ _MACOS_TCC_PROTECTED_HOME_DIRS = ( ) +_FILENAME_SEARCH_ADMISSION = threading.Condition() +_ACTIVE_FILENAME_SEARCH_ROOTS: set[tuple[str, str, str]] = set() +_FILENAME_SEARCH_WAIT_SECONDS = 0.05 + + +def _normalized_filename_search_root(env: Any, root: str, fallback_cwd: str) -> str: + """Normalize a filename-walk root without resolving remote paths locally.""" + from tools.environments.local import LocalEnvironment, _IS_WINDOWS, _msys_to_windows_path + + cwd = getattr(env, "cwd", None) or fallback_cwd + if isinstance(env, LocalEnvironment): + if _IS_WINDOWS: + root = _msys_to_windows_path(root) + cwd = _msys_to_windows_path(cwd) + if not os.path.isabs(root): + root = os.path.join(cwd, root) + return os.path.normcase(os.path.abspath(os.path.normpath(root))) + + if not posixpath.isabs(root): + root = posixpath.join(cwd, root) + return posixpath.normpath(root) + + +def _filename_search_root_keys( + env: Any, roots: List[str], fallback_cwd: str +) -> tuple[tuple[str, str, str], ...]: + """Return unique backend/root admission keys in deterministic order.""" + env_type = type(env) + return tuple(sorted({ + ( + env_type.__module__, + env_type.__qualname__, + _normalized_filename_search_root(env, root, fallback_cwd), + ) + for root in roots + })) + + +def _acquire_filename_search_roots( + keys: tuple[tuple[str, str, str], ...], +) -> bool: + """Atomically claim every key, polling for thread-scoped interruption.""" + with _FILENAME_SEARCH_ADMISSION: + while any(key in _ACTIVE_FILENAME_SEARCH_ROOTS for key in keys): + if tool_interrupt.is_interrupted(): + return False + _FILENAME_SEARCH_ADMISSION.wait(_FILENAME_SEARCH_WAIT_SECONDS) + if tool_interrupt.is_interrupted(): + return False + if tool_interrupt.is_interrupted(): + return False + return tool_interrupt.run_if_not_interrupted( + lambda: _ACTIVE_FILENAME_SEARCH_ROOTS.update(keys) + ) + + +def _release_filename_search_roots( + keys: tuple[tuple[str, str, str], ...], +) -> None: + """Release a completed walk and leave no idle per-root state behind.""" + with _FILENAME_SEARCH_ADMISSION: + _ACTIVE_FILENAME_SEARCH_ROOTS.difference_update(keys) + _FILENAME_SEARCH_ADMISSION.notify_all() + + def _macos_protected_search_exclusions( path: str, *, @@ -368,6 +441,7 @@ class SearchResult: result["counts"] = self.counts if self.truncated: result["truncated"] = True + result["total_count_is_lower_bound"] = True if self.limit_reason: result["limit_reason"] = self.limit_reason if self.warning: @@ -623,7 +697,8 @@ class FileOperations(ABC): @abstractmethod def search(self, pattern: str, path: str = ".", target: str = "content", file_glob: Optional[str] = None, limit: int = 50, offset: int = 0, - output_mode: str = "content", context: int = 0) -> SearchResult: + output_mode: str = "content", context: int = 0, + order: str = "discovery") -> SearchResult: """Search for content or files.""" ... @@ -848,6 +923,36 @@ DEFAULT_SEARCH_LIMIT = 50 # `wc -c` prints only digits, so this can never collide with a real size. NOT_REGULAR_SENTINEL = "__hermes_not_regular__" +# Echoed by the compound read/write probes when the path does not exist. +# A compound command only reports its *last* exit status, so the missing-file +# signal that ``_size_probe_cmd`` carries in ``exit 1`` has to travel in-band. +MISSING_SENTINEL = "__hermes_missing__" + +_READ_SENTINEL_PREFIX = "__HERMES_RF_" +_WRITE_SENTINEL_PREFIX = "__HERMES_WF_" + + +def _new_sentinel(prefix: str) -> str: + """Per-call separator line for a compound shell probe. + + 128 random bits make a collision with file content negligible, and the + underscores keep the token outside the base64 alphabet, so a sentinel + that ever leaked into a sample segment fails base64 validation instead + of decoding into bytes. + """ + return f"{prefix}{secrets.token_hex(16)}__" + + +def _split_segments(output: str, sentinel: str) -> List[str]: + """Split compound-probe stdout on its sentinel lines. + + Every producer (``wc``, ``base64``, ``cut``) newline-terminates its + output or prints nothing, so the separator is always ``sentinel + "\\n"`` + on a line of its own. The text after the final sentinel is the status + segment. + """ + return output.split(sentinel + "\n") + def _coerce_int(value: Any, default: int) -> int: """Best-effort integer coercion for tool pagination inputs.""" @@ -963,8 +1068,13 @@ class ShellFileOperations(FileOperations): self.cwd = cwd or getattr(terminal_env, 'cwd', None) or \ getattr(getattr(terminal_env, 'config', None), 'cwd', None) or "/" - # Cache for command availability checks + # Preserve the historical bool cache for ordinary executables: both + # hits and misses stay cached. Ripgrep is special because it has an + # off-PATH resolver and may be installed while this object is alive; + # only successful rg resolutions are cached. self._command_cache: Dict[str, bool] = {} + self._rg_resolution_cache: Dict[str, str] = {} + self._rg_modified_capability: Dict[str, Optional[str]] = {} def _exec(self, command: str, cwd: str = None, timeout: int = None, stdin_data: str = None) -> ExecuteResult: @@ -1005,12 +1115,96 @@ class ShellFileOperations(FileOperations): exit_code=exit_code ) + def _resolve_command(self, cmd: str) -> Optional[str]: + """Resolve an executable in the command host's namespace. + + Ordinary commands retain the original bool hit/miss cache. Ripgrep + alone caches successful resolved paths and re-probes misses so a + mid-session install becomes visible. + """ + if cmd != "rg": + return cmd if self._has_command(cmd) else None + + cached = self._rg_resolution_cache.get(cmd) + if cached: + return cached + + result = self._exec("command -v rg 2>/dev/null") + if result.exit_code == 0 and result.stdout.strip(): + resolved = result.stdout.strip().splitlines()[0] + # Compatibility with old boolean-probe fakes. + if resolved == "yes": + resolved = "rg" + self._rg_resolution_cache[cmd] = resolved + return resolved + + from tools.environments.local import LocalEnvironment, _IS_WINDOWS + + if _IS_WINDOWS and isinstance(self.env, LocalEnvironment): + user_profile = os.environ.get("USERPROFILE") or str(Path.home()) + local_app_data = os.environ.get("LOCALAPPDATA") + scoop = os.environ.get("SCOOP") or os.path.join(user_profile, "scoop") + candidates = [ + os.path.join(user_profile, ".cargo", "bin", "rg.exe"), + os.path.join(scoop, "shims", "rg.exe"), + ] + if local_app_data: + candidates.append( + os.path.join(local_app_data, "Microsoft", "WinGet", "Links", "rg.exe") + ) + for candidate in candidates: + if os.path.isfile(candidate): + resolved = candidate.replace("\\", "/") + self._rg_resolution_cache[cmd] = resolved + return resolved + return None + def _has_command(self, cmd: str) -> bool: - """Check if a command exists in the environment (cached).""" + """Check command availability with rg-specific resolution semantics.""" + if cmd == "rg": + return self._resolve_command(cmd) is not None if cmd not in self._command_cache: - result = self._exec(f"command -v {cmd} >/dev/null 2>&1 && echo 'yes'") - self._command_cache[cmd] = result.stdout.strip() == 'yes' + result = self._exec( + f"command -v {cmd} >/dev/null 2>&1 && echo 'yes'" + ) + self._command_cache[cmd] = result.stdout.strip() == "yes" return self._command_cache[cmd] + + def _modified_rg_capability_error(self, executable: str) -> Optional[str]: + """Return a cached actionable error unless rg can sort exactly.""" + if executable in self._rg_modified_capability: + return self._rg_modified_capability[executable] + quoted = self._quote_executable(executable) + result = self._exec(f"{quoted} --version", timeout=10) + match = re.search( + r"(?m)^ripgrep\s+((?:0|[1-9]\d*))\." + r"(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)" + r"(?:-(?:(?:0|[1-9]\d*)|(?:[0-9A-Za-z-]*[A-Za-z-]" + r"[0-9A-Za-z-]*))(?:\.(?:(?:0|[1-9]\d*)|" + r"(?:[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)))*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" + r"(?:\s+\(rev [^)]+\))?\s*$", + result.stdout or "", + ) + if result.exit_code == 0 and match and int(match.group(1)) >= 14: + error = None + else: + error = ( + "Exact modification-time order requires ripgrep 14 or newer; " + "upgrade ripgrep or use order='discovery'." + ) + self._rg_modified_capability[executable] = error + return error + + def _quote_executable(self, executable: str) -> str: + """Quote an executable without leaking controller path semantics.""" + if re.fullmatch(r"[A-Za-z0-9_.-]+", executable): + return executable + from tools.environments.local import LocalEnvironment + + if isinstance(self.env, LocalEnvironment): + return self._escape_native_tool_arg(executable) + return "'" + executable.replace("'", "'\"'\"'") + "'" def _sample_file_bytes(self, path: str, length: int = 1000): """Fetch the first ``length`` raw bytes of a file through the terminal. @@ -1032,7 +1226,18 @@ class ShellFileOperations(FileOperations): ) if result.exit_code != 0: return None - encoded = _strip_terminal_fence_leaks(result.stdout) + return self._decode_base64_sample(result.stdout) + + @staticmethod + def _decode_base64_sample(text: str) -> Optional[bytes]: + """Decode one base64 sample as emitted by ``head -c N | base64``. + + Whitespace-joins the whole text first (``base64`` wraps at 76 + columns), so callers must hand over exactly one segment; anything + else in the text fails validation and yields ``None``, which sends + the caller to the legacy text-sample heuristic. + """ + encoded = _strip_terminal_fence_leaks(text) encoded = "".join(encoded.split()) if not encoded: return b"" @@ -1549,42 +1754,343 @@ class ShellFileOperations(FileOperations): def read_file(self, path: str, offset: int = 1, limit: int = 2000) -> ReadResult: """ Read a file with pagination, binary detection, and line numbers. - + Args: path: File path (absolute or relative to cwd) offset: Line number to start from (1-indexed, default 1) limit: Maximum lines to return (default 500, max 2000) - + Returns: ReadResult with content, metadata, or error info + + One shell round-trip answers every question the read needs: + existence, size, binary sample, the page, line count, trailing + newline (see ``_read_probe_cmd``). A reply that cannot be parsed + falls back to ``_read_file_sequential``, the one-probe-per-call + form, so an exotic shell can never do worse than before. On a local + POSIX environment the read never touches the shell at all; see + ``_read_file_native``. """ # Expand ~ and other shell paths path = self._expand_path(path) - + offset, limit = normalize_read_pagination(offset, limit) - + + if self._native_read_enabled(): + return self._read_file_native(path, offset, limit) + + # Images and known-binary extensions never inline content; the + # sequential path stops at the probes for them, so nothing is gained + # by streaming their bytes through the page pipeline. + if self._is_image(path) or os.path.splitext(path)[1].lower() in BINARY_EXTENSIONS: + return self._read_file_sequential(path, offset, limit) + + from tools.tool_output_limits import get_max_line_length + line_clamp_bytes = 4 * get_max_line_length() + 1 + end_line = offset + limit - 1 + sentinel = _new_sentinel(_READ_SENTINEL_PREFIX) + probe = self._exec( + self._read_probe_cmd(path, offset, end_line, line_clamp_bytes, sentinel) + ) + output = probe.stdout or "" + + if sentinel not in output: + # Single-line replies: the path is missing or not a regular file. + marker = _strip_terminal_fence_leaks(output).strip() + if marker == MISSING_SENTINEL: + return self._read_file_missing(path, offset, limit) + if marker == NOT_REGULAR_SENTINEL: + return self._not_regular_error(path) + logger.debug( + "read_file: compound probe reply for %s has no sentinel " + "(exit %s, %d chars); falling back to sequential probes", + path, probe.exit_code, len(output), + ) + return self._read_file_sequential(path, offset, limit) + + segments = _split_segments(output, sentinel) + if probe.exit_code != 0 or len(segments) != 6: + logger.debug( + "read_file: compound probe for %s returned exit %s with %d " + "segments (want 6); falling back to sequential probes", + path, probe.exit_code, len(segments), + ) + return self._read_file_sequential(path, offset, limit) + size_seg, sample_seg, page_seg, wc_seg, tail_seg, status_seg = segments + + status = _strip_terminal_fence_leaks(status_seg).split() + try: + sample_rc, read_rc = int(status[0]), int(status[1]) + except (IndexError, ValueError): + logger.debug( + "read_file: compound probe for %s has unparseable status %r; " + "falling back to sequential probes", + path, status_seg[-40:], + ) + return self._read_file_sequential(path, offset, limit) + + try: + file_size = int(_strip_terminal_fence_leaks(size_seg).strip()) + except ValueError: + file_size = 0 + + # Byte-layer binary detection when base64 was available, else the + # legacy text heuristic over a plain sample: one extra round-trip, + # paid only on shells without base64. + sample_bytes = self._decode_base64_sample(sample_seg) if sample_rc == 0 else None + if sample_bytes is not None: + is_binary = self._is_likely_binary_bytes(sample_bytes) + else: + logger.debug( + "read_file: no usable base64 sample for %s (base64 exit %s); " + "paying one extra round-trip for the text heuristic", + path, sample_rc, + ) + sample_cmd = f"head -c 1000 {self._escape_shell_arg(path)} 2>/dev/null" + sample_result = self._exec(sample_cmd) + sample_output = _strip_terminal_fence_leaks(sample_result.stdout) + is_binary = self._is_likely_binary(path, sample_output) + + if is_binary: + return self._read_binary_file(path, offset, limit, file_size, sample_bytes) + + if read_rc != 0: + return ReadResult( + error=f"Failed to read file: {_strip_terminal_fence_leaks(page_seg)}" + ) + + read_output = _strip_terminal_fence_leaks(page_seg) + try: + total_lines = int(_strip_terminal_fence_leaks(wc_seg).strip()) + except ValueError: + total_lines = 0 + tail_flag = _strip_terminal_fence_leaks(tail_seg).strip() + file_ends_with_newline = tail_flag == "1" if tail_flag in ("0", "1") else None + + return self._assemble_read_result( + read_output, + offset=offset, + end_line=end_line, + total_lines=total_lines, + file_size=file_size, + file_ends_with_newline=file_ends_with_newline, + ) + + def _native_read_enabled(self) -> bool: + """Whether ``read_file`` may bypass the shell and read from this host. + + Only on POSIX with a ``LocalEnvironment``: the file is on this + machine and the path is already in native form. Windows keeps the + shell path, since file_operations holds Git-Bash-style paths there. + ``HERMES_NATIVE_FILE_READ=0`` turns the fast path off. + """ + flag = os.environ.get("HERMES_NATIVE_FILE_READ", "1").strip().lower() + if flag in ("0", "false", "no", "off"): + return False + # Same "is this env the local host" test the LSP path already uses; + # ``self.env`` is bound once in __init__ and never rebound, and the + # isinstance check is microseconds, so there is nothing to memoize. + return sys.platform != "win32" and self._lsp_local_only() + + def _read_file_native(self, path: str, offset: int, limit: int) -> ReadResult: + """``read_file`` without a shell: the file lives on this host. + + Same contract as the shell path, byte for byte. ``os.stat`` is the + ``[ -f ]`` guard (a stat, never an open, so FIFOs and devices are + refused before anything touches their contents); the first 1000 + bytes drive the byte-layer binary check; the page is produced + exactly as ``sed -n 'a,bp' | cut -b1-N`` prints it (every line + clamped to N bytes and newline-terminated), then decoded with + errors="replace" like the terminal transport. One chunked pass + counts lines and collects the page, so neither the file nor a + single pathological line is ever held in memory whole. + + ``path`` is already expanded and ``offset``/``limit`` normalized. + Anything unexpected from the OS hands over to the shell path. + """ + import stat as _stat + + full = path if os.path.isabs(path) else os.path.join( + getattr(self.env, "cwd", None) or self.cwd, path + ) + try: + st = os.stat(full) + except (FileNotFoundError, NotADirectoryError): + return self._read_file_missing(path, offset, limit) + except OSError: + return self._read_file_sequential(path, offset, limit) + if not _stat.S_ISREG(st.st_mode): + return self._not_regular_error(path) + file_size = st.st_size + + # Images are never inlined: redirect to the vision tool + if self._is_image(path): + return self._image_redirect_result(file_size) + + from tools.tool_output_limits import get_max_line_length + clamp = 4 * get_max_line_length() + 1 + end_line = offset + limit - 1 + + page: List[bytes] = [] + total_lines = 0 + lineno = 1 # the line currently being scanned + kept = bytearray() # first ``clamp`` bytes of that line + have_partial = False # that line has bytes but no newline yet + last_byte = b"" + try: + with open(full, "rb") as fh: + sample = fh.read(1000) + ext_binary = os.path.splitext(path)[1].lower() in BINARY_EXTENSIONS + if ext_binary or self._is_likely_binary_bytes(sample): + return self._read_binary_file(path, offset, limit, file_size, sample) + fh.seek(0) + while True: + chunk = fh.read(1 << 20) + if not chunk: + break + last_byte = chunk[-1:] + if lineno > end_line: + # Past the requested window: only the line count and + # trailing byte are still needed, so let memchr do the + # rest instead of the per-line bookkeeping below. + total_lines += chunk.count(b"\n") + have_partial = chunk[-1:] != b"\n" + continue + pos, n = 0, len(chunk) + while pos < n: + nl = chunk.find(b"\n", pos) + in_page = offset <= lineno <= end_line + if nl < 0: + if in_page and len(kept) < clamp: + kept += chunk[pos:pos + (clamp - len(kept))] + have_partial = True + break + if in_page: + if len(kept) < clamp: + kept += chunk[pos:min(nl, pos + (clamp - len(kept)))] + page.append(bytes(kept) + b"\n") + kept = bytearray() + have_partial = False + total_lines += 1 + lineno += 1 + pos = nl + 1 + except OSError: + return self._read_file_sequential(path, offset, limit) + if have_partial and offset <= lineno <= end_line: + # ``sed`` prints a final line that lacks a newline; ``cut`` adds one. + page.append(bytes(kept) + b"\n") + + read_output = _strip_terminal_fence_leaks( + b"".join(page).decode("utf-8", errors="replace") + ) + return self._assemble_read_result( + read_output, + offset=offset, + end_line=end_line, + total_lines=total_lines, + file_size=file_size, + file_ends_with_newline=(last_byte == b"\n") if file_size else None, + ) + + @staticmethod + def _image_redirect_result(file_size: int) -> ReadResult: + return ReadResult( + is_image=True, + is_binary=True, + file_size=file_size, + hint=( + "Image file detected. Automatically redirected to vision_analyze tool. " + "Use vision_analyze with this file path to inspect the image contents." + ), + ) + + def _read_probe_cmd(self, path: str, offset: int, end_line: int, + line_clamp_bytes: int, sentinel: str) -> str: + """One shell command answering every question ``read_file`` asks. + + Six segments, each closed by a ``sentinel`` line: byte size, base64 + of the first 1000 bytes, the ``sed | cut`` page, ``wc -l``, whether + the last byte is a newline, then the base64 and page pipeline + statuses. The probes run only inside ``[ -f ]``, the same + stat-not-open guard as ``_size_probe_cmd``, so a FIFO or device + never reaches ``head``/``sed``. A missing path echoes + ``MISSING_SENTINEL`` instead of exiting non-zero, because a compound + command only reports its last status. Every stage silences stderr: + the local backend merges stderr into stdout and a stray diagnostic + would otherwise land inside a segment. + + The page clamp is byte-based on purpose; see ``_read_file_sequential`` + for why it is ``4 * max_line_length + 1``. + """ + arg = self._escape_shell_arg(path) + mark = f"echo {sentinel}" + return ( + f"if [ -f {arg} ]; then " + f"wc -c < {arg} 2>/dev/null; {mark}; " + f"head -c 1000 {arg} 2>/dev/null | base64 2>/dev/null; __hs=$?; {mark}; " + f"sed -n '{offset},{end_line}p' {arg} 2>/dev/null" + f" | cut -b1-{line_clamp_bytes} 2>/dev/null; __hr=$?; {mark}; " + f"wc -l < {arg} 2>/dev/null; {mark}; " + f"tail -c 1 {arg} 2>/dev/null | wc -l; {mark}; " + f'echo "$__hs $__hr"; ' + f"elif [ -e {arg} ]; then echo {NOT_REGULAR_SENTINEL}; " + f"else echo {MISSING_SENTINEL}; fi" + ) + + def _read_file_missing(self, path: str, offset: int, limit: int) -> ReadResult: + """Not-found recovery shared by every read path. + + Before failing, try unicode-equivalent spellings: NFC/NFD, narrow + no-break space, curly quotes render identically in a terminal, so + the model retyping a visually-correct path can never discover the + byte mismatch on its own (retrying is the tool's job, not the + model's). No equivalent spelling → suggest similar files. + """ + variant = self._unicode_variant_match(path) + if variant is not None: + result = self.read_file(variant, offset=offset, limit=limit) + note = ( + f"Note: '{path}' not found byte-for-byte; resolved to " + f"the unicode-equivalent file '{variant}' (invisible " + "encoding difference: NFC/NFD or special space/quote " + "characters)." + ) + result.hint = f"{note} {result.hint}" if result.hint else note + return result + return self._suggest_similar_files(path) + + def _read_binary_file(self, path: str, offset: int, limit: int, + file_size: int, sample_bytes: Optional[bytes]) -> ReadResult: + """Binary branch shared by every read path. + + UTF-16 rescue (ported from MoonshotAI/kimi-code#2647): the terminal + env decodes stdout as UTF-8 with errors="replace", so a UTF-16 text + file (Windows Notepad .txt, PowerShell `>` redirects) arrives + mangled with U+FFFD and trips the binary guard. Probe the raw bytes + via the backend's Python and transcode to UTF-8 when a BOM or the + zero-byte parity heuristic identifies UTF-16. + """ + utf16_result = self._try_read_utf16(path, offset, limit, file_size) + if utf16_result is not None: + return utf16_result + return ReadResult( + is_binary=True, + file_size=file_size, + error=describe_binary_file(sample_bytes, file_size), + ) + + def _read_file_sequential(self, path: str, offset: int, limit: int) -> ReadResult: + """One-probe-per-call read: the pre-compound form, kept as fallback. + + ``read_file`` lands here for image / known-binary extensions (only + the probes matter) and whenever the compound reply cannot be parsed. + ``path`` is already expanded and ``offset``/``limit`` normalized. + """ # Check if file exists and get size (POSIX, works on Linux + macOS) stat_result = self._exec(self._size_probe_cmd(path)) if stat_result.exit_code != 0: - # File not found. Before failing, try unicode-equivalent - # spellings — NFC/NFD, narrow no-break space, curly quotes - # render identically in a terminal, so the model retyping a - # visually-correct path can never discover the byte mismatch - # on its own (retrying is the tool's job, not the model's). - variant = self._unicode_variant_match(path) - if variant is not None: - result = self.read_file(variant, offset=offset, limit=limit) - note = ( - f"Note: '{path}' not found byte-for-byte; resolved to " - f"the unicode-equivalent file '{variant}' (invisible " - "encoding difference: NFC/NFD or special space/quote " - "characters)." - ) - result.hint = f"{note} {result.hint}" if result.hint else note - return result - # No equivalent spelling — suggest similar files - return self._suggest_similar_files(path) + return self._read_file_missing(path, offset, limit) stat_output = _strip_terminal_fence_leaks(stat_result.stdout) if stat_output.strip() == NOT_REGULAR_SENTINEL: @@ -1593,24 +2099,16 @@ class ShellFileOperations(FileOperations): file_size = int(stat_output.strip()) except ValueError: file_size = 0 - + # Check if file is too large if file_size > MAX_FILE_SIZE: # Still try to read, but warn pass - + # Images are never inlined — redirect to the vision tool if self._is_image(path): - return ReadResult( - is_image=True, - is_binary=True, - file_size=file_size, - hint=( - "Image file detected. Automatically redirected to vision_analyze tool. " - "Use vision_analyze with this file path to inspect the image contents." - ), - ) - + return self._image_redirect_result(file_size) + # Read a sample to check for binary content — at the byte layer when # the transport allows, falling back to the legacy text heuristic. sample_bytes = self._sample_file_bytes(path) @@ -1624,22 +2122,8 @@ class ShellFileOperations(FileOperations): is_binary = self._is_likely_binary(path, sample_output) if is_binary: - # UTF-16 rescue (ported from MoonshotAI/kimi-code#2647): the - # terminal env decodes stdout as UTF-8 with errors="replace", so - # a UTF-16 text file (Windows Notepad .txt, PowerShell `>` - # redirects) arrives mangled with U+FFFD and trips the binary - # guard. Probe the raw bytes via the backend's Python and - # transcode to UTF-8 when a BOM or the zero-byte parity - # heuristic identifies UTF-16. - utf16_result = self._try_read_utf16(path, offset, limit, file_size) - if utf16_result is not None: - return utf16_result - return ReadResult( - is_binary=True, - file_size=file_size, - error=describe_binary_file(sample_bytes, file_size), - ) - + return self._read_binary_file(path, offset, limit, file_size, sample_bytes) + # Read with pagination using sed, clamping each line to a byte # budget IN THE SHELL so a pathological single-line file (e.g. one # 400MB minified line) never crosses the exec transport. The Python @@ -1672,16 +2156,11 @@ class ShellFileOperations(FileOperations): f" | cut -b1-{line_clamp_bytes}" ) read_result = self._exec(read_cmd) - + if read_result.exit_code != 0: return ReadResult(error=f"Failed to read file: {read_result.stdout}") read_output = _strip_terminal_fence_leaks(read_result.stdout) - # Strip a leading UTF-8 BOM so the model never sees a phantom U+FEFF - # before the first real character. Only meaningful on the first - # chunk (the marker lives at byte 0); later pages can't carry it. - if offset == 1: - read_output, _ = _strip_bom(read_output) - + # Get total line count wc_cmd = f"wc -l < {self._escape_shell_arg(path)}" wc_result = self._exec(wc_cmd) @@ -1690,7 +2169,50 @@ class ShellFileOperations(FileOperations): total_lines = int(wc_output.strip()) except ValueError: total_lines = 0 - + + # Only the page that reaches the file's final line can carry the + # ``cut`` newline artifact (see _assemble_read_result); probe the + # last byte just for that case, exactly as before. + file_ends_with_newline: Optional[bool] = None + if not total_lines > end_line and read_output.endswith('\n'): + tail_cmd = f"tail -c 1 {self._escape_shell_arg(path)} | wc -l" + tail_result = self._exec(tail_cmd) + tail_output = _strip_terminal_fence_leaks(tail_result.stdout) + if tail_result.exit_code == 0: + file_ends_with_newline = tail_output.strip() != "0" + + return self._assemble_read_result( + read_output, + offset=offset, + end_line=end_line, + total_lines=total_lines, + file_size=file_size, + file_ends_with_newline=file_ends_with_newline, + ) + + def _assemble_read_result( + self, + read_output: str, + *, + offset: int, + end_line: int, + total_lines: int, + file_size: int, + file_ends_with_newline: Optional[bool], + ) -> ReadResult: + """Turn a raw ``sed | cut`` page into the final ``ReadResult``. + + Shared by every read path so the BOM strip, pagination hint, the + ``cut`` newline artifact fix and the ambiguous-silence guards can + never drift apart. ``file_ends_with_newline`` is ``None`` when the + caller could not tell (the artifact is then left alone, as before). + """ + # Strip a leading UTF-8 BOM so the model never sees a phantom U+FEFF + # before the first real character. Only meaningful on the first + # chunk (the marker lives at byte 0); later pages can't carry it. + if offset == 1: + read_output, _ = _strip_bom(read_output) + # Check if truncated truncated = total_lines > end_line hint = None @@ -1700,13 +2222,13 @@ class ShellFileOperations(FileOperations): # ``cut`` (unlike sed -n p) always newline-terminates its output, # so a file whose final line has no trailing newline would grow a # phantom empty last line. Only possible when this page reaches the - # file's final line; probe the last byte and strip the artifact. - if not truncated and read_output.endswith('\n'): - tail_cmd = f"tail -c 1 {self._escape_shell_arg(path)} | wc -l" - tail_result = self._exec(tail_cmd) - tail_output = _strip_terminal_fence_leaks(tail_result.stdout) - if tail_result.exit_code == 0 and tail_output.strip() == "0": - read_output = read_output[:-1] + # file's final line; strip the artifact when the last byte says so. + if ( + not truncated + and read_output.endswith('\n') + and file_ends_with_newline is False + ): + read_output = read_output[:-1] # Ambiguous-silence guards: an empty content string is # indistinguishable, from inside the model, from a broken tool — @@ -2002,6 +2524,115 @@ class ShellFileOperations(FileOperations): # WRITE Implementation # ========================================================================= + def _write_probe_cmd(self, path: str, sentinel: str, body: Optional[str]) -> str: + """One shell command for the on-disk questions ``write_file`` asks. + + Two segments closed by a ``sentinel`` line: base64 of the first three + bytes (BOM detection at the byte layer, the same on-disk truth as + ``_file_has_bom``), then ``body``: ``"cat"`` for the full text when + pre-content is wanted, ``"sample"`` for the 4 KB line-ending sample, + or ``None`` for nothing. Gated on ``[ -f ]`` so a FIFO or device never + reaches ``head``/``cat``; a missing path echoes ``MISSING_SENTINEL``. + """ + arg = self._escape_shell_arg(path) + if body == "cat": + body_cmd = f"cat {arg} 2>/dev/null" + elif body == "sample": + body_cmd = f"head -c 4096 {arg} 2>/dev/null" + else: + body_cmd = ":" + return ( + f"if [ -f {arg} ]; then " + f"head -c 3 {arg} 2>/dev/null | base64 2>/dev/null; echo {sentinel}; " + f"{body_cmd}; " + f"else echo {MISSING_SENTINEL}; fi" + ) + + def _probe_write_target( + self, path: str, pre_content: Optional[str], want_pre: bool, + ) -> tuple[bool, Optional[str], Optional[str]]: + """Return ``(has_bom, pre_content, original_line_ending)`` for ``path``. + + Replaces three probes (``cat`` when pre-content is wanted, a + ``head -c 4096`` line-ending sample, a ``head -c 3`` BOM check) with + one round-trip. Semantics are unchanged: pre-content is only read + when wanted and not supplied; the line ending comes from pre-content + when there is any, else from the sample; the BOM always comes from + the bytes on disk. A reply that cannot be parsed falls back to the + separate probes. + """ + if want_pre and pre_content is None: + body_mode: Optional[str] = "cat" + elif not pre_content: + body_mode = "sample" + else: + body_mode = None + + sentinel = _new_sentinel(_WRITE_SENTINEL_PREFIX) + probe = self._exec(self._write_probe_cmd(path, sentinel, body_mode)) + output = probe.stdout or "" + + if sentinel not in output: + if _strip_terminal_fence_leaks(output).strip() == MISSING_SENTINEL: + ending = _detect_line_ending(pre_content) if pre_content else None + return False, pre_content, ending + logger.debug( + "write_file: pre-write probe reply for %s has no sentinel " + "(exit %s, %d chars); falling back to sequential probes", + path, probe.exit_code, len(output), + ) + return self._probe_write_target_sequential(path, pre_content, want_pre) + + segments = _split_segments(output, sentinel) + if probe.exit_code != 0 or len(segments) != 2: + logger.debug( + "write_file: pre-write probe for %s returned exit %s with %d " + "segments (want 2); falling back to sequential probes", + path, probe.exit_code, len(segments), + ) + return self._probe_write_target_sequential(path, pre_content, want_pre) + head_seg, body = segments + + head_bytes = self._decode_base64_sample(head_seg) + if head_bytes is None: + # No clean base64 on this shell; ask the way we used to. + logger.debug( + "write_file: no usable base64 head for %s; paying one extra " + "round-trip for the BOM probe", + path, + ) + has_bom = self._file_has_bom(path, pre_content) + else: + has_bom = head_bytes.startswith(_UTF8_BOM.encode("utf-8")) + + if body_mode == "cat" and body: + pre_content = body + + if pre_content: + ending = _detect_line_ending(pre_content) + elif body_mode == "sample" and body: + ending = _detect_line_ending(body) + else: + ending = None + return has_bom, pre_content, ending + + def _probe_write_target_sequential( + self, path: str, pre_content: Optional[str], want_pre: bool, + ) -> tuple[bool, Optional[str], Optional[str]]: + """The pre-compound form of ``_probe_write_target``: one exec per question.""" + if want_pre and pre_content is None: + # Best-effort read; failure (file missing, permission) leaves + # pre_content as None which makes both downstream consumers + # degrade gracefully (lint reports all errors; LSP skips the + # shift map). + read_cmd = f"cat {self._escape_shell_arg(path)} 2>/dev/null" + read_result = self._exec(read_cmd) + if read_result.exit_code == 0 and read_result.stdout: + pre_content = read_result.stdout + ending = self._detect_file_line_ending(path, pre_content) + has_bom = self._file_has_bom(path, pre_content) + return has_bom, pre_content, ending + def write_file(self, path: str, content: str, pre_content: Optional[str] = None) -> WriteResult: """ @@ -2121,29 +2752,20 @@ class ShellFileOperations(FileOperations): # extensions outside both sets (binaries, opaque formats), # skipping the read keeps the hot path fast. want_pre = ext in LINTERS_INPROC or self._lsp_handles_extension(ext) - if want_pre: - if pre_content is not None: - # Caller already has file content (e.g. patch_replace read it - # for fuzzy matching) — reuse directly, skip redundant cat. - pass - else: - # Best-effort read; failure (file missing, permission) leaves - # pre_content as None which makes both downstream consumers - # degrade gracefully (lint reports all errors; LSP skips the - # shift map). - read_cmd = f"cat {self._escape_shell_arg(path)} 2>/dev/null" - read_result = self._exec(read_cmd) - if read_result.exit_code == 0 and read_result.stdout: - pre_content = read_result.stdout + # One shell round-trip answers every on-disk question the write + # needs (pre-content when wanted, line endings, BOM); see + # _probe_write_target. A caller that already has the file content + # (e.g. patch_replace read it for fuzzy matching) skips the read; + # the BOM is still taken from disk, never from pre_content. + has_bom, pre_content, original_ending = self._probe_write_target( + path, pre_content, want_pre + ) # ── Line-ending preservation (Roo Code pattern) ────────────── # If the file existed with CRLF endings and the agent's content # has bare LFs, convert to CRLF before writing. Otherwise the # write silently normalizes a Windows-line-ending file (and patch # produces mixed endings when only a substituted region changes). - # Detect from a small head sample to avoid reading the full file - # for line-ending purposes alone. - original_ending = self._detect_file_line_ending(path, pre_content) if original_ending == "\r\n": content = _normalize_line_endings(content, "\r\n") @@ -2156,7 +2778,7 @@ class ShellFileOperations(FileOperations): # toolchains key on it). Only prepend when the original had a BOM # and the new content doesn't already carry one (guards against # double-BOM if a caller passed raw bytes). - if self._file_has_bom(path, pre_content) and not _has_bom(content): + if has_bom and not _has_bom(content): content = _UTF8_BOM + content # Snapshot LSP diagnostics for this file (best-effort) so the @@ -2851,7 +3473,8 @@ class ShellFileOperations(FileOperations): def search(self, pattern: str, path: str = ".", target: str = "content", file_glob: Optional[str] = None, limit: int = 50, offset: int = 0, - output_mode: str = "content", context: int = 0) -> SearchResult: + output_mode: str = "content", context: int = 0, + order: str = "discovery") -> SearchResult: """ Search for content or files. @@ -2864,12 +3487,19 @@ class ShellFileOperations(FileOperations): offset: Skip first N results output_mode: "content", "files_only", or "count" context: Lines of context around matches + order: File-search ordering: fast discovery or exact modified time Returns: SearchResult with matches or file list """ offset, limit = normalize_search_pagination(offset, limit) + if target == "files" and order not in {"discovery", "modified"}: + return SearchResult( + error=(f"Invalid file search order {order!r}; expected " + "'discovery' or 'modified'.") + ) + # Expand ~ and other shell paths path = self._expand_path(path) @@ -2881,7 +3511,8 @@ class ShellFileOperations(FileOperations): # failing the whole call, split, search every path that exists, # merge the results, and report the skipped parts. multi = self._try_multi_path_search( - pattern, path, target, file_glob, limit, offset, output_mode, context + pattern, path, target, file_glob, limit, offset, output_mode, context, + order, ) if multi is not None: return multi @@ -2916,7 +3547,7 @@ class ShellFileOperations(FileOperations): ) if target == "files": - result = self._search_files(pattern, path, limit, offset) + result = self._search_files(pattern, path, limit, offset, order) else: result = self._search_content(pattern, path, file_glob, limit, offset, output_mode, context) @@ -2950,19 +3581,69 @@ class ShellFileOperations(FileOperations): return _macos_protected_search_exclusions( path, cwd=cwd, home=_HOME, platform=sys.platform ) + + def _effective_macos_search_exclusions( + self, roots: List[str] + ) -> List[tuple[str, str, str]]: + """Return unique exclusions without pruning an explicitly chosen root.""" + cwd = getattr(self.env, "cwd", None) or self.cwd + use_posix_paths = sys.platform == "darwin" and all( + not re.match(r"^[A-Za-z]:[\\/]", root) and "\\" not in root + for root in roots + ) + + def normalized(root: str) -> str: + if use_posix_paths: + if not posixpath.isabs(root): + root = posixpath.join(cwd, root) + return posixpath.normpath(root) + return os.path.normcase(os.path.abspath(os.path.normpath(root))) + + normalized_roots = [normalized(root) for root in roots] + explicit_roots = set(normalized_roots) + seen = set() + effective = [] + for root, normalized_root in zip(roots, normalized_roots): + for relative in self._macos_search_exclusions(root): + if use_posix_paths: + absolute = posixpath.normpath(posixpath.join(normalized_root, relative)) + key = absolute + else: + absolute = os.path.normpath(os.path.join(root, relative)) + key = os.path.normcase(os.path.abspath(absolute)) + if key in explicit_roots or key in seen: + continue + seen.add(key) + effective.append((root, relative, absolute)) + return effective + + @staticmethod + def _macos_protected_search_warning(paths: List[str]) -> str: + skipped = ", ".join(os.path.basename(item) for item in paths) + return ( + "Skipped macOS protected folders during broad search to avoid " + f"an unattended privacy prompt: {skipped}. Search a protected " + "folder directly when access is intentional." + ) def _try_multi_path_search(self, pattern: str, path: str, target: str, file_glob: Optional[str], limit: int, offset: int, - output_mode: str, context: int) -> Optional[SearchResult]: + output_mode: str, context: int, + order: str = "discovery") -> Optional[SearchResult]: """Recover a not-found ``path`` that is really several paths in one string. Production trajectories show models passing "dir1 dir2 dir3" (or - comma-separated lists) as ``path``. Split on whitespace/commas; when - at least one candidate exists and at least two candidates were given, - search every existing path, merge results, and note skipped parts. - Returns None when this doesn't look like a multi-path string. + comma-separated lists) as ``path``. Commas explicitly delimit paths and + therefore preserve internal spaces; without commas, retain the legacy + whitespace-separated recovery. When at least one candidate exists and + at least two candidates were given, search every existing path, merge + results, and note skipped parts. Returns None when this doesn't look + like a multi-path string. """ - parts = [p for chunk in path.split(",") for p in chunk.split() if p.strip()] + if "," in path: + parts = [part.strip() for part in path.split(",") if part.strip()] + else: + parts = path.split() if len(parts) < 2: return None existing, missing = [], [] @@ -2975,43 +3656,77 @@ class ShellFileOperations(FileOperations): if not existing: return None - merged = SearchResult() - for p in existing: - if target == "files": - sub = self._search_files(pattern, p, limit, offset) - else: - sub = self._search_content(pattern, p, file_glob, limit, offset, - output_mode, context) - if sub.error: - continue - merged.matches.extend(sub.matches) - merged.files.extend(sub.files) - merged.counts.update(sub.counts) - merged.total_count += sub.total_count - merged.truncated = merged.truncated or sub.truncated - # Respect the caller's limit across the merged set. - merged.matches = merged.matches[:limit] - merged.files = merged.files[:limit] + if target == "files": + # A file search across several roots is one global traversal so + # modified ordering and pagination are exact across the whole set. + # Route every engine through _search_files so root admission wraps + # the actual rg/find invocation for this multi-root request. + merged = self._search_files(pattern, existing, limit, offset, order) + else: + merged = SearchResult() + for root in existing: + sub = self._search_content( + pattern, root, file_glob, limit, offset, output_mode, context + ) + if sub.error: + return sub + merged.matches.extend(sub.matches) + merged.files.extend(sub.files) + merged.counts.update(sub.counts) + merged.total_count += sub.total_count + merged.truncated = merged.truncated or sub.truncated + merged.matches = merged.matches[:limit] + merged.files = merged.files[:limit] + note = f"path contained {len(parts)} entries; searched {len(existing)} that exist" if missing: note += "; skipped missing: " + ", ".join(missing[:3]) if len(missing) > 3: note += f" (+{len(missing) - 3} more)" - merged.warning = note + warning_parts = [note] + if not merged.error: + protected_paths = [ + absolute + for _root, _relative, absolute + in self._effective_macos_search_exclusions(existing) + ] + if protected_paths: + warning_parts.append( + self._macos_protected_search_warning(protected_paths) + ) + merged.warning = " ".join(warning_parts) return merged + def _search_prune_glob_args(self) -> str: + """Return rg globs that prune known heavyweight recursive subtrees. + + The two forms cover both a root whose basename is a protected name and + protected descendants. Globs are relative to each rg search root, so a + single ``**/name/**`` pattern does not cover an explicitly selected + ``name/`` root. The directory names come from the shared scan policy; + this method deliberately does not maintain a second search-only list. + """ + globs = [] + for dirname in sorted(SEARCH_PRUNE_DIR_NAMES): + for prefix in ("", "**/"): + pattern = f"!{prefix}{dirname}/**" + globs.extend(("--glob", self._escape_shell_arg(pattern))) + return " ".join(globs) + def _zero_match_probe(self, pattern: str, path: str, file_glob: Optional[str]) -> Optional[str]: """Return a hint for a 0-match content search, or None. 13.9% of production content searches return zero matches and give - the model nothing to steer by. Run ONE cheap case-insensitive count - probe; if it hits, say so. If the pattern contains regex - metacharacters, also probe it as a fixed string. Bounded: two rg - invocations max, count-only output. + the model nothing to steer by. Run cheap count-only probes for near + misses (wrong casing, hidden-only matches, unescaped regex + metacharacters). The hidden/ignored probe is bounded with the shared + dependency, cache, VCS, vendor, and build-tree pruning policy. """ - if not self._has_command('rg'): + rg_executable = self._resolve_command('rg') + if not rg_executable: return None + rg = self._quote_executable(rg_executable) def _tally(stdout: str): """Parse ``path:count`` lines from rg --count-matches.""" @@ -3031,8 +3746,8 @@ class ShellFileOperations(FileOperations): glob_expr = f" --glob {self._escape_shell_arg(file_glob, translate_path=False)}" if file_glob else "" probe = self._exec( - f"rg -i --count-matches{glob_expr} " - f"{self._escape_shell_arg(pattern, translate_path=False)} {self._escape_native_tool_arg(path)} " + f"{rg} -i --count-matches{glob_expr} " + f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", timeout=30, ) @@ -3046,10 +3761,13 @@ class ShellFileOperations(FileOperations): # Hidden/ignored probe: rg skips dotdirs and .gitignore'd files by # default. When the pattern exists only there, say so instead of # returning a bare zero (bench case: match in .hidden/ silently - # missing from results). + # missing from results). Keep --no-ignore so project-local ignored + # files remain diagnosable, but prune heavyweight trees before rg can + # recurse into them. hidden = self._exec( - f"rg --hidden --no-ignore --count-matches{glob_expr} " - f"{self._escape_shell_arg(pattern, translate_path=False)} {self._escape_native_tool_arg(path)} " + f"{rg} --hidden --no-ignore --count-matches{glob_expr}" + f" {self._search_prune_glob_args()} " + f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", timeout=30, ) @@ -3062,8 +3780,8 @@ class ShellFileOperations(FileOperations): ) if re.search(r"[.\[\](){}?*+^$\\|]", pattern): fixed = self._exec( - f"rg -F --count-matches{glob_expr} " - f"{self._escape_shell_arg(pattern, translate_path=False)} {self._escape_native_tool_arg(path)} " + f"{rg} -F --count-matches{glob_expr} " + f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", timeout=30, ) @@ -3077,7 +3795,36 @@ class ShellFileOperations(FileOperations): ) return None - def _search_files(self, pattern: str, path: str, limit: int, offset: int) -> SearchResult: + def _is_broad_local_search_root(self, path: str) -> bool: + """Whether a no-rg local root is unsafe for recursive find.""" + from tools.environments.local import ( + LocalEnvironment, _IS_WINDOWS, _msys_to_windows_path, + ) + + if not isinstance(self.env, LocalEnvironment): + return False + + def normalized(value: str) -> str: + if _IS_WINDOWS: + value = _msys_to_windows_path(value).replace("\\", "/") + if not os.path.isabs(value): + value = os.path.join(getattr(self.env, "cwd", None) or self.cwd, value) + return os.path.normcase(os.path.abspath(value)) + + root = normalized(path) + home = normalized(_HOME) + drive = os.path.splitdrive(root)[0] + anchor = drive + os.sep if drive else os.path.abspath(os.sep) + if root == os.path.normcase(anchor): + return True + try: + common = os.path.commonpath([root, home]) + except ValueError: + return False + return root == home or common == root + + def _search_files(self, pattern: str, path: str | List[str], limit: int, offset: int, + order: str = "discovery") -> SearchResult: """Search for files by name pattern (glob-like).""" # Auto-prepend **/ for recursive search if not already present if not pattern.startswith('**/') and '/' not in pattern: @@ -3085,104 +3832,148 @@ class ShellFileOperations(FileOperations): else: search_pattern = pattern.split('/')[-1] - search_root = Path(path) - has_hidden_path_ancestor = any( - part not in {".", ".."} and part.startswith(".") - for part in search_root.parts - ) + roots = [path] if isinstance(path, str) else path + if not roots: + return SearchResult( + error="File search requires at least one search root in 'path'." + ) - # Prefer ripgrep: respects .gitignore, excludes hidden dirs by - # default, and has parallel directory traversal (~200x faster than - # find on wide trees). Mirrors _search_content which already uses rg. - if self._has_command('rg'): - return self._search_files_rg(search_pattern, path, limit, offset) + # Prefer ripgrep: bounded parallel traversal with ignore semantics. + # Resolve the engine and exact-order capability before admission so a + # queued request does not occupy a root while doing command discovery. + if self._has_command("rg"): + rg_executable = self._resolve_command("rg") or "rg" + if order == "modified": + capability_error = self._modified_rg_capability_error(rg_executable) + if capability_error: + return SearchResult(error=capability_error) + keys = _filename_search_root_keys(self.env, roots, self.cwd) + if not _acquire_filename_search_roots(keys): + return SearchResult(error=( + "File search was interrupted while waiting for another filename " + "search on the same root. Retry when ready." + )) + try: + return self._search_files_rg( + search_pattern, path, limit, offset, order, + rg_executable=rg_executable, + ) + finally: + _release_filename_search_roots(keys) - # Fallback: find (slower, no .gitignore awareness) - if not self._has_command('find'): + # A local find traversal rooted at/above the user's home or at a + # filesystem root can consume minutes and prompt on protected paths. + # Refuse before invoking find. Controller paths never classify remotes. + if any(self._is_broad_local_search_root(root) for root in roots): + return SearchResult(error=( + "Broad local file search without ripgrep is disabled because " + "find cannot keep this traversal safely bounded. Install " + "ripgrep or search a narrower directory." + )) + + if not self._has_command("find"): return SearchResult( error="File search requires 'rg' (ripgrep) or 'find'. " "Install ripgrep for best results: " "https://github.com/BurntSushi/ripgrep#installation" ) - # Exclude hidden directories (matching ripgrep's default behavior). - hidden_exclude = "-not -path '*/.*'" if not has_hidden_path_ancestor else "" - hidden_filter_expr = f" {hidden_exclude}" if hidden_exclude else "" - - # Use shell pagination for standard roots. For hidden roots, gather full - # output so we can re-apply hidden-descendant filtering while allowing - # explicit hidden-root searches. - pagination_expr = "" - if not has_hidden_path_ancestor: - pagination_expr = f" | tail -n +{offset + 1} | head -n {limit}" - - # Prune protected directories before traversal so macOS never receives - # an access attempt (filtering matched paths after descent is too late). - protected_paths = [ - os.path.normpath(os.path.join(path, item)) - for item in self._macos_search_exclusions(path) + # Prune hidden descendant directories while still allowing an + # explicitly selected hidden root. Hidden files are excluded too, + # matching rg's default semantics. + find_roots = [ + f"./{root}" if root.startswith("-") else root + for root in roots ] - prune_expr = "" + q_roots = [self._escape_shell_arg(root) for root in find_roots] + root_exemptions = "".join(f" ! -path {root}" for root in q_roots) + hidden_prune = ( + f" \\( -type d -name '.*'{root_exemptions} \\) -prune -o" + ) + protected_paths = [ + absolute + for _root, _relative, absolute + in self._effective_macos_search_exclusions(roots) + ] + protected_prune = "" if protected_paths: - prune_terms = " -o ".join( + terms = " -o ".join( f"-path {self._escape_shell_arg(item)}" for item in protected_paths ) - prune_expr = f" \\( {prune_terms} \\) -prune -o" + protected_prune = f" \\( {terms} \\) -prune -o" - cmd = f"find {self._escape_shell_arg(path)}{prune_expr}{hidden_filter_expr} -type f -name {self._escape_shell_arg(search_pattern)} " \ - f"-printf '%T@ %p\\n' 2>/dev/null | sort -rn{pagination_expr}" + fetch_limit = offset + limit + 1 + base = ( + f"find {' '.join(q_roots)}{protected_prune}{hidden_prune} -type f " + f"! -name '.*' -name {self._escape_shell_arg(search_pattern)}" + ) + if order == "modified": + cmd = ( + "set -o pipefail; " + base + + f" -printf '%T@ %p\\n' 2>/dev/null | sort -rn | head -n {fetch_limit}" + ) + else: + cmd = ( + "set -o pipefail; " + base + + f" -print 2>/dev/null | head -n {fetch_limit}" + ) - result = self._exec(cmd, timeout=60) + keys = _filename_search_root_keys(self.env, roots, self.cwd) + if not _acquire_filename_search_roots(keys): + return SearchResult(error=( + "File search was interrupted while waiting for another filename " + "search on the same root. Retry when ready." + )) + try: + result = self._exec(cmd, timeout=60) + finally: + _release_filename_search_roots(keys) stdout, limit_reason = _search_stdout_and_limit(result) - if not stdout.strip() and not limit_reason: - # Try without -printf (BSD find compatibility -- macOS) - cmd_simple = f"find {self._escape_shell_arg(path)}{prune_expr}{hidden_filter_expr} -type f -name {self._escape_shell_arg(search_pattern)} " \ - f"2>/dev/null | sort -rn{pagination_expr}" - result = self._exec(cmd_simple, timeout=60) - stdout, limit_reason = _search_stdout_and_limit(result) - - files = [] - for line in stdout.strip().split('\n'): - if not line: - continue - parts = line.split(' ', 1) - if len(parts) == 2 and parts[0].replace('.', '').isdigit(): - files.append(parts[1]) - else: - files.append(line) - - # For explicit hidden roots, find's path-based filtering excludes every - # file under the hidden path. Apply descendant filtering after command - # execution so only the explicit root ancestry is bypassed. - if has_hidden_path_ancestor: - normalized_root = search_root.resolve() - filtered_files = [] - for file_path in files: - try: - rel_parts = Path(file_path).resolve().relative_to(normalized_root).parts - except ValueError: - rel_parts = Path(file_path).parts - if any(part not in {".", ".."} and part.startswith(".") for part in rel_parts): + # Parse before classifying exit 141: with pipefail, a bounded producer + # can receive SIGPIPE when head intentionally closes after fetch_limit + # rows. It is benign only when the parsed payload proves that bound was + # reached; a shorter payload remains a hard failure. + raw_files: List[str] = [] + for line in stdout.splitlines(): + if order == "modified": + parts = line.split(" ", 1) + if len(parts) != 2 or not parts[0].replace(".", "", 1).isdigit(): continue - filtered_files.append(file_path) - files = filtered_files[offset:offset + limit] - # pagination for standard roots is already applied in shell + raw_files.append(parts[1]) + elif line: + raw_files.append(line) + bounded_sigpipe = result.exit_code == 141 and len(raw_files) >= fetch_limit + if order == "modified" and result.exit_code not in {0, 124} and not bounded_sigpipe: + return SearchResult(error=( + "Exact modification-time order requires GNU find with " + "-printf support; install ripgrep 14+ or use order='discovery'." + )) + if order == "discovery" and result.exit_code not in {0, 124} and not bounded_sigpipe: + return SearchResult(error="File search failed while running bounded find traversal.") + + from tools.environments.local import LocalEnvironment, _IS_WINDOWS, _msys_to_windows_path + if _IS_WINDOWS and isinstance(self.env, LocalEnvironment): + raw_files = [_msys_to_windows_path(file_path) for file_path in raw_files] + + page = raw_files[offset:offset + limit] return SearchResult( - files=files, - total_count=len(files), - truncated=bool(limit_reason), + files=page, + total_count=len(raw_files), + truncated=len(raw_files) > offset + limit or bool(limit_reason), limit_reason=limit_reason, ) - def _search_files_rg(self, pattern: str, path: str, limit: int, offset: int) -> SearchResult: + def _search_files_rg(self, pattern: str, path: str | List[str], limit: int, offset: int, + order: str = "discovery", + rg_executable: Optional[str] = None) -> SearchResult: """Search for files by name using ripgrep's --files mode. rg --files respects .gitignore and excludes hidden directories by default, and uses parallel directory traversal for ~200x speedup - over find on wide trees. Results are sorted by modification time - (most recently edited first) when rg >= 13.0 supports --sortr. + over find on wide trees. Discovery order stays bounded and fast; + exact modification-time ordering is explicit because it scans globally. """ # rg --files -g uses glob patterns; wrap bare names so they match # at any depth (equivalent to find -name). @@ -3191,41 +3982,79 @@ class ShellFileOperations(FileOperations): else: glob_pattern = pattern - fetch_limit = limit + offset - exclusion_globs = " ".join( - f"--glob {self._escape_shell_arg(f'!{item}/**')}" - for item in self._macos_search_exclusions(path) + roots = [path] if isinstance(path, str) else path + fetch_limit = limit + offset + 1 + effective_exclusions = self._effective_macos_search_exclusions(roots) + scoped_common = None + command_roots = roots + use_posix_paths = sys.platform == "darwin" and all( + not re.match(r"^[A-Za-z]:[\\/]", root) and "\\" not in root + for root in roots ) + if len(roots) > 1 and effective_exclusions and use_posix_paths: + cwd = getattr(self.env, "cwd", None) or self.cwd + absolute_roots = [ + posixpath.normpath( + root if posixpath.isabs(root) else posixpath.join(cwd, root) + ) + for root in roots + ] + scoped_common = posixpath.commonpath(absolute_roots) + command_roots = [ + posixpath.relpath(root, scoped_common) for root in absolute_roots + ] + exclusion_terms = [ + f"--glob {self._escape_shell_arg(f'!{posixpath.relpath(absolute, scoped_common)}/**')}" + for _root, _relative, absolute in effective_exclusions + ] + else: + exclusion_terms = [ + f"--glob {self._escape_shell_arg(f'!{relative}/**')}" + for _root, relative, _absolute in effective_exclusions + ] + exclusion_globs = " ".join(dict.fromkeys(exclusion_terms)) exclusion_args = f" {exclusion_globs}" if exclusion_globs else "" - # Try mtime-sorted first (rg 13+); fall back to unsorted if not supported. - cmd_sorted = ( - f"rg --files --sortr=modified -g {self._escape_shell_arg(glob_pattern)}" - f"{exclusion_args} " - f"{self._escape_native_tool_arg(path)} 2>/dev/null " - f"| head -n {fetch_limit}" + rg_executable = rg_executable or self._resolve_command("rg") + if not rg_executable: + return SearchResult(error="File search requires ripgrep (rg).") + if order == "modified": + capability_error = self._modified_rg_capability_error(rg_executable) + if capability_error: + return SearchResult(error=capability_error) + rg = self._quote_executable(rg_executable) + sort_arg = " --sortr=modified" if order == "modified" else "" + root_args = " ".join(self._escape_native_tool_arg(root) for root in command_roots) + cd_prefix = ( + f"cd {self._escape_shell_arg(scoped_common)} && " if scoped_common else "" ) - result = self._exec(cmd_sorted, timeout=60) + cmd = ( + f"set -o pipefail; {cd_prefix}{rg} --files{sort_arg} -g {self._escape_shell_arg(glob_pattern)}" + f"{exclusion_args} -- {root_args} 2>/dev/null | head -n {fetch_limit}" + ) + result = self._exec(cmd, timeout=60) stdout, limit_reason = _search_stdout_and_limit(result) - all_files = [f for f in stdout.strip().split('\n') if f] + all_files = [f for f in stdout.splitlines() if f] + if scoped_common: + all_files = [ + file_path if posixpath.isabs(file_path) + else posixpath.normpath(posixpath.join(scoped_common, file_path)) + for file_path in all_files + ] + bounded_sigpipe = result.exit_code == 141 and len(all_files) >= fetch_limit - if not all_files and not limit_reason: - # --sortr may have failed on older rg; retry without it. - cmd_plain = ( - f"rg --files -g {self._escape_shell_arg(glob_pattern)}" - f"{exclusion_args} " - f"{self._escape_native_tool_arg(path)} 2>/dev/null " - f"| head -n {fetch_limit}" - ) - result = self._exec(cmd_plain, timeout=60) - stdout, limit_reason = _search_stdout_and_limit(result) - all_files = [f for f in stdout.strip().split('\n') if f] + if result.exit_code not in {0, 1, 124} and not bounded_sigpipe: + if order == "modified": + return SearchResult(error=( + "Exact modification-time order failed; ripgrep 14+ is " + "required. Upgrade ripgrep or use order='discovery'." + )) + return SearchResult(error="File search failed while running ripgrep.") page = all_files[offset:offset + limit] - return SearchResult( files=page, total_count=len(all_files), - truncated=len(all_files) >= fetch_limit or bool(limit_reason), + truncated=len(all_files) > offset + limit or bool(limit_reason), limit_reason=limit_reason, ) @@ -3237,7 +4066,8 @@ class ShellFileOperations(FileOperations): if self._has_command('rg'): used_rg = True result = self._search_with_rg(pattern, path, file_glob, limit, offset, - output_mode, context) + output_mode, context, + rg_executable=self._resolve_command("rg") or "rg") elif self._has_command('grep'): result = self._search_with_grep(pattern, path, file_glob, limit, offset, output_mode, context) @@ -3268,9 +4098,28 @@ class ShellFileOperations(FileOperations): return _maybe_warn_line_oriented_newline_pattern(result, pattern) def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], - limit: int, offset: int, output_mode: str, context: int) -> SearchResult: + limit: int, offset: int, output_mode: str, context: int, + rg_executable: Optional[str] = None) -> SearchResult: """Search using ripgrep.""" - cmd_parts = ["rg", "--line-number", "--no-heading", "--with-filename"] + rg_executable = rg_executable or self._resolve_command("rg") + if not rg_executable: + return SearchResult(error="Content search requires ripgrep (rg).") + cmd_parts = [self._quote_executable(rg_executable), "--line-number", "--no-heading", "--with-filename"] + + # Giant-single-line containment (ported from cline/cline#13525): a + # match inside a serialized dump (multi-MB single-line JSON/minified + # bundle) makes rg emit the ENTIRE line into stdout. `head -n` counts + # lines, so a 40MB match line sails through untruncated, gets buffered + # whole into Python, and only THEN hits the per-match [:500] clamp — + # measured 42MB across the transport / ~180MB peak alloc for one + # match on main. --max-columns bounds each printed line at the rg + # layer; --max-columns-preview keeps a truncated prefix (instead of + # omitting the match) so the model still sees the hit. 2000 cols + # comfortably exceeds the 500-char content clamp below, so no + # previously-visible content is lost. Both flags predate rg 11; the + # engine floor here is already rg 13 (--sortr). + if output_mode not in ("files_only", "count"): + cmd_parts.extend(["--max-columns", "2000", "--max-columns-preview"]) # Auto-multiline: a regex `\n` (or a literal newline in the pattern) # cannot match in rg's default line-oriented mode — it used to hard @@ -3476,6 +4325,12 @@ class ShellFileOperations(FileOperations): # Fetch generously so we can compute total before slicing fetch_limit = limit + offset + (200 if context > 0 else 0) cmd_parts.extend(["|", "head", "-n", str(fetch_limit)]) + # grep has no --max-columns: bound giant single-line matches (see the + # rg branch's containment comment) at the pipe layer instead. Safe for + # the file:line:content parser — truncation only ever drops content + # tail. Skipped for files_only/count where lines are paths/counts. + if output_mode not in ("files_only", "count"): + cmd_parts.extend(["|", "cut", "-c1-2000"]) # `set -o pipefail` so grep's exit status propagates through `| head` # (without it the pipeline reports head's 0, masking grep's error 2). @@ -3524,9 +4379,12 @@ class ShellFileOperations(FileOperations): find_parts.extend(["-name", self._escape_shell_arg(file_glob)]) find_parts.extend(["-exec", *grep_parts, "{}", "+"]) fetch_limit = limit + offset + (200 if context > 0 else 0) + # Same giant-single-line bound as the plain grep path (grep lacks + # --max-columns); see the rg branch's containment comment. + line_cap = " | cut -c1-2000" if output_mode not in ("files_only", "count") else "" cmd = ( "set -o pipefail; " + " ".join(find_parts) - + f" 2>/dev/null | head -n {fetch_limit}" + + f" 2>/dev/null | head -n {fetch_limit}{line_cap}" ) result = self._exec(cmd, timeout=60) return self._parse_grep_search_output(result, output_mode, limit, offset, context) diff --git a/tools/file_state.py b/tools/file_state.py index f22a966e1d..5b1e2e7e81 100644 --- a/tools/file_state.py +++ b/tools/file_state.py @@ -63,6 +63,7 @@ class FileStateRegistry: self._reads: Dict[str, Dict[str, ReadStamp]] = defaultdict(dict) self._last_writer: Dict[str, Tuple[str, float]] = {} self._path_locks: Dict[str, threading.Lock] = {} + self._path_lock_users: Dict[str, int] = {} self._meta_lock = threading.Lock() # guards _path_locks self._state_lock = threading.Lock() # guards _reads + _last_writer @@ -73,6 +74,7 @@ class FileStateRegistry: if lock is None: lock = threading.Lock() self._path_locks[resolved] = lock + self._path_lock_users[resolved] = self._path_lock_users.get(resolved, 0) + 1 return lock @contextmanager @@ -88,6 +90,13 @@ class FileStateRegistry: yield finally: lock.release() + with self._meta_lock: + users = self._path_lock_users[resolved] - 1 + if users: + self._path_lock_users[resolved] = users + else: + self._path_lock_users.pop(resolved, None) + self._path_locks.pop(resolved, None) # ── Read/write accounting ─────────────────────────────────────── def record_read( @@ -248,6 +257,11 @@ class FileStateRegistry: with self._state_lock: return list(self._reads.get(task_id, {}).keys()) + def forget_task(self, task_id: str) -> None: + """Release read stamps owned by a task after its lifecycle ends.""" + with self._state_lock: + self._reads.pop(task_id, None) + # ── Testing hooks ─────────────────────────────────────────────── def clear(self) -> None: """Reset all state. Intended for tests only.""" @@ -256,6 +270,7 @@ class FileStateRegistry: self._last_writer.clear() with self._meta_lock: self._path_locks.clear() + self._path_lock_users.clear() # ── Module-level singleton + helpers ───────────────────────────────── diff --git a/tools/file_tools.py b/tools/file_tools.py index ddcb060301..a68655d59f 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -256,7 +256,11 @@ def _configured_terminal_cwd() -> str | None: relative to, which is exactly the ambiguity that misroutes worktree edits. Only an absolute, sentinel-free value is honored. """ - return _sentinel_free_abs_cwd(os.environ.get("TERMINAL_CWD")) + # Scope-aware: under gateway multiplexing the routed profile's cwd lives in + # the per-turn terminal scope, not the process env (#68559). + from agent.runtime_cwd import scope_terminal_cwd + + return _sentinel_free_abs_cwd(scope_terminal_cwd() or None) def _registered_task_cwd_override(task_id: str = "default") -> str | None: @@ -1128,9 +1132,13 @@ _file_ops_cache: dict = {} # "consecutive": how many times that exact call has been repeated in a row # "read_history": set of (path, offset, limit) tuples for get_read_files_summary # "dedup": dict mapping (resolved_path, offset, limit) → mtime float -# Used to skip re-reads of unchanged files. Reset on -# context compression (the original content is summarised -# away so the model needs the full content again). +# Used to skip re-reads of unchanged files. Survives +# context compression so unchanged files can resume +# returning lightweight stubs after one recovery read. +# "dedup_generation_reads": set of dedup keys whose full content has been +# served since the latest compaction boundary. Cleared on +# compression so the first post-compaction read can recover +# exact bytes that the summary may have omitted. # "read_timestamps": dict mapping resolved_path → modification-time float # recorded when the file was last read (or written) by # this task. Used by write_file and patch to detect @@ -1236,6 +1244,15 @@ def _cap_read_tracker_data(task_data: dict) -> None: except (StopIteration, KeyError): break + generation_reads = task_data.get("dedup_generation_reads") + if generation_reads is not None and len(generation_reads) > _DEDUP_CAP: + excess = len(generation_reads) - _DEDUP_CAP + for _ in range(excess): + try: + generation_reads.pop() + except KeyError: + break + ts = task_data.get("read_timestamps") if ts is not None and len(ts) > _READ_TIMESTAMPS_CAP: excess = len(ts) - _READ_TIMESTAMPS_CAP @@ -1580,13 +1597,30 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: def clear_file_ops_cache(task_id: str = None): - """Clear the file operations cache.""" + """Clear file-operation state for a finished task, or all tasks.""" with _file_ops_lock: if task_id: _file_ops_cache.pop(task_id, None) else: _file_ops_cache.clear() + with _read_tracker_lock: + if task_id: + _read_tracker.pop(task_id, None) + else: + _read_tracker.clear() + + with _patch_failure_lock: + if task_id: + _patch_failure_tracker.pop(task_id, None) + else: + _patch_failure_tracker.clear() + + if task_id: + file_state.get_registry().forget_task(task_id) + else: + file_state.get_registry().clear() + def _special_file_kind(path) -> str | None: """Return a human name for non-regular file types that block reads. @@ -1795,7 +1829,8 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = task_data = _read_tracker.setdefault(task_id, { "last_key": None, "consecutive": 0, "read_history": set(), "dedup": {}, - "dedup_hits": {}, "read_timestamps": {}, + "dedup_hits": {}, "dedup_generation_reads": set(), + "read_timestamps": {}, }) # Backward-compat for pre-existing tracker entries that predate # dedup_hits/read_timestamps (long-lived task or crossed an @@ -1804,12 +1839,14 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = task_data["dedup_hits"] = {} if "read_timestamps" not in task_data: task_data["read_timestamps"] = {} + generation_reads = task_data.setdefault("dedup_generation_reads", set()) cached_mtime = task_data.get("dedup", {}).get(dedup_key) + content_served_in_generation = dedup_key in generation_reads if cached_mtime is not None: try: current_mtime = os.path.getmtime(resolved_str) - if current_mtime == cached_mtime: + if current_mtime == cached_mtime and content_served_in_generation: # Count repeated stub returns so weak tool-followers that # ignore the "refer to earlier result" hint don't burn # their iteration budget in an infinite read loop. After @@ -1933,6 +1970,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = # reset its hit counter. (File either changed or stat failed # earlier and we fell through.) task_data["dedup_hits"].pop(dedup_key, None) + task_data.setdefault("dedup_generation_reads", set()).add(dedup_key) task_data["read_history"].add((path, offset, limit)) if task_data["last_key"] == read_key: task_data["consecutive"] += 1 @@ -2011,30 +2049,29 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = def reset_file_dedup(task_id: str = None): - """Clear the deduplication cache for file reads. + """Advance the read-dedup generation after context compression. - Called after context compression — the original read content has been - summarised away, so the model needs the full content if it reads the - same file again. Without this, reads after compression would return - a "file unchanged" stub pointing at content that no longer exists in - context. + Called after context compression. The per-key ``dedup`` mtime map is + preserved, but the generation-read set is cleared. The first unchanged + read of each key after compaction therefore returns full content that may + have been summarized away; later reads in the same generation return the + lightweight stub. Stub-hit counters are also cleared so the hard block + restarts fresh (issue #84857). - Call with a task_id to clear just that task, or without to clear all. + Call with a task_id to reset just that task, or without to reset all. """ with _read_tracker_lock: if task_id: task_data = _read_tracker.get(task_id) if task_data: - if "dedup" in task_data: - task_data["dedup"].clear() if "dedup_hits" in task_data: task_data["dedup_hits"].clear() + task_data.setdefault("dedup_generation_reads", set()).clear() else: for task_data in _read_tracker.values(): - if "dedup" in task_data: - task_data["dedup"].clear() if "dedup_hits" in task_data: task_data["dedup_hits"].clear() + task_data.setdefault("dedup_generation_reads", set()).clear() def notify_other_tool_call(task_id: str = "default"): @@ -2550,6 +2587,7 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, def search_tool(pattern: str, target: str = "content", path: str = ".", file_glob: str = None, limit: int = 50, offset: int = 0, output_mode: str = "content", context: int = 0, + order: str = "discovery", task_id: str = "default") -> str: """Search for content or files.""" try: @@ -2566,6 +2604,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", file_glob or "", limit, offset, + order, ) with _read_tracker_lock: task_data = _read_tracker.setdefault(task_id, { @@ -2611,7 +2650,8 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", file_ops = _get_file_ops(task_id) result = file_ops.search( pattern=pattern, path=path, target=target, file_glob=file_glob, - limit=limit, offset=offset, output_mode=output_mode, context=context + limit=limit, offset=offset, output_mode=output_mode, context=context, + order=order, ) omitted = _filter_read_blocked_search_results(result, task_id) if hasattr(result, 'matches'): @@ -2803,7 +2843,7 @@ def _is_openai_family_main() -> bool: SEARCH_FILES_SCHEMA = { "name": "search_files", - "description": "Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents. On macOS, broad searches above the user home automatically skip TCC-protected folders (Desktop, Documents, Downloads, Library, Movies, Music, Pictures); target one directly when access is intentional.\n\nContent search (target='content'): Regex search inside files. Output modes: full matches with line numbers, file paths only, or match counts.\n\nFile search (target='files'): Find files by glob pattern (e.g., '*.py', '*config*'). Also use this instead of ls — results sorted by modification time.", + "description": "Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents. On macOS, broad searches above the user home automatically skip TCC-protected folders (Desktop, Documents, Downloads, Library, Movies, Music, Pictures); target one directly when access is intentional.\n\nContent search (target='content'): Regex search inside files. Output modes: full matches with line numbers, file paths only, or match counts.\n\nFile search (target='files'): Find files by glob pattern (e.g., '*.py', '*config*'). Also use this instead of ls. Discovery order is the fast bounded default; exact global newest-first order is an explicit opt-in and may scan the full tree.", "parameters": { "type": "object", "properties": { @@ -2813,6 +2853,7 @@ SEARCH_FILES_SCHEMA = { "file_glob": {"type": "string", "description": "Filter files by pattern in grep mode (e.g., '*.py' to only search Python files)"}, "limit": {"type": "integer", "description": "Maximum number of results to return (default: 50)", "default": 50}, "offset": {"type": "integer", "description": "Skip first N results for pagination (default: 0)", "default": 0}, + "order": {"type": "string", "enum": ["discovery", "modified"], "description": "File-search order: 'discovery' is fast bounded traversal order; 'modified' is exact global newest-first and may scan the full tree; ignored for content", "default": "discovery"}, "output_mode": {"type": "string", "enum": ["content", "files_only", "count"], "description": "Output format for grep mode: 'content' shows matching lines with line numbers, 'files_only' lists file paths, 'count' shows match counts per file", "default": "content"}, "context": {"type": "integer", "description": "Number of context lines before and after each match (grep mode only)", "default": 0} }, @@ -2872,7 +2913,8 @@ def _handle_search_files(args, **kw): return search_tool( pattern=args.get("pattern", ""), target=target, path=args.get("path", "."), file_glob=args.get("file_glob"), limit=args.get("limit", 50), offset=args.get("offset", 0), - output_mode=args.get("output_mode", "content"), context=args.get("context", 0), task_id=tid) + output_mode=args.get("output_mode", "content"), context=args.get("context", 0), + order=args.get("order", "discovery"), task_id=tid) def _read_file_schema_overrides(): diff --git a/tools/interrupt.py b/tools/interrupt.py index aaa4b1081d..230d7643fe 100644 --- a/tools/interrupt.py +++ b/tools/interrupt.py @@ -17,6 +17,7 @@ Usage in tools: import logging import os import threading +from collections.abc import Callable logger = logging.getLogger(__name__) @@ -95,6 +96,21 @@ def is_thread_interrupted(thread_id: int | None) -> bool: return thread_id in _interrupted_threads +def run_if_not_interrupted(callback: Callable[[], None]) -> bool: + """Run a state transition atomically with current-thread interruption. + + Returns ``False`` without calling ``callback`` when the current thread is + already interrupted. The callback runs under the interrupt lock and must + not block or re-enter any interrupt API. + """ + tid = threading.current_thread().ident + with _lock: + if tid in _interrupted_threads: + return False + callback() + return True + + def get_interrupt_reason() -> str | None: """Return the user-safe interrupt cause for the current thread, if known.""" tid = threading.current_thread().ident diff --git a/tools/kanban_tools.py b/tools/kanban_tools.py index d49b53a221..dd1db3ed3d 100644 --- a/tools/kanban_tools.py +++ b/tools/kanban_tools.py @@ -251,10 +251,16 @@ def _goal_judge_available() -> bool: return client is not None and bool(model) -def _goal_mode_handoff_rejection(task, evidence: str) -> Optional[str]: - """Return a rejection reason when a goal-mode terminal handoff is premature.""" +def _goal_mode_handoff_rejection(task, evidence: str): + """Return ``(verdict, reason_or_None)`` for a goal-mode terminal handoff. + + ``{"done", None}`` means the judge allows the handoff; anything else is + a rejection whose verdict disambiguates the guidance the caller gives + the worker (``continue`` = not done yet, ``blocked`` = judged + unachievable — see #100954). + """ if not task or not task.goal_mode or not _goal_judge_available(): - return None + return ("done", None) verdict = "done" reason = "" try: @@ -270,7 +276,7 @@ def _goal_mode_handoff_rejection(task, evidence: str) -> Optional[str]: judge_exc, exc_info=True, ) - return reason if verdict != "done" else None + return (verdict, None if verdict == "done" else reason) # --------------------------------------------------------------------------- @@ -752,10 +758,18 @@ def _handle_complete(args: dict, **kw) -> str: # Only enforce when a judge is actually reachable — see # _goal_judge_available for why an unavailable judge fails open. task = kb.get_task(conn, tid) - rejection = _goal_mode_handoff_rejection( + gate_verdict, rejection = _goal_mode_handoff_rejection( task, (summary or result or "").strip(), ) + if gate_verdict == "blocked": + return tool_error( + f"Goal completion rejected: judge ruled the goal " + f"unachievable — {rejection}. The task will NOT complete " + f"silently. Either re-scope the task with kanban_edit, " + f"or record the block with kanban_block and hand the " + f"decision to a human / reviewer." + ) if rejection is not None: return tool_error( f"Goal completion rejected by judge: {rejection}. " @@ -937,7 +951,13 @@ def _handle_request_review(args: dict, **kw) -> str: kb, conn = _connect(board=board) try: task = kb.get_task(conn, tid) - rejection = _goal_mode_handoff_rejection(task, summary) + gate_verdict, rejection = _goal_mode_handoff_rejection(task, summary) + if gate_verdict == "blocked": + return tool_error( + f"Goal review handoff rejected: judge ruled the goal " + f"unachievable — {rejection}. Record the block with " + f"kanban_block instead of requesting review." + ) if rejection is not None: return tool_error( f"Goal review handoff rejected by judge: {rejection}. " diff --git a/tools/mcp_death_supervisor.py b/tools/mcp_death_supervisor.py new file mode 100644 index 0000000000..87302ae685 --- /dev/null +++ b/tools/mcp_death_supervisor.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""One parent-death supervisor per Hermes process, shared by all stdio MCP servers. + +Why this exists +--------------- +When Hermes dies without running its cleanup path (SIGKILL, OOM killer, a hard +crash), stdio MCP servers it spawned are reparented to init and keep running +forever. macOS has no ``PR_SET_PDEATHSIG``, so something has to outlive Hermes +and reap them. + +This module is deliberately standard-library-only and must not import anything +from ``tools/``: it runs after Hermes may already be dead, and pulling in +``mcp_tool`` would drag the whole agent with it. The TERM -> grace -> KILL +``killpg`` sweep in ``_reap`` therefore duplicates similar sweeps elsewhere in +the tree on purpose. + +The predecessor (``mcp_stdio_watchdog.py``) solved this with one CPython +*per MCP server*, wrapping each server command and polling ``getppid()`` every +two seconds. That costs ~10 MB of resident memory per server and detects death +up to one poll interval late. This module replaces the whole fleet of pollers +with a single supervisor per Hermes process: + +* **Death detection is a blocking read on a pipe.** Hermes holds the only write + end. When Hermes dies -- by any means, including SIGKILL -- the write end + closes and the read returns EOF. Exact, instant, and free. +* **Servers are spawned unwrapped.** The MCP SDK already spawns stdio children + with ``start_new_session=True``, so each one is its own process-group leader + and ``killpg`` still reaches its descendants. Removing the wrapper also + removes the signal-forwarding layer the wrapper needed to avoid inverting the + bug it fixed. + +Protocol (line-based, on stdin) +------------------------------- + register \n start reaping this process group on parent death + unregister \n stop reaping it (its server shut down cleanly) + +On EOF the supervisor SIGTERMs every still-registered process group, waits a +short grace period, SIGKILLs the survivors, and exits. A registered group that +Hermes never unregistered *is* the orphan set, so a clean Hermes shutdown -- +which unregisters as it tears each server down -- ends with nothing to kill. + +Unparseable lines are ignored rather than fatal: a corrupted byte on the control +pipe must not cost us the reaping guarantee for every other server. + +Residual risk: process-group reuse +---------------------------------- +We reap by pgid, so a registration is only as meaningful as the group's +identity. A group we deliberately keep registered -- an orphan that teardown +failed to kill, such as the ``node`` ``mcp-remote`` leaves behind -- can +eventually exit on its own, after which the kernel is free to hand that pgid to +an unrelated process owned by the same user. If Hermes then dies ungracefully +while the registration is still stale, we would signal a stranger. +``_is_safe_target`` cannot catch this: the value is stale, not invalid. + +Two things narrow the window. Hermes prunes registrations whose group has no +members left (``_prune_dead_supervised_pgids``) on every registration change, +and the orphan sweep unregisters whatever it reaps. Neither closes it -- a +group can die and its pgid be recycled between two probes -- so the exposure is +real but bounded to that gap, and requires an ungraceful death inside it. + +Closing it completely means proving group identity at reap time, e.g. stamping +MCP children with a boot-unique env marker and checking that some member still +carries it before signalling. That was judged not worth putting a ``ps`` parse +into the one process whose job is to stay simple enough to always work; it is +the obvious next step if this class of bug ever actually bites. Note the same +exposure already exists in Hermes's own killpg-based orphan cleanup, which this +module did not introduce (see upstream issue #88350). +""" + +from __future__ import annotations + +import argparse +import os +import signal +import sys +import time + +# Matches the grace period the per-server watchdog used before it escalated. +_TERM_GRACE_S = 3.0 +# How often we re-check for survivors during that grace period. +_REAP_POLL_S = 0.1 +# A command is "unregister " -- around 20 characters. The cap only has to +# be generous enough for a legitimate line; see _serve for why it exists. +_MAX_LINE_CHARS = 256 + + +def _is_safe_target(pgid: int, *, own_pgid: int, parent_pgid: int) -> bool: + """Return True if ``pgid`` is a process group we may signal. + + Defensive only -- Hermes already filters non-MCP children before it + registers anything (see ``_filter_mcp_children`` in ``tools/mcp_tool.py``). + But this process signals whole process *groups*, so a bad value here is + unusually expensive: ``killpg(0, ...)`` signals our own group, and pgid 1 + is init. A caller bug should cost us one unreaped server, never the + Hermes process tree or the session. + """ + if pgid <= 1: + return False + if pgid == own_pgid or pgid == parent_pgid: + return False + return True + + +def _reap(pgids: set[int]) -> None: + """SIGTERM every group, then SIGKILL whatever is still alive. + + Every process-group call below is POSIX-only by construction: this whole + module only ever runs as a child of ``_update_death_supervisor``, which + returns early unless ``os.name == "posix"``, so the supervisor is never + spawned on Windows in the first place. + """ + if not pgids: + return + + alive = set() + for pgid in pgids: + try: + os.killpg(pgid, signal.SIGTERM) # windows-footgun: ok — POSIX-only process + alive.add(pgid) + except (ProcessLookupError, PermissionError, OSError): + # Already gone, or not ours to signal. Either way, nothing to reap. + pass + + deadline = time.monotonic() + _TERM_GRACE_S + while alive and time.monotonic() < deadline: + time.sleep(_REAP_POLL_S) + for pgid in list(alive): + try: + # Signal 0 probes liveness: succeeds iff some member survives. + os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only process + except (ProcessLookupError, PermissionError, OSError): + alive.discard(pgid) + + for pgid in alive: + try: + os.killpg(pgid, signal.SIGKILL) # windows-footgun: ok — POSIX-only + except (ProcessLookupError, PermissionError, OSError): + pass + + +def _serve(stream, *, own_pgid: int, parent_pgid: int) -> set[int]: + """Read control lines until EOF; return the groups still registered. + + Reads are length-capped rather than newline-terminated. Iterating the + stream instead lets a writer that never sends a newline grow this process + without bound -- feeding it ``/dev/zero`` reached 15 GB before it was + stopped. Nothing in Hermes can produce that today, but this process is the + last line of defense against leaked servers, so it must not be the thing + that dies under memory pressure. A line truncated by the cap fails to parse + and is skipped; the remainder resyncs at the next newline. + """ + registered: set[int] = set() + while True: + line = stream.readline(_MAX_LINE_CHARS) + if not line: + break # EOF: the parent is gone. + if not line.endswith("\n"): + # Truncated by the cap, or an unterminated tail at EOF. Either way + # it is not a command we are willing to act on. + continue + parts = line.split() + if len(parts) != 2: + continue + verb, raw = parts + try: + pgid = int(raw) + except ValueError: + continue + if verb == "register": + if _is_safe_target(pgid, own_pgid=own_pgid, parent_pgid=parent_pgid): + registered.add(pgid) + elif verb == "unregister": + registered.discard(pgid) + return registered + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Reap registered process groups when the parent dies." + ) + parser.add_argument( + "--parent-pgid", + type=int, + required=True, + help="Process group of the spawning Hermes process; never signalled.", + ) + args = parser.parse_args(argv) + + # The parent may be torn down with killpg on its own group. We are spawned + # with start_new_session=True precisely so that sweep cannot take us with + # it before we have reaped -- assert that here rather than trust the caller. + own_pgid = os.getpgid(0) + if own_pgid == args.parent_pgid: + print( + "mcp_death_supervisor: refusing to run inside the parent's process " + "group (a killpg of the parent would kill us before we can reap)", + file=sys.stderr, + ) + return 2 + + # A dying parent's SIGINT/SIGHUP must not preempt the reap; the pipe's EOF + # is our only shutdown signal. SIGHUP is POSIX-only, which is fine here -- + # this process is never spawned on Windows (see _reap's docstring). + for sig in (signal.SIGINT, signal.SIGHUP): # windows-footgun: ok — POSIX-only process + try: + signal.signal(sig, signal.SIG_IGN) + except (ValueError, OSError): + pass + + registered = _serve(sys.stdin, own_pgid=own_pgid, parent_pgid=args.parent_pgid) + _reap(registered) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/mcp_stdio_watchdog.py b/tools/mcp_stdio_watchdog.py deleted file mode 100644 index a39f36d6fe..0000000000 --- a/tools/mcp_stdio_watchdog.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -"""Parent-death watchdog supervisor for stdio MCP subprocesses. - -Problem this fixes (#TBD): a stdio MCP server (e.g. ``npx -y mcp-remote -``) is spawned as a direct child of the Hermes process. Hermes's own -teardown path (``MCPServerTask.shutdown()`` / ``_kill_orphaned_mcp_children`` -at final exit) reaps it cleanly on a *graceful* exit. But if the spawning -Hermes process dies hard — ``kill -9``, an OS-level crash, a force-quit of -the TUI/desktop app — that teardown code never runs, and the child (plus any -of its own descendants, e.g. mcp-remote's spawned ``node`` process) is -orphaned. macOS has no direct equivalent of Linux's -``prctl(PR_SET_PDEATHSIG)`` to make the kernel auto-kill a child when its -parent dies, so nothing reaps these until the next Hermes startup's opt-in -``_kill_orphaned_mcp_children()`` sweep — which only runs if something calls -it. Repeated ungraceful session restarts can pile up N orphaned processes, -all racing to hold the same upstream SSE session, producing errors like -"Invalid request parameters" / "Received request before initialization was -complete" on the *legitimate* new connection. - -Fix: don't spawn the MCP server command directly. Spawn this supervisor -instead, which: - 1. execs the real command as its own child (own process group via - ``start_new_session``, so it doesn't inherit the supervisor's - controlling terminal weirdly and so we can killpg it cleanly); - 2. transparently passes stdin/stdout/stderr through — the MCP stdio - protocol talks directly over those pipes, so the supervisor must be a - no-op relay, not a bytes-in-the-middle proxy; - 3. runs a background thread that polls the direct POSIX parent identity: - compare current ``getppid()`` against the parent PID recorded when the - wrapper was created; - 4. the instant the original parent is gone, terminates the real child's - process group (SIGTERM, grace period, then SIGKILL) and exits. - -This is intentionally a thin, standard-library-only script so it starts fast -and can't itself become a resource leak. - -Usage (see ``tools/mcp_tool.py::_run_stdio``):: - - python3 -m tools.mcp_stdio_watchdog \\ - --ppid -- ... -""" - -from __future__ import annotations - -import argparse -import os -import signal -import subprocess -import sys -import threading -import time - -_POLL_INTERVAL_S = 2.0 -_TERM_GRACE_S = 3.0 - - -def _is_orphaned(original_ppid: int, getppid=os.getppid) -> bool: - """Return whether this process no longer has its original POSIX parent.""" - return getppid() != original_ppid - - -def _terminate_process_group(proc: subprocess.Popen) -> None: - """Best-effort SIGTERM-then-SIGKILL of the child's process group. - - This module only ever runs on POSIX (the wrap site in tools/mcp_tool.py - gates on ``os.name == "posix"``), but guard the POSIX-only primitives - anyway so an accidental Windows import/execute degrades to a plain - child kill instead of AttributeError. - """ - killpg = getattr(os, "killpg", None) - if killpg is None: # windows-footgun: ok — non-POSIX fallback - try: - proc.terminate() - proc.wait(timeout=_TERM_GRACE_S) - except (OSError, subprocess.TimeoutExpired): - proc.kill() - return - try: - pgid = os.getpgid(proc.pid) - except (ProcessLookupError, OSError): - return - sigkill = getattr(signal, "SIGKILL", signal.SIGTERM) - for sig in (signal.SIGTERM, sigkill): - try: - killpg(pgid, sig) - except (ProcessLookupError, PermissionError, OSError): - return - try: - proc.wait(timeout=_TERM_GRACE_S) - return - except subprocess.TimeoutExpired: - continue - - -def _watchdog_loop(proc: subprocess.Popen, original_ppid: int) -> None: - while proc.poll() is None: - if _is_orphaned(original_ppid): - _terminate_process_group(proc) - return - time.sleep(_POLL_INTERVAL_S) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Parent-death watchdog for a stdio MCP subprocess.", - ) - parser.add_argument("--ppid", type=int, required=True) - parser.add_argument("command", nargs=argparse.REMAINDER) - args = parser.parse_args(argv) - - real_argv = list(args.command) - if real_argv and real_argv[0] == "--": - real_argv = real_argv[1:] - if not real_argv: - print("mcp_stdio_watchdog: no command given after '--'", file=sys.stderr) - return 2 - - # New process group so we can killpg() the whole tree the real command - # may spawn (e.g. mcp-remote's own child `node` process), without - # touching our own group or the (already-gone) original parent's. - proc = subprocess.Popen( - real_argv, - stdin=sys.stdin, - stdout=sys.stdout, - stderr=sys.stderr, - start_new_session=True, - ) - - # Because the real server lives in its OWN process group (above), the - # parent's graceful-shutdown killpg of *our* group no longer reaches it. - # Forward SIGTERM/SIGINT to the child's group so graceful teardown - # (`_kill_orphaned_mcp_children`, shutdown sweeps) still kills a wedged - # server that ignores stdin EOF — otherwise the watchdog wrap would - # invert the bug it fixes. - def _forward_shutdown(signum, frame): # noqa: ARG001 - _terminate_process_group(proc) - sys.exit(128 + signum) - - signal.signal(signal.SIGTERM, _forward_shutdown) - signal.signal(signal.SIGINT, _forward_shutdown) - - watchdog = threading.Thread( - target=_watchdog_loop, - args=(proc, args.ppid), - daemon=True, - ) - watchdog.start() - - try: - return proc.wait() - except KeyboardInterrupt: - _terminate_process_group(proc) - return 130 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index 3ab60301ea..f7cf11213a 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -588,6 +588,13 @@ _MAX_BACKOFF_SECONDS = 60 # can ever reach the circuit-breaker half-open probe or _signal_reconnect. _PARKED_RETRY_INTERVAL = 300 # seconds between parked self-probes _RECYCLED_RECONNECT_TIMEOUT = 15.0 +# How long a tool call waits for a respawned stdio child after its subprocess +# was found dead — a gateway restart kills every MCP stdio child, +# and the next call from a still-live session would otherwise fail for no real +# reason). Bounded: when the wait elapses the call reports the dead transport +# instead of looping, so a genuinely broken server still parks via the +# rapid-drop budget in run() rather than hot-cycling respawns. +_STDIO_RESPAWN_WAIT_SEC = 15.0 # Jitter applied to reconnect backoff sleeps. Without it, every server that # lost the same backend retries in lockstep (thundering herd) and log lines # from N servers land in synchronized bursts. @@ -1066,32 +1073,281 @@ def _resolve_stdio_command(command: str, env: dict) -> tuple[str, dict]: return resolved_command, resolved_env -def _wrap_command_with_watchdog(command: str, args: list) -> tuple[str, list]: - """Wrap a stdio MCP server command in the parent-death watchdog supervisor. +def _npx_bin_candidates(bin_dir: str, name: str, *, windows: Optional[bool] = None) -> list: + """Launcher paths to try for *name* inside an npx cache's ``.bin``, in order. - On POSIX, the watchdog records this process's PID and later detects parent - death directly through ``getppid()``. Returns the (command, args) unchanged - on non-POSIX platforms or if the PID cannot be read. + On Windows that directory holds three siblings per bin — the extensionless + sh script, ``.cmd`` and ``.ps1``. Spawning the sh one from a + Windows process fails, and ``os.access(X_OK)`` there is effectively an + existence check, so it cannot tell them apart. Select by extension instead, + the same precedence ``hermes_constants._candidate_node_command_names`` + already uses for npm/npx/node; when no launcher exists the caller falls + back to npx rather than spawning something that will not run. + + ``windows`` is injectable so the platform branch is testable without + monkeypatching ``os.name`` (which breaks path handling process-wide). + """ + is_windows = os.name == "nt" if windows is None else windows + if is_windows: + return [os.path.join(bin_dir, name + ext) for ext in (".cmd", ".exe")] + return [os.path.join(bin_dir, name)] + + +def _npx_cached_bin(args: list) -> Optional[tuple]: + """Resolve ``npx -y `` to the already-installed binary, or None. + + ``npx`` resolves the package and then FORKS: it stays resident as the + parent of the real server for the whole process lifetime, doing no work. + Measured on a 4-agent host, that is ~48 MB of private memory per MCP + server — and it buys nothing here, because Hermes already supervises the + child itself (the shared death supervisor), so npx's supervision is a + second parent nobody reads. + + When the package is already in npx's cache we can spawn its binary + directly and drop the middle process. A cache miss returns None and the + caller falls back to ``npx`` unchanged, so the first run still installs + and nothing regresses on a cold machine. + + Deliberately conservative — returns None for anything unusual: + a version-pinned spec (``pkg@1.2.3``), extra npx flags, a package whose + manifest declares no single obvious bin, or any unreadable cache entry. + + Returns ``(binary_path, remaining_args)`` or None. + """ + if not isinstance(args, list) or not args: + return None + + rest = list(args) + while rest and rest[0] in ("-y", "--yes"): + rest.pop(0) + if not rest: + return None + + # `npx pkg -y` (flag AFTER the spec) is an unusual shape: those args are + # forwarded verbatim to the resolved binary, which would hand the server a + # flag npx would have eaten. Leave anything like that to npx. + if any(str(a) in ("-y", "--yes") for a in rest[1:]): + return None + + spec = str(rest[0]) + # A version pin means the user asked for a specific build; npx owns that + # resolution and the cache key may not match. Scoped names keep their + # leading '@', so only an '@' AFTER the scope is a version separator. + if "@" in (spec[1:] if spec.startswith("@") else spec): + return None + if not spec or spec.startswith("-"): + return None + + cache_root = os.environ.get("npm_config_cache") or os.path.join( + os.path.expanduser("~"), ".npm" + ) + npx_root = os.path.join(cache_root, "_npx") + if not os.path.isdir(npx_root): + return None + + try: + entries = os.listdir(npx_root) + except OSError: + return None + + for entry in entries: + manifest = os.path.join(npx_root, entry, "package.json") + try: + with open(manifest, "r", encoding="utf-8") as fh: + deps = (json.load(fh) or {}).get("dependencies") or {} + except (OSError, ValueError, TypeError): + continue + if spec not in deps: + continue + + pkg_json = os.path.join(npx_root, entry, "node_modules", spec, "package.json") + try: + with open(pkg_json, "r", encoding="utf-8") as fh: + bin_field = (json.load(fh) or {}).get("bin") + except (OSError, ValueError, TypeError): + continue + + if isinstance(bin_field, str): + names = [os.path.basename(spec)] + elif isinstance(bin_field, dict) and len(bin_field) == 1: + names = list(bin_field.keys()) + else: + # Zero or several bins: which one npx would pick is not ours to + # guess. Let npx decide. + continue + + bin_dir = os.path.join(npx_root, entry, "node_modules", ".bin") + for candidate in _npx_bin_candidates(bin_dir, names[0]): + if os.path.exists(candidate) and os.access(candidate, os.X_OK): + return candidate, rest[1:] + + return None + + +# --------------------------------------------------------------------------- +# Shared parent-death supervisor +# --------------------------------------------------------------------------- +# If this Hermes process dies without running its cleanup path (kill -9, OOM, +# crash, force-quit), stdio MCP children reparent to init and run forever. +# macOS has no PR_SET_PDEATHSIG, so something has to outlive us and reap them. +# +# We keep ONE supervisor process for all stdio servers and tell it which process +# groups to reap over a pipe. It detects our death as EOF on that pipe -- exact +# and instant -- rather than by polling getppid(). This replaced a design that +# wrapped every server command in its own poller, which cost ~10 MB resident per +# server (measured 9.8 MB physical footprint on macOS/arm64) and needed a signal +# forwarding layer, because wrapping put the real server in a different session +# from the pgid we tracked for killpg. See tools/mcp_death_supervisor.py. +# +# POSIX-only (relies on process groups), matching the platform scope of the +# killpg-based orphan cleanup below. +_death_supervisor = None # Optional[subprocess.Popen] +_death_supervisor_lock = threading.Lock() +# Process groups the supervisor is currently reaping on our behalf. Replayed +# verbatim if the supervisor has to be respawned, so a respawn never silently +# drops coverage for servers that are still running. +_supervised_pgids: set = set() + + +def _spawn_death_supervisor(): + """Start the shared supervisor, or return None if it cannot be started.""" + import subprocess + + supervisor = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "mcp_death_supervisor.py" + ) + try: + # start_new_session=True is load-bearing, not hygiene: shutdown paths + # killpg this process's own group, which would kill the supervisor + # before it could reap anything. + return subprocess.Popen( + [sys.executable, supervisor, "--parent-pgid", str(os.getpgid(0))], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=_get_mcp_stderr_log(), + start_new_session=True, + close_fds=True, + text=True, + ) + except Exception: + # Never let supervisor bookkeeping failure block a real MCP connection. + # The graceful shutdown paths still reap normally; we only lose the + # ungraceful-exit safety net. + logger.debug("Could not start the MCP parent-death supervisor", exc_info=True) + return None + + +def _prune_dead_supervised_pgids() -> set: + """Forget supervised groups that have no members left; return what went. + + Caller must hold ``_death_supervisor_lock``. Probing with signal 0 is a + pure existence question -- it cannot terminate anything -- so this is safe + to run on every registration change. It narrows, but cannot close, the + window where a group dies and its pgid is recycled before we notice; see + the residual-risk note in ``tools/mcp_death_supervisor.py``. + """ + killpg = getattr(os, "killpg", None) + if killpg is None: # windows-footgun: ok - POSIX-only, guarded + return set() + stale = set() + for pgid in list(_supervised_pgids): + try: + killpg(pgid, 0) + except ProcessLookupError: + stale.add(pgid) + except (PermissionError, OSError): + # Exists but is not ours to signal, or the probe itself failed. + # Keep it: dropping coverage on an ambiguous answer is the more + # expensive mistake of the two. + pass + _supervised_pgids.difference_update(stale) + return stale + + +def _update_death_supervisor(verb: str, pgids) -> None: + """Register or unregister process groups with the shared supervisor. + + ``verb`` is ``"register"`` or ``"unregister"``. Failures are swallowed: + losing the ungraceful-exit safety net must never fail a live MCP session. """ if os.name != "posix": - # Relies on process groups (os.getpgid/os.killpg); no POSIX - # equivalent wired up here yet, matching the existing killpg-based - # orphan cleanup's platform scope (Windows falls back to plain - # os.kill there too). - return command, args - try: - my_pid = os.getpid() - except Exception: - # Never let watchdog bookkeeping failure block a real MCP connection. - return command, args - watchdog_args = [ - os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp_stdio_watchdog.py"), - "--ppid", str(my_pid), - "--", - command, - *args, - ] - return sys.executable, watchdog_args + return + wanted = {int(pgid) for pgid in pgids} + if not wanted: + return + + global _death_supervisor + with _death_supervisor_lock: + if verb == "register": + _supervised_pgids.update(wanted) + else: + _supervised_pgids.difference_update(wanted) + + # Drop groups with nothing left alive. A registration outlives the + # server only while some member survives -- e.g. an orphaned grandchild + # that teardown failed to kill, which we deliberately keep registered. + # Once that group is finally empty the pgid can be recycled by an + # unrelated process, and a stale registration would have us reap a + # stranger. The orphan sweep already unregisters what it reaps, but it + # is not guaranteed to run in a given process, so prune here too -- + # signal 0 cannot kill anything, it only asks whether the group exists. + stale = _prune_dead_supervised_pgids() + + proc = _death_supervisor + if proc is None or proc.poll() is not None: + if not _supervised_pgids: + # Nothing left to cover, so there is nothing to tell -- and + # nothing to respawn a supervisor for. Keyed on the SET, not + # on the verb: after a broken-pipe write dropped the + # supervisor while groups were still registered, an + # unregister of one of them must still rebuild coverage for + # the survivors (review finding on #93517). + return + proc = _spawn_death_supervisor() + _death_supervisor = proc + if proc is None: + return + # A fresh supervisor knows nothing. Replay live coverage, which + # already reflects this call's mutation and the prune above, so + # pruned groups simply never reach the replacement. + payload = "".join(f"register {pgid}\n" for pgid in _supervised_pgids) + else: + payload = "".join(f"{verb} {pgid}\n" for pgid in wanted) + payload += "".join(f"unregister {pgid}\n" for pgid in stale) + + try: + proc.stdin.write(payload) + proc.stdin.flush() + except (BrokenPipeError, ValueError, OSError): + # It exited between poll() and write(). Drop it so the next call + # respawns and replays, rather than writing into a dead pipe. + # Recovery is deliberately two-step: this call gives up, and the + # next one sees ``poll()`` non-None and rebuilds coverage from + # ``_supervised_pgids``. Nothing is lost in between because that + # set, not the pipe, is the record of what needs reaping. + _death_supervisor = None + return + + if not _supervised_pgids: + # Nothing left to reap: release the supervisor instead of keeping + # a ~15 MB process and a pipe resident for the life of a gateway + # that once connected a stdio server. Closing our write end is + # the same EOF signal parent death sends; with an empty set the + # supervisor reaps nothing and exits. The next register respawns + # and replays from ``_supervised_pgids`` as it already does. + try: + proc.stdin.close() + except (BrokenPipeError, ValueError, OSError): + pass + # Reap it, or the exited supervisor stays a zombie until the next + # Popen in this process (CPython only collects abandoned children + # opportunistically). It exits on EOF with nothing to do, so this + # returns promptly; the timeout keeps a wedged one from stalling us. + try: + proc.wait(timeout=5) + except Exception: # noqa: BLE001 - timeout or already gone; either way we drop it + pass + _death_supervisor = None # --------------------------------------------------------------------------- @@ -1273,6 +1529,35 @@ def _cache_mcp_audio_block(block) -> str: return f"MEDIA:{audio_path}" +def _render_mcp_dropped_block_notice(block, block_type: str) -> str: + """Render an inline notice for an unsupported MCP content block. + + Ported from MoonshotAI/kimi-code#3227: silently dropping a block leaves + the model unaware content went missing, with no way to recover it. The + notice carries whatever handles the block exposes — mime type, size, + uri — so the agent can fetch or reason about the missing content (for + link-shaped blocks the uri lets it retrieve the data itself). + """ + details = [f"type={block_type}"] + mime = mcp_field(block, "mime_type", "mimeType", None) + if mime: + details.append(f"mimeType={mime}") + uri = getattr(block, "uri", None) or getattr( + getattr(block, "resource", None), "uri", None + ) + if uri: + details.append(f"uri={uri}") + for size_attr in ("size", "sizeInBytes"): + size = getattr(block, size_attr, None) + if isinstance(size, int): + details.append(f"size={size}") + break + name = getattr(block, "name", None) + if name and isinstance(name, str): + details.append(f"name={name}") + return f"[MCP content dropped: unsupported block ({', '.join(details)})]" + + def _render_mcp_resource_block(block, server_name: str = "") -> str: """Render an MCP ``ResourceLink`` or ``EmbeddedResource`` block as text. @@ -2800,7 +3085,7 @@ class MCPServerTask: # is currently owned by another server. if registry.get_toolset_for_tool(tool_name) != toolset_name: continue - registry.deregister(tool_name) + registry.deregister(tool_name, scope=_server_registry_scope(self.name)) _forget_mcp_tool_server(tool_name) # 3. Re-register with the fresh list. The helper may skip names that @@ -2818,7 +3103,7 @@ class MCPServerTask: for tool_name in old_tool_names - registered_name_set: if registry.get_toolset_for_tool(tool_name) != toolset_name: continue - registry.deregister(tool_name) + registry.deregister(tool_name, scope=_server_registry_scope(self.name)) _forget_mcp_tool_server(tool_name) self._registered_tool_names = registered_names @@ -2864,21 +3149,44 @@ class MCPServerTask: await asyncio.wait_for(self.session.send_ping(), timeout=30.0) return except Exception as exc: - # Only a "method not found" means ping is unsupported. Any - # other error (timeout, closed transport, session expired) is - # a real liveness failure — propagate so we reconnect. - if not _is_method_not_found_error(exc): + if _is_method_not_found_error(exc): + # Structural -32601 or "Unknown method" — ping is + # definitively unsupported. + if not self._advertises_tools(): + raise + self._ping_unsupported = True + logger.info( + "MCP server '%s': does not implement the optional " + "'ping' utility (-32601); using 'list_tools' for " + "keepalive on this connection.", + self.name, + ) + elif isinstance(exc, (TimeoutError, asyncio.TimeoutError)) and self._advertises_tools(): + # A server that silently drops ping (no response at all) + # produces a TimeoutError indistinguishable from a dead + # transport. Before declaring it dead, try list_tools as + # a confirmation probe (#97245). If the transport is + # genuinely broken, list_tools will also fail and we + # propagate that failure. + try: + await asyncio.wait_for(self.session.list_tools(), timeout=30.0) + except Exception: + # Both probes failed — genuine liveness failure. + raise exc from None + # Transport alive, ping just isn't answered. Latch the + # fallback so subsequent keepalives skip the 30s wait. + self._ping_unsupported = True + logger.info( + "MCP server '%s': ping timed out but list_tools " + "succeeded — server silently drops ping; using " + "'list_tools' for keepalive on this connection.", + self.name, + ) + return + else: + # Any other error (closed transport, session expired, + # etc.) is a real liveness failure — propagate. raise - if not self._advertises_tools(): - # No ping, no tools → no cheaper probe to fall back to. - raise - self._ping_unsupported = True - logger.info( - "MCP server '%s': does not implement the optional 'ping' " - "utility (-32601); using 'list_tools' for keepalive on " - "this connection.", - self.name, - ) # Fallback probe for servers without ping support. await asyncio.wait_for(self.session.list_tools(), timeout=30.0) @@ -3213,9 +3521,10 @@ class MCPServerTask: # it with a wall-clock timeout so a stalled SSL handshake can't freeze # MCP discovery / gateway startup (#29184). The check is fail-open, so # on timeout we log and proceed rather than blocking indefinitely. - # NOTE: must run against the REAL command/args — the watchdog wrap - # below rewrites argv to `python -m tools.mcp_stdio_watchdog …`, - # which would silently turn the preflight into a no-op. + # NOTE: must run against the REAL command/args. Anything that rewrites + # argv to point at a wrapper or a resolved binary has to happen AFTER + # this call, or the preflight silently inspects the wrapper instead of + # the package and becomes a no-op. from tools.osv_check import check_package_for_malware try: malware_error = await asyncio.wait_for( @@ -3234,17 +3543,24 @@ class MCPServerTask: f"MCP server '{self.name}': {malware_error}" ) - # Wrap the real command in a parent-death watchdog supervisor so an - # ungraceful exit of this Hermes process (kill -9, crash, force-quit) - # can't leave the stdio MCP child (and its own descendants, e.g. - # mcp-remote's spawned `node`) running forever. On a clean exit, - # MCPServerTask.shutdown() / _kill_orphaned_mcp_children() still do - # the reaping as before -- this only covers the case where that code - # never gets to run. POSIX-only (relies on process groups); no-op - # elsewhere, matching existing killpg-based cleanup's platform scope. - # Applied AFTER the OSV preflight so the check inspects the real - # package, not the watchdog wrapper. - command, args = _wrap_command_with_watchdog(command, args) + # npx resolves the package and then FORKS, staying resident as the + # real server's parent for nothing (~48 MB per MCP server, measured). + # Hermes already supervises the child (shared death supervisor), so + # when the package is cached we spawn its binary directly and drop + # that middle process. + # Deliberately AFTER the OSV preflight: the check keys off the command + # basename being `npx`, so swapping first would silently turn the + # malware gate into a no-op. Cache miss leaves npx untouched. + if os.path.basename(command).lower().startswith("npx"): + cached = _npx_cached_bin(args) + if cached: + direct_command, direct_args = cached + logger.debug( + "MCP server '%s': using cached npx binary %s (skipping the " + "resident `npm exec` parent)", + self.name, direct_command, + ) + command, args = direct_command, direct_args server_params = StdioServerParameters( command=command, @@ -3310,9 +3626,17 @@ class MCPServerTask: for _pid in new_pids: try: new_pgids[_pid] = os.getpgid(_pid) - except (AttributeError, ProcessLookupError, OSError): + except ProcessLookupError: + # The child raced and already exited. The MCP SDK + # spawns stdio servers with start_new_session=True, + # so the child was its own group leader (pgid == + # pid); keep that group covered rather than drop + # it -- any descendant it left behind still has + # to be reaped, and the prune forgets the group + # once nothing in it is alive. + new_pgids[_pid] = _pid + except (AttributeError, OSError): # AttributeError: Windows (os.getpgid is POSIX-only) - # ProcessLookupError: child raced and already exited pass with _lock: for _pid in new_pids: @@ -3335,6 +3659,14 @@ class MCPServerTask: _pid, exc_info=True, ) + # Hand the pgroups to the shared parent-death supervisor so + # an ungraceful exit of this process (kill -9, crash, + # force-quit) can't leave this server -- or its own + # descendants, e.g. mcp-remote's spawned `node` -- running + # forever. The graceful paths (MCPServerTask.shutdown, + # _kill_orphaned_mcp_children) still reap as before; this + # only covers the case where they never get to run. + _update_death_supervisor("register", new_pgids.values()) # Track the spawned children on the connection object for # fast-fail of in-flight calls when the subprocess dies # (#81995). @@ -3387,6 +3719,11 @@ class MCPServerTask: if new_pids: from gateway.status import _pid_exists _killpg = getattr(os, "killpg", None) + # Groups with nothing left alive; the supervisor is told to + # forget them after the lock is released. Groups that ARE still + # alive stay registered on purpose, so the supervisor still + # reaps them if this process dies before the orphan sweep runs. + released_pgids: list = [] with _lock: for _pid in new_pids: _stdio_pids.pop(_pid, None) @@ -3412,7 +3749,10 @@ class MCPServerTask: else: # Nothing left to reap — drop the pgid entry so # PID-reuse can't surface stale pgroup state later. - _stdio_pgids.pop(pid, None) + dropped = _stdio_pgids.pop(pid, None) + if dropped is not None: + released_pgids.append(dropped) + _update_death_supervisor("unregister", released_pgids) # Content types a real MCP Streamable-HTTP endpoint may return on the # initial POST/GET. Anything else on a 2xx response means the URL is not @@ -4451,7 +4791,7 @@ class MCPServerTask: from tools.registry import registry for tool_name in list(getattr(self, "_registered_tool_names", [])): - registry.deregister(tool_name) + registry.deregister(tool_name, scope=_server_registry_scope(self.name)) _forget_mcp_tool_server(tool_name) self._registered_tool_names = [] @@ -4479,6 +4819,10 @@ class MCPServerTask: # --------------------------------------------------------------------------- _servers: Dict[str, MCPServerTask] = {} +# Profile registry scope that owns each live connection (None outside +# multiplex). A multiplexed /reload-mcp tears down only its own profile's +# servers; process shutdown still takes everything. +_server_scope_keys: Dict[str, Optional[str]] = {} _server_connecting: set[str] = set() _server_connect_errors: Dict[str, str] = {} # Lazy MCP startup (#56832): servers whose tools were registered from the @@ -5234,6 +5578,123 @@ def _handle_session_expired_and_retry( return None +class _StdioChildExited(RuntimeError): + """A server's stdio subprocess was gone when (or while) a call ran. + + Deliberately NOT a TimeoutError: nothing timed out — the child was + already dead, usually because a gateway restart killed every MCP stdio + subprocess out from under a still-live agent session. The old wording + ("failing the call fast instead of waiting 300s") sent an investigation + into the remote server for an afternoon; the server was healthy. + + Handled by :func:`_handle_stdio_child_exited_and_retry`, which respawns + and retries the call once before any error reaches the model. + """ + + +def _handle_stdio_child_exited_and_retry( + server_name: str, + exc: Exception, + retry_call, + op_description: str, +): + """Respawn a dead stdio child and retry the call once. + + A gateway restart kills every MCP stdio subprocess. An agent session that + outlives the restart still holds the dead child, so its next tool call + used to fail in 0.00s — before anything reached the network — while the + subprocess was respawned seconds later. Cron runs spanning a restart lost + tool calls this way, silently. + + Why retrying here cannot hot-cycle respawns: this function never spawns + anything. It sets ``_reconnect_event`` (one signal, same as before) and + waits for the server task to publish a fresh session. Spawn frequency + stays governed entirely by ``run()``'s rapid-drop budget, which parks a + transport that keeps dropping without proving healthy (#62212). The retry + is single-shot: a child that dies again immediately reports and stops, + so a genuinely broken server converges on the park instead of looping. + + Returns: + A JSON string when this was a dead-stdio failure (retry result, or a + clean error), or ``None`` when ``exc`` is something else and the + caller should use its generic error path. + """ + if not isinstance(exc, _StdioChildExited): + return None + + with _lock: + srv = _servers.get(server_name) + + reconnected = False + if srv is not None and hasattr(srv, "_reconnect_event"): + logger.info( + "MCP server '%s': %s found the stdio subprocess dead (%s); " + "respawning and retrying once.", + server_name, op_description, exc, + ) + loop = _mcp_loop + if loop is not None and loop.is_running(): + reconnected = _signal_reconnect_and_wait( + server_name, + srv, + op_description=op_description, + timeout=_STDIO_RESPAWN_WAIT_SEC, + ) + else: + # No MCP loop to wait on (non-async adapters, tests) — still ask + # for the respawn so the next call lands on a live transport. + _signal_reconnect(srv) + + if reconnected: + try: + result = retry_call() + except _StdioChildExited as retry_exc: + # Respawned and died again straight away: this is a broken + # server, not a restart artifact. Stop here — run()'s budget + # takes it to the park. + logger.warning( + "MCP server '%s': %s stdio subprocess exited again right " + "after respawn (%s); not retrying further.", + server_name, op_description, retry_exc, + ) + _bump_server_error(server_name) + return tool_error( + f"MCP server '{server_name}' respawned its stdio subprocess " + f"and it exited again immediately. The server is not " + f"starting cleanly — do NOT retry this tool; ask the user to " + f"check the server's command and its stderr log." + ) + except Exception as retry_exc: + logger.warning( + "MCP %s/%s retry after stdio respawn failed: %s", + server_name, op_description, retry_exc, + ) + _bump_server_error(server_name) + return tool_error(_sanitize_error( + f"MCP call failed after respawning the stdio subprocess for " + f"'{server_name}': {type(retry_exc).__name__}: " + f"{_exc_str(retry_exc)}" + )) + try: + parsed = json.loads(result) + if "error" not in parsed: + _reset_server_error(server_name) + else: + _bump_server_error(server_name) + except (json.JSONDecodeError, TypeError): + _reset_server_error(server_name) + return result + + _bump_server_error(server_name) + return tool_error( + f"MCP server '{server_name}' stdio subprocess had exited (this is " + f"not a timeout — the call never reached the server). A respawn was " + f"requested but no fresh session came back within " + f"{_STDIO_RESPAWN_WAIT_SEC:.0f}s. Wait a few seconds before retrying; " + f"if it keeps failing the server is not starting and needs the user." + ) + + # Exact raw server names whose ``supports_parallel_tool_calls`` config is True. # Raw identity matters: distinct names such as ``foo-bar`` and ``foo_bar`` both # sanitize to ``foo_bar`` but must not share policy. @@ -5253,6 +5714,36 @@ _mcp_thread: Optional[threading.Thread] = None # _parallel_safe_servers, _mcp_tool_server_names, and _stdio_pids. _lock = threading.Lock() + +def _mcp_registry_scope() -> Optional[str]: + """Registry scope owning MCP registrations made from the current context. + + Under a profile multiplexer each profile's MCP tools live in that + profile's registry overlay (the same overlay its plugins use) so two + profiles' servers never share one process-global slot. Single-profile + processes keep MCP tools process-global (``None``). + """ + from agent.secret_scope import is_multiplex_active + + if not is_multiplex_active(): + return None + from tools.registry import registry + + return registry.current_scope_key() + + +def _server_registry_scope(name: str) -> Optional[str]: + """Scope owning server *name*'s tools: recorded at connect, else current. + + Teardown paths run on the MCP loop (process exit, reconnect exhaustion), + which does not carry the discovering profile's context, so the scope + captured when the server was adopted into ``_servers`` is authoritative. + """ + if name in _server_scope_keys: + return _server_scope_keys[name] + return _mcp_registry_scope() + + # --------------------------------------------------------------------------- # Cross-process MCP discovery guard # --------------------------------------------------------------------------- @@ -5410,11 +5901,26 @@ def _snapshot_child_pids() -> set: """ my_pid = os.getpid() - # Linux: read from /proc + # Linux: read from /proc. ``/proc//task//children`` is + # per-THREAD — a child forked from thread T is listed only under T's + # task dir. stdio_client() spawns from the background MCP loop thread, + # so reading only the main thread's file (``task//children``) + # returned an empty set on every Linux install and left + # ``_stdio_child_pids`` / ``_stdio_pids`` empty: the #81995 dead-child + # fast-fail, the #96452 respawn signal, and the killpg shutdown sweep + # never saw the subprocess. Union the children of every task instead. try: - children_path = f"/proc/{my_pid}/task/{my_pid}/children" - with open(children_path, encoding="utf-8-sig") as f: - return {int(p) for p in f.read().split() if p.strip()} + task_dir = f"/proc/{my_pid}/task" + tids = os.listdir(task_dir) + found: set = set() + for tid in tids: + try: + with open(f"{task_dir}/{tid}/children", encoding="utf-8-sig") as f: + found.update(int(p) for p in f.read().split() if p.strip()) + except (FileNotFoundError, OSError, ValueError): + # Thread exited between listdir and open — skip it. + continue + return found except (FileNotFoundError, OSError, ValueError): pass @@ -5969,7 +6475,7 @@ def _ensure_lazy_server_connected(server_name: str) -> bool: from tools.registry import registry for tool_name in phantom_names: - registry.deregister(tool_name) + registry.deregister(tool_name, scope=_server_registry_scope(server_name)) _forget_mcp_tool_server(tool_name) logger.info( "MCP server '%s': deregistered %d phantom cached tool(s) not " @@ -6166,27 +6672,19 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): and _stdio_dead_result ): # Dead children but stale server.session, so the - # transport-down path above never fired — signal the - # server task to respawn and return a clean - # reconnecting error. No explicit _bump_server_error: - # the error return flows through the handler's JSON - # parse, which already bumps once. - if _signal_reconnect(server): - return tool_error( - f"MCP server '{server_name}' stdio subprocess is " - f"dead and reconnect was requested. Do NOT retry " - f"immediately — give it a few seconds to respawn." - ) - raise TimeoutError( - f"MCP stdio subprocess for '{server_name}' has " - f"exited; failing the call fast instead of " - f"waiting {float(tool_timeout):.0f}s" + # transport-down path above never fired. Hand this to + # the handler's respawn-and-retry path — + # it is not a timeout, and a gateway restart that + # killed the child must not cost the caller a call. + raise _StdioChildExited( + f"MCP stdio subprocess for '{server_name}' had " + f"already exited when the call was dispatched" ) _call_coro = server.session.call_tool(tool_name, arguments=args) _watch_children = getattr(server, "_watch_stdio_children", None) _watch_ok = ( _watch_children is not None - and inspect.isawaitable(_watch_children()) + and inspect.iscoroutinefunction(_watch_children) and asyncio.iscoroutine(_call_coro) ) if not _watch_ok: @@ -6216,16 +6714,13 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): # Same stale-session problem as the pre-call # gate above: the subprocess died mid-call but # nothing clears server.session, so without a - # reconnect signal the server would stay dead - # until the idle keepalive probe notices. - _signal_reconnect(server) - raise TimeoutError( - f"MCP stdio subprocess for '{server_name}' " - f"exited mid-call; failing the call fast " - f"instead of waiting " - f"{float(tool_timeout):.0f}s; reconnect " - f"requested — give it a few seconds to " - f"respawn before retrying" + # reconnect the server would stay dead until + # the idle keepalive probe notices. The + # handler's respawn-and-retry path owns the + # reconnect signal. + raise _StdioChildExited( + f"MCP stdio subprocess for " + f"'{server_name}' exited mid-call" ) result = await rpc_task finally: @@ -6275,17 +6770,29 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): # Hermes' MEDIA tag + cache_image_from_bytes) was the cleaner of # the two — plugs into existing infrastructure. parts: List[str] = [] + # Count only *real* rendered content toward the + # content-vs-structuredContent arbitration below — drop notices + # for unsupported block types are appended to ``parts`` so the + # model knows content went missing, but they must not suppress + # a structuredContent fallback on their own. + usable_parts = 0 for block in (result.content or []): if hasattr(block, "text") and block.text: parts.append(strip_unicode_tags(block.text)) + if block.text.strip(): + # Whitespace-only text renders but is not usable + # content for arbitration purposes (kimi-code#3234). + usable_parts += 1 continue image_tag = _cache_mcp_image_block(block) if image_tag: parts.append(image_tag) + usable_parts += 1 continue audio_tag = _cache_mcp_audio_block(block) if audio_tag: parts.append(audio_tag) + usable_parts += 1 continue # ResourceLink / EmbeddedResource blocks (PDFs, archives, # office docs, ...). Previously these were silently dropped, @@ -6294,6 +6801,7 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): resource_text = _render_mcp_resource_block(block, server_name) if resource_text: parts.append(resource_text) + usable_parts += 1 continue # Benign empty renders (empty text blocks, empty text # resources, audio in a process without the gateway cache) @@ -6310,16 +6818,31 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): "MCP %s: dropping unsupported content block type %r", server_name, block_type, ) + # Surface the drop to the MODEL, not just the log + # (ported from MoonshotAI/kimi-code#3227): a silent + # drop leaves the agent believing the tool returned + # less than it did, with no way to recover. Carry + # whatever handles the block exposes (mime, uri) so + # the agent can fetch the content itself. + parts.append(_render_mcp_dropped_block_notice(block, block_type)) text_result = "\n".join(parts) if parts else "" # Hard-cap pathological payloads before they propagate (#56059); # ordinary large results pass untouched to the spillover layer. text_result = _truncate_mcp_text_result(text_result) - # Combine content + structuredContent when both are present. - # MCP spec: content is model-oriented (text), structuredContent - # is machine-oriented (JSON metadata). For an AI agent, content - # is the primary payload; structuredContent supplements it. + # content and structuredContent are ALTERNATIVES — never both + # forwarded (ported from MoonshotAI/kimi-code#3234). Spec-following + # servers already render their data into content (the verbatim + # dual-emit SHOULD, or a faithful human reorganisation), so + # forwarding both sent the same information to the model twice. + # content wins whenever it rendered anything usable; there is no + # reliable signal that the structured payload is richer than what + # the server put in content (semantic equality misses faithful + # reorganisations, size ratios misjudge both directions), so no + # heuristic is attempted. structuredContent fills in only when + # the content blocks rendered effectively empty, which keeps + # structuredContent-only servers working. # # Server-level `_meta` is also surfaced (ported from # MoonshotAI/kimi-code#2596): servers return namespaced metadata @@ -6346,6 +6869,11 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): if _structured_json is not None and len(_structured_json) > _MCP_HARD_RESULT_CAP_CHARS: structured = _truncate_mcp_text_result(_structured_json) meta = _strip_reserved_meta_keys(mcp_field(result, "meta", "meta")) + # Arbitration (kimi-code#3234): forward structuredContent only + # when the content blocks rendered nothing usable. Drop notices + # appended above do not count as usable content. + if structured is not None and usable_parts > 0: + structured = None if structured is not None or meta is not None: payload: Dict[str, Any] = {} if text_result: @@ -6385,6 +6913,16 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): except InterruptedError: return _interrupted_call_result() except Exception as exc: + # Dead stdio child: respawn and retry once before any + # error reaches the model — a gateway restart kills every MCP + # subprocess, and the call it lands on is not really a failure. + recovered = _handle_stdio_child_exited_and_retry( + server_name, exc, _call_once, + f"tools/call {tool_name}", + ) + if recovered is not None: + return recovered + # Auth-specific recovery path: consult the manager, signal # reconnect if viable, retry once. Returns None to fall # through for non-auth exceptions. @@ -7362,6 +7900,7 @@ def _register_server_tools(name: str, server: MCPServerTask, config: dict) -> Li check_fn=candidate["check_fn"], is_async=False, description=candidate["schema"]["description"], + scope=_server_registry_scope(name), ) # The pre-check above is advisory only. Multiple servers connect in @@ -7519,6 +8058,7 @@ def _register_from_cache_sync(name: str, config: dict, entry: dict) -> List[str] check_fn=check_fn, is_async=False, description=schema["description"], + scope=_mcp_registry_scope(), ) if registry.get_toolset_for_tool(registry_name) != toolset_name: continue @@ -7552,6 +8092,7 @@ def _register_from_cache_sync(name: str, config: dict, entry: dict) -> List[str] check_fn=check_fn, is_async=False, description=schema.get("description") or "", + scope=_mcp_registry_scope(), ) if registry.get_toolset_for_tool(util_name) != toolset_name: continue @@ -7609,6 +8150,7 @@ async def _discover_and_register_server(name: str, config: dict) -> List[str]: # self-probe, so adopt it into the registry for shutdown/revival. with _lock: _servers[name] = server + _server_scope_keys[name] = _mcp_registry_scope() elif server is not None: await server.shutdown() raise @@ -7619,6 +8161,7 @@ async def _discover_and_register_server(name: str, config: dict) -> List[str]: _server_connecting.discard(name) _server_connect_errors.pop(name, None) _servers[name] = server + _server_scope_keys[name] = _mcp_registry_scope() registered_names = _register_server_tools(name, server, config) server._registered_tool_names = list(registered_names) @@ -7850,7 +8393,7 @@ def register_mcp_servers(servers: Dict[str, dict]) -> List[str]: return _existing_tool_names() -def discover_mcp_tools() -> List[str]: +def discover_mcp_tools(allowed_mcp_names: Optional[List[str]] = None) -> List[str]: """Entry point: load config, connect to MCP servers, register tools. Called from ``model_tools`` after ``discover_builtin_tools()``. Safe to call even when @@ -7859,6 +8402,19 @@ def discover_mcp_tools() -> List[str]: Idempotent for already-connected servers. If some servers failed on a previous call, only the missing ones are retried. + Args: + allowed_mcp_names: If provided, only spawn MCP servers whose names + appear in this list. Built-in toolset names (e.g. "web", "memory") + in the list are ignored — only matching MCP-server names trigger + spawning. Pass ``None`` (default) to spawn all configured servers + for backwards compatibility. + + This is used by ``hermes -z -t `` to skip cold-starting + MCP subprocesses that the caller doesn't need — saving 10-60s of + startup wait per non-needed server. The full set of MCP names is + still discoverable via the ``-t`` validation path; this filter + only affects which servers are actually started. + Returns: List of all registered MCP tool names. """ @@ -7867,8 +8423,28 @@ def discover_mcp_tools() -> List[str]: logger.debug("No MCP servers configured") return [] + if allowed_mcp_names is not None: + # Filter by MCP-server-name match. Built-in toolset names that aren't + # MCP servers will simply not match — that's fine; they don't need + # MCP spawning anyway. + allowed_set = {str(n) for n in allowed_mcp_names} + filtered = {name: cfg for name, cfg in servers.items() if name in allowed_set} + skipped_count = len(servers) - len(filtered) + if skipped_count: + logger.debug( + "MCP discovery filter: spawning %d/%d configured server(s) per --toolsets filter " + "(skipped: %s)", + len(filtered), len(servers), + ",".join(sorted(set(servers) - set(filtered))), + ) + servers = filtered + if not servers: + logger.debug("No MCP servers in --toolsets filter; skipping MCP load entirely") + return [] + # SDK import is deferred to HERE so a config with zero MCP servers (the - # default) never pays the ~260ms `mcp` import on CLI startup. + # default) — or a -t/--toolsets filter that keeps none — never pays the + # ~260ms `mcp` import on CLI startup. if not _ensure_mcp_sdk(): logger.debug("MCP SDK not available -- skipping MCP tool discovery") return [] @@ -8144,6 +8720,7 @@ def refresh_agent_mcp_tools( disabled_override=None, quiet_mode: bool = True, content_aware: bool = False, + preserve_prefix: bool = False, ) -> set: """Re-derive an already-built agent's tool snapshot from the live registry. @@ -8172,6 +8749,22 @@ def refresh_agent_mcp_tools( under ``_agent_tools_lock`` so a concurrent reader never sees a cross-attribute half-swap. + ``preserve_prefix`` is for the callers that rebuild inside a live + conversation (the between-turns prologue). There the tool array is a + cached request prefix: every provider that renders ``tools`` ahead of the + messages re-prefills the entire history behind any byte that moves. A + plain rebuild moves two kinds of bytes — it drops a tool whose ``check_fn`` + merely flapped (a headless browser probe, an expired credential, a docker + blip), and it splices a late-landing tool into sorted position, which can + be index 0. With ``preserve_prefix`` the live order is authoritative: + existing tools keep their slot (schemas still refresh), a tool that is + still *registered* but momentarily unavailable is carried forward, a tool + that genuinely left the registry is still dropped, and new tools are + appended at the tail so the prefix only ever grows. Carrying an + unavailable tool forward changes nothing about dispatch — ``check_fn`` + gates exposure at snapshot time, never invocation, and every handler + already owns its own unavailability error. + Returns the set of newly-added tool names (empty when nothing changed), so callers can decide whether to notify the user / re-emit session info. The caller owns the prompt-cache contract: this helper does NOT check turn state, @@ -8227,6 +8820,18 @@ def refresh_agent_mcp_tools( # this rebuild actually appended (matching agent_init's dedup-aware add). staged_engine_names = _reinject_post_build_tools(agent, new_defs, new_names) + # Snapshot registry membership OUTSIDE ``_agent_tools_lock`` — it is the + # only input ``preserve_prefix`` needs beyond the two tool lists, and + # taking ``registry._lock`` under the tools lock would be the first place + # in the process to nest those two. + registered_names: set = set() + if preserve_prefix: + try: + registered_names = {entry.name for entry in registry.get_all_entries()} + except Exception: # noqa: BLE001 + # Fail open to the plain rebuild rather than pinning a stale list. + preserve_prefix = False + # Single atomic read-diff-publish so the returned ``added`` is consistent # with what was actually published, even under concurrent callers, and a # stale (older-generation) rebuild can't overwrite a newer published one. @@ -8240,10 +8845,12 @@ def refresh_agent_mcp_tools( if snapshot_generation < published_gen: # A newer snapshot already won; our set is stale — drop it. return set() - current = { - t["function"]["name"] - for t in (getattr(agent, "tools", None) or []) - } + current_defs = list(getattr(agent, "tools", None) or []) + current = {t["function"]["name"] for t in current_defs} + if preserve_prefix: + new_defs, new_names = _merge_preserving_prefix( + current_defs, new_defs, registered_names, + ) if new_names == current: # Same NAME set. For MCP-reload callers that is "no change" — # leave the live snapshot untouched (no churn). Content-aware @@ -8278,7 +8885,117 @@ def refresh_agent_mcp_tools( engine_names.clear() engine_names.update(staged_engine_names) agent._tool_snapshot_generation = max(published_gen, snapshot_generation) - return new_names - current + added = new_names - current + # Every published snapshot re-pins the session's tool order so a later + # rebuild-for-existing-session (gateway agent-cache eviction) restores + # exactly these names — see ``restore_agent_tool_prefix``. + persist_agent_tool_names(agent) + return added + + +def reprobe_tool_availability() -> None: + """Explicit ``/reload-mcp`` hatch out of the tools[] freeze. + + Availability-gated tools (``check_fn``: Docker, HASS_TOKEN, OAuth…) are + frozen for the life of a session; a credential or daemon that appears + mid-session is only picked up when the user consciously asks. Drop the + ``check_fn`` verdict cache AND the ``get_tool_definitions`` memo (keyed on + registry generation, so it would otherwise replay the stale verdicts). + """ + from model_tools import _clear_tool_defs_cache + from tools.registry import invalidate_check_fn_cache + + invalidate_check_fn_cache() + _clear_tool_defs_cache() + + +def persist_agent_tool_names(agent) -> None: + """Best-effort: write ``agent.tools`` names to the session row (freeze pin).""" + db = getattr(agent, "_session_db", None) + session_id = getattr(agent, "session_id", None) + if not db or not session_id: + return + try: + db.update_session_tool_names( + session_id, + [t["function"]["name"] for t in (getattr(agent, "tools", None) or [])], + ) + except Exception: # noqa: BLE001 + logger.debug("tool_names persist skipped", exc_info=True) + + +def restore_agent_tool_prefix(agent, saved_names: list) -> bool: + """Fold a freshly built agent's ``tools`` onto the session's saved order. + + Closes the second door on the tools[] freeze: the gateway rebuilds a NEW + ``AIAgent`` for an existing session after agent-cache eviction, and + ``agent_init`` re-derives ``agent.tools`` from live ``check_fn`` probes + with no predecessor to preserve. The saved name list stands in for that + predecessor: a saved tool that is still registered but failed its probe + this time is carried forward from the registry's schema, a deregistered + one is dropped, and genuinely new tools append at the tail — the same + ``_merge_preserving_prefix`` rule the between-turns refresh uses. + Returns True when the snapshot was changed. + """ + if not saved_names: + return False + from tools.registry import registry + + fresh_defs = list(getattr(agent, "tools", None) or []) + fresh = {t["function"]["name"]: t for t in fresh_defs} + saved_defs = [] + for name in saved_names: + entry_def = fresh.get(name) + if entry_def is None: + entry = registry.get_entry(name) + if entry is None: + continue + entry_def = {"type": "function", "function": {**entry.schema, "name": entry.name}} + saved_defs.append(entry_def) + registered_names = {entry.name for entry in registry.get_all_entries()} + merged, merged_names = _merge_preserving_prefix(saved_defs, fresh_defs, registered_names) + with _agent_tools_lock: + if merged == fresh_defs: + return False + agent.tools = merged + agent.valid_tool_names = merged_names + if [t["function"]["name"] for t in merged] != list(saved_names): + persist_agent_tool_names(agent) + return True + + +def _merge_preserving_prefix( + current_defs: list, new_defs: list, registered_names: set, +) -> tuple[list, set]: + """Fold a fresh tool snapshot into a live one without moving existing bytes. + + The live tool array is a cached request prefix, so the merge is ordered by + ``current_defs``, not by the fresh list: + + * a name in both keeps its slot and takes the fresh schema (dynamic + overrides — delegate_task limits, execute_code stubs — still land); + * a name only in the live list is carried forward when it is still + registered (its ``check_fn`` flapped) and dropped when it is not (the + MCP server or plugin genuinely went away); + * a name only in the fresh list is appended at the tail, so a late-landing + MCP tool extends the prefix instead of splicing into sorted position. + """ + fresh = {} + for entry in new_defs: + name = (entry.get("function") or {}).get("name", "") + if name: + fresh[name] = entry + + merged = [] + for entry in current_defs: + name = (entry.get("function") or {}).get("name", "") + replacement = fresh.pop(name, None) + if replacement is not None: + merged.append(replacement) + elif name and name in registered_names: + merged.append(entry) + merged.extend(fresh.values()) + return merged, {(t.get("function") or {}).get("name", "") for t in merged} def _reinject_post_build_tools(agent, tools_list: list, name_set: set) -> set: @@ -8350,15 +9067,24 @@ def _reinject_post_build_tools(agent, tools_list: list, name_set: set) -> set: return staged_engine_names -def shutdown_mcp_servers(): - """Close all MCP server connections and stop the background loop. +def shutdown_mcp_servers(*, scope: Optional[str] = None): + """Close MCP server connections and stop the background loop. Each server Task is signalled to exit its ``async with`` block so that the anyio cancel-scope cleanup happens in the same Task that opened it. All servers are shut down in parallel via ``asyncio.gather``. + + ``scope`` (a registry scope key) restricts teardown to the servers one + multiplexed profile owns — its ``/reload-mcp`` must not kill the other + profiles' connections — and leaves the shared loop running when anything + else is still connected. Without it every server goes, as before. """ with _lock: - servers_snapshot = list(_servers.values()) + selected = [ + name for name in _servers + if scope is None or _server_scope_keys.get(name) == scope + ] + servers_snapshot = [_servers[name] for name in selected] # Fast path: nothing to shut down. The connect-cooldown maps can still # be populated here — a server that failed to connect is never recorded @@ -8370,7 +9096,7 @@ def shutdown_mcp_servers(): with _lock: _server_connect_retry_after.clear() _server_connect_failures.clear() - _stop_mcp_loop() + _stop_mcp_loop(only_if_idle=scope is not None) return async def _shutdown(): @@ -8384,7 +9110,9 @@ def shutdown_mcp_servers(): "Error closing MCP server '%s': %s", server.name, result, ) with _lock: - _servers.clear() + for name in selected: + _servers.pop(name, None) + _server_scope_keys.pop(name, None) # Drop connect-retry cooldowns too: a full shutdown/restart # should re-attempt every server immediately, not honour a # stale per-server backoff from before the restart (#50394). @@ -8414,7 +9142,7 @@ def shutdown_mcp_servers(): _server_connect_retry_after.clear() _server_connect_failures.clear() - _stop_mcp_loop() + _stop_mcp_loop(only_if_idle=scope is not None) def _kill_orphaned_mcp_children( @@ -8544,6 +9272,10 @@ def _kill_orphaned_mcp_children( pid, server_name, ) + # These groups are reaped. Release them last, so a crash partway through + # the SIGTERM/SIGKILL dance still leaves the supervisor holding them. + _update_death_supervisor("unregister", pgids.values()) + def _stop_mcp_loop_if_idle() -> bool: """Stop the MCP loop only when no registered server still owns it. diff --git a/tools/memory_tool.py b/tools/memory_tool.py index 54dcaa71a6..eccbe82674 100644 --- a/tools/memory_tool.py +++ b/tools/memory_tool.py @@ -1271,10 +1271,12 @@ MEMORY_SCHEMA = { "reports current/limit chars and confirms completion; one batch call finishes the " "update, so don't repeat it. Use the bare action/content/old_text fields only for a " "single lone change.\n\n" - "WHEN: save proactively when the user states a preference, correction, or personal " - "detail, or you learn a stable fact about their environment, conventions, or workflow. " - "Priority: user preferences & corrections > environment facts > procedures. The best " - "memory stops the user repeating themselves.\n\n" + "WHEN: only for facts that apply to EVERY session regardless of task: who the user " + "is, stable environment facts, standing conventions with no task home. Anything " + "learned while doing a task (procedures, pitfalls, and the user's preferences and " + "corrections for that kind of work) belongs in the task's skill via skill_manage, " + "where it loads only when relevant; memory is injected into every turn and must " + "stay small.\n\n" "IF FULL: an add is rejected with the current entries shown. Reissue as ONE batch that " "removes or shortens enough stale entries and adds the new one together.\n\n" "TARGETS: 'user' = who the user is (name, role, preferences, style). 'memory' = your " diff --git a/tools/osv_check.py b/tools/osv_check.py index b543dd7f33..19acdfb065 100644 --- a/tools/osv_check.py +++ b/tools/osv_check.py @@ -9,7 +9,6 @@ Fail-open: network errors allow the package to proceed. Inspired by Block/goose's extension malware check. """ - import json import logging import os @@ -17,36 +16,150 @@ import re import threading import time import urllib.request +from pathlib import Path from typing import Optional, Tuple - logger = logging.getLogger(__name__) _OSV_ENDPOINT = os.getenv("OSV_ENDPOINT", "https://api.osv.dev/v1/query") _TIMEOUT = 10 # seconds -# Result cache: (ecosystem, package, version) -> (expiry_monotonic, result). -# MCP reconnect ladders, stdio recycles, and parked-server self-probes re-run -# the preflight for the SAME package on every spawn attempt. Without a cache, -# a flapping server turns into a sustained OSV query/DNS stream — the #75485 -# incident logged 779K api.osv.dev DNS queries in 16h from revival loops. -# Malware advisories don't appear or vanish on second-to-second timescales, -# so a successful verdict (clean OR blocked) is reusable. Network failures -# are NOT cached: fail-open already covers them, and caching a failure could -# mask a real advisory once connectivity returns. +# Result cache: (ecosystem, package, version) -> (expiry_timestamp, result). +# MCP reconnect ladders, stdio recycles, parked-server self-probes, and +# repeated `hermes mcp test` invocations re-run the preflight for the SAME +# package on every spawn attempt. Without a cache, a flapping server turns +# into a sustained OSV query/DNS stream — the #75485 incident logged 779K +# api.osv.dev DNS queries in 16h from revival loops. Malware advisories don't +# appear or vanish on second-to-second timescales, so a successful verdict +# (clean OR blocked) is reusable. Network failures are NOT cached: fail-open +# already covers them, and caching a failure could mask a real advisory once +# connectivity returns. +# +# The cache is also persisted to disk inside the Hermes home so that separate +# `hermes mcp test` processes (and gateway restarts) reuse a warm verdict +# instead of re-querying OSV. Expiry is stored as absolute wall-clock time so +# it survives process restarts and monotonic-clock skew. +# +# Trade-off: persisting *clean* verdicts means a MAL advisory published right +# after a clean query is noticed at TTL expiry (<= 1h by default) instead of +# at the next process start. The window is the same one the in-process cache +# already accepted; it just now spans restarts. Lower OSV_CHECK_CACHE_TTL to +# tighten it. _CACHE_TTL_S = float(os.getenv("OSV_CHECK_CACHE_TTL", "3600")) _CACHE_MAX_ENTRIES = 256 _cache: dict = {} _cache_lock = threading.Lock() +_disk_cache_loaded = False +_DISK_CACHE_VERSION = 1 + + +def _disk_cache_path() -> Optional[Path]: + """Return the path for the persistent OSV verdict cache. + + Uses ``hermes_constants.get_hermes_home()`` so the cache follows the + active profile and is isolated across Hermes homes. The cache directory + is created on demand. Returns ``None`` when Hermes home cannot be + resolved, in which case only the in-process cache is used. + """ + try: + from hermes_constants import get_hermes_home + + home = get_hermes_home() + except Exception: + return None + try: + cache_dir = home / "cache" + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir / "osv_check.json" + except Exception: + return None + + +def _load_disk_cache() -> None: + """Load persistent cache entries from disk into the in-process cache. + + Invoked under ``_cache_lock`` from every get/put but does real work only + once per process (``_disk_cache_loaded`` latch); a transient ``OSError`` + leaves the latch unset so the next call retries. Skips expired or + malformed entries. Only adds missing keys so an in-memory overwrite + (e.g. a test forcing expiry) is not silently reversed by the disk copy. + """ + global _disk_cache_loaded + if _disk_cache_loaded: + return + + path = _disk_cache_path() + if path is None: + _disk_cache_loaded = True + return + + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + except FileNotFoundError: + data = None + except OSError: + # Transient I/O (file busy, brief permission flap). Retry next call. + return + except Exception: + # Malformed JSON or anything else: unrecoverable, don't spin on it. + data = None + + _disk_cache_loaded = True + if not isinstance(data, dict) or data.get("version") != _DISK_CACHE_VERSION: + return + + now = time.time() + for key_str, entry in data.get("entries", {}).items(): + if not isinstance(entry, dict): + continue + expiry = entry.get("expiry") + result = entry.get("result") + if expiry is None or expiry <= now: + continue + parts = key_str.split("|", 2) + if len(parts) != 3: + continue + key = (parts[0], parts[1], parts[2] or None) + if key not in _cache: + _cache[key] = (expiry, result) + + +def _save_disk_cache() -> None: + """Persist the in-process cache to disk. + + Caller must hold ``_cache_lock`` for consistency. Writes atomically to + a sibling file then renames into place. + """ + path = _disk_cache_path() + if path is None: + return + + entries: dict = {} + for key, (expiry, result) in _cache.items(): + key_str = "|".join(str(k) if k is not None else "" for k in key) + entries[key_str] = {"expiry": expiry, "result": result} + + data = {"version": _DISK_CACHE_VERSION, "entries": entries} + + try: + # Shared atomic writer (temp file + fsync + rename); mkstemp's 0600 + # is kept on create, so verdicts never sit in a world-readable file. + from utils import atomic_write_text + + atomic_write_text(path, json.dumps(data)) + except Exception as exc: + logger.debug("Failed to save OSV disk cache to %s: %s", path, exc) def _cache_get(key) -> Tuple[bool, Optional[str]]: """Return (hit, result) for a fresh cache entry.""" with _cache_lock: + _load_disk_cache() entry = _cache.get(key) if entry is None: return False, None expiry, result = entry - if time.monotonic() >= expiry: + if time.time() >= expiry: del _cache[key] return False, None return True, result @@ -54,13 +167,15 @@ def _cache_get(key) -> Tuple[bool, Optional[str]]: def _cache_put(key, result: Optional[str]) -> None: with _cache_lock: + _load_disk_cache() if len(_cache) >= _CACHE_MAX_ENTRIES: - now = time.monotonic() + now = time.time() for k in [k for k, (exp, _) in _cache.items() if exp <= now]: del _cache[k] if len(_cache) >= _CACHE_MAX_ENTRIES: _cache.clear() # tiny working set in practice; safe reset - _cache[key] = (time.monotonic() + _CACHE_TTL_S, result) + _cache[key] = (time.time() + _CACHE_TTL_S, result) + _save_disk_cache() def check_package_for_malware( diff --git a/tools/preview_tool.py b/tools/preview_tool.py index 2c2181a39e..e3890c02d3 100644 --- a/tools/preview_tool.py +++ b/tools/preview_tool.py @@ -50,7 +50,7 @@ def _handle_preview(args, **kw): PREVIEW_SCHEMA = { "name": "desktop_preview", "description": ( - "The preview pane beside the chat in the Hermes desktop app. open: show " + "Open, close, or read the preview pane beside the chat. open: show " "a web URL (bare domains fine), a localhost dev server, or a file path " "(HTML renders live) — opens for the current window only. close: dismiss " "the whole pane, or one tab via url. read: what the pane currently shows " diff --git a/tools/process_registry.py b/tools/process_registry.py index 3f36fe34ca..9fd3058080 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -323,6 +323,35 @@ def _build_systemd_scope_argv( ] +def restart_safe_gateway_child_argv( + command: List[str], *, unit_suffix: str +) -> List[str]: + """Place a managed-systemd gateway child outside the gateway cgroup. + + Children that must survive an intentional gateway restart cannot rely on + ``start_new_session`` alone: systemd still kills every process in the + service cgroup. In that topology, require a transient user scope and fail + closed if it cannot be established. Standalone processes, non-systemd + supervisors, and non-Linux hosts retain the direct command. + """ + if not _IS_LINUX: + return command + if not _is_supervised_gateway_process() or not os.environ.get("INVOCATION_ID"): + return command + if not _systemd_run_user_scope_available(): + raise RuntimeError( + "cannot create restart-safe systemd scope for gateway child: " + "systemd-run --user --scope is unavailable" + ) + scoped = _build_systemd_scope_argv(command, unit_suffix=unit_suffix) + if scoped == command: + raise RuntimeError( + "cannot create restart-safe systemd scope for gateway child: " + "systemd-run disappeared after the availability probe" + ) + return scoped + + def _stop_systemd_unit(unit_name: str) -> bool: """Stop a transient systemd user scope by unit name. @@ -1522,6 +1551,51 @@ class ProcessRegistry: session.completion_reason = "exited" self._move_to_finished(session) + @staticmethod + def _log_delta_command(quoted_log_path: str, offset: int) -> str: + """Build the shell command that reads only new bytes from a log file. + + The old version ran ``cat`` on the whole file every poll, so a job + that keeps writing pays for its entire output again and again. Over a + long run that turns into a lot of wasted traffic on the docker/SSH + channel, since only the new part is ever used. + + The command prints one header line, ``" "``, then the + bytes between ``offset`` and ``size``. Reading the size first and + cutting the tail at that same size keeps the two numbers in step, so + a file that grows while the command runs never sends a byte twice. + A file that shrank was rotated or truncated, so the offset drops back + to 0 and the reader starts over. + + The end of the window is pulled back to a UTF-8 character boundary: + the backend decodes each ``execute()`` result on its own, so a + multibyte character straddling two polls would otherwise come back + as replacement characters (and break watch patterns near the seam). + Up to 3 trailing continuation bytes are held for the next poll; the + header reports the trimmed size so the offset stays consistent. + """ + return ( + f"O={offset}; " + f"S=$({{ wc -c < {quoted_log_path}; }} 2>/dev/null | tr -dc '0-9'); " + f"S=${{S:-0}}; " + f'if [ "$S" -lt "$O" ]; then O=0; fi; ' + # Hold back an INCOMPLETE trailing UTF-8 sequence for the next + # poll. Scan back up to 3 continuation bytes (octal 200-277) to + # the lead byte; if the lead byte's declared length (3xx=2, 34x-35x + # =3, 36x-37x=4) exceeds the bytes present, trim to before it. + # Complete sequences and ASCII tails are left untouched. + f'N=0; P=$S; while [ "$P" -gt "$O" ] && [ "$N" -lt 3 ]; do ' + f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); " + f'case "$B" in 2[0-7][0-7]) P=$((P-1)); N=$((N+1));; *) break;; esac; done; ' + f'if [ "$N" -gt 0 ] || [ "$P" -eq "$S" ]; then ' + f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); " + f'case "$B" in 3[0-3][0-7]) L=2;; 3[4-5][0-7]) L=3;; 3[6-7][0-7]) L=4;; *) L=1;; esac; ' + f'if [ "$L" -gt $((N+1)) ]; then S=$((P-1)); fi; fi; ' + f'echo "$S $O"; ' + f'if [ "$S" -gt "$O" ]; then ' + f"tail -c +$((O+1)) {quoted_log_path} 2>/dev/null | head -c $((S-O)); fi" + ) + def _env_poller_loop( self, session: ProcessSession, env: Any, log_path: str, pid_path: str, exit_path: str ): @@ -1529,24 +1603,44 @@ class ProcessRegistry: quoted_log_path = shlex.quote(log_path) quoted_pid_path = shlex.quote(pid_path) quoted_exit_path = shlex.quote(exit_path) - prev_output_len = 0 # track delta for watch pattern scanning + # Byte offset already read from the log. Bytes, not characters: the + # shell counts bytes, and a log with non-ASCII text has more bytes + # than characters. + prev_output_bytes = 0 while not session.exited: time.sleep(2) # Poll every 2 seconds try: - # Read new output from the log file - result = env.execute(f"cat {quoted_log_path} 2>/dev/null", timeout=10) - new_output = result.get("output", "") - if new_output: - # Compute delta for watch pattern scanning - delta = new_output[prev_output_len:] if len(new_output) > prev_output_len else "" - prev_output_len = len(new_output) + # Read only the bytes written since the last poll. + result = env.execute( + self._log_delta_command(quoted_log_path, prev_output_bytes), + timeout=10, + ) + raw = result.get("output", "") + header, _, delta = raw.partition("\n") + try: + size_str, offset_str = header.split() + new_size = int(size_str) + used_offset = int(offset_str) + except ValueError: + # No usable header (command failed, shell missing a tool). + # Skip this poll rather than act on a half-read value. + new_size = None + used_offset = None + delta = "" + if new_size is not None: + if used_offset < prev_output_bytes: + # The log was rotated or truncated, so what we hold no + # longer lines up with the file. Drop it and restart. + with session._lock: + session.output_buffer = "" + prev_output_bytes = new_size + if delta: with session._lock: - session.output_buffer = new_output + session.output_buffer += delta if len(session.output_buffer) > session.max_output_chars: session.output_buffer = session.output_buffer[-session.max_output_chars:] - if delta: - self._check_watch_patterns(session, delta) - self._emit_output(session, delta) + self._check_watch_patterns(session, delta) + self._emit_output(session, delta) # Check if process is still running check = env.execute( @@ -3366,13 +3460,14 @@ def format_process_notification(evt: dict) -> "str | None": from tools.registry import registry, tool_error PROCESS_SCHEMA = { - "name": "process", + "name": "process_manage", # Dieted (#95681): the action enum names the verbs; the description # keeps only non-obvious semantics. write-vs-submit is the tool's one # real trap (a lone \n on a Windows PTY is not a line terminator) — # that teaching gains emphasis rather than losing it. "description": ( - "Manage background processes started with terminal(background=true). " + "Poll, wait on, or kill background terminal processes (from " + "terminal(background=true)). " "poll: status + new output. log: full output, paged. wait: block " "until exit or timeout (partial output on timeout). write vs " "submit: submit appends Enter — use it to answer prompts; write " @@ -3487,7 +3582,7 @@ def _handle_process(args, **kw): registry.register( - name="process", + name="process_manage", toolset="terminal", schema=PROCESS_SCHEMA, handler=_handle_process, diff --git a/tools/project_tools.py b/tools/project_tools.py index dc4642a0ba..e7ff1fc90b 100644 --- a/tools/project_tools.py +++ b/tools/project_tools.py @@ -160,7 +160,7 @@ registry.register( schema={ "name": "desktop_project", "description": ( - "Desktop Projects (named workspaces). create: make one and switch " + "Create or switch desktop Projects (named workspaces). create: one and switch " "this chat into it — pass path to anchor it to a repo/folder (the " "chat's workspace moves there, the sidebar follows). switch: move " "this chat into an existing project by name/slug/id — the " diff --git a/tools/registry.py b/tools/registry.py index 0daa26461b..5261b25cdd 100644 --- a/tools/registry.py +++ b/tools/registry.py @@ -348,8 +348,33 @@ def check_fn_cache_scope() -> Optional[str]: def _run_check_fn_uncached(fn: Callable, *, unresolved_scope: bool = False) -> bool: """Run an availability check without cache/grace handling.""" + from agent.secret_scope import UnscopedSecretError + try: return bool(fn()) + except UnscopedSecretError: + if unresolved_scope: + # Expected fail-closed probe: with multiplexing on, boot-time + # check_fns run before any profile secret scope exists, so + # get_secret raises by design. The tool re-probes on the first + # scoped turn — log without a traceback so this cannot be + # mistaken for a crashed check_fn (#100697). + logger.debug( + "check_fn %s hit the multiplex fail-closed path with no " + "profile secret scope active; dependent tools re-probe on " + "the first scoped turn", + getattr(fn, "__qualname__", fn), + ) + return False + # The scope resolved but the read still failed closed: a genuinely + # lost scope. Keep the loud crash-style report. + logger.warning( + "check_fn %s raised UnscopedSecretError while the profile cache " + "scope was resolved; dependent tools will be unavailable this turn", + getattr(fn, "__qualname__", fn), + exc_info=True, + ) + return False except Exception: detail = " while profile cache scope was unresolved" if unresolved_scope else "" logger.warning( @@ -886,13 +911,18 @@ class ToolRegistry: self._toolset_checks[toolset] = check_fn self._generation += 1 - def deregister(self, name: str) -> None: + def deregister(self, name: str, *, scope: Optional[str] = None) -> None: """Remove a tool from the registry. Also cleans up the toolset check if no other tools remain in the same toolset. Used by MCP dynamic tool discovery to nuke-and-repave when a server sends ``notifications/tools/list_changed``. + ``scope`` selects a profile overlay explicitly (multiplexed MCP tools + live in the owning profile's overlay). Plugin callers keep their own + scope and may not name another one; non-plugin callers without + ``scope`` keep the historical process-global target. + Gated by the same operator opt-in policy ``register(override=True)`` enforces. Without this, a plugin could bypass that gate entirely by deregistering a tool it doesn't own and then calling plain @@ -910,14 +940,21 @@ class ToolRegistry: if caller_owner is not None else None ) + if caller_owner is not None and scope is not None and scope != caller_scope: + raise PermissionError( + f"Plugin module {caller_mod!r} cannot deregister tools " + "outside its own profile scope." + ) + if scope is None: + scope = caller_scope target = ( - self._scoped_tools.get(caller_scope, {}) - if caller_scope is not None + self._scoped_tools.get(scope, {}) + if scope is not None else self._tools ) entry = target.get(name) - if entry is None and caller_scope is not None: - if name in self._tools: + if entry is None and scope is not None: + if caller_owner is not None and name in self._tools: raise PermissionError( f"Scoped plugin module {caller_mod!r} cannot deregister " f"process-global tool {name!r}; register a scoped " @@ -957,13 +994,13 @@ class ToolRegistry: f"opt-in (allow_tool_override)." ) del target[name] - if caller_scope is not None and not target: - self._scoped_tools.pop(caller_scope, None) + if scope is not None and not target: + self._scoped_tools.pop(scope, None) # Drop the toolset check and aliases if this was the last tool in # that toolset. toolset_still_exists = any( e.toolset == entry.toolset - for e in self._merged_tools(caller_scope).values() + for e in self._merged_tools(scope).values() ) if not toolset_still_exists: self._toolset_checks.pop(entry.toolset, None) diff --git a/tools/session_search_tool.py b/tools/session_search_tool.py index 865e7f7e6c..6ac28a10f0 100644 --- a/tools/session_search_tool.py +++ b/tools/session_search_tool.py @@ -485,18 +485,24 @@ def _read_session(db, session_id: str, head: int = 20, tail: int = 10, link_prof def _list_recent_sessions(db, limit: int, current_session_id: str = None, link_profile: str = None) -> str: """Return metadata for the most recent sessions (no LLM calls, no FTS5).""" try: - # list_sessions_rich (include_children=False) already applies the - # canonical child classifier (_LISTABLE_CHILD_SQL): roots, /branch - # children, and /new-reset children are admitted (stable markers plus - # the legacy same-key heuristic), while delegation/compression - # children are hidden. Re-classifying rows here in Python duplicated - # that predicate and re-hid legacy pre-marker reset children the SQL - # deliberately admits — trust the query instead (#85756). - sessions = db.list_sessions_rich( + # Never use list_sessions_rich(order_by_last_active=True) here. That + # generic query walks every compression chain and derives activity and + # previews from messages before LIMIT; on a multi-GB state.db it can + # monopolise a gateway callback for minutes. The dedicated browse + # query preselects an indexed, bounded candidate set and carries a + # cooperative SQLite VM cancellation deadline. + bounded_list = getattr(db, "list_recent_sessions_bounded", None) + if bounded_list is None: + # Fail closed rather than silently returning to the exact + # whole-database query shape this path exists to eliminate. + raise RuntimeError( + "session database does not support bounded recent-session browse" + ) + sessions = bounded_list( limit=limit + 15, exclude_sources=list(_HIDDEN_SESSION_SOURCES), - order_by_last_active=True, - ) # fetch extra so we can skip current / compression roots + timeout_seconds=3.0, + ) current_root, has_compression_hop = ( _resolve_to_parent(db, current_session_id) @@ -1146,7 +1152,7 @@ def check_session_search_requirements() -> bool: SESSION_SEARCH_SCHEMA = { "name": "session_search", "description": ( - "Search past Hermes sessions (FTS5 over the local session DB), or read/" + "Recall past conversations: search or read old Hermes sessions (FTS5), or " "scroll inside one. Four shapes, picked by args: `query` = discovery " "(top-N matching sessions, top result fully hydrated); `session_id` + " "`around_message_id` = scroll (window of messages around an anchor); " diff --git a/tools/subagent_worktree.py b/tools/subagent_worktree.py index b09b1a55be..54611295f0 100644 --- a/tools/subagent_worktree.py +++ b/tools/subagent_worktree.py @@ -44,6 +44,8 @@ import uuid from pathlib import Path from typing import Any, Dict, Optional +from hermes_cli._subprocess_compat import harden_git_argv, noninteractive_git_env + logger = logging.getLogger(__name__) _GIT_TIMEOUT = 30 @@ -52,15 +54,24 @@ _BRANCH_NAMESPACE = "hermes-subagent" def _run_git(args, cwd: str, timeout: int = _GIT_TIMEOUT): - """Run a git command, capturing output. Never raises on non-zero exit.""" + """Run a git command, capturing output. Never raises on non-zero exit. + + Runs under :func:`noninteractive_git_env` (GHSA-7x36-8jrh-v4pw): worktree + isolation runs automatically for delegated subagents against whatever repo + the parent sits in, and ``worktree add`` runs checkout hooks. Disabling the + fsmonitor/hooks/pager/credential config sinks keeps a malicious ``.git/config`` + from executing on the host. + """ return subprocess.run( - ["git", *args], + ["git", *harden_git_argv(args)], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, + stdin=subprocess.DEVNULL, + env=noninteractive_git_env(), ) diff --git a/tools/terminal_scope.py b/tools/terminal_scope.py new file mode 100644 index 0000000000..da79538ab7 --- /dev/null +++ b/tools/terminal_scope.py @@ -0,0 +1,298 @@ +"""Per-turn terminal scope: profile-scoped TERMINAL_* policy. + +The multiplexing gateway (and the unified dashboard/TUI, and cron) serve +several Hermes profiles from one process. Terminal settings were historically +mirrored into the process-global ``os.environ`` (first writer wins), so the +first profile to touch the terminal after startup pinned its backend — and +every other setting — onto all later turns: a ``local`` profile silently +executing inside another profile's docker sandbox, or the reverse (a sandbox +escape). Mirrors the isolation seam that ``agent/secret_scope.py`` provides +for credentials: a ContextVar holds the active profile's COMPLETE effective +``TERMINAL_*`` policy, installed at each in-process profile boundary. + +Two contracts distinguish this from a plain override dict: + +- **Authoritative projection.** While a scope is bound, ``terminal_env`` + resolves ONLY from that policy (built from defined defaults + the profile's + ``.env`` + its ``config.yaml`` explicit keys). Omitted keys resolve to the + defined default — never to ambient ``os.environ`` — so a routed profile can + neither inherit nor be escaped onto the launch process's mounts, SSH + targets, or resource policy (#68559). +- **Fail closed.** If the profile's policy cannot be resolved (unreadable or + malformed ``.env``/``config.yaml``), the install raises + :class:`TerminalPolicyUnavailable` and callers must install a *refusal* + scope; terminal execution under a refusal scope is rejected outright + rather than falling back to ambient authority. +""" + +from __future__ import annotations + +import logging +from contextlib import contextmanager +from contextvars import ContextVar, Token +from pathlib import Path +from typing import Any, Dict, Iterator, Optional + +logger = logging.getLogger(__name__) + +# ``None`` = no scope bound in this context; readers use the historical +# process-env behavior (single-process CLI/TUI, unaffected surfaces). +# A dict = the active profile's complete effective terminal policy. +# A TerminalPolicyRefusal = resolution failed; terminal execution must refuse. +_terminal_scope_var: ContextVar = ContextVar("hermes_terminal_scope", default=None) + + +class TerminalPolicyUnavailable(Exception): + """The routed profile's terminal policy could not be resolved. + + Raised when the profile's ``.env`` or ``config.yaml`` exists but cannot be + read/parsed. Callers must install the returned refusal scope instead of + continuing without a scope — executing under ambient process authority is + exactly the leak this module exists to close. + """ + + +class TerminalPolicyRefusal(Dict[str, str]): + """Marker scope installed when policy resolution failed. + + An (empty) dict subclass so existing dict-typed checks keep working, with + a flag that makes ``terminal_env`` raise before any value is served. + """ + + refused = True + + def __init__(self, reason: str) -> None: + super().__init__() + self.reason = reason + + +def set_terminal_scope(mapping: Optional[Dict[str, str]]) -> Token: + """Install *mapping* as the current context's terminal policy.""" + return _terminal_scope_var.set(mapping) + + +def install_refusal_scope(reason: str) -> Token: + """Install a refusal scope after :class:`TerminalPolicyUnavailable`. + + Terminal execution under this scope is rejected (fail closed) instead of + running under the launch process's ambient policy. + """ + return _terminal_scope_var.set(TerminalPolicyRefusal(reason)) + + +def reset_terminal_scope(token: Token) -> None: + _terminal_scope_var.reset(token) + + +def get_terminal_scope() -> Optional[Dict[str, str]]: + """The active scope mapping/refusal, or ``None`` when no scope is bound.""" + return _terminal_scope_var.get() + + +@contextmanager +def terminal_scope(mapping: Optional[Dict[str, str]]) -> Iterator[None]: + """Context manager form of set/reset_terminal_scope.""" + token = set_terminal_scope(mapping) + try: + yield + finally: + reset_terminal_scope(token) + + +def terminal_env(name: str, default: str = "") -> str: + """Authoritative read of a ``TERMINAL_*`` variable. + + - No scope bound: process env, then *default* (historical single-process + behavior — CLI/TUI surfaces that never route profiles are unchanged). + - Refusal scope bound: raise — policy is unavailable and execution must + fail closed, not fall back to ambient authority. + - Policy scope bound: resolve ONLY from the policy; a missing key yields + the *default* (which callers derive from defined defaults), never + ``os.environ``. + """ + scope = _terminal_scope_var.get() + if scope is None: + import os + + return os.environ.get(name, default) + if isinstance(scope, TerminalPolicyRefusal): + raise TerminalPolicyUnavailable( + f"terminal policy unavailable for this profile: {scope.reason}" + ) + value = scope.get(name) + if value is not None: + return str(value) + return default + + +def build_profile_terminal_scope(hermes_home: "Any") -> Dict[str, str]: + """Build the COMPLETE effective ``TERMINAL_*`` policy for a profile home. + + Projection order: defined defaults (``DEFAULT_CONFIG['terminal']``) ← the + profile's ``.env`` TERMINAL_* selections ← its ``config.yaml`` explicit + ``terminal:`` keys. The result is total: every key the terminal stack can + ask for resolves from this mapping, so a bound scope never widens back to + ambient process authority. Raises :class:`TerminalPolicyUnavailable` when + either file exists but cannot be read/parsed (fail closed). + """ + home = Path(hermes_home) + + from hermes_cli.config_defaults import DEFAULT_CONFIG + + defaults = DEFAULT_CONFIG.get("terminal") if isinstance( + DEFAULT_CONFIG, dict) else None + defaults = dict(defaults) if isinstance(defaults, dict) else {} + # Terminal keys whose env mirror exists but whose config default lives in + # the consuming tool rather than DEFAULT_CONFIG. These are the documented + # tool-level defaults (tools/terminal_tool.py); without them the + # projection would not be total and reads could observe nothing (which is + # correct) OR fall back ambiently (which is not). + defaults.setdefault("cwd", ".") # per-surface placeholder + defaults.setdefault("ssh_host", "") # remote backends: unset = none + defaults.setdefault("ssh_user", "") + defaults.setdefault("ssh_port", 22) + defaults.setdefault("ssh_key", "") + defaults.setdefault("docker_orphan_reaper", True) + defaults.setdefault("docker_persist_across_processes", True) + defaults.setdefault("sandbox_dir", "") # tool derives HERMES_HOME path + defaults.setdefault("lifetime_seconds", 300) + defaults.setdefault("docker_shared_container_key", "") + defaults.setdefault("home_mode", "auto") + + scope: Dict[str, str] = {} + + def _apply(cfg_key: str, value: Any) -> None: + if value is None: + return + # cwd placeholders (".", "auto", "cwd") are resolved per-surface + # later; they are not a policy value. + if cfg_key == "cwd" and str(value).strip() in {".", "auto", "cwd"}: + return + from hermes_cli.config import TERMINAL_CONFIG_ENV_MAP + + env_var = TERMINAL_CONFIG_ENV_MAP.get(cfg_key) + if env_var: + scope[env_var] = str(value) + + # 1) Defined defaults — the total baseline. + for cfg_key, value in defaults.items(): + _apply(cfg_key, value) + + # 2) The profile's .env TERMINAL_* selections. Fail closed on unreadable + # files (missing file = no selections, fine). + env_path = home / ".env" + if env_path.exists(): + # Pre-flight readability: load_env_file swallows OSError/UnicodeError + # by design (secret scope fails soft), but an unreadable profile .env + # is a policy-resolution failure here and must fail closed. + try: + env_path.read_bytes() + except Exception as exc: + raise TerminalPolicyUnavailable( + f"cannot read {env_path}: {exc}" + ) from exc + from agent.secret_scope import load_env_file + + selections = load_env_file(env_path) + for key, value in selections.items(): + if key.startswith("TERMINAL_"): + scope[key] = str(value) + + # 3) The profile's config.yaml explicit terminal keys. Read through the + # HERMES_HOME override so the profile's own file is consulted; a + # present-but-unparseable file fails closed (matches the gateway's + # _warn_config_parse_failure posture of refusing to guess policy). + from hermes_constants import ( + get_hermes_home_override, + reset_hermes_home_override, + set_hermes_home_override, + ) + + override_token = None + if get_hermes_home_override() != str(home): + override_token = set_hermes_home_override(home) + try: + config_path = home / "config.yaml" + if config_path.exists(): + # Parse the profile's file directly rather than through + # read_raw_config(): that helper collapses "missing" and + # "unparseable" into the same {} result. Here the file's existence + # is already established, so {} can only mean a parse failure — + # which must fail closed rather than silently projecting defaults. + from hermes_cli.config import fast_safe_load + + try: + with open(config_path, encoding="utf-8") as f: + raw = fast_safe_load(f) + except Exception as exc: + raise TerminalPolicyUnavailable( + f"cannot parse {config_path}: {exc}" + ) from exc + raw_terminal = raw.get("terminal") if isinstance(raw, dict) else None + if isinstance(raw_terminal, dict): + for cfg_key, value in raw_terminal.items(): + _apply(cfg_key, value) + except TerminalPolicyUnavailable: + raise + except Exception as exc: + raise TerminalPolicyUnavailable( + f"cannot resolve terminal config in {home}: {exc}" + ) from exc + finally: + if override_token is not None: + reset_hermes_home_override(override_token) + + return scope + + +def install_profile_terminal_scope(hermes_home: "Any") -> Token: + """Build AND install a profile's policy in one call. + + The single entry point for every profile boundary (gateway turn, TUI/ + dashboard turn, cron fire). On resolution failure this installs the + refusal scope instead of raising — the turn continues only in the sense + that terminal tools will refuse execution with the typed reason; it never + falls back to ambient process policy. + + Returns the token for ``reset_terminal_scope``. + """ + try: + return set_terminal_scope(build_profile_terminal_scope(hermes_home)) + except TerminalPolicyUnavailable as exc: + logger.warning("terminal policy unavailable: %s", exc) + return install_refusal_scope(str(exc)) + + +def enforce_no_refusal() -> None: + """Raise when the active scope is a refusal scope (fail closed). + + Execution paths (terminal tool, execute_code) call this before spawning + anything: under a refusal scope the profile's terminal policy could not be + resolved, and running with the launch process's ambient policy is exactly + the authority leak this module closes (#68559 requires refusal, not + fallback). Non-scoped and policy-scoped contexts pass silently. + """ + scope = _terminal_scope_var.get() + if isinstance(scope, TerminalPolicyRefusal): + raise TerminalPolicyUnavailable( + f"terminal policy unavailable for this profile: {scope.reason}" + ) + + +@contextmanager +def install_and_reset_profile_terminal_scope( + hermes_home: "Any", +) -> Iterator[None]: + """Install the profile's terminal policy for a bounded turn/fire. + + Single call for every in-process profile boundary (gateway turn, + dashboard/TUI turn, cron fire): builds the complete effective policy and + resets it on exit. Resolution failure installs the refusal scope for the + same duration — terminal execution inside the block raises (fail closed) + instead of inheriting the launch process's ambient policy. Never raises. + """ + token = install_profile_terminal_scope(hermes_home) + try: + yield + finally: + reset_terminal_scope(token) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index d4b8e5082a..2add39879e 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -839,7 +839,7 @@ def _sudo_nopasswd_works() -> bool: cache) so an expired sudo timestamp cannot make a later command silently block waiting for a password. """ - terminal_env = os.getenv("TERMINAL_ENV", "local").strip().lower() or "local" + terminal_env = _tenv("TERMINAL_ENV", "local").strip().lower() or "local" if terminal_env != "local": return False @@ -1017,7 +1017,19 @@ def _rewrite_compound_background(command: str) -> str: suffix = result[amp_pos + 1 :] # `{` needs a trailing space in bash; the closing `}` needs to be # preceded by `;` or `&` — we're providing `&` from the backgrounding. - result = prefix + "{ " + middle + "& }" + suffix + # + # The consumed `&` also separated the compound from any statement + # that followed on the same line (`A && B & C`); `{ B & } C` is a + # syntax error, so restore a `;` when the suffix resumes with command + # text. No separator when the suffix already starts with a + # terminator (`;` `&` `|` newline `)` `}`) — except `&>`, which is a + # redirect prefix for the NEXT command, not a terminator. + tail = suffix.lstrip(" \t") + needs_separator = bool(tail) and ( + tail[0] not in ";\n&|)}" or tail.startswith("&>") + ) + separator = " ;" if needs_separator else "" + result = prefix + "{ " + middle + "& }" + separator + suffix return result @@ -1195,7 +1207,7 @@ def _maybe_reap_docker_orphans(container_config: Dict[str, Any]) -> None: # ``container_config`` only carries container_* keys, so read # lifetime_seconds from the env var the rest of the module uses. try: - lifetime = int(os.getenv("TERMINAL_LIFETIME_SECONDS", "300")) + lifetime = int(_tenv("TERMINAL_LIFETIME_SECONDS", "300")) except (TypeError, ValueError): lifetime = 300 lifetime = max(60, lifetime) @@ -1384,12 +1396,12 @@ def _session_isolation_enabled() -> bool: attach one live VM and delete it out from under each other). """ _ensure_terminal_env_bridged() - env_type = os.getenv("TERMINAL_ENV", "local") + env_type = _tenv("TERMINAL_ENV", "local") if env_type != "docker" and not _plugin_env_flag( env_type, "session_isolated_when_nonpersistent" ): return False - return os.getenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() not in {"true", "1", "yes"} + return _tenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() not in {"true", "1", "yes"} def _docker_session_isolation_enabled() -> bool: @@ -1399,7 +1411,7 @@ def _docker_session_isolation_enabled() -> bool: selection, session-scoped container teardown) key off it; those must not fire for other backends. """ - if os.getenv("TERMINAL_ENV", "local") != "docker": + if _tenv("TERMINAL_ENV", "local") != "docker": return False return _session_isolation_enabled() @@ -1419,9 +1431,9 @@ def _docker_persistent_profile_scoped() -> bool: keep the session-scoped cache key that fixed the original leak. """ _ensure_terminal_env_bridged() - if os.getenv("TERMINAL_ENV", "local") != "docker": + if _tenv("TERMINAL_ENV", "local") != "docker": return False - return os.getenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() in {"true", "1", "yes"} + return _tenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() in {"true", "1", "yes"} def _current_session_profile() -> str: @@ -1515,7 +1527,7 @@ def _resolve_container_task_id(task_id: Optional[str]) -> str: # Explicit opt-in: trusted profiles configuring the same # terminal.docker_shared_container_key share ONE container/cache # slot (and sandbox dir) regardless of profile name (#84671). - shared = os.getenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() + shared = _tenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() if shared: return f"shared:{shared}" profile = _current_session_profile() or "default" @@ -1528,7 +1540,7 @@ def _resolve_container_task_id(task_id: Optional[str]) -> str: # sessions land in "shared:" — splitting the very container the # setting exists to unify. if _docker_persistent_profile_scoped(): - shared = os.getenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() + shared = _tenv("TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "").strip() if shared: return f"shared:{shared}" return "default" @@ -1607,6 +1619,10 @@ def _parse_env_var(name: str, default: str, converter: Any = int, type_label: st causes an unhandled ValueError that kills every terminal command. """ raw = os.getenv(name, default) + if name.startswith("TERMINAL_"): + # Scope-aware: under gateway multiplexing the active profile's + # per-turn scope overrides the process env. + raw = _tenv(name, default) try: return converter(raw) except (ValueError, json.JSONDecodeError): @@ -1632,7 +1648,7 @@ def _safe_getcwd() -> str: try: return os.getcwd() except (FileNotFoundError, PermissionError): - return os.getenv("TERMINAL_CWD") or os.path.expanduser("~") + return _tenv("TERMINAL_CWD") or os.path.expanduser("~") # Path prefixes that identify a *host* working directory which cannot exist @@ -1703,6 +1719,20 @@ def _is_unusable_container_cwd(cwd: str) -> bool: return False +def _tenv(name: str, default: str = "") -> str: + """Scope-aware read of a ``TERMINAL_*`` variable. + + Every terminal setting read in this module must go through this helper: + under gateway multiplexing the active profile's terminal config arrives + via a per-turn scope (``tools.terminal_scope``), and a raw ``os.getenv`` + would read whatever profile's config a previous turn pinned into the + process env (the cross-profile backend leak fixed here). + """ + from tools.terminal_scope import terminal_env + + return terminal_env(name, default) + + # One-shot guard for the config-fallback bridge below. Purely an # optimization: after the first attempt either TERMINAL_ENV is set (bridge # succeeded — merged config always carries terminal.backend) or the import @@ -1728,7 +1758,17 @@ def _ensure_terminal_env_bridged() -> None: be stale from ``hermes setup``). Environment values for omitted terminal keys are preserved. When no terminal section exists, exported/.env values keep working unchanged. + + A per-turn terminal scope (multiplexed gateway / profile-scoped cron) + suppresses this bridge entirely: the scope holds the active profile's + authoritative values and reads fall through ``_tenv`` — writing them into + the process-global ``os.environ`` would re-create the first-writer-wins + cross-profile leak the scope exists to fix. """ + from tools.terminal_scope import get_terminal_scope + + if get_terminal_scope() is not None: + return global _terminal_config_bridge_attempted if _terminal_config_bridge_attempted: return @@ -1762,9 +1802,9 @@ def _get_env_config() -> Dict[str, Any]: # Default image with Python and Node.js for maximum compatibility default_image = "nikolaik/python-nodejs:python3.11-nodejs20" _ensure_terminal_env_bridged() - env_type = os.getenv("TERMINAL_ENV", "local") + env_type = _tenv("TERMINAL_ENV", "local") - mount_docker_cwd = os.getenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").lower() in {"true", "1", "yes"} + mount_docker_cwd = _tenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false").lower() in {"true", "1", "yes"} container_backend = _is_container_backend(env_type) docker_backend = env_type == "docker" @@ -1786,7 +1826,7 @@ def _get_env_config() -> Dict[str, Any]: docker_volumes = _parse_env_var("TERMINAL_DOCKER_VOLUMES", "[]", json.loads, "valid JSON") docker_env = _parse_env_var("TERMINAL_DOCKER_ENV", "{}", json.loads, "valid JSON") docker_extra_args = _parse_env_var("TERMINAL_DOCKER_EXTRA_ARGS", "[]", json.loads, "valid JSON") - docker_shm_size = os.getenv("TERMINAL_DOCKER_SHM_SIZE", "1g") + docker_shm_size = _tenv("TERMINAL_DOCKER_SHM_SIZE", "1g") else: docker_forward_env = [] docker_volumes = [] @@ -1810,13 +1850,13 @@ def _get_env_config() -> Dict[str, Any]: # If Docker cwd passthrough is explicitly enabled, remap the host path to # /workspace and track the original host path separately. Otherwise keep the # normal sandbox behavior and discard host paths. - cwd = os.getenv("TERMINAL_CWD", default_cwd) + cwd = _tenv("TERMINAL_CWD", default_cwd) from hermes_cli.config import _is_ssh_remote_tilde_cwd if cwd and not _is_ssh_remote_tilde_cwd(env_type, cwd): cwd = os.path.expanduser(cwd) host_cwd = None if env_type == "docker" and mount_docker_cwd: - docker_cwd_source = os.getenv("TERMINAL_CWD") or _safe_getcwd() + docker_cwd_source = _tenv("TERMINAL_CWD") or _safe_getcwd() candidate = os.path.abspath(os.path.expanduser(docker_cwd_source)) if ( any(candidate.startswith(p) for p in _HOST_CWD_PREFIXES) @@ -1834,41 +1874,41 @@ def _get_env_config() -> Dict[str, Any]: return { "env_type": env_type, - "modal_mode": coerce_modal_mode(os.getenv("TERMINAL_MODAL_MODE", "auto")), - "docker_image": os.getenv("TERMINAL_DOCKER_IMAGE", default_image), + "modal_mode": coerce_modal_mode(_tenv("TERMINAL_MODAL_MODE", "auto")), + "docker_image": _tenv("TERMINAL_DOCKER_IMAGE", default_image), "docker_forward_env": docker_forward_env, - "singularity_image": os.getenv("TERMINAL_SINGULARITY_IMAGE", f"docker://{default_image}"), - "modal_image": os.getenv("TERMINAL_MODAL_IMAGE", default_image), - "daytona_image": os.getenv("TERMINAL_DAYTONA_IMAGE", default_image), - "vercel_runtime": os.getenv("TERMINAL_VERCEL_RUNTIME", "").strip(), + "singularity_image": _tenv("TERMINAL_SINGULARITY_IMAGE", f"docker://{default_image}"), + "modal_image": _tenv("TERMINAL_MODAL_IMAGE", default_image), + "daytona_image": _tenv("TERMINAL_DAYTONA_IMAGE", default_image), + "vercel_runtime": _tenv("TERMINAL_VERCEL_RUNTIME", "").strip(), "cwd": cwd, "host_cwd": host_cwd, "docker_mount_cwd_to_workspace": mount_docker_cwd, "timeout": _parse_env_var("TERMINAL_TIMEOUT", "180"), "lifetime_seconds": _parse_env_var("TERMINAL_LIFETIME_SECONDS", "300"), # SSH-specific config - "ssh_host": os.getenv("TERMINAL_SSH_HOST", ""), - "ssh_user": os.getenv("TERMINAL_SSH_USER", ""), + "ssh_host": _tenv("TERMINAL_SSH_HOST", ""), + "ssh_user": _tenv("TERMINAL_SSH_USER", ""), "ssh_port": _parse_env_var("TERMINAL_SSH_PORT", "22"), - "ssh_key": os.getenv("TERMINAL_SSH_KEY", ""), + "ssh_key": _tenv("TERMINAL_SSH_KEY", ""), # Persistent shell: SSH defaults to the config-level persistent_shell # setting (true by default for non-local backends); local is always opt-in. # Per-backend env vars override if explicitly set. - "ssh_persistent": os.getenv( + "ssh_persistent": _tenv( "TERMINAL_SSH_PERSISTENT", - os.getenv("TERMINAL_PERSISTENT_SHELL", "true"), + _tenv("TERMINAL_PERSISTENT_SHELL", "true"), ).lower() in {"true", "1", "yes"}, - "local_persistent": os.getenv("TERMINAL_LOCAL_PERSISTENT", "false").lower() in {"true", "1", "yes"}, + "local_persistent": _tenv("TERMINAL_LOCAL_PERSISTENT", "false").lower() in {"true", "1", "yes"}, # Container resource config (applies to docker, singularity, modal, # daytona, and vercel_sandbox -- ignored for local/ssh) "container_cpu": container_cpu, "container_memory": container_memory, # MB (default 5GB) "container_disk": container_disk, # MB (default 50GB) - "container_persistent": os.getenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() in {"true", "1", "yes"}, + "container_persistent": _tenv("TERMINAL_CONTAINER_PERSISTENT", "true").lower() in {"true", "1", "yes"}, "docker_volumes": docker_volumes, "docker_env": docker_env, - "docker_run_as_host_user": os.getenv("TERMINAL_DOCKER_RUN_AS_HOST_USER", "false").lower() in {"true", "1", "yes"}, - "docker_network": os.getenv("TERMINAL_DOCKER_NETWORK", "true").lower() in {"true", "1", "yes"}, + "docker_run_as_host_user": _tenv("TERMINAL_DOCKER_RUN_AS_HOST_USER", "false").lower() in {"true", "1", "yes"}, + "docker_network": _tenv("TERMINAL_DOCKER_NETWORK", "true").lower() in {"true", "1", "yes"}, "docker_extra_args": docker_extra_args, "docker_shm_size": docker_shm_size, # Cross-process container reuse (issue #20561). The docs claim @@ -1877,17 +1917,17 @@ def _get_env_config() -> Dict[str, Any]: # attaching to it instead of always starting a fresh one. Set to # ``false`` for hard per-process isolation (no reuse, container is # removed on exit). - "docker_persist_across_processes": os.getenv( + "docker_persist_across_processes": _tenv( "TERMINAL_DOCKER_PERSIST_ACROSS_PROCESSES", "true" ).lower() in {"true", "1", "yes"}, - "docker_shared_container_key": os.getenv( + "docker_shared_container_key": _tenv( "TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "" ).strip(), # Startup orphan reaper for hermes-tagged containers left behind by # crashed / SIGKILL'd previous processes that bypassed atexit. # Conservative: only sweeps Exited containers older than 2× the # idle-reap window AND scoped to the current profile. Issue #20561. - "docker_orphan_reaper": os.getenv( + "docker_orphan_reaper": _tenv( "TERMINAL_DOCKER_ORPHAN_REAPER", "true" ).lower() in {"true", "1", "yes"}, } @@ -2403,6 +2443,26 @@ def cleanup_all_environments(): return cleaned +def _cleanup_env(env, *, force_remove: bool = False) -> None: + """Tear down one environment, passing ``force_remove`` only when accepted. + + ``DockerEnvironment.cleanup(force_remove=...)`` (issue #20561) diverges + from the base ``cleanup(self)``; other backends expose ``stop`` / + ``terminate`` instead. Shared by ``cleanup_vm`` and the prompt-time + backend probe so the signature check lives in one place. + """ + if hasattr(env, 'cleanup'): + import inspect + if "force_remove" in inspect.signature(env.cleanup).parameters: + env.cleanup(force_remove=force_remove) + else: + env.cleanup() + elif hasattr(env, 'stop'): + env.stop() + elif hasattr(env, 'terminate'): + env.terminate() + + def cleanup_vm(task_id: str, *, force_remove: bool = False): """Manually clean up a specific environment by task_id. @@ -2447,19 +2507,7 @@ def cleanup_vm(task_id: str, *, force_remove: bool = False): return try: - if hasattr(env, 'cleanup'): - # Pass force_remove only if the env's cleanup() accepts it - # (DockerEnvironment after issue #20561; other backends don't). - import inspect - sig = inspect.signature(env.cleanup) - if "force_remove" in sig.parameters: - env.cleanup(force_remove=force_remove) - else: - env.cleanup() - elif hasattr(env, 'stop'): - env.stop() - elif hasattr(env, 'terminate'): - env.terminate() + _cleanup_env(env, force_remove=force_remove) logger.info("Manually cleaned up environment for task: %s", task_id) @@ -2877,6 +2925,15 @@ def terminal_tool( config = _get_env_config() env_type = "local" if _host_local else config["env_type"] + # Fail closed under a refusal scope (#68559): the routed profile's + # terminal policy could not be resolved, so executing with the launch + # process's ambient policy is forbidden — refuse with a typed, + # model-actionable error instead. + if not _host_local: + from tools.terminal_scope import enforce_no_refusal + + enforce_no_refusal() + # Use task_id for environment isolation. By default all subagent # task_ids collapse back to "default" so the top-level agent and # every delegate_task child share one container; only task_ids with @@ -3089,8 +3146,15 @@ def terminal_tool( _MAX_REFERENCED_SCRIPT_BYTES, contains_gateway_lifecycle_command_or_referenced_script, contains_launchctl_submit_command, + lifecycle_scan_root_within_budget, ) - if contains_launchctl_submit_command(command): + # Keep the specific launchctl diagnostic when this optional + # pre-scan fits the budget. The full fail-closed guard below still + # runs when it does not, so oversized roots never reach shlex here. + if ( + lifecycle_scan_root_within_budget(command) + and contains_launchctl_submit_command(command) + ): return json.dumps({ "output": "", "exit_code": 1, @@ -3868,7 +3932,7 @@ def terminal_tool( # warn (default) — return a structured degraded result the model # can act on (reason + retry hint, no traceback). # fail — preserve the historical error+traceback result. - degraded_mode = os.getenv("TERMINAL_DEGRADED_MODE", "warn").strip().lower() + degraded_mode = _tenv("TERMINAL_DEGRADED_MODE", "warn").strip().lower() if degraded_mode == "fail": import traceback tb_str = traceback.format_exc() @@ -4089,18 +4153,18 @@ if __name__ == "__main__": default_img = "nikolaik/python-nodejs:python3.11-nodejs20" print( " TERMINAL_ENV: " - f"{os.getenv('TERMINAL_ENV', 'local')} " + f"{_tenv('TERMINAL_ENV', 'local')} " "(local/docker/singularity/modal/daytona/vercel_sandbox/ssh)" ) - print(f" TERMINAL_DOCKER_IMAGE: {os.getenv('TERMINAL_DOCKER_IMAGE', default_img)}") - print(f" TERMINAL_SINGULARITY_IMAGE: {os.getenv('TERMINAL_SINGULARITY_IMAGE', f'docker://{default_img}')}") - print(f" TERMINAL_MODAL_IMAGE: {os.getenv('TERMINAL_MODAL_IMAGE', default_img)}") - print(f" TERMINAL_DAYTONA_IMAGE: {os.getenv('TERMINAL_DAYTONA_IMAGE', default_img)}") - print(f" TERMINAL_CWD: {os.getenv('TERMINAL_CWD', _safe_getcwd())}") + print(f" TERMINAL_DOCKER_IMAGE: {_tenv('TERMINAL_DOCKER_IMAGE', default_img)}") + print(f" TERMINAL_SINGULARITY_IMAGE: {_tenv('TERMINAL_SINGULARITY_IMAGE', f'docker://{default_img}')}") + print(f" TERMINAL_MODAL_IMAGE: {_tenv('TERMINAL_MODAL_IMAGE', default_img)}") + print(f" TERMINAL_DAYTONA_IMAGE: {_tenv('TERMINAL_DAYTONA_IMAGE', default_img)}") + print(f" TERMINAL_CWD: {_tenv('TERMINAL_CWD', _safe_getcwd())}") from hermes_constants import display_hermes_home as _dhh - print(f" TERMINAL_SANDBOX_DIR: {os.getenv('TERMINAL_SANDBOX_DIR', f'{_dhh()}/sandboxes')}") - print(f" TERMINAL_TIMEOUT: {os.getenv('TERMINAL_TIMEOUT', '60')}") - print(f" TERMINAL_LIFETIME_SECONDS: {os.getenv('TERMINAL_LIFETIME_SECONDS', '300')}") + print(f" TERMINAL_SANDBOX_DIR: {_tenv('TERMINAL_SANDBOX_DIR', f'{_dhh()}/sandboxes')}") + print(f" TERMINAL_TIMEOUT: {_tenv('TERMINAL_TIMEOUT', '60')}") + print(f" TERMINAL_LIFETIME_SECONDS: {_tenv('TERMINAL_LIFETIME_SECONDS', '300')}") # --------------------------------------------------------------------------- diff --git a/tools/tip_tool.py b/tools/tip_tool.py index 57bd72e508..293c10185c 100644 --- a/tools/tip_tool.py +++ b/tools/tip_tool.py @@ -59,7 +59,7 @@ def tip_tool(text: str, selector: str, title: str = "", side: str = "") -> str: TIP_SCHEMA = { - "name": "tip", + "name": "show_tip", "description": ( "Point at one thing in the desktop UI with a small arrow bubble (no " "dimming, no tour chrome) — for when a sentence is clearer with a " @@ -100,7 +100,7 @@ def check_tips_enabled() -> bool: registry.register( - name="tip", + name="show_tip", toolset="desktop_ui", schema=TIP_SCHEMA, handler=lambda args, **kw: tip_tool( diff --git a/tools/todo_tool.py b/tools/todo_tool.py index bd487c974e..11056736a4 100644 --- a/tools/todo_tool.py +++ b/tools/todo_tool.py @@ -378,12 +378,12 @@ def check_todo_requirements() -> bool: # static tool schema (cached, never changes mid-conversation). TODO_SCHEMA = { - "name": "todo", + "name": "todo_list", # Dieted (#95681): the item shape and merge semantics live ONLY in the # parameter schema below — the description teaches behavior, not # structure the params already define. "description": ( - "Manage your task list for the current session. Use for complex tasks " + "Track a task list for multi-step work (3+ steps). Use for complex tasks " "with 3+ steps or when the user provides multiple tasks. " "For 'all N items' tasks, enumerate every instance as its own checklist " "item so none are silently dropped. " @@ -440,7 +440,7 @@ TODO_SCHEMA = { from tools.registry import registry, tool_error registry.register( - name="todo", + name="todo_list", toolset="todo", schema=TODO_SCHEMA, handler=lambda args, **kw: todo_tool( diff --git a/tools/tool_search.py b/tools/tool_search.py index 66f1b198c5..b509be2350 100644 --- a/tools/tool_search.py +++ b/tools/tool_search.py @@ -42,6 +42,7 @@ for the full rationale): from __future__ import annotations +import copy import functools import json import logging @@ -57,6 +58,8 @@ from tools.registry import tool_error logger = logging.getLogger("tools.tool_search") +_SCHEMA_LITERAL_KEYS = frozenset({"const", "default", "enum", "example", "examples"}) + # Bridge tool names. These names are reserved and may not collide with a # user/plugin/MCP tool — registration of any tool with these names is @@ -110,6 +113,14 @@ class ToolSearchConfig: # Absolute cap on the embedded listing, regardless of context size. # Effective budget = min(listing_max_tokens, threshold_pct% of context). listing_max_tokens: int = 4000 + # Core/GUI tool names deferred behind the bridge. None = use the curated + # default (_DEFAULT_DEFERRED_TOOLS); an explicit list from config + # replaces the default wholesale ([] = defer no core tools — legacy). + defer_tools: Optional[frozenset] = None + + @property + def effective_defer_tools(self) -> frozenset: + return _DEFAULT_DEFERRED_TOOLS if self.defer_tools is None else self.defer_tools @classmethod def from_raw(cls, raw: Any) -> "ToolSearchConfig": @@ -159,6 +170,14 @@ class ToolSearchConfig: listing = "auto" listing_max_tokens = max(200, min(60000, _safe_int(raw.get("listing_max_tokens"), 4000))) + defer_raw = raw.get("defer") + if isinstance(defer_raw, (list, tuple, set)): + defer_tools = frozenset( + str(n).strip() for n in defer_raw if str(n).strip() + ) + else: + defer_tools = None # curated default + return cls( enabled=enabled, threshold_pct=threshold_pct, @@ -166,6 +185,7 @@ class ToolSearchConfig: max_search_limit=max_search_limit, listing=listing, listing_max_tokens=listing_max_tokens, + defer_tools=defer_tools, ) @@ -230,21 +250,54 @@ def _core_tool_names() -> frozenset[str]: # Session-gated GUI toolsets. Off ``_HERMES_CORE_TOOLS`` so non-GUI clients -# never pay their schema; once a session enables them they stay direct. +# never pay their schema; once a session enables them they stay direct +# UNLESS the deferral list (below) names them. _DIRECT_SURFACE_TOOLSETS = frozenset({"desktop_ui", "project"}) +# Core-tool deferral (2026-08, maintainer-directed): the curated set of +# event-triggered tools that hide behind the bridge BY DEFAULT. These are +# tools a session reaches for when something specific happens (user asks +# for a tour / a cron job / a screenshot / a clarification), not tools in +# the every-turn working set — so a catalog stub is enough to find them. +# Config override: ``tools.tool_search.defer`` (list of tool names); +# ``[]`` restores the legacy everything-eager behavior, any other list +# replaces this default wholesale. Names here are POST-rename. +# +# ``clarify`` was in the original curated set but was pulled back to eager +# after the maintainer A/B (PR #97979, 288 runs × 3 model tiers): with the +# schema visible models used structured clarify 18/18 on ambiguous tasks; +# deferred, usage collapsed to 7/18 (gpt-terra 0/6) — models fell back to +# plain-text questions, losing the structured-choice UX and costing an +# extra user round-trip. The ask-the-user affordance has to be ambient to +# fire; a catalog stub is not enough. (~250 tok to keep it eager.) +_DEFAULT_DEFERRED_TOOLS = frozenset({ + "computer_use", "session_search", "image_generate", + "todo_list", "process_manage", "cronjob_manage", + # Desktop GUI surface (desktop_ui + project toolsets) + "drive_preview", "gui_tour", "desktop_preview", "annotate_preview", + "show_tip", "setup_mcp", "desktop_project", "close_terminal", + "apply_layout", "read_terminal", "read_window_below", "focus_pane", +}) -def is_deferrable_tool_name(name: str) -> bool: + +def is_deferrable_tool_name(name: str, defer_tools: Optional[frozenset] = None) -> bool: """Return True if a tool with this name is *eligible* for deferral. - A tool is deferrable iff it is registered with an MCP toolset prefix - OR it is neither in ``_HERMES_CORE_TOOLS`` nor a session-gated GUI - surface toolset. Core and direct surface tools are never deferred even - when their toolset is technically plugin-provided (this protects - against accidental shadowing). + A tool is deferrable iff: + * it is named in ``defer_tools`` (the maintainer-curated core-deferral + set, or the user's ``tools.tool_search.defer`` override) — this is + the 2026-08 revision of the old "core never defers" rule: core tools + in the WORKING set (terminal, files, memory, ...) still never defer, + but the curated event-triggered set (computer_use, the GUI + surface, ...) hides behind the bridge by default; OR + * it is registered with an MCP toolset prefix; OR + * it is neither in ``_HERMES_CORE_TOOLS`` nor a session-gated GUI + surface toolset (plugin tools). """ if name in BRIDGE_TOOL_NAMES: return False + if defer_tools is not None and name in defer_tools: + return True if name in _core_tool_names(): return False # Check registry toolset for MCP prefix. @@ -265,6 +318,7 @@ def is_deferrable_tool_name(name: str) -> bool: def _describe_classification( name: str, + defer_tools: Optional[frozenset] = None, ) -> Literal["available", "not_found", "not_deferrable"]: """Classify a describe name without treating unknown names as errors.""" try: @@ -274,6 +328,8 @@ def _describe_classification( return "not_found" if entry is None: return "not_found" + if defer_tools is not None and name in defer_tools: + return "available" if ( name in BRIDGE_TOOL_NAMES or name in _core_tool_names() @@ -283,12 +339,15 @@ def _describe_classification( return "available" -def classify_tools(tool_defs: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: +def classify_tools( + tool_defs: List[Dict[str, Any]], + defer_tools: Optional[frozenset] = None, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: """Split a tool-defs list into (visible, deferrable). - ``visible`` retains every tool that must stay in the model-facing array: - every core tool, every session-gated GUI surface tool, plus any tool we - can't classify. ``deferrable`` is the candidate set for catalog entry. + ``visible`` retains every tool that must stay in the model-facing array. + ``deferrable`` is the candidate set for catalog entry — MCP/plugin tools + plus any core/GUI tool named in ``defer_tools``. """ visible: List[Dict[str, Any]] = [] deferrable: List[Dict[str, Any]] = [] @@ -299,7 +358,7 @@ def classify_tools(tool_defs: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any] # Should never happen — bridge tools are added after classification — # but be defensive. continue - if is_deferrable_tool_name(name): + if is_deferrable_tool_name(name, defer_tools): deferrable.append(td) else: visible.append(td) @@ -927,7 +986,7 @@ def assemble_tool_defs( incoming = [td for td in tool_defs if (td.get("function") or {}).get("name") not in BRIDGE_TOOL_NAMES] - visible, deferrable = classify_tools(incoming) + visible, deferrable = classify_tools(incoming, config.effective_defer_tools) if not deferrable: return AssemblyResult(tool_defs=incoming, activated=False) @@ -1078,7 +1137,9 @@ def dispatch_tool_search(args: Dict[str, Any], else: limit = max(1, min(config.max_search_limit, _safe_int(raw_limit, config.search_default_limit))) - _, deferrable = classify_tools(current_tool_defs) + _, deferrable = classify_tools( + current_tool_defs, load_config_readonly().effective_defer_tools + ) catalog = build_catalog(deferrable) results: List[Dict[str, Any]] = [] @@ -1151,7 +1212,9 @@ def dispatch_tool_describe(args: Dict[str, Any], "Retry with fewer names per call." ) - _, deferrable = classify_tools(current_tool_defs) + _, deferrable = classify_tools( + current_tool_defs, load_config_readonly().effective_defer_tools + ) by_name: Dict[str, Dict[str, Any]] = {} for td in deferrable: fn = td.get("function") or {} @@ -1168,7 +1231,9 @@ def dispatch_tool_describe(args: Dict[str, Any], "description": fn.get("description", ""), "parameters": fn.get("parameters", {}), } - elif _describe_classification(name) == "not_deferrable": + elif _describe_classification( + name, load_config_readonly().effective_defer_tools + ) == "not_deferrable": errors[name] = ( f"'{name}' is not a deferrable tool. If you see it in the tools list " "already, call it directly; otherwise check the spelling against tool_search." @@ -1198,15 +1263,93 @@ def scoped_deferrable_names(tool_defs: List[Dict[str, Any]]) -> frozenset[str]: an out-of-scope tool via the bridge. """ names: set[str] = set() + defer_tools = load_config_readonly().effective_defer_tools for td in tool_defs: name = (td.get("function") or {}).get("name", "") - if name and is_deferrable_tool_name(name): + if name and is_deferrable_tool_name(name, defer_tools): names.add(name) return frozenset(names) +def _schema_for_local_validation(node: Any) -> Any: + """Return a JSON-Schema-compatible copy that honors ``nullable: true``. + + Some MCP/plugin schemas use OpenAPI's ``nullable`` extension instead of a + JSON Schema null union. Hermes' normal coercion path accepts that shape; + mirror it here so local validation never rejects a value dispatch would + intentionally accept. + """ + if isinstance(node, list): + return [_schema_for_local_validation(item) for item in node] + if not isinstance(node, dict): + return node + + normalized = {} + for key, value in node.items(): + if key == "nullable": + continue + # These keywords contain instance data, not nested schemas. An enum + # value such as {"nullable": true} must remain byte-for-byte data. + normalized[key] = ( + copy.deepcopy(value) + if key in _SCHEMA_LITERAL_KEYS + else _schema_for_local_validation(value) + ) + if node.get("nullable") is not True: + return normalized + + schema_type = normalized.get("type") + if isinstance(schema_type, str): + if schema_type != "null": + normalized["type"] = [schema_type, "null"] + return normalized + if isinstance(schema_type, list): + if "null" not in schema_type: + normalized["type"] = [*schema_type, "null"] + return normalized + + # ``nullable`` alongside a $ref/combinator has no ``type`` to extend. + # Wrap the original constraint so local references keep resolving from the + # parameters schema's root while null remains an explicit alternative. + return {"anyOf": [normalized, {"type": "null"}]} + + +def _schema_has_external_ref(node: Any) -> bool: + """Return whether *node* contains a non-local ``$ref``. + + Local validation must never turn a tool call into an implicit network + fetch. Schemas with remote/file references remain the underlying tool's + responsibility and therefore follow the existing fail-open contract. + """ + if isinstance(node, list): + return any(_schema_has_external_ref(item) for item in node) + if not isinstance(node, dict): + return False + ref = node.get("$ref") + if isinstance(ref, str) and not ref.startswith("#"): + return True + return any( + _schema_has_external_ref(value) + for key, value in node.items() + if key not in _SCHEMA_LITERAL_KEYS + ) + + +def _validation_path(error: Any) -> str: + """Format a jsonschema error path as a compact argument path.""" + path = "arguments" + for part in getattr(error, "absolute_path", ()): + if isinstance(part, int): + path += f"[{part}]" + elif isinstance(part, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", part): + path += f".{part}" + else: + path += f"[{json.dumps(part, ensure_ascii=False)}]" + return path + + def validate_deferred_call_args(name: str, args: Dict[str, Any]) -> Optional[str]: - """Probe-validate ``tool_call`` arguments against the deferred tool's schema. + """Validate ``tool_call`` arguments against the deferred tool's schema. A deferred tool's parameter schema is invisible to the model until it calls ``tool_describe`` — so models routinely invoke deferred tools @@ -1215,17 +1358,16 @@ def validate_deferred_call_args(name: str, args: Dict[str, Any]) -> Optional[str that tells the model nothing about what the tool expects, and cheap models loop on it until the iteration budget dies. - Port of the describe-first probe-validation fix from nearai/ironclaw#5149: - when required arguments are missing, return the tool's parameter schema - instead of dispatching blind — the model repairs the call in one - round-trip. Valid calls (and any call we can't confidently validate) - dispatch untouched, so this can never block a legitimate invocation. + Keep the original describe-first required-field probe from + nearai/ironclaw#5149, then run the same schema-guided coercion used by + normal dispatch and validate the repaired copy. This restores the + concrete-schema checks that the provider cannot perform through the + generic ``arguments: object`` bridge. - Only *key absence* of schema-``required`` fields counts as invalid. - No type checking, no null rejection — nullable/typed edge cases are the - tool's own business, and ``coerce_tool_args`` already handles type repair - downstream. Returns a JSON error string when invalid, ``None`` when the - call should dispatch. + Missing/malformed schemas, unavailable validators, and external references + fail open so validation cannot make a previously callable tool unavailable. + Returns a JSON error string when invalid, ``None`` when the call should + dispatch through the existing middleware/hook/approval pipeline. """ try: from tools.registry import registry as _registry @@ -1239,14 +1381,68 @@ def validate_deferred_call_args(name: str, args: Dict[str, Any]) -> Optional[str if not isinstance(params, dict): return None required = params.get("required") - if not isinstance(required, list) or not required: + if isinstance(required, list) and required: + missing = [r for r in required if isinstance(r, str) and r not in args] + if missing: + return tool_error( + f"tool_call to '{name}' is missing required argument(s): " + f"{', '.join(missing)}. The tool was NOT invoked.", + path="arguments", + constraint="required", + parameters=params, + hint=( + "Retry tool_call with 'arguments' matching the parameters " + "schema above." + ), + ) + + validation_schema = _schema_for_local_validation(params) + if _schema_has_external_ref(validation_schema): + logger.debug( + "Skipping local deferred-argument validation for %s: external $ref", + name, + ) return None - missing = [r for r in required if isinstance(r, str) and r not in args] - if not missing: + + # Validate the same repaired shape normal dispatch will receive. Work on + # a copy because coerce_tool_args may normalize values in place; actual + # dispatch performs the canonical coercion again after this probe. + candidate_args = dict(args) + try: + from model_tools import coerce_tool_args + candidate_args = coerce_tool_args(name, candidate_args) + except Exception: + logger.debug("Deferred-argument coercion failed for %s", name, exc_info=True) + candidate_args = dict(args) + + try: + from jsonschema.exceptions import best_match + from jsonschema.validators import validator_for + except ImportError: + logger.debug( + "jsonschema unavailable; keeping required-only validation for %s", + name, + ) return None + + validator_cls = validator_for(validation_schema) + validator_cls.check_schema(validation_schema) + validation_error = best_match( + validator_cls(validation_schema).iter_errors(candidate_args) + ) + if validation_error is None: + return None + + path = _validation_path(validation_error) + constraint = str(getattr(validation_error, "validator", None) or "schema") + detail = re.sub(r"\s+", " ", str(validation_error.message)).strip() + if len(detail) > 600: + detail = detail[:597] + "..." return tool_error( - f"tool_call to '{name}' is missing required argument(s): " - f"{', '.join(missing)}. The tool was NOT invoked.", + f"tool_call to '{name}' failed argument validation at {path} " + f"({constraint}): {detail}. The tool was NOT invoked.", + path=path, + constraint=constraint, parameters=params, hint=( "Retry tool_call with 'arguments' matching the parameters " @@ -1283,7 +1479,7 @@ def resolve_underlying_call(args: Dict[str, Any]) -> Tuple[Optional[str], Dict[s return None, {}, f"tool_call 'arguments' is not valid JSON: {e}" if not isinstance(raw_args, dict): return None, {}, "tool_call 'arguments' must be an object" - if not is_deferrable_tool_name(name): + if not is_deferrable_tool_name(name, load_config_readonly().effective_defer_tools): return None, {}, ( f"'{name}' is not a deferrable tool. If it appears in the model-facing tools " "list already, call it directly instead of via tool_call." diff --git a/tools/tour_tool.py b/tools/tour_tool.py index 5f6b204a57..88cde969c8 100644 --- a/tools/tour_tool.py +++ b/tools/tour_tool.py @@ -129,7 +129,7 @@ _STEP_SCHEMA = { } TOUR_SCHEMA = { - "name": "tour", + "name": "gui_tour", # Dieted (#95681): targets-first flow + stable-selector preference kept # (pre-effect: skipping them means guessed selectors on re-rendering UI). "description": ( @@ -189,7 +189,7 @@ def check_tours_enabled() -> bool: registry.register( - name="tour", + name="gui_tour", toolset="desktop_ui", schema=TOUR_SCHEMA, handler=lambda args, **kw: tour_tool( diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 5b2f9229d4..02bfa3b97e 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -262,6 +262,26 @@ def _get_default_output_dir() -> str: return str(get_hermes_dir("cache/audio", "audio_cache")) DEFAULT_OUTPUT_DIR = _get_default_output_dir() +_DEFAULT_OUTPUT_DIR_AT_IMPORT = DEFAULT_OUTPUT_DIR + +def _default_output_dir() -> str: + """Return the active profile's audio output dir at call time. + + Same bug class as skills_tool (f8723c478) and skills_sync (#65828): + long-lived multi-profile runtimes (dashboard console, TUI/Desktop backend, + cron, kanban workers) import this module once under the launch + HERMES_HOME and later scope requests to a different profile via + ``hermes_constants.set_hermes_home_override()`` — a frozen module + constant keeps writing synthesized audio into the launch profile's + cache instead of the active profile's (#98749). Keep the legacy + ``DEFAULT_OUTPUT_DIR`` module attribute for tests and external patchers; + when it has not been patched, re-resolve from the live profile-scoped + HERMES_HOME on every call. + """ + configured = DEFAULT_OUTPUT_DIR + if configured != _DEFAULT_OUTPUT_DIR_AT_IMPORT: + return configured + return _get_default_output_dir() # --------------------------------------------------------------------------- # Per-provider input-character limits (from official provider docs). @@ -2889,10 +2909,240 @@ def _tts_cache_get_or_load(cache: Dict[str, Any], key: str, load: Callable[[], A return value +# =========================================================================== +# Local-engine lifecycle: warm-up / release driven by TTS-output toggles +# =========================================================================== +# +# Local engines (Piper, KittenTTS) load their model lazily on the first +# synthesis call, so the first spoken reply after a user turns on "read +# replies aloud" / a voice conversation pays the whole load (plus a voice +# download on a fresh install) as dead air before the first word. And once +# loaded, the model stays resident for the process lifetime even after every +# TTS-output toggle is off again. +# +# The toggles ARE the intent signal. Every surface that flips speech output +# on holds a *lease* here (warming the configured engine as a side effect); +# flipping it off releases the lease, and when the last lease is gone the +# local model caches are dropped. Lease-counting instead of a bare +# on/off keeps one surface's "off" from unloading a model another surface +# (TUI /voice tts, desktop read-aloud, desktop conversation) still needs — +# they share this process's caches. +# +# Cloud providers have no resident model; warming them is a no-op beyond +# making sure the lazily-installed SDK is importable (edge-tts), which is +# also first-use latency users see as silence. + +# Provider name → local model cache it populates. The single registry both +# warm_tts_provider() and the release path consult — a new local engine adds +# one row here (at its cache declaration) plus a loader in +# _local_tts_warmers() and gets warm/release for free. +_LOCAL_TTS_MODEL_CACHES: Dict[str, Dict[str, Any]] = {} + + +def _local_tts_warmers() -> Dict[str, Callable[[Dict[str, Any]], Any]]: + # Resolved lazily: the loader functions are defined later in this module. + return { + "piper": lambda cfg: _load_piper_voice_for_config(cfg)[0], + "kittentts": lambda cfg: _load_kittentts_model_for_config(cfg)[0], + } + + +def _lazy_sdk_feature_for_provider(provider: str) -> Optional[str]: + """tools.lazy_deps feature key for providers whose SDK installs on first use.""" + return { + "edge": "tts.edge", + "elevenlabs": "tts.elevenlabs", + "mistral": "tts.mistral", + }.get(provider) + + +_tts_lease_lock = threading.Lock() +_tts_leases: set = set() + + +def _signal_user_tts_provider(name: str, tts_config: Dict[str, Any], hook: str) -> Optional[str]: + """Forward a lease ``hook`` (``"warm"`` / ``"release"``) to a user-declared provider. + + Command providers run their optional ``warm_command`` / ``release_command`` + (same template/env/timeout rules as ``command``; output discarded) on a + background thread so a toggle never waits on a model server. Plugin + providers get :meth:`TTSProvider.warm` / :meth:`TTSProvider.release`. + Best-effort: failures are logged at debug. Returns the action taken. + """ + if not name or name in BUILTIN_TTS_PROVIDERS: + return None + cfg = _get_named_provider_config(tts_config, name) + try: + if _is_command_provider_config(cfg): + template = str(cfg.get(f"{hook}_command") or "").strip() + if not template: + return None + command = _render_command_tts_template(template, { + "voice": str(cfg.get("voice", "")), + "model": str(cfg.get("model", "")), + "speed": str(cfg.get("speed", tts_config.get("speed", ""))), + }) + + def _run() -> None: + try: + _run_command_tts(command, _get_command_tts_timeout(cfg), + env_passthrough=_command_provider_env_passthrough(cfg)) + except Exception as exc: # noqa: BLE001 — best-effort hook + logger.debug("[TTS] %s_command for %s failed: %s", hook, name, exc) + + threading.Thread(target=_run, name=f"tts-{hook}-{name}", daemon=True).start() + return hook + from agent.tts_registry import get_provider + from hermes_cli.plugins import _ensure_plugins_discovered + + _ensure_plugins_discovered() + plugin_provider = get_provider(name) + if plugin_provider is None: + return None + getattr(plugin_provider, hook)() + return hook + except Exception as exc: # noqa: BLE001 — best-effort hook + logger.debug("[TTS] %s hook for %s failed: %s", hook, name, exc) + return "error" + + +def warm_tts_provider( + tts_config: Optional[Dict[str, Any]] = None, + provider: Optional[str] = None, +) -> Dict[str, Any]: + """Pre-load the configured TTS provider so the next synthesis starts hot. + + * Local engines (Piper, KittenTTS): resolve the configured voice/model + exactly as synthesis would (including first-use voice download) and + load it into the same LRU cache slot synthesis reads. + * Lazily-installed cloud SDKs (edge-tts, ElevenLabs, Mistral): make sure + the SDK is importable, installing it if lazy installs are allowed. + * User-declared providers: command providers run ``warm_command`` when + set; plugin providers get :meth:`TTSProvider.warm`. + * Everything else: nothing to warm — reported as ``action: "noop"``. + + Never raises; the result dict carries ``warmed`` / ``action`` / ``error`` + so callers on a toggle path can log and move on. Blocking — callers on a + UI thread should run it in the background. + """ + if tts_config is None: + tts_config = _load_tts_config() + name = (provider or _get_provider(tts_config) or "").lower().strip() + result: Dict[str, Any] = {"provider": name, "warmed": False, "action": "noop"} + + warmer = _local_tts_warmers().get(name) + if warmer is not None: + cache = _LOCAL_TTS_MODEL_CACHES.get(name) + before = len(cache) if cache is not None else 0 + started = time.monotonic() + try: + warmer(tts_config) + except Exception as exc: # engine missing, download failed, bad voice… + logger.warning("[TTS] warm-up for %s failed: %s", name, exc) + result.update(action="error", error=str(exc)) + return result + after = len(cache) if cache is not None else 0 + result.update( + warmed=True, + action="loaded" if after > before else "cached", + elapsed_ms=int((time.monotonic() - started) * 1000), + ) + logger.info("[TTS] warm-up %s: %s in %dms", name, result["action"], result["elapsed_ms"]) + return result + + signalled = _signal_user_tts_provider(name, tts_config, "warm") + if signalled is not None: + result.update(warmed=signalled != "error", action="warmed" if signalled != "error" else "error") + return result + + feature = _lazy_sdk_feature_for_provider(name) + if feature is not None: + try: + from tools.lazy_deps import ensure, is_available + + if is_available(feature): + result.update(warmed=True, action="cached") + else: + ensure(feature, prompt=False) + result.update(warmed=True, action="installed") + except Exception as exc: + logger.debug("[TTS] SDK warm-up for %s skipped: %s", name, exc) + result.update(action="error", error=str(exc)) + return result + + +def release_tts_provider(provider: Optional[str] = None) -> Dict[str, Any]: + """Drop resident local TTS models so their memory is returned. + + With ``provider`` given, only that engine's cache is cleared; otherwise + every local engine cache is and the configured user-declared provider + (plugin ``release()`` / command ``release_command``) is signalled. + Cloud providers hold nothing to release. + Returns ``{"released": }``. The next + synthesis simply reloads (or a warm-up does it ahead of time). + """ + name = (provider or "").lower().strip() + if not name: + tts_config = _load_tts_config() + _signal_user_tts_provider(_get_provider(tts_config), tts_config, "release") + released = 0 + for cache_name, cache in _LOCAL_TTS_MODEL_CACHES.items(): + if name and cache_name != name: + continue + released += len(cache) + cache.clear() + if released: + logger.info("[TTS] released %d resident local model(s)", released) + return {"released": released} + + +def acquire_tts_lease(lease: str, tts_config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Register ``lease`` as a live TTS-output consumer and warm the provider. + + ``lease`` names the surface/toggle (e.g. ``"desktop:read-aloud"``, + ``"tui:voice-tts"``). Re-acquiring an existing lease is idempotent (still + re-warms — cheap on a cache hit, and heals a cache cleared elsewhere). + """ + with _tts_lease_lock: + _tts_leases.add(lease) + holders = len(_tts_leases) + result = warm_tts_provider(tts_config) + result["leases"] = holders + return result + + +def release_tts_lease(lease: str) -> Dict[str, Any]: + """Drop ``lease``; when it was the last one, unload resident local models. + + Releasing a lease that was never acquired is a no-op (still reports the + live holder count) so surfaces can call it unconditionally on their + "off" path. + """ + with _tts_lease_lock: + _tts_leases.discard(lease) + holders = len(_tts_leases) + result: Dict[str, Any] = {"leases": holders, "released": 0} + if holders == 0: + result["released"] = release_tts_provider()["released"] + return result + + +def tts_lease_holders() -> List[str]: + """Snapshot of live lease names (diagnostics / tests).""" + with _tts_lease_lock: + return sorted(_tts_leases) + + +def _reset_tts_leases_for_tests() -> None: + with _tts_lease_lock: + _tts_leases.clear() + + # Module-level cache for Piper voice instances. Voices are keyed on their # absolute .onnx model path so switching voices doesn't invalidate older # cached voices. _piper_voice_cache: Dict[str, Any] = {} +_LOCAL_TTS_MODEL_CACHES["piper"] = _piper_voice_cache def _check_piper_available() -> bool: @@ -2969,15 +3219,16 @@ def _resolve_piper_voice_path(voice: str, download_dir: Path) -> str: return str(cached) -def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: - """Generate speech using the local Piper engine. +def _load_piper_voice_for_config(tts_config: Dict[str, Any]) -> Tuple[Any, Dict[str, Any]]: + """Resolve + load (or fetch from cache) the Piper voice ``tts_config`` selects. - Loads the voice model once per process (cached by absolute path) and - writes a WAV file. Caller is responsible for converting to MP3/Opus - via ffmpeg when a different output format is required. + Shared by synthesis and :func:`warm_tts_provider` so a warm-up populates + exactly the cache slot the next synthesis call will hit — same voice + resolution, same download-on-first-use, same cache key. + + Returns ``(voice, piper_config)``. """ PiperVoice = _import_piper() - import wave piper_config = tts_config.get("piper") or {} if isinstance(tts_config, dict) else {} voice_name = piper_config.get("voice") or DEFAULT_PIPER_VOICE @@ -2987,15 +3238,6 @@ def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) model_path = _resolve_piper_voice_path(voice_name, download_dir) - # Tolerant speaker_id parse: drop bad input (non-int strings, lists, dicts) - # to 0 (Piper's own default). Booleans are rejected outright — True/False - # would silently coerce to 1/0 and hide a config mistake. - _raw_speaker = piper_config.get("speaker_id", 0) - if isinstance(_raw_speaker, bool) or not isinstance(_raw_speaker, int): - speaker_id = 0 - else: - speaker_id = _raw_speaker - # speaker_id is applied per-call via syn_config.speaker_id — the same # PiperVoice instance serves all speakers, so it stays out of the cache # key. Multi-speaker workflows share one model load. @@ -3008,6 +3250,28 @@ def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) return v voice = _tts_cache_get_or_load(_piper_voice_cache, cache_key, _load_piper_voice) + return voice, piper_config + + +def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: + """Generate speech using the local Piper engine. + + Loads the voice model once per process (cached by absolute path) and + writes a WAV file. Caller is responsible for converting to MP3/Opus + via ffmpeg when a different output format is required. + """ + import wave + + voice, piper_config = _load_piper_voice_for_config(tts_config) + + # Tolerant speaker_id parse: drop bad input (non-int strings, lists, dicts) + # to 0 (Piper's own default). Booleans are rejected outright — True/False + # would silently coerce to 1/0 and hide a config mistake. + _raw_speaker = piper_config.get("speaker_id", 0) + if isinstance(_raw_speaker, bool) or not isinstance(_raw_speaker, int): + speaker_id = 0 + else: + speaker_id = _raw_speaker # Optional synthesis knobs — only pass a SynthesisConfig when at least # one advanced knob is configured, so we don't depend on a newer Piper @@ -3075,6 +3339,28 @@ def _generate_piper_tts(text: str, output_path: str, tts_config: Dict[str, Any]) # Module-level cache for KittenTTS model instance _kittentts_model_cache: Dict[str, Any] = {} +_LOCAL_TTS_MODEL_CACHES["kittentts"] = _kittentts_model_cache + + +def _load_kittentts_model_for_config(tts_config: Dict[str, Any]) -> Tuple[Any, Dict[str, Any]]: + """Load (or fetch from cache) the KittenTTS model ``tts_config`` selects. + + Shared by synthesis and :func:`warm_tts_provider` — same model name, + same cache key. Returns ``(model, kittentts_config)``. + """ + KittenTTS = _import_kittentts() + kt_config = tts_config.get("kittentts", {}) if isinstance(tts_config, dict) else {} + kt_config = kt_config or {} + model_name = kt_config.get("model", DEFAULT_KITTENTTS_MODEL) + + def _load_kittentts_model(): + logger.info("[KittenTTS] Loading model: %s", model_name) + m = KittenTTS(model_name) + logger.info("[KittenTTS] Model loaded successfully") + return m + + model = _tts_cache_get_or_load(_kittentts_model_cache, model_name, _load_kittentts_model) + return model, kt_config def _generate_kittentts(text: str, output_path: str, tts_config: Dict[str, Any]) -> str: @@ -3091,22 +3377,11 @@ def _generate_kittentts(text: str, output_path: str, tts_config: Dict[str, Any]) Returns: Path to the saved audio file. """ - KittenTTS = _import_kittentts() - kt_config = tts_config.get("kittentts", {}) - model_name = kt_config.get("model", DEFAULT_KITTENTTS_MODEL) + model, kt_config = _load_kittentts_model_for_config(tts_config) voice = kt_config.get("voice", DEFAULT_KITTENTTS_VOICE) speed = kt_config.get("speed", 1.0) clean_text = kt_config.get("clean_text", True) - # Use cached model instance if available - def _load_kittentts_model(): - logger.info("[KittenTTS] Loading model: %s", model_name) - m = KittenTTS(model_name) - logger.info("[KittenTTS] Model loaded successfully") - return m - - model = _tts_cache_get_or_load(_kittentts_model_cache, model_name, _load_kittentts_model) - # Generate audio (returns numpy array at 24kHz) audio = model.generate(text, voice=voice, speed=speed, clean_text=clean_text) @@ -3239,7 +3514,7 @@ def _text_to_speech_single( }, ensure_ascii=False) else: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") - out_dir = Path(DEFAULT_OUTPUT_DIR) + out_dir = Path(_default_output_dir()) out_dir.mkdir(parents=True, exist_ok=True) if command_provider_config is not None: fmt = _get_command_tts_output_format(command_provider_config) @@ -3597,7 +3872,7 @@ def text_to_speech_tool( }, ensure_ascii=False) else: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") - out_dir = Path(DEFAULT_OUTPUT_DIR) + out_dir = Path(_default_output_dir()) out_dir.mkdir(parents=True, exist_ok=True) if command_provider_config is not None: fmt = _get_command_tts_output_format(command_provider_config) @@ -4480,7 +4755,7 @@ if __name__ == "__main__": print(f" MiniMax: {minimax_status}") print(f" Piper: {'installed' if _check_piper_available() else 'not installed (pip install piper-tts)'}") print(f" ffmpeg: {'✅ found' if _has_ffmpeg() else '❌ not found (needed for Telegram Opus)'}") - print(f"\n Output dir: {DEFAULT_OUTPUT_DIR}") + print(f"\n Output dir: {_default_output_dir()}") provider = _get_provider(config) print(f" Configured provider: {provider}") diff --git a/tools/vision_tools.py b/tools/vision_tools.py index 7a9a600885..f11630b166 100644 --- a/tools/vision_tools.py +++ b/tools/vision_tools.py @@ -1077,6 +1077,22 @@ def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None, # --------------------------------------------------------------------------- +def _profile_rejects_tool_media(provider: str) -> bool: + """Hard veto: the provider's ``ProviderProfile`` declares + ``supports_vision_tool_messages=False`` — images are accepted in user + messages but list-type tool-result content is rejected with 400 + (xiaomi/MiMo "text is not set"). ``supports_vision`` alone must not + override this, or the multimodal tool-result envelope 400s every turn + and the image never enters context (#89981). + """ + try: + from providers import get_provider_profile + profile = get_provider_profile(str(provider or "").strip().lower()) + return profile is not None and profile.supports_vision_tool_messages is False + except Exception: + return False + + def _supports_media_in_tool_results(provider: str, model: str) -> bool: """Whether the given provider+model combination accepts image content inside a tool-result message. @@ -1100,7 +1116,7 @@ def _supports_media_in_tool_results(provider: str, model: str) -> bool: if not isinstance(provider, str): return False p = provider.strip().lower() - if not p: + if not p or _profile_rejects_tool_media(p): return False # Aggregators that route to multiple vendors — assume support since @@ -1170,6 +1186,11 @@ def _should_use_native_vision_fast_path() -> bool: cfg = load_config() if decide_image_input_mode(provider, model, cfg) != "native": return False + # The profile veto applies ahead of the capability lookup too: a + # model marked vision-capable by models.dev / custom_providers must + # not re-open the multimodal-envelope route the profile rejects. + if _profile_rejects_tool_media(provider): + return False return ( _supports_media_in_tool_results(provider, model) or _lookup_supports_vision(provider, model, cfg) is True diff --git a/tools/working_diff.py b/tools/working_diff.py index b4886144f8..fd68529c2a 100644 --- a/tools/working_diff.py +++ b/tools/working_diff.py @@ -25,6 +25,8 @@ import shutil import subprocess from typing import Dict, List, Optional +from hermes_cli._subprocess_compat import harden_git_argv, noninteractive_git_env + _GIT_TIMEOUT = 15 _MAX_UNTRACKED_FILES = 50 # sanity cap so a node_modules explosion can't hang us @@ -56,14 +58,22 @@ def _git_command() -> Optional[List[str]]: def _run(args: List[str], cwd: str, timeout: int = _GIT_TIMEOUT): - """Run git, returning (returncode, stdout). Never raises on git failure.""" + """Run git, returning (returncode, stdout). Never raises on git failure. + + Hardened against a malicious repo's ``.git/config`` (GHSA-7x36-8jrh-v4pw): + ``noninteractive_git_env`` disables fsmonitor/hooks/pager/editor/credential + sinks, and ``harden_git_argv`` appends ``--no-ext-diff --no-textconv`` to + the diff-rendering subcommands so attribute-scoped diff/textconv drivers + can't execute either. + """ command = _git_command() if command is None: return 127, "" proc = subprocess.run( - [*command, "-c", "core.quotePath=false", *args], + [*command, "-c", "core.quotePath=false", *harden_git_argv(args)], cwd=cwd, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", + stdin=subprocess.DEVNULL, env=noninteractive_git_env(), ) return proc.returncode, proc.stdout diff --git a/toolsets.py b/toolsets.py index d205c0c02e..a40d803538 100644 --- a/toolsets.py +++ b/toolsets.py @@ -32,7 +32,7 @@ _HERMES_CORE_TOOLS = [ # Web "web_search", "web_extract", # Terminal + process management - "terminal", "process", + "terminal", "process_manage", # NOTE: the desktop GUI affordances (read_terminal, open_preview, …) are # deliberately NOT here, for the same reason as the `project` tools below: # they only work where a GUI renderer can answer them. They live in the @@ -56,7 +56,7 @@ _HERMES_CORE_TOOLS = [ # Text-to-speech "text_to_speech", # Planning & memory - "todo", "memory", + "todo_list", "memory", # NOTE: the desktop Project tools (project_list/create/switch) are # deliberately NOT here. They only make sense where a GUI can follow the # move, so they live in the `project` toolset and are enabled solely by the @@ -69,7 +69,7 @@ _HERMES_CORE_TOOLS = [ # Code execution + delegation "execute_code", "delegate_task", # Cronjob management - "cronjob", + "cronjob_manage", # Home Assistant smart home control (gated on HASS_TOKEN via check_fn) "ha_list_entities", "ha_get_state", "ha_list_services", "ha_call_service", # Kanban multi-agent coordination — only in schema when the agent is @@ -169,7 +169,7 @@ TOOLSETS = { "terminal": { "description": "Terminal/command execution and process management tools", - "tools": ["terminal", "process"], + "tools": ["terminal", "process_manage"], "includes": [] }, @@ -193,7 +193,7 @@ TOOLSETS = { "cronjob": { "description": "Cronjob management tool - create, list, update, pause, resume, remove, and trigger scheduled tasks", - "tools": ["cronjob"], + "tools": ["cronjob_manage"], "includes": [] }, @@ -212,7 +212,7 @@ TOOLSETS = { "todo": { "description": "Task planning and tracking for multi-step work", - "tools": ["todo"], + "tools": ["todo_list"], "includes": [] }, @@ -262,7 +262,7 @@ TOOLSETS = { "desktop_preview", "drive_preview", "annotate_preview", "read_window_below", "focus_pane", "react_to_message", - "setup_mcp", "tour", "tip", + "setup_mcp", "gui_tour", "show_tip", ], "includes": [] }, @@ -369,7 +369,7 @@ TOOLSETS = { "debugging": { "description": "Debugging and troubleshooting toolkit", - "tools": ["terminal", "process"], + "tools": ["terminal", "process_manage"], "includes": ["web", "file"] # For searching error messages and solutions, and file operations }, @@ -392,7 +392,7 @@ TOOLSETS = { "description": "Coding-focused toolset: files, terminal, search, web docs, skills, todo, delegate, vision, browser", "tools": [ "web_search", "web_extract", - "terminal", "process", + "terminal", "process_manage", "read_file", "write_file", "patch", "search_files", "vision_analyze", "skills_list", "skill_view", "skill_manage", @@ -401,7 +401,7 @@ TOOLSETS = { "browser_press", "browser_get_images", "browser_vision", "browser_console", "browser_cdp", "browser_dialog", "browser_exec", - "todo", "memory", + "todo_list", "memory", "session_search", "clarify", "execute_code", "delegate_task", ], @@ -425,7 +425,7 @@ TOOLSETS = { "description": "Editor integration (VS Code, Zed, JetBrains) — coding-focused tools without messaging, audio, or clarify UI", "tools": [ "web_search", "web_extract", - "terminal", "process", + "terminal", "process_manage", "read_file", "write_file", "patch", "search_files", "vision_analyze", "skills_list", "skill_view", "skill_manage", @@ -434,7 +434,7 @@ TOOLSETS = { "browser_press", "browser_get_images", "browser_vision", "browser_console", "browser_cdp", "browser_dialog", "browser_exec", - "todo", "memory", + "todo_list", "memory", "session_search", "execute_code", "delegate_task", ], @@ -447,7 +447,7 @@ TOOLSETS = { # Web "web_search", "web_extract", # Terminal + process management - "terminal", "process", + "terminal", "process_manage", # File manipulation "read_file", "write_file", "patch", "search_files", # Vision + image generation @@ -461,13 +461,13 @@ TOOLSETS = { "browser_vision", "browser_console", "browser_cdp", "browser_dialog", "browser_exec", # Planning & memory - "todo", "memory", + "todo_list", "memory", # Session history search "session_search", # Code execution + delegation "execute_code", "delegate_task", # Cronjob management - "cronjob", + "cronjob_manage", # Home Assistant smart home control (gated on HASS_TOKEN via check_fn) "ha_list_entities", "ha_get_state", "ha_list_services", "ha_call_service", diff --git a/tui_gateway/host_supervisor.py b/tui_gateway/host_supervisor.py index 0b826e4abe..9f8a7bd4a6 100644 --- a/tui_gateway/host_supervisor.py +++ b/tui_gateway/host_supervisor.py @@ -47,6 +47,11 @@ MUTATOR_ROUTE_TABLE: dict[str, str] = { _REGISTRY_NAME = "dashboard-compute-host.json" _RESPAWN_WINDOW_SECS = 300.0 _SHUTDOWN_TIMEOUT_SECS = 10.0 +# Late control-ack handlers (#97948): a compress that outlives its RPC waiter +# can legitimately run for the full compression ceiling plus a stall-fallback +# retry, so keep registrations around well past that — but bounded. +_LATE_CONTROL_TTL_SECS = 1800.0 +_LATE_CONTROL_MAX = 64 def append_log_record(path: str | Path, record: str) -> None: @@ -167,6 +172,11 @@ class HostSupervisor: self._restart_times: list[float] = [] self._pending_turns: dict[str, tuple[str, Callable[[dict], None] | None]] = {} self._pending_controls: dict[str, queue.Queue[dict]] = {} + # request_id -> (registered_at, handler) for control waiters that timed + # out but whose host work is still running (#97948). The host emits + # its control.ack whenever it finishes; without this the ack matched + # no queue and was silently dropped. + self._late_control_handlers: dict[str, tuple[float, Callable[[dict], None]]] = {} self._stderr_tail: list[str] = [] self._last_progress_counter = 0 @@ -307,7 +317,17 @@ class HostSupervisor: payload: dict[str, Any] | None = None, wait: bool = True, timeout: float = 30.0, + on_late_ack: Callable[[dict], None] | None = None, ) -> dict: + """Send a control frame; with ``wait`` block up to ``timeout`` for its ack. + + ``on_late_ack`` (only meaningful with ``wait``) keeps the request + adoptable after the waiter gives up: when the host's ``control.ack`` / + ``control.error`` / ``error`` for this ``request_id`` eventually + arrives, the handler fires once instead of the frame being dropped. + Registrations are bounded by ``_LATE_CONTROL_TTL_SECS`` / + ``_LATE_CONTROL_MAX``. + """ if route_name not in MUTATOR_ROUTE_TABLE: raise ValueError(f"unclassified host mutator route: {route_name}") self.start() @@ -327,10 +347,47 @@ class HostSupervisor: return {"status": "sent", "request_id": request_id} try: return q.get(timeout=timeout) + except queue.Empty: + if on_late_ack is not None: + self._register_late_control_handler(request_id, on_late_ack) + raise finally: with self._lock: self._pending_controls.pop(request_id, None) + def _register_late_control_handler(self, request_id: str, handler: Callable[[dict], None]) -> None: + now = time.monotonic() + with self._lock: + expired = [ + rid + for rid, (registered_at, _cb) in self._late_control_handlers.items() + if now - registered_at > _LATE_CONTROL_TTL_SECS + ] + for rid in expired: + self._late_control_handlers.pop(rid, None) + while len(self._late_control_handlers) >= _LATE_CONTROL_MAX: + oldest = min(self._late_control_handlers, key=lambda rid: self._late_control_handlers[rid][0]) + self._late_control_handlers.pop(oldest, None) + self._late_control_handlers[request_id] = (now, handler) + + def _deliver_control_frame(self, request_id: str, frame: dict[str, Any]) -> None: + with self._lock: + q = self._pending_controls.get(request_id) + late = None if q is not None else self._late_control_handlers.pop(request_id, None) + if q is not None: + try: + q.put_nowait(frame) + except queue.Full: + pass + return + if late is None: + return + _registered_at, handler = late + try: + handler(frame) + except Exception: + logger.exception("compute host late control ack handler failed (request_id=%s)", request_id) + def _spawn_locked(self, *, reason: str) -> None: if self._stopped_respawning: raise RuntimeError("compute host respawn disabled after crash loop") @@ -452,24 +509,10 @@ class HostSupervisor: self._complete_turn(frame) return if ftype in {"control.ack", "control.error", "respond.ack", "respond.error", "interrupt.ack", "reload_mcp.ack", "shutdown.ack"}: - request_id = str(frame.get("request_id") or "") - with self._lock: - q = self._pending_controls.get(request_id) - if q is not None: - try: - q.put_nowait(frame) - except queue.Full: - pass + self._deliver_control_frame(str(frame.get("request_id") or ""), frame) return if ftype == "error" and frame.get("request_id"): - request_id = str(frame.get("request_id") or "") - with self._lock: - q = self._pending_controls.get(request_id) - if q is not None: - try: - q.put_nowait(frame) - except queue.Full: - pass + self._deliver_control_frame(str(frame.get("request_id") or ""), frame) def _complete_turn(self, frame: dict[str, Any]) -> None: request_id = str(frame.get("request_id") or "") @@ -524,6 +567,17 @@ class HostSupervisor: cb(frame) except Exception: logger.exception("compute host error callback failed") + # A crashed host will never emit the late acks the timed-out control + # waiters are still expecting; fail them the same way so the client's + # "still running in the background" notice does not hang forever. + with self._lock: + late = self._late_control_handlers + self._late_control_handlers = {} + for request_id, (_registered_at, handler) in late.items(): + try: + handler({"type": "control.error", "request_id": request_id, "reason": reason, "message": message}) + except Exception: + logger.exception("compute host late control error handler failed") def _maybe_respawn_after_crash(self) -> None: now = time.monotonic() diff --git a/tui_gateway/methods_bot_relay.py b/tui_gateway/methods_bot_relay.py index 992bbc1704..5044518759 100644 --- a/tui_gateway/methods_bot_relay.py +++ b/tui_gateway/methods_bot_relay.py @@ -109,6 +109,41 @@ def _(rid, params: dict) -> dict: if resolved not in known: return _err(rid, 4092, f"no profile '{profile}' on this gateway") + # #100523: when THIS gateway already hosts the target's Bot Chat live + # (the Desktop has it open), the subprocess transport is fenced out by + # the single-owner lease ("already has a live owner") and the payload + # is dropped. Land the DM in the live session as a normal user turn + # via prompt.submit instead — same choke point the composer uses, so + # role alternation, persistence and streaming all behave as a typed + # message would. (Nested per method_ctx rebinding.) + def _live_bot_chat_sid(profile_name: str) -> str: + from tools.bot_mode_probe import BOT_CHAT_TITLE + + live_home = _profile_home(profile_name) + want_home = str(live_home) if live_home is not None else None + for live_sid, record in list(_sessions.items()): + if not isinstance(record, dict): + continue + if (record.get("profile_home") or None) != want_home: + continue + key = _session_lookup_key(record, fallback=live_sid) + if _session_live_title(record, key) == BOT_CHAT_TITLE: + return live_sid + return "" + + live_sid = _live_bot_chat_sid(resolved) + if live_sid: + # queued=True: a teammate's DM runs as the NEXT turn. It must never + # interrupt or steer a turn already in flight (the default busy + # mode does); hundreds of arrivals simply queue in arrival order. + submitted = _methods["prompt.submit"](rid, {"session_id": live_sid, "text": message, "queued": True}) + if "error" in submitted: + return submitted + return _ok( + rid, + {"reply": f"Delivered into @{resolved}'s open Bot Chat; the reply will appear there."}, + ) + fd, tmp = tempfile.mkstemp(prefix="hermes-relay-dm-", suffix=".txt", text=True) try: with os.fdopen(fd, "w", encoding="utf-8") as f: diff --git a/tui_gateway/methods_config.py b/tui_gateway/methods_config.py index 916abfc182..d0ffb2396e 100644 --- a/tui_gateway/methods_config.py +++ b/tui_gateway/methods_config.py @@ -123,6 +123,9 @@ def _(rid, params: dict) -> dict: Lanes carry no session rows here; drill-in uses ``projects.project_sessions``. """ try: + from tui_gateway.project_tree import stamp_profile + from tui_gateway.server import _response_profile_name + with _profile_db(params) as db: if db is None: return _ok( @@ -136,6 +139,9 @@ def _(rid, params: dict) -> dict: session_limit=int(params.get("session_limit") or 2000), include_discovered=True, ) + stamp_profile( + tree["projects"], _response_profile_name(params.get("profile")) + ) return _ok( rid, { @@ -155,6 +161,9 @@ def _(rid, params: dict) -> dict: built from the same authoritative grouping as ``projects.tree`` so ids and membership match exactly. Used when the user enters a project.""" try: + from tui_gateway.project_tree import stamp_profile + from tui_gateway.server import _response_profile_name + project_id = str(params.get("project_id") or "") if not project_id: return _err(rid, 5063, "project_id required") @@ -172,6 +181,9 @@ def _(rid, params: dict) -> dict: session_limit=int(params.get("session_limit") or 5000), include_discovered=False, ) + stamp_profile( + tree["projects"], _response_profile_name(params.get("profile")) + ) proj = next((p for p in tree["projects"] if p["id"] == project_id), None) return _ok(rid, {"project": proj}) except Exception as e: @@ -377,12 +389,49 @@ def _(rid, params: dict) -> dict: return _err(rid, 4002, f"unknown config key: {key}") +def _readiness_profile_scope(params: dict): + """Resolve the optional ``profile`` param of the setup readiness RPCs. + + Returns ``(profile, scope)`` where ``scope`` is a context manager binding + that profile's HERMES_HOME and ``.env`` secret scope (ContextVars, so + concurrent checks for different profiles stay isolated). The launch + profile / no param yields ``("", nullcontext())``. A profile unknown to + this host raises ``FileNotFoundError`` — a readiness check must never + quietly answer for the launch profile instead (#94071). + """ + import contextlib + + profile = str(params.get("profile") or "").strip() if isinstance(params, dict) else "" + if not profile: + return "", contextlib.nullcontext() + from hermes_cli import profiles as profiles_mod + from tui_gateway import server as _server + + if not profiles_mod.profile_exists(profile): + raise FileNotFoundError(f"Profile '{profile}' does not exist on this backend.") + home = _server._profile_home(profile) + if home is None: + return profile, contextlib.nullcontext() + return profile, _server._session_profile_runtime_scope({"profile_home": str(home)}) + + @method("setup.status") def _(rid, params: dict) -> dict: + """Loose provider check; ``profile`` (optional) scopes it to that profile's home.""" try: from hermes_cli.main import _has_any_provider_configured + from tui_gateway.methods_config import _readiness_profile_scope - return _ok(rid, {"provider_configured": bool(_has_any_provider_configured())}) + try: + profile, scope = _readiness_profile_scope(params) + except FileNotFoundError as e: + return _ok(rid, {"ok": False, "profile": params.get("profile"), "error": str(e)}) + with scope: + configured = bool(_has_any_provider_configured(strict_profile_scope=bool(profile))) + payload = {"provider_configured": configured} + if profile: + payload["profile"] = profile + return _ok(rid, payload) except Exception as e: return _err(rid, 5016, str(e)) @@ -397,15 +446,27 @@ def _(rid, params: dict) -> dict: uses on session creation. It returns ok=False with the auth error message when the user's configured model cannot actually be served, so UIs can surface onboarding before the user submits a doomed prompt. + + ``profile`` (optional): answer for THAT profile's home on this host — its + config.yaml model pin and its ``.env`` — instead of the launch profile's + (#94071). A profile unknown to this backend answers ``ok=False`` rather + than reporting the launch profile's readiness. """ try: from hermes_cli.runtime_provider import resolve_runtime_provider from hermes_cli.auth import has_usable_secret from hermes_cli.main import _has_any_provider_configured + from tui_gateway.methods_config import _readiness_profile_scope requested = str(params.get("provider") or "").strip() or None - runtime = resolve_runtime_provider(requested=requested) - provider_configured = bool(_has_any_provider_configured()) + try: + profile, scope = _readiness_profile_scope(params) + except FileNotFoundError as e: + return _ok(rid, {"ok": False, "profile": params.get("profile"), "error": str(e)}) + with scope: + runtime = resolve_runtime_provider(requested=requested) + provider_configured = bool(_has_any_provider_configured(strict_profile_scope=bool(profile))) + scoped = {"profile": profile} if profile else {} provider = runtime.get("provider") or "provider" source = str(runtime.get("source") or "") if ( @@ -425,6 +486,7 @@ def _(rid, params: dict) -> dict: "model": runtime.get("model"), "source": source, "error": "No Hermes provider is configured.", + **scoped, }, ) @@ -446,6 +508,7 @@ def _(rid, params: dict) -> dict: "model": runtime.get("model"), "source": runtime.get("source"), "error": f"No usable credentials found for {provider}.", + **scoped, }, ) @@ -456,6 +519,7 @@ def _(rid, params: dict) -> dict: "provider": runtime.get("provider"), "model": runtime.get("model"), "source": runtime.get("source"), + **scoped, }, ) except Exception as e: diff --git a/tui_gateway/methods_profiles.py b/tui_gateway/methods_profiles.py index db894586bf..3cc757d3b5 100644 --- a/tui_gateway/methods_profiles.py +++ b/tui_gateway/methods_profiles.py @@ -467,6 +467,15 @@ def _(rid, params: dict) -> dict: os.chmod(str(dst_auth), 0o600) except OSError: pass + # Mirroring must not fork single-use OAuth grants (Anthropic / + # Codex / xAI): the first profile to refresh strands every + # sibling (#100339). API keys stay; OAuth rows are dropped + # and read from the root grant via the pool fallback. + try: + from hermes_cli.auth import strip_cloned_single_use_oauth_grants + strip_cloned_single_use_oauth_grants(path) + except Exception: + pass mirrored["auth"] = True except Exception: pass diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 1711f36a10..2b1d0d10fe 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -119,6 +119,82 @@ def _(rid, params: dict) -> dict: # behind for every launch the user never typed into. The row is now created # lazily on the first prompt (see _ensure_session_db_row + prompt.submit), # and the AIAgent's own INSERT-OR-IGNORE persists it on the first turn too. + # + # EXCEPTION — seeded branch children (#93959): a desktop branch carries + # parent_session_id AND a seeded transcript, which is explicit user intent, + # not an abandoned draft. The row MUST exist immediately: the renderer's + # post-create resume re-fetches the child through REST + defer_history + # hydration, both of which read the DB — an unpersisted child 404s, the + # fail-latch then refuses to bind a "transcript-less" session, and the user + # sees an infinite spinner whose optimistic row vanishes on restart. + # Persisting up front also means a restart keeps the branch (both reports + # lost it) and the title lands in the parent's lineage instead of falling + # back to a message-preview name. Title mirrors the TUI /branch naming. + if parent_session_id and history: + try: + with _session_db(_sessions[sid]) as db: + if db is not None: + parent_key = parent_session_id + current = db.get_session_title(parent_key) or "branch" + branch_title = ( + db.get_next_title_in_lineage(current) + if hasattr(db, "get_next_title_in_lineage") + else f"{current} (branch)" + ) + db.create_session( + key, + source=source, + model=_resolve_model(), + model_config={"_branched_from": parent_key}, + parent_session_id=parent_key, + cwd=_sessions[sid]["cwd"], + profile_name=( + Path(profile_home).name if profile_home else None + ), + ) + # Compensation guard (#93959 review): if the transcript + # copy or title write fails AFTER the row committed, the + # durable-but-empty row would defeat the lazy first-prompt + # fallback (_ensure_session_db_row is INSERT OR IGNORE — + # the row exists, so the seed never lands and the renderer + # fail-latches on a "transcript-less" session again). + # Roll back just this child so the seed path can retry + # cleanly on first submit. + try: + db.append_messages_batch( + key, + [ + {"role": m.get("role", "user"), "content": m.get("content")} + for m in history + ], + chunk_rows=500, + ) + db.set_session_title(key, branch_title) + except Exception as exc: + from hermes_state import is_disk_full_error + + if is_disk_full_error(exc): + raise + try: + db.delete_session(key) + except Exception: + logger.debug( + "branch seed compensation delete failed for %s", + key, + exc_info=True, + ) + raise + _sessions[sid]["pending_title"] = None + except Exception: + # Persistence is best-effort here: a failed write must not break + # session.create itself — the lazy first-prompt path remains as the + # fallback, exactly as for plain drafts. + logger.warning( + "seeded-branch persistence failed for %s; falling back to " + "lazy row creation", + key, + exc_info=True, + ) # Return the lightweight session immediately so Ink can paint the composer # + skeleton panel, then build the real AIAgent just after this response is @@ -583,20 +659,37 @@ def _(rid, params: dict) -> dict: # (see _todo_state_from_history) — no extra transcript read here. # Every interactive resume path materializes the model history, even when - # omit_messages suppresses the response copy. Count the complete lineage - # before any reopen/history read so a runaway transcript cannot exhaust - # the dashboard. The metadata fallback keeps lightweight test/adaptor DBs - # that predate the shared SessionDB guard compatible. The limit resolves - # from config (sessions.max_resume_messages, 0 disables). + # omit_messages suppresses the response copy. Count what THIS path will + # actually load before any reopen/history read so a runaway transcript + # cannot exhaust the dashboard. Only the non-deferred, non-omitted + # resume reads the whole compression lineage (ancestors → tip) into + # memory; the deferred Desktop resume (display transcript paged over + # REST), the omit_messages resume, and the lazy watch resume all load + # the TIP segment only — guarding those against the full-lineage count + # rejected exactly the well-compressed conversations compaction is + # meant to produce (85 segments / ~29k lineage rows / ~700-row tip → + # 4130 and a Bot Chat stuck on "Waking up…"). The metadata fallback + # keeps lightweight test/adaptor DBs that predate the shared SessionDB + # guard compatible. The limit resolves from config + # (sessions.max_resume_messages, 0 disables). from hermes_state import ( SessionResumeTooLargeError, resolved_max_resume_messages, ) + eager_build = is_truthy_value(params.get("eager_build", False)) + guard_tip_only = ( + is_truthy_value(params.get("lazy", False)) + or omit_messages + or (defer_history and not eager_build) + ) safety_check = getattr(db, "assert_resume_safe", None) try: if callable(safety_check): - safety_check(target) + if guard_tip_only: + safety_check(target, tip_only=True) + else: + safety_check(target) else: resume_limit = resolved_max_resume_messages() stored_message_count = int(found.get("message_count") or 0) @@ -659,9 +752,11 @@ def _(rid, params: dict) -> dict: _cancel_ws_orphan_reap(sid) return _ok(rid, _reuse_live_payload(sid, session)) - # Fast path: if the session is already live, reuse it under the lock. + # Fast path: if the session is already live IN THIS PROFILE, reuse it + # under the lock. Never another profile's runtime of the same stored id + # — that ran profile B's turn on profile A's agent/memory (#100029). with _session_resume_lock: - live = _find_live_session_by_key(target) + live = _find_live_session_by_key(target, profile_home) if live is not None: return _reuse_live_response(*live) @@ -984,7 +1079,7 @@ def _(rid, params: dict) -> dict: # live session while we were building. Re-check under the lock; if it won, # discard our just-built agent and reuse theirs (no worker/poller wired yet). with _session_resume_lock: - live = _find_live_session_by_key(target) + live = _find_live_session_by_key(target, profile_home) if live is not None: try: if hasattr(agent, "close"): @@ -1614,7 +1709,8 @@ def _(rid, params: dict) -> dict: except (ValueError, KeyError): return _err(rid, 4024, f"unknown platform '{platform_name}'") try: - gw_config = load_gateway_config() + with _session_profile_runtime_scope(session): + gw_config = load_gateway_config() except Exception as e: return _err(rid, 5021, f"could not load gateway config: {e}") pcfg = gw_config.platforms.get(platform) @@ -2913,13 +3009,37 @@ def _(rid, params: dict) -> dict: sid = str(params.get("session_id") or "") focus_topic = str(params.get("focus_topic", "") or "").strip() command = "/compress" + (f" {focus_topic}" if focus_topic else "") + _late_session = session + + def _on_late_ack(late: dict, _sid=sid) -> None: + _adopt_late_compute_host_compress_ack(_sid, _late_session, late, route_name="session.compress") + try: ack = _send_compute_host_control( sid, route_name="session.compress", command=command, wait=True, - timeout=120.0, + # Follows compression.context_total_ceiling_seconds instead of + # a fixed 120s: the host legitimately runs that long (#97948). + timeout=_compute_host_compress_wait_seconds(), + on_late_ack=_on_late_ack, + ) + except queue.Empty: + # The waiter gave up but the host is still compressing; the late + # ack handler adopts the rotated session and pushes session.info + # when it lands. Not an error — the old 5019 made Desktop/TUI + # report a timeout while compression later succeeded silently. + return _ok( + rid, + { + "status": "pending", + "turn_isolation": True, + "message": ( + "compression still running in the background; " + "the transcript will refresh when it finishes" + ), + }, ) except Exception as exc: return _err(rid, 5019, f"compute-host compress failed: {exc}") diff --git a/tui_gateway/methods_tools.py b/tui_gateway/methods_tools.py index 7aeea32949..f54310f2b6 100644 --- a/tui_gateway/methods_tools.py +++ b/tui_gateway/methods_tools.py @@ -133,7 +133,7 @@ def _(rid, params: dict) -> dict: return _err(rid, 5019, f"compute-host reload_mcp failed: {exc}") return _ok(rid, {"status": "reloaded", "turn_isolation": True, "host_ack": ack}) - from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools + from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, reprobe_tool_availability def _refresh_session_agent() -> None: """Rebuild THIS session's cached tool snapshot from the live @@ -184,6 +184,7 @@ def _(rid, params: dict) -> dict: loaded = _compute_mcp_rev() for _ in range(_MCP_RELOAD_MAX_PASSES): shutdown_mcp_servers() + reprobe_tool_availability() discover_mcp_tools() after = _compute_mcp_rev() if after == loaded: @@ -1064,12 +1065,31 @@ def _(rid, params: dict) -> dict: sid = params.get("session_id", "") if _session_uses_compute_host(session): command = f"/{name}" + (f" {arg}" if arg else "") + _late_session = session + + def _on_late_ack(late: dict, _sid=sid) -> None: + _adopt_late_compute_host_compress_ack(_sid, _late_session, late, route_name="slash.compress") + try: ack = _send_compute_host_control( sid, route_name="slash.compress", command=command, wait=True, + timeout=_compute_host_compress_wait_seconds(), + on_late_ack=_on_late_ack, + ) + except queue.Empty: + return _ok( + rid, + { + "type": "exec", + "status": "pending", + "output": ( + "compression still running in the background; " + "the transcript will refresh when it finishes" + ), + }, ) except Exception as exc: return _err(rid, 5019, f"compute-host slash.compress failed: {exc}") diff --git a/tui_gateway/project_tree.py b/tui_gateway/project_tree.py index 4f2d1eaaae..0b4310ddaa 100644 --- a/tui_gateway/project_tree.py +++ b/tui_gateway/project_tree.py @@ -63,6 +63,21 @@ NO_PROJECT_LABEL = "Home" _MAX_SIBLING_PROBES = 4 +def stamp_profile(projects: list[dict], profile: str) -> None: + """Make every session row self-describing for cross-profile routing. + + A scoped project tree is built from one profile's state.db, so the request + scope is authoritative even for legacy rows whose ``profile_name`` is NULL. + """ + for project in projects: + for session in project.get("previewSessions") or []: + session["profile"] = profile + for repo in project.get("repos") or []: + for group in repo.get("groups") or []: + for session in group.get("sessions") or []: + session["profile"] = profile + + def _branch_lane_id(repo_root: str, branch: str = "") -> str: """The one definition of a main-checkout lane id (must match the desktop).""" return f"{repo_root}::branch::{(branch or '').strip()}" diff --git a/tui_gateway/server.py b/tui_gateway/server.py index f51db6d9db..d805401153 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -771,6 +771,17 @@ def _release_active_session_slot(session: dict | None) -> bool: return False +def _own_live_lease_ids(*, exclude=None) -> set[str]: + """Snapshot leases still backed by this process's live session records.""" + with _sessions_lock: + return { + str(lease.lease_id) + for session in _sessions.values() + if (lease := session.get("active_session_lease")) is not None + and lease is not exclude + } + + @contextlib.contextmanager def _other_runtime_lease_guard(session_id: str, session: dict): """Release this runtime and lock sibling ownership through the DB write.""" @@ -791,13 +802,20 @@ def _other_runtime_lease_guard(session_id: str, session: dict): last_error: Exception | None = None stack = contextlib.ExitStack() + own_live_lease_ids = _own_live_lease_ids(exclude=lease) for attempt in range(3): try: if lease is not None and getattr(lease, "enabled", False): - guard = release_active_session_liveness_guard(lease, session_id) + guard = release_active_session_liveness_guard( + lease, + session_id, + own_live_lease_ids=own_live_lease_ids, + ) else: guard = active_session_liveness_guard( - session_id, registry_home=session.get("profile_home") + session_id, + registry_home=session.get("profile_home"), + own_live_lease_ids=own_live_lease_ids, ) active = stack.enter_context(guard) break @@ -1937,12 +1955,7 @@ def _reclaim_orphaned_leases() -> None: try: from hermes_cli.active_sessions import release_orphaned_leases - with _sessions_lock: - live = { - lease.lease_id - for session in _sessions.values() - if (lease := session.get("active_session_lease")) is not None - } + live = _own_live_lease_ids() if dropped := release_orphaned_leases(live): logger.info("Reclaimed %d orphaned active-session lease(s)", dropped) except Exception: @@ -2464,7 +2477,18 @@ def _profile_home(profile: str | None) -> Path | None: # Already the launch profile? No override needed. if home.resolve() == Path(_hermes_home).resolve(): return None - return home if (home / "state.db").exists() or home.exists() else None + if (home / "state.db").exists() or home.exists(): + # Remember every sibling home this backend was asked to serve so the + # change watcher stats its store too (#99333 class). + _served_profile_homes.add(home) + return home + return None + + +# Profile homes served by this process besides the launch home — the only +# extra stores the sessions watcher must probe. Empty on single-profile +# installs, so their watcher stays byte-identical (two stats per tick). +_served_profile_homes: set[Path] = set() def _profile_scoped(handler): @@ -2663,6 +2687,11 @@ def _broadcast_global_event(event: str, payload: dict | None = None) -> None: _compute_host_supervisor = None _compute_host_supervisor_lock = threading.Lock() +# Hard cap on how long session.compress blocks its RPC waiting for the compute +# host (#97948). Must stay below the desktop's SESSION_COMPRESS_TIMEOUT_MS +# (660s) so the client receives the `pending` answer instead of its own +# timeout error; the late-ack path covers anything slower. +_COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS = 630.0 def _inside_compute_host_child() -> bool: @@ -2938,6 +2967,7 @@ def _send_compute_host_control( payload: dict | None = None, wait: bool = True, timeout: float = 30.0, + on_late_ack=None, ) -> dict: frame = dict(payload or {}) frame.setdefault("type", "control") @@ -2948,9 +2978,68 @@ def _send_compute_host_control( payload=frame, wait=wait, timeout=timeout, + on_late_ack=on_late_ack, ) +def _compute_host_compress_wait_seconds(cfg: dict | None = None) -> float: + """RPC wait budget for a compute-host compress control (#97948). + + Manual compression legitimately runs up to the configured + ``compression.context_total_ceiling_seconds`` (default 600s), so a fixed + 120s waiter reported a false timeout while the host kept working. Follow + the ceiling with a little slack, but cap the blocking wait so it stays + below the desktop's own RPC timeout; anything longer is adopted through + the late-ack path instead of failing. + """ + from agent.conversation_compression import resolve_context_compression_timeouts + + try: + compression_cfg = (cfg if cfg is not None else _load_cfg()).get("compression", {}) + except Exception: + compression_cfg = {} + _idle, ceiling = resolve_context_compression_timeouts( + compression_cfg if isinstance(compression_cfg, dict) else {} + ) + return float(min(max(ceiling + 30.0, 120.0), _COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS)) + + +def _announce_compute_host_compress_done(sid: str, session: dict, ack: dict) -> None: + """Mirror a compute-host compress ack and push the client-visible edges. + + Emits the same ``session.info`` the in-process /compress path does plus + the ``compacted`` status edge, so a client whose own RPC wait already + expired still learns the transcript changed. + """ + _apply_compute_host_metadata_mirror(session, ack) + try: + info = _session_info(session.get("agent"), session) + except TypeError: + info = _session_info(session.get("agent")) + _emit("session.info", sid, info) + _status_update(sid, "compacted", "✓ Context compression complete") + + +def _adopt_late_compute_host_compress_ack(sid: str, session: dict, ack: dict, *, route_name: str) -> None: + """Adopt a compute-host compress ack that arrived after its RPC waiter gave up. + + The RPC already answered ``status: pending``; this is the only place the + rotated session_key / history_version / session_info mirror can land, and + the only signal the client gets that the transcript changed. A late + ``control.error`` surfaces through the existing ``error`` event path. + """ + with _sessions_lock: + live = _sessions.get(sid) + if live is not session: + return + if not isinstance(ack, dict) or ack.get("type") in {"control.error", "error"}: + message = str((ack or {}).get("message") or f"compute-host {route_name} failed") + _emit("error", sid, {"message": f"compression failed: {message}"}) + _status_update(sid, "ready") + return + _announce_compute_host_compress_done(sid, session, ack) + + def _approval_request_payload(data: dict | None) -> dict: """Build the client-safe representation of a pending approval.""" payload = dict(data or {}) @@ -3357,6 +3446,7 @@ def _start_agent_build(sid: str, session: dict) -> None: notify_registered = False home_token = None secret_token = None + build_terminal_token = None session_db = None owns_db = False profile_home = current.get("profile_home") @@ -3383,6 +3473,21 @@ def _start_agent_build(sid: str, session: dict) -> None: secret_token = set_secret_scope(build_profile_secret_scope(Path(profile_home))) except Exception: pass + # Bind the profile's COMPLETE terminal policy for the agent + # build (fail-closed: malformed policy → refusal scope) so + # _make_agent's terminal probing / cwd hints resolve the + # routed profile, never the launch process (#98581 class). + try: + from tools.terminal_scope import ( + install_profile_terminal_scope, + reset_terminal_scope, + ) + + build_terminal_token = install_profile_terminal_scope( + Path(profile_home) + ) + except Exception: + build_terminal_token = None # DEDICATED handle — ours until _transfer_db_to_agent hands # it to the built agent in the finally below. Every path # that leaves this build without that transfer (the except @@ -3534,6 +3639,13 @@ def _start_agent_build(sid: str, session: dict) -> None: reset_secret_scope(secret_token) except Exception: pass + if build_terminal_token is not None: + try: + from tools.terminal_scope import reset_terminal_scope + + reset_terminal_scope(build_terminal_token) + except Exception: + pass # _attach_worker already closed the worker if this session was # reaped mid-build; only the late notify registration can still # leak (session.close unregistered before _build registered it). @@ -5122,15 +5234,17 @@ def _sessions_sig(): """Newest mtime across state.db and its WAL — the cross-process change signal. Messaging-gateway turns and cron runs are written by OTHER processes that never touch this gateway's transports; the shared SQLite - file is the one thing they all move (#58671).""" - home = _watcher_home() + file is the one thing they all move (#58671). A backend serving several + profiles owns one store per profile, so every served sibling home is + probed too — otherwise a routed profile's Bot Chat never refreshes.""" sig = None - for name in ("state.db", "state.db-wal"): - try: - mtime = (home / name).stat().st_mtime_ns - except OSError: - continue - sig = mtime if sig is None else max(sig, mtime) + for root in (_watcher_home(), *_served_profile_homes): + for name in ("state.db", "state.db-wal"): + try: + mtime = (root / name).stat().st_mtime_ns + except OSError: + continue + sig = mtime if sig is None else max(sig, mtime) return sig @@ -6055,6 +6169,8 @@ def _load_service_tier() -> str | None: return None if raw in {"fast", "priority", "on"}: return "priority" + if raw in {"auto", "cold"}: + return raw return None @@ -6397,9 +6513,20 @@ def _session_profile_runtime_scope(session: dict): return home_token = set_hermes_home_override(profile_home) secret_token = set_secret_scope(build_profile_secret_scope(Path(profile_home))) + # Same authoritative terminal policy the gateway binds per turn (#68559): + # a docker-configured dashboard profile must never resolve the launch + # process's pinned env. Failure → refusal scope (fail closed). + from tools.terminal_scope import ( + install_profile_terminal_scope as _install_term_scope, + ) + + terminal_token = _install_term_scope(Path(profile_home)) try: yield finally: + from tools.terminal_scope import reset_terminal_scope + + reset_terminal_scope(terminal_token) reset_secret_scope(secret_token) reset_hermes_home_override(home_token) @@ -7977,7 +8104,7 @@ def _todo_state_from_history(history) -> dict | None: if not isinstance(msg, dict): continue for call in msg.get("tool_calls") or []: - if (call.get("function") or {}).get("name") == "todo": + if (call.get("function") or {}).get("name") in ("todo_list", "todo"): cid = call.get("id") if cid: todo_call_ids.add(cid) @@ -8062,7 +8189,7 @@ def _on_tool_complete(sid: str, tool_call_id: str, name: str, args: dict, result if result_text: payload["result_text"] = result_text todo_state = None - if name == "todo": + if name in ("todo_list", "todo"): # legacy alias: pre-rename replays todo_state = _normalize_todo_state(payload.get("result")) if todo_state is not None: payload.update(todo_state) @@ -8088,7 +8215,7 @@ def _on_tool_complete(sid: str, tool_call_id: str, name: str, args: dict, result _tool_progress_enabled(sid) or payload.get("inline_diff") or _tool_lifecycle_required_for_ui(name) - or name == "todo" + or name in ("todo_list", "todo") ): _emit("tool.complete", sid, payload) # Task state is application data, not optional tool-progress chrome. A @@ -8201,6 +8328,8 @@ def _on_tool_progress( payload["parent_id"] = str(_kwargs["parent_id"]) if _kwargs.get("child_session_id"): payload["child_session_id"] = str(_kwargs["child_session_id"]) + if _kwargs.get("delegation_id"): + payload["delegation_id"] = str(_kwargs["delegation_id"]) if _kwargs.get("depth") is not None: payload["depth"] = int(_kwargs["depth"]) if _kwargs.get("model"): @@ -9984,6 +10113,128 @@ def _fail_inflight_turn( session["inflight_turn"] = turn +_TURN_FAILURE_DETAIL_LIMIT = 240 +# Shortest run of the submitted prompt that counts as the provider quoting it +# back. Long enough that shared boilerplate ("Invalid request for model ") does +# not trip it, short enough to catch a quoted sentence. +_TURN_PROMPT_ECHO_WINDOW = 24 +# Ceiling on the prompt we shingle. An @-expanded prompt can carry a whole +# file; the failure path must stay cheap. +_TURN_PROMPT_ECHO_MAX_PROMPT = 65536 + + +def _strip_prompt_echo(message: str, prompt: Any) -> str: + """Blank runs of the submitted prompt that ``message`` quotes back. + + Secret redaction and prompt omission are different contracts, and only the + first one is pattern-based. A provider 4xx that echoes the request carries + ordinary private prose -- a paragraph about a person, a pasted file from an + ``@`` reference -- that matches no credential pattern and would otherwise + reach the log intact. This closes that path directly: anything the message + shares with the prompt for ``_TURN_PROMPT_ECHO_WINDOW`` characters or more + becomes ````. + + Shingle-set matching, not a diff: cost is linear in both strings, which + matters because this runs on every failed turn and an ``@`` reference can + make the prompt arbitrarily long. The JSON-escaped form of the prompt is + shingled too, since a provider that hands back its own request body often + hands it back escaped. + + Verbatim echo is what this stops. A paraphrase, a re-encoding (base64, a + different unicode normalization) or a summary of the prompt would survive, + so this is a floor and not a proof; the guarantee it does give is that the + prompt cannot reach the record by being quoted. + """ + if not message or not prompt: + return message + needle = " ".join(str(prompt).split())[:_TURN_PROMPT_ECHO_MAX_PROMPT] + window = _TURN_PROMPT_ECHO_WINDOW + if len(needle) < window or len(message) < window: + return message + shingles = {needle[i:i + window] for i in range(len(needle) - window + 1)} + try: + escaped = json.dumps(needle)[1:-1] + except Exception: + escaped = "" + if escaped and escaped != needle: + shingles.update( + escaped[i:i + window] for i in range(len(escaped) - window + 1) + ) + out: list[str] = [] + i = 0 + n = len(message) + while i <= n - window: + if message[i:i + window] in shingles: + j = i + window + while j < n and message[j - window + 1:j + 1] in shingles: + j += 1 + out.append("") + i = j + else: + out.append(message[i]) + i += 1 + out.append(message[i:]) + return "".join(out) + + +def _turn_failure_detail(error: Any, reason: Any = None, prompt: Any = None) -> str: + """Render why a turn failed, for the ``tui turn finished`` bookend. + + Returns ``""`` when there is nothing to say, otherwise a fragment that + already carries its own leading space, so the caller can append it to the + record unconditionally. + + #86865 added the bookend to trace compression rotations, so it logs + identities and a coarse ``status`` and deliberately logs no content. + #89117 is what the missing cause costs: a report consisting of two lines + reading ``status=error error_retained=True duration=0.9s`` with no way to + tell a provider 4xx from a budget wall from a crashed finalizer. The + returned-error path -- the one a 0.9 s failure almost always takes -- + emits no other log line at all; only the exception path prints to stderr, + which is why the quiet failures are the ones that get filed. + + Content discipline follows #86865's, and it takes two separate steps + because it is two separate contracts. ``redact_sensitive_text`` removes + credentials, which are pattern-shaped. It does nothing about a 4xx body + that quotes the request back, because ordinary private prose is not + pattern-shaped -- so ``_strip_prompt_echo`` removes that separately, using + the submitted ``prompt`` itself as the thing to look for. The invariant the + two of them keep is: this record may gain failure classification and + provider detail, and may not newly persist the user's own content. + + ``prompt`` is optional so the helper stays callable from a path that has no + prompt in scope, but the turn paths always pass it; without it, only the + secret contract is enforced. + """ + reason_text = str(reason or "").strip() + message = str(error or "").strip() + if isinstance(error, BaseException): + message = message or type(error).__name__ + if not message and not reason_text: + return "" + try: + from agent.redact import redact_sensitive_text + + message = redact_sensitive_text(message, force=True) + except Exception: + # A redactor that cannot run must not be able to leak the raw + # message into the log by failing open. + message = "" + message = " ".join(message.split()) + # After the collapse, so both sides are compared in the same shape, and + # before the truncation, so a quote that starts inside the kept prefix + # cannot survive by being cut mid-run. + message = _strip_prompt_echo(message, prompt) + if len(message) > _TURN_FAILURE_DETAIL_LIMIT: + message = message[:_TURN_FAILURE_DETAIL_LIMIT] + "\u2026" + out = "" + if reason_text: + out += " failure_reason=%s" % " ".join(reason_text.split()) + if message: + out += " cause=%r" % message + return out + + # ── Auto-continue: resume a turn killed by a process/machine death ──── # # A turn that concludes — success, handled error, interrupt — clears its @@ -10747,14 +10998,33 @@ def _deferred_session_record( } +_ANY_PROFILE = object() # default: match a live session regardless of profile + + +def _live_profile_matches(session: dict, profile_home) -> bool: + """True when ``session`` belongs to ``profile_home`` (None = launch profile). + + Same string compare as session.resume's ``_find_live_unpersisted``: a + record with no ``profile_home`` is the launch profile's. ``_ANY_PROFILE`` + disables the check for callers that have no profile to scope by. + """ + if profile_home is _ANY_PROFILE: + return True + want = str(profile_home) if profile_home else None + return (session.get("profile_home") or None) == want + + def _claim_or_reuse_live( sid: str, session_key: str, record: dict, lease ) -> tuple[str, dict] | None: """Register ``record`` as the live session for ``session_key`` under the resume lock, or — if a concurrent resume already won — release ``lease`` and return the winner for the caller to reuse.""" + # The record carries the home this resume resolved; a live runtime of the + # same stored id under ANOTHER profile is not a winner to reuse (#100029). + profile_home = record.get("profile_home") with _session_resume_lock: - live = _find_live_session_by_key(session_key) + live = _find_live_session_by_key(session_key, profile_home) if live is not None: if lease is not None: lease.release() @@ -10772,7 +11042,7 @@ def _claim_or_reuse_live( # those quietly so the reap doesn't later broadcast session.reclaimed # for a session the client just re-resumed (auto-re-resume storm). _cancel_ws_orphan_reap(sid) - stale = _claim_parked_runtimes(session_key, keep_sid=sid) + stale = _claim_parked_runtimes(session_key, keep_sid=sid, profile_home=profile_home) # Slow finalization work stays OUTSIDE _session_resume_lock (see # _pop_session_by_id) — the stale records are already claimed above. _finalize_superseded_runtimes(stale) @@ -10780,7 +11050,7 @@ def _claim_or_reuse_live( def _claim_parked_runtimes( - session_key: str, *, keep_sid: str + session_key: str, *, keep_sid: str, profile_home=_ANY_PROFILE ) -> list[tuple[str, dict]]: """Claim sentinel-parked stale runtimes of ``session_key`` for supersession. @@ -10799,6 +11069,7 @@ def _claim_parked_runtimes( if old_sid != keep_sid and not old.get("_finalized") and _session_lookup_key(old, fallback=old_sid) == session_key + and _live_profile_matches(old, profile_home) and old.get("transport") is _detached_ws_transport ] for old_sid, _old in candidates: @@ -10858,8 +11129,39 @@ def _schedule_resume_hydration( {"phase": "history", "status": "loading"}, ) db.reopen_session(stored_id) - raw_history, display_history = db.get_resume_conversations(stored_id) - prefix = db.get_ancestor_display_prefix(stored_id) + from hermes_state import SessionResumeTooLargeError + + # The deferred resume is guarded tip-only (session.resume): the + # display transcript is REST-paginated, so the ancestor prefix is + # an in-memory convenience (rewind ordinal translation, branch + # snapshots), not a requirement. Materialize the full lineage only + # while it fits sessions.max_resume_messages; past that, hydrate + # the tip alone instead of loading the runaway lineage the guard + # exists to keep out of memory (the omit_messages resume already + # runs with an empty prefix, so this is an existing shape). + prefix_fits = True + guard = getattr(db, "assert_resume_safe", None) + if callable(guard): + try: + guard(stored_id) + except SessionResumeTooLargeError as exc: + prefix_fits = False + logger.info( + "resume %s: compression lineage exceeds the resume " + "limit (%s); hydrating the tip segment only", + stored_id, exc, + ) + except Exception: + logger.debug("resume lineage guard failed; loading full lineage", exc_info=True) + if prefix_fits: + raw_history, display_history = db.get_resume_conversations(stored_id) + prefix = db.get_ancestor_display_prefix(stored_id) + else: + raw_history = db.get_messages_as_conversation( + stored_id, repair_alternation=True, include_row_ids=True + ) + display_history = raw_history + prefix = [] history = sanitize_replay_history(raw_history) if _sessions.get(sid) is not session: @@ -10996,11 +11298,19 @@ def _session_lookup_key(session: dict, *, fallback: str = "") -> str: ) -def _find_live_session_by_key(session_key: str) -> tuple[str, dict] | None: +def _find_live_session_by_key( + session_key: str, profile_home=_ANY_PROFILE +) -> tuple[str, dict] | None: + # Stored session ids are timestamp-based and can legitimately exist in more + # than one profile's store, so a bare-id match can hand profile B's resume + # profile A's live runtime (#100029). Profile-aware callers pass the home + # they resolved; the match must then be on (profile_home, session_key). for sid, session in list(_sessions.items()): if session.get("_finalized"): continue - if _session_lookup_key(session, fallback=sid) == session_key: + if _session_lookup_key(session, fallback=sid) == session_key and _live_profile_matches( + session, profile_home + ): return sid, session return None @@ -11098,7 +11408,16 @@ def _live_visible_history(session: dict, db, in_memory_fallback: list[dict]) -> key = session.get("session_key") if db is not None and key: try: - display = db.get_messages_as_conversation(key, include_ancestors=True, include_row_ids=True) + display = db.get_messages_as_conversation( + key, + include_ancestors=True, + include_row_ids=True, + # Display read: a compacted session's archived turns are still + # the user's conversation. Without them a warm switch repainted + # the chat as just the summary + tail while the REST transcript + # showed everything (#92080). + include_compacted=True, + ) return _reconcile_display_with_live(display, in_memory_fallback) except Exception: logger.debug("live display projection read failed", exc_info=True) @@ -12964,6 +13283,16 @@ def _run_prompt_submit( # True once a failed turn's snapshot was retained for resume replay — # tells the finally below to skip the normal inflight clear. turn_error_retained = False + # One-line cause for the "tui turn finished" bookend below. The record + # fires from a `finally`, where neither `result` nor the caught + # exception is reliably in scope, so both failure paths stash their + # cause here on the way past. + turn_error_detail = "" + # What this turn actually submitted, kept only so the cause can be + # checked for quoting it back (see _strip_prompt_echo). Bound here + # rather than read from the turn body because the exception path can + # fire before the prompt is resolved. + turn_prompt_text = "" # Durable crash marker: written before the turn runs, retired the # moment its outcome reaches the client (see _retire_turn_marker). # Any concluded turn — success, handled error, interrupt — retires @@ -13001,6 +13330,20 @@ def _run_prompt_submit( if _profile_home_str: home_token = set_hermes_home_override(_profile_home_str) secret_token = set_secret_scope(build_profile_secret_scope(Path(_profile_home_str))) + # Fourth profile seam: bind the session profile's COMPLETE + # terminal policy for this turn (dashboard/TUI analogue of the + # gateway's per-turn scope). #98581's unified-desktop + # reproduction ran a docker-configured profile on the host + # because terminal_tool read the launch process's pinned env. + # Failure installs a refusal scope → terminal tools raise + # (fail closed) instead of inheriting ambient policy. + from tools.terminal_scope import ( + install_profile_terminal_scope as _install_term_scope, + ) + + _terminal_scope_token = _install_term_scope(Path(_profile_home_str)) + else: + _terminal_scope_token = None # The sudo password callback is thread-local (tools.terminal_tool # _callback_tls), so wiring it on the build thread doesn't reach this # turn thread — terminal sudo prompts would fall through to /dev/tty @@ -13070,6 +13413,11 @@ def _run_prompt_submit( return prompt = ctx.message + # After @-expansion on purpose: an injected file's contents are + # exactly the kind of private material a provider echo would carry + # back, and they are not in `text`. + turn_prompt_text = prompt if isinstance(prompt, str) else "" + # Decide image routing per-turn based on active provider/model. # "native" → pass pixels to the main model as OpenAI-style content # parts (adapters translate for Anthropic/Gemini/Bedrock/etc.). @@ -13483,6 +13831,11 @@ def _run_prompt_submit( error_surface=_error_surface, ) turn_error_retained = True + turn_error_detail = _turn_failure_detail( + (result.get("error") if isinstance(result, dict) else raw), + (result.get("failure_reason") if isinstance(result, dict) else None), + turn_prompt_text, + ) else: _clear_inflight_turn(session) if status == "error": @@ -13711,6 +14064,9 @@ def _run_prompt_submit( retire_marker=terminal_receipt_committed, ) turn_error_retained = True + turn_error_detail = _turn_failure_detail( + e, type(e).__name__, turn_prompt_text + ) except Exception as emit_exc: print( f"[gateway-turn] terminal error emit failed: " @@ -13765,6 +14121,10 @@ def _run_prompt_submit( reset_hermes_home_override(home_token) if secret_token is not None: reset_secret_scope(secret_token) + if _terminal_scope_token is not None: + from tools.terminal_scope import reset_terminal_scope + + reset_terminal_scope(_terminal_scope_token) _clear_session_context(session_tokens) _current_runtime_session_record.reset(runtime_session_token) reset_transport(transport_token) @@ -13786,7 +14146,8 @@ def _run_prompt_submit( # without reaching this finally. logger.info( "tui turn finished: ui_session=%s session_key=%s " - "agent_session_id=%s status=%s error_retained=%s duration=%.1fs", + "agent_session_id=%s status=%s error_retained=%s duration=%.1fs" + "%s", sid, session.get("session_key") or "", getattr(agent, "session_id", "") or "", @@ -13801,6 +14162,7 @@ def _run_prompt_submit( else ("error" if turn_error_retained else "complete"), turn_error_retained, time.monotonic() - _turn_started_monotonic, + turn_error_detail, ) # Backstop for turns that never reached a terminal frame (the # frame paths retire the marker as they emit). @@ -14406,19 +14768,20 @@ def _(rid, params: dict) -> dict: raw = str(value or "").strip().lower() agent = session.get("agent") if session else None if agent is not None: - current_fast = getattr(agent, "service_tier", None) == "priority" + current_tier = getattr(agent, "service_tier", None) elif session is not None and session.get("create_service_tier_override") is not None: # Pre-build session with a pinned tier (desktop draft pick or an # earlier session-scoped toggle) — report/toggle from the pin, not # the global default. - current_fast = session["create_service_tier_override"] == "priority" + current_tier = session["create_service_tier_override"] or None else: - current_fast = _load_service_tier() == "priority" + current_tier = _load_service_tier() + current_fast = current_tier == "priority" if raw in {"status"}: return _ok( rid, - {"key": key, "value": "fast" if current_fast else "normal"}, + {"key": key, "value": {"priority": "fast", None: "normal"}.get(current_tier, current_tier)}, ) if raw in {"", "toggle"}: @@ -14427,6 +14790,8 @@ def _(rid, params: dict) -> dict: nv = "fast" elif raw in {"normal", "off"}: nv = "normal" + elif raw in {"auto", "cold"}: + nv = raw else: return _err(rid, 4002, f"unknown fast mode: {value}") @@ -14452,7 +14817,11 @@ def _(rid, params: dict) -> dict: 4002, "fast mode is not available without a selected model", ) - overrides = resolve_fast_mode_overrides(target_model) + overrides = resolve_fast_mode_overrides( + target_model, + provider=getattr(agent, "provider", None), + base_url=getattr(agent, "base_url", None), + ) if overrides is None: return _err( rid, @@ -14469,13 +14838,11 @@ def _(rid, params: dict) -> dict: # build ("switch one session, switches everywhere"). Pin the # create override so lazily-built sessions and rebuilds (/new, # deferred resume) keep the choice; "" pins normal explicitly. - session["create_service_tier_override"] = ( - "priority" if nv == "fast" else "" - ) + session["create_service_tier_override"] = {"fast": "priority", "normal": ""}.get(nv, nv) else: _write_config_key("agent.service_tier", nv) if agent is not None: - agent.service_tier = "priority" if nv == "fast" else None + agent.service_tier = {"fast": "priority", "normal": None}.get(nv, nv) current_overrides = dict(getattr(agent, "request_overrides", {}) or {}) current_overrides.pop("service_tier", None) current_overrides.pop("speed", None) @@ -15474,6 +15841,7 @@ def _project_tree_row(r: dict) -> dict: return { "id": r.get("id"), "_lineage_root_id": r.get("_lineage_root_id"), + "_lineage_ids": r.get("_lineage_ids"), # The sidebar nests branch/fork sessions under their parent # (flattenSessionsWithBranches keys on this); without it, lane rows can't # draw the └─ connector the flat Recents list shows. @@ -16535,13 +16903,28 @@ def _mirror_slash_side_effects(sid: str, session: dict, command: str) -> str: _MUTATES_WHILE_RUNNING = {"model", "personality", "prompt", "compress"} if _session_uses_compute_host(session) and name in _MUTATES_WHILE_RUNNING: route_name = f"slash.{name}" + is_compress = name == "compress" + _late_session = session + + def _on_late_ack(late: dict, _sid=sid) -> None: + _adopt_late_compute_host_compress_ack(_sid, _late_session, late, route_name=route_name) + try: ack = _send_compute_host_control( sid, route_name=route_name, command=command, wait=True, + **( + {"timeout": _compute_host_compress_wait_seconds(), "on_late_ack": _on_late_ack} + if is_compress + else {} + ), ) + except queue.Empty: + if is_compress: + return "compression still running in the background; the transcript will refresh when it finishes" + return f"compute-host {route_name} failed: timed out" except Exception as exc: return f"compute-host {route_name} failed: {exc}" if ack.get("type") in {"control.error", "error"}: @@ -16644,6 +17027,8 @@ def _mirror_slash_side_effects(sid: str, session: dict, command: str) -> str: agent.service_tier = "priority" elif mode in {"normal", "off"}: agent.service_tier = None + elif mode in {"auto", "cold"}: + agent.service_tier = mode _emit("session.info", sid, _session_info(agent, session)) elif name == "reload-mcp" and agent and hasattr(agent, "reload_mcp_tools"): agent.reload_mcp_tools() @@ -16709,6 +17094,30 @@ def _voice_tts_enabled() -> bool: return os.environ.get("HERMES_VOICE_TTS", "").strip() == "1" +def _tts_lease_async(lease: str, active: bool) -> None: + """Acquire/release a TTS engine lease off the RPC thread. + + Speech-output toggles are the signal that TTS is about to be needed (or + no longer is). Acquiring warms the configured provider — for local + engines that is a model load, possibly a voice download — so it must not + block the toggle's RPC reply. Release is cheap but rides the same thread + for symmetry. Best-effort: a failure here never affects the toggle. + """ + + def _run(): + try: + from tools.tts_tool import acquire_tts_lease, release_tts_lease + + if active: + acquire_tts_lease(lease) + else: + release_tts_lease(lease) + except Exception as e: + logger.debug("voice: tts lease %s active=%s failed: %s", lease, active, e) + + threading.Thread(target=_run, name=f"tts-lease-{lease}", daemon=True).start() + + def _any_session_running() -> bool: """True while any session's agent turn is in flight. @@ -17558,6 +17967,12 @@ def _(rid, params: dict) -> dict: except Exception: stop_hint = "" + # Voice mode with speech output already on (voice.auto_tts / + # prior /voice tts) means replies will be spoken — warm the + # engine now rather than on the first reply. + if _voice_tts_enabled(): + _tts_lease_async("tui:voice-tts", True) + if not enabled: # Disabling the mode must tear the continuous loop down; the # loop holds the microphone and would otherwise keep running. @@ -17574,6 +17989,7 @@ def _(rid, params: dict) -> dict: # and silence any in-flight streaming speech. os.environ["HERMES_VOICE_TTS"] = "0" _tts_stream_stop(user_barge=False) + _tts_lease_async("tui:voice-tts", False) return _ok( rid, @@ -17593,6 +18009,10 @@ def _(rid, params: dict) -> dict: os.environ["HERMES_VOICE_TTS"] = "1" if new_value else "0" if not new_value: _tts_stream_stop(user_barge=False) + # The TTS toggle is the "speech is about to be needed" signal: on → + # pre-load the configured engine so the first reply starts hot; off → + # release the lease (last holder gone = resident local model freed). + _tts_lease_async("tui:voice-tts", new_value) # Include ``record_key`` on every branch so a /voice tts toggle # doesn't reset the TUI's cached shortcut to the default when a # user has a custom binding configured (Copilot review, round 2 diff --git a/tui_gateway/ws.py b/tui_gateway/ws.py index 988733a9b1..e0b8bbe225 100644 --- a/tui_gateway/ws.py +++ b/tui_gateway/ws.py @@ -33,10 +33,47 @@ import time from typing import Any from tui_gateway import server +from agent.message_sanitization import _sanitize_surrogates from tui_gateway.event_replay import replay_epoch _log = logging.getLogger(__name__) +# Scale-to-zero: tell the (separate) gateway process that a dashboard/desktop/ +# TUI client is attached, via the mtime of a marker file it reads in its idle +# predicate. Clients ping every 15s; one mtime write per 5s per process is +# plenty and keeps the volume quiet. See gateway/scale_to_zero.py. +_DASHBOARD_CLIENT_TOUCH_MIN_INTERVAL_S = 5.0 +_dashboard_client_touched_at = 0.0 +_dashboard_client_touch_lock = threading.Lock() + + +def _note_dashboard_client_activity(*, force: bool = False) -> None: + """Refresh the dashboard-client liveness marker (throttled, best-effort).""" + global _dashboard_client_touched_at + now = time.monotonic() + with _dashboard_client_touch_lock: + if not force and now - _dashboard_client_touched_at < _DASHBOARD_CLIENT_TOUCH_MIN_INTERVAL_S: + return + _dashboard_client_touched_at = now + try: + from gateway.scale_to_zero import touch_dashboard_client_heartbeat + + touch_dashboard_client_heartbeat() + except Exception: # noqa: BLE001 - liveness garnish must never break the WS + _log.debug("dashboard client heartbeat touch failed", exc_info=True) + + +def _sanitize_ws_text(text: str) -> str: + """Return *text* that can be UTF-8 encoded for a WebSocket frame. + + ``json.dumps(..., ensure_ascii=False)`` happily emits lone UTF-16 + surrogates; Starlette's ``send_text`` then raises ``UnicodeEncodeError``, + which used to latch the whole connection closed (#97288). Same U+FFFD + replacement every other Hermes transport applies. + """ + return _sanitize_surrogates(text) if text else text + + # Max seconds a pool-dispatched handler will block waiting for the event loop # to flush a WS frame before we mark the transport dead. Protects handler # threads from a wedged socket. @@ -253,20 +290,30 @@ class WSTransport: async with self._send_lock: if self._closed: return - try: - for line in lines: - if self._closed: - return - await self._ws.send_text(line) - except Exception as exc: - # Latch while still holding the writer lock so queued batches - # observe the failure before they get a chance to touch the - # socket. - self._closed = True - _log.warning( - "ws send failed peer=%s error_type=%s error=%s", - self._peer, type(exc).__name__, exc, - ) + for line in lines: + if self._closed: + return + payload = _sanitize_ws_text(line) + try: + await self._ws.send_text(payload) + except UnicodeEncodeError as exc: + # A single illegal UTF-8 frame (lone surrogate in a + # status/ready payload) must not tear down the socket. + # Fresh Desktop installs looped on this (#97288). + _log.warning( + "ws send skipped invalid utf-8 frame peer=%s error=%s", + self._peer, exc, + ) + continue + except Exception as exc: + # Latch while still holding the writer lock so queued + # batches observe the failure before they touch the socket. + self._closed = True + _log.warning( + "ws send failed peer=%s error_type=%s error=%s", + self._peer, type(exc).__name__, exc, + ) + return def close(self) -> None: self._closed = True @@ -346,6 +393,9 @@ async def handle_ws( else: await ws.accept() disconnect_reason = "connected" + # A client is attached from the moment the upgrade is accepted — mark it + # before the (possibly slow) ready/skin setup so scale-to-zero sees it. + _note_dashboard_client_activity(force=True) # Push small streamed frames out immediately instead of letting Nagle # batch them — keeps the live token cadence intact for GUI clients. _disable_nagle(ws) @@ -420,6 +470,7 @@ async def handle_ws( while True: try: raw = await ws.receive_text() + _note_dashboard_client_activity() except _WebSocketDisconnect as exc: disconnect_reason = ( "client_disconnect(" diff --git a/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/index.ts b/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/index.ts index a62a4bae16..4b99ee2ec2 100644 --- a/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/index.ts +++ b/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/index.ts @@ -436,6 +436,20 @@ export class Node { _cGen = -1 _cN = 0 _cWr = 0 + _rSubtreeLayoutGen = -1 + _rValid = false + _rLeft = NaN + _rTop = NaN + _rWidth = NaN + _rHeight = NaN + _rRoundedLeft = NaN + _rRoundedTop = NaN + _rRoundedWidth = NaN + _rRoundedHeight = NaN + _rParentAbsLeft = NaN + _rParentAbsTop = NaN + _rScale = NaN + _rIsText = false constructor(config?: Config) { this.style = defaultStyle() this.layout = { @@ -509,6 +523,7 @@ export class Node { this._cN = 0 this._cWr = 0 this._fbBasis = NaN + this._rValid = false } markDirty(): void { this.isDirty_ = true @@ -533,26 +548,26 @@ export class Node { this.markDirty() } getComputedLeft(): number { - return this.layout.left + return this._rValid ? this._rRoundedLeft : this.layout.left } getComputedTop(): number { - return this.layout.top + return this._rValid ? this._rRoundedTop : this.layout.top } getComputedWidth(): number { - return this.layout.width + return this._rValid ? this._rRoundedWidth : this.layout.width } getComputedHeight(): number { - return this.layout.height + return this._rValid ? this._rRoundedHeight : this.layout.height } getComputedRight(): number { const p = this.parent - return p ? p.layout.width - this.layout.left - this.layout.width : 0 + return p ? p.getComputedWidth() - this.getComputedLeft() - this.getComputedWidth() : 0 } getComputedBottom(): number { const p = this.parent - return p ? p.layout.height - this.layout.top - this.layout.height : 0 + return p ? p.getComputedHeight() - this.getComputedTop() - this.getComputedHeight() : 0 } getComputedLayout(): { left: number @@ -563,12 +578,12 @@ export class Node { height: number } { return { - left: this.layout.left, - top: this.layout.top, + left: this.getComputedLeft(), + top: this.getComputedTop(), right: this.getComputedRight(), bottom: this.getComputedBottom(), - width: this.layout.width, - height: this.layout.height + width: this.getComputedWidth(), + height: this.getComputedHeight() } } getComputedBorder(edge: Edge): number { @@ -842,6 +857,8 @@ export class Node { _yogaNodesVisited = 0 _yogaMeasureCalls = 0 _yogaCacheHits = 0 + _yogaRoundedNodes = 0 + _yogaRoundSkips = 0 _generation++ const w = ownerWidth === undefined ? NaN : ownerWidth const h = ownerHeight === undefined ? NaN : ownerHeight @@ -924,18 +941,24 @@ let _yogaNodesVisited = 0 let _yogaMeasureCalls = 0 let _yogaCacheHits = 0 let _yogaLiveNodes = 0 +let _yogaRoundedNodes = 0 +let _yogaRoundSkips = 0 export function getYogaCounters(): { visited: number measured: number cacheHits: number live: number + rounded: number + roundSkips: number } { return { visited: _yogaNodesVisited, measured: _yogaMeasureCalls, cacheHits: _yogaCacheHits, - live: _yogaLiveNodes + live: _yogaLiveNodes, + rounded: _yogaRoundedNodes, + roundSkips: _yogaRoundSkips } } @@ -1020,6 +1043,15 @@ function layoutNode( } } + if (performLayout) { + let ancestor: Node | null = node + + while (ancestor && ancestor._rSubtreeLayoutGen !== _generation) { + ancestor._rSubtreeLayoutGen = _generation + ancestor = ancestor.parent + } + } + const wasDirty = node.isDirty_ if (performLayout) { @@ -2191,28 +2223,63 @@ function collectLayoutChildren(node: Node, flow: Node[], abs: Node[]): void { } function roundLayout(node: Node, scale: number, absLeft: number, absTop: number): void { - if (scale === 0) { + const l = node.layout + const isText = node.measureFunc !== null + + if ( + node._rValid && + node._rSubtreeLayoutGen !== _generation && + sameFloat(node._rParentAbsLeft, absLeft) && + sameFloat(node._rParentAbsTop, absTop) && + sameFloat(node._rScale, scale) && + node._rIsText === isText && + sameFloat(node._rLeft, l.left) && + sameFloat(node._rTop, l.top) && + sameFloat(node._rWidth, l.width) && + sameFloat(node._rHeight, l.height) + ) { + _yogaRoundSkips++ + return } - const l = node.layout + _yogaRoundedNodes++ + const nodeLeft = l.left const nodeTop = l.top const nodeWidth = l.width const nodeHeight = l.height const absNodeLeft = absLeft + nodeLeft const absNodeTop = absTop + nodeTop - const isText = node.measureFunc !== null - l.left = roundValue(nodeLeft, scale, false, isText) - l.top = roundValue(nodeTop, scale, false, isText) - const absRight = absNodeLeft + nodeWidth - const absBottom = absNodeTop + nodeHeight - const hasFracW = !isWholeNumber(nodeWidth * scale) - const hasFracH = !isWholeNumber(nodeHeight * scale) - l.width = - roundValue(absRight, scale, isText && hasFracW, isText && !hasFracW) - roundValue(absNodeLeft, scale, false, isText) - l.height = - roundValue(absBottom, scale, isText && hasFracH, isText && !hasFracH) - roundValue(absNodeTop, scale, false, isText) + node._rValid = true + node._rLeft = nodeLeft + node._rTop = nodeTop + node._rWidth = nodeWidth + node._rHeight = nodeHeight + node._rParentAbsLeft = absLeft + node._rParentAbsTop = absTop + node._rScale = scale + node._rIsText = isText + + if (scale === 0) { + node._rRoundedLeft = nodeLeft + node._rRoundedTop = nodeTop + node._rRoundedWidth = nodeWidth + node._rRoundedHeight = nodeHeight + } else { + node._rRoundedLeft = roundValue(nodeLeft, scale, false, isText) + node._rRoundedTop = roundValue(nodeTop, scale, false, isText) + const absRight = absNodeLeft + nodeWidth + const absBottom = absNodeTop + nodeHeight + const hasFracW = !isWholeNumber(nodeWidth * scale) + const hasFracH = !isWholeNumber(nodeHeight * scale) + node._rRoundedWidth = + roundValue(absRight, scale, isText && hasFracW, isText && !hasFracW) - + roundValue(absNodeLeft, scale, false, isText) + node._rRoundedHeight = + roundValue(absBottom, scale, isText && hasFracH, isText && !hasFracH) - + roundValue(absNodeTop, scale, false, isText) + } for (const c of node.children) { roundLayout(c, scale, absNodeLeft, absNodeTop) diff --git a/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/round-layout.test.ts b/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/round-layout.test.ts new file mode 100644 index 0000000000..41f78feea3 --- /dev/null +++ b/ui-tui/packages/hermes-ink/src/native-ts/yoga-layout/round-layout.test.ts @@ -0,0 +1,321 @@ +import { describe, expect, it } from 'vitest' + +import Yoga, { FlexDirection, getYogaCounters, type Node } from './index.js' + +const snapshot = (node: Node): number[] => { + const result = [node.getComputedLeft(), node.getComputedTop(), node.getComputedWidth(), node.getComputedHeight()] + + for (let index = 0; index < node.getChildCount(); index++) { + result.push(...snapshot(node.getChild(index))) + } + + return result +} + +const buildTree = (rootWidth: number, widths: number[], scale: number) => { + const config = Yoga.Config.create() + config.setPointScaleFactor(scale) + const root = Yoga.Node.create(config) + root.setFlexDirection(FlexDirection.Column) + root.setWidth(rootWidth) + root.setHeight(20) + const leaves: Node[] = [] + + for (let groupIndex = 0; groupIndex < 4; groupIndex++) { + const group = Yoga.Node.create(config) + group.setFlexDirection(FlexDirection.Row) + group.setHeight(3.125) + root.insertChild(group, groupIndex) + + for (let leafIndex = 0; leafIndex < 8; leafIndex++) { + const leaf = Yoga.Node.create(config) + leaf.setWidth(widths[groupIndex * 8 + leafIndex]!) + leaf.setHeight(1.125 + (leafIndex % 3) * 0.25) + group.insertChild(leaf, leafIndex) + leaves.push(leaf) + } + } + + return { config, leaves, root } +} + +interface NodeSpec { + w?: number + h?: number + pad?: number + mar?: number + row: boolean + grow?: number + kids: NodeSpec[] +} + +const buildMutationTree = (spec: NodeSpec) => { + const all: Node[] = [] + + const makeNode = (value: NodeSpec): Node => { + const node = Yoga.Node.create() + + if (value.w !== undefined) { + node.setWidth(value.w) + } + + if (value.h !== undefined) { + node.setHeight(value.h) + } + + if (value.pad !== undefined) { + node.setPadding(1, value.pad) + } + + if (value.mar !== undefined) { + node.setMargin(1, value.mar) + } + + if (value.row) { + node.setFlexDirection(FlexDirection.Row) + } + + if (value.grow !== undefined) { + node.setFlexGrow(value.grow) + } + + all.push(node) + value.kids.forEach((child, index) => node.insertChild(makeNode(child), index)) + + return node + } + + return { all, root: makeNode(spec) } +} + +describe('incremental layout rounding', () => { + it('keeps rounding work flat when only the clock changes', () => { + const results = [50, 500, 5000].map(rowCount => { + const config = Yoga.Config.create() + config.setPointScaleFactor(2) + + const root = Yoga.Node.create(config) + root.setWidth(80) + root.setHeight(40) + + const transcript = Yoga.Node.create(config) + transcript.setHeight(39) + root.insertChild(transcript, 0) + + for (let index = 0; index < rowCount; index++) { + const row = Yoga.Node.create(config) + row.setWidth(20.25) + row.setHeight(0.25) + transcript.insertChild(row, index) + } + + const clock = Yoga.Node.create(config) + clock.setWidth(5.25) + clock.setHeight(1) + root.insertChild(clock, 1) + + root.calculateLayout(80, 40) + const transcriptWidth = transcript.getComputedWidth() + + clock.setWidth(6.25) + root.calculateLayout(80, 40) + + const counters = getYogaCounters() + expect(clock.getComputedWidth()).toBe(6.5) + expect(transcript.getComputedWidth()).toBe(transcriptWidth) + root.freeRecursive() + Yoga.Config.destroy(config) + + return { rounded: counters.rounded, roundSkips: counters.roundSkips } + }) + + expect(results).toEqual([ + { rounded: 2, roundSkips: 1 }, + { rounded: 2, roundSkips: 1 }, + { rounded: 2, roundSkips: 1 } + ]) + }) + + it('re-rounds cached raw geometry when the point scale changes', () => { + const config = Yoga.Config.create() + config.setPointScaleFactor(2) + + const root = Yoga.Node.create(config) + root.setWidth(20) + root.setHeight(10) + + const child = Yoga.Node.create(config) + child.setWidth(10.25) + child.setHeight(1) + root.insertChild(child, 0) + + root.calculateLayout(20, 10) + expect(child.getComputedWidth()).toBe(10.5) + + config.setPointScaleFactor(4) + child.setWidth(10.125) + root.calculateLayout(20, 10) + + expect(child.getComputedWidth()).toBe(10.25) + + config.setPointScaleFactor(0) + root.calculateLayout(20, 10) + + expect(child.getComputedWidth()).toBe(10.125) + + root.freeRecursive() + Yoga.Config.destroy(config) + }) + + it('matches a fresh full layout across leaf, root, and scale changes', () => { + const widths = Array.from({ length: 32 }, (_, index) => 1.125 + (index % 5) * 0.375) + let rootWidth = 40.25 + let scale = 2 + const incremental = buildTree(rootWidth, widths, scale) + + for (let step = 0; step < 24; step++) { + if (step % 6 === 0) { + scale = scale === 2 ? 4 : 2 + incremental.config.setPointScaleFactor(scale) + } else if (step % 5 === 0) { + rootWidth += 0.375 + incremental.root.setWidth(rootWidth) + } else { + const leafIndex = (step * 7) % widths.length + widths[leafIndex]! += 0.125 + incremental.leaves[leafIndex]!.setWidth(widths[leafIndex]!) + } + + incremental.root.calculateLayout(rootWidth, 20) + const fresh = buildTree(rootWidth, widths, scale) + fresh.root.calculateLayout(rootWidth, 20) + + expect(snapshot(incremental.root), `step ${step}`).toEqual(snapshot(fresh.root)) + + fresh.root.freeRecursive() + Yoga.Config.destroy(fresh.config) + } + + incremental.root.freeRecursive() + Yoga.Config.destroy(incremental.config) + }) + + it('rounds a fractional child after its whole-pixel parent hits the layout cache', () => { + const root = Yoga.Node.create() + root.setWidth(120) + root.setHeight(40) + const row = Yoga.Node.create() + row.setWidth(120) + row.setHeight(2) + const leaf = Yoga.Node.create() + leaf.setWidth(10.4) + leaf.setHeight(1.6) + row.insertChild(leaf, 0) + root.insertChild(row, 0) + root.calculateLayout(120, 40) + + leaf.setWidth(11.4) + leaf.setHeight(2.6) + root.calculateLayout(120, 40) + + expect(leaf.getComputedLayout()).toMatchObject({ width: 11, height: 3 }) + root.freeRecursive() + }) + + it('does not mix cached rounded coordinates with raw dimensions', () => { + const spec: NodeSpec = { + h: 4.917283, + pad: 1.934923, + row: false, + kids: [ + { + w: 11.007846, + row: false, + kids: [ + { + mar: 1.306334, + row: true, + kids: [ + { + mar: 0.199071, + row: true, + grow: 1.522763, + kids: [{ w: 4.6117, h: 5.852709, row: false, kids: [] }] + } + ] + }, + { + w: 40.14272, + pad: 1.043468, + mar: 1.495136, + row: false, + kids: [ + { + w: 8.335441, + row: false, + kids: [ + { h: 5.249866, mar: 0.997316, row: true, grow: 1.554555, kids: [] }, + { h: 6.335704, mar: 0.817136, row: false, grow: 0.079169, kids: [] } + ] + }, + { + w: 32.935694, + row: false, + grow: 1.781838, + kids: [ + { w: 38.068879, pad: 0.696862, row: true, kids: [] }, + { pad: 1.66287, row: false, kids: [] } + ] + } + ] + } + ] + } + ] + } + + const mutations = [ + { index: 10, kind: 'height', value: 2.823024 }, + { index: 6, kind: 'margin', value: 0.858021 }, + { index: 3, kind: 'grow', value: 1.321691 }, + { index: 9, kind: 'grow', value: 0.022362 }, + { index: 6, kind: 'grow', value: 0.812981 }, + { index: 7, kind: 'width', value: 5.644585 }, + { index: 2, kind: 'width', value: 20.498063 }, + { index: 4, kind: 'grow', value: 1.653743 }, + { index: 1, kind: 'height', value: 2.709275 }, + { index: 1, kind: 'margin', value: 2.411063 } + ] as const + + const applyMutation = (all: Node[], mutation: (typeof mutations)[number]) => { + const node = all[mutation.index]! + + if (mutation.kind === 'width') { + node.setWidth(mutation.value) + } else if (mutation.kind === 'height') { + node.setHeight(mutation.value) + } else if (mutation.kind === 'margin') { + node.setMargin(1, mutation.value) + } else { + node.setFlexGrow(mutation.value) + } + } + + const incremental = buildMutationTree(spec) + incremental.root.calculateLayout(96, 5) + + for (const mutation of mutations) { + applyMutation(incremental.all, mutation) + incremental.root.calculateLayout(96, 5) + } + + const fresh = buildMutationTree(spec) + mutations.forEach(mutation => applyMutation(fresh.all, mutation)) + fresh.root.calculateLayout(96, 5) + + expect(incremental.all[11]!.getComputedHeight()).toBe(fresh.all[11]!.getComputedHeight()) + expect(fresh.all[11]!.getComputedHeight()).toBe(2) + incremental.root.freeRecursive() + fresh.root.freeRecursive() + }) +}) diff --git a/ui-tui/src/__tests__/createSlashHandler.test.ts b/ui-tui/src/__tests__/createSlashHandler.test.ts index 6afd025587..67f2609f91 100644 --- a/ui-tui/src/__tests__/createSlashHandler.test.ts +++ b/ui-tui/src/__tests__/createSlashHandler.test.ts @@ -1072,6 +1072,43 @@ describe('createSlashHandler', () => { expect(ctx.transcript.sys).toHaveBeenCalledWith('no active session — nothing to rollback') }) + // A pasted PR thread / diff / log reaches a skill command as its argument. + // parseSlashCommand used to split the whole line on `\s+` and rejoin with a + // single space, so every line break was gone before the skill ran — and the + // fallback command.dispatch carried that flattened text. + it('carries a multi-line argument to the backend without flattening it', async () => { + patchUiState({ sid: 'sid-abc' }) + + const arg = 'line one\nline two\n\n indented tail' + + const ctx = buildCtx({ + gateway: { + gw: { + getLogTail: vi.fn(() => ''), + kill: vi.fn(), + request: vi.fn((method: string) => + method === 'slash.exec' ? Promise.reject(new Error('skill command')) : Promise.resolve({}) + ) + }, + rpc: vi.fn(() => Promise.resolve({})) + } + }) + + createSlashHandler(ctx)(`/pr-triage ${arg}`) + + expect(ctx.gateway.gw.request).toHaveBeenCalledWith('slash.exec', { + command: `pr-triage ${arg}`, + session_id: 'sid-abc' + }) + await vi.waitFor(() => { + expect(ctx.gateway.gw.request).toHaveBeenCalledWith('command.dispatch', { + arg, + name: 'pr-triage', + session_id: 'sid-abc' + }) + }) + }) + it('/title uses session.title RPC and bypasses slash.exec', async () => { patchUiState({ sid: 'sid-abc' }) const rpc = vi.fn(() => Promise.resolve({ pending: false, title: 'my title' })) diff --git a/ui-tui/src/__tests__/queueSubmission.test.ts b/ui-tui/src/__tests__/queueSubmission.test.ts index 149811fad6..62f43366b3 100644 --- a/ui-tui/src/__tests__/queueSubmission.test.ts +++ b/ui-tui/src/__tests__/queueSubmission.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' import type { ComposerToken } from '../app/interfaces.js' -import { expandPasteTokens, queueItemFromSlash } from '../app/useSubmission.js' +import { expandPasteTokens, prepareSlashSubmission, queueItemFromSlash } from '../app/useSubmission.js' import { imageToken } from '../domain/attachments.js' describe('/queue collapsed paste submission', () => { @@ -29,3 +29,30 @@ describe('/queue collapsed paste submission', () => { expect(expandPasteTokens([paste, image])(`${paste.label} and ${image.label}`)).toBe(`one\ntwo and ${image.label}`) }) }) + +describe('prepareSlashSubmission', () => { + const label = '[[ Done — verified.. [412 lines] .. already on it. ]]' + const text = 'Done — verified through the real resolver\nline two\nline three' + const tokens: ComposerToken[] = [{ kind: 'paste', label, text }] + + // The reported bug: `/pr-triage ` dispatched the LABEL, so the skill + // received "[412 lines]" as its argument and the agent reported the paste as + // truncated. The command has to carry the full text; only the transcript + // stays collapsed. + it('dispatches the full paste while the transcript keeps the collapsed label', () => { + expect(prepareSlashSubmission(`/pr-triage ${label}`, tokens)).toEqual({ + command: `/pr-triage ${text}`, + display: `/pr-triage ${label}` + }) + }) + + it('leaves image tokens as labels — the gateway already holds the file', () => { + const image: ComposerToken = { kind: 'image', index: 1, label: imageToken(1), path: '/tmp/shot.png' } + + expect(prepareSlashSubmission(`/pr-triage ${image.label}`, [image]).command).toBe(`/pr-triage ${image.label}`) + }) + + it('is a no-op on a token-free command', () => { + expect(prepareSlashSubmission('/model opus', [])).toEqual({ command: '/model opus', display: '/model opus' }) + }) +}) diff --git a/ui-tui/src/__tests__/slashParity.test.ts b/ui-tui/src/__tests__/slashParity.test.ts index f54fa629a6..c76be8efca 100644 --- a/ui-tui/src/__tests__/slashParity.test.ts +++ b/ui-tui/src/__tests__/slashParity.test.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' import { findSlashCommand, SLASH_COMMANDS } from '../app/slash/registry.js' +import { parseSlashCommand } from '../domain/slash.js' type CommandRoute = 'fallback' | 'local' | 'native' @@ -122,3 +123,29 @@ describe('slash parity matrix', () => { expect(cmd!.name).toBe('queue') }) }) + +describe('parseSlashCommand argument fidelity', () => { + it('keeps a multi-line argument byte-for-byte', () => { + const arg = 'first line\nsecond line\n\n indented tail' + + expect(parseSlashCommand(`/pr-triage ${arg}`)).toEqual({ + arg, + cmd: `/pr-triage ${arg}`, + name: 'pr-triage' + }) + }) + + it('preserves runs of spaces inside the argument', () => { + expect(parseSlashCommand('/goal ship it').arg).toBe('ship it') + }) + + it('still splits the command name off a single separator', () => { + expect(parseSlashCommand('/cron add daily')).toEqual({ + arg: 'add daily', + cmd: '/cron add daily', + name: 'cron' + }) + expect(parseSlashCommand('/exit')).toEqual({ arg: '', cmd: '/exit', name: 'exit' }) + expect(parseSlashCommand('/exit ')).toEqual({ arg: '', cmd: '/exit ', name: 'exit' }) + }) +}) diff --git a/ui-tui/src/app/createGatewayEventHandler.ts b/ui-tui/src/app/createGatewayEventHandler.ts index 33f8096bd0..9e24da7517 100644 --- a/ui-tui/src/app/createGatewayEventHandler.ts +++ b/ui-tui/src/app/createGatewayEventHandler.ts @@ -420,7 +420,16 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: const { rpc } = ctx.gateway const { STARTUP_RESUME_ID, newSession, recoverSidRef, resumeById, setCatalog } = ctx.session - const { bellOnComplete, stdout, sys } = ctx.system + const { bellOnComplete, bellOnPrompt, stdout, sys } = ctx.system + + // display.bell_on_prompt — BEL whenever a blocking prompt modal opens + // (same mechanism as bell_on_complete; works over SSH, triggers tmux bell-action). + const ringPromptBell = () => { + if (bellOnPrompt && stdout?.isTTY) { + stdout.write('\x07') + } + } + const { appendMessage, panel, setHistoryItems } = ctx.transcript const { setInput } = ctx.composer const { submitLiteralRef, submitRef } = ctx.submission @@ -1250,6 +1259,7 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: } }) setStatus('waiting for input…') + ringPromptBell() return } @@ -1269,6 +1279,7 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: } }) setStatus('approval needed') + ringPromptBell() return } @@ -1276,6 +1287,7 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: case 'sudo.request': patchOverlayState({ sudo: { requestId: ev.payload.request_id } }) setStatus('sudo password needed') + ringPromptBell() return @@ -1284,6 +1296,7 @@ export function createGatewayEventHandler(ctx: GatewayEventHandlerContext): (ev: secret: { envVar: ev.payload.env_var, prompt: ev.payload.prompt, requestId: ev.payload.request_id } }) setStatus('secret input needed') + ringPromptBell() return diff --git a/ui-tui/src/app/interfaces.ts b/ui-tui/src/app/interfaces.ts index 0f4b9cb6ac..1aecedd70b 100644 --- a/ui-tui/src/app/interfaces.ts +++ b/ui-tui/src/app/interfaces.ts @@ -498,6 +498,7 @@ export interface GatewayEventHandlerContext { } system: { bellOnComplete: boolean + bellOnPrompt?: boolean stdout?: NodeJS.WriteStream sys: (text: string) => void } diff --git a/ui-tui/src/app/turnController.ts b/ui-tui/src/app/turnController.ts index ac7f5ca541..f6a1c53ef3 100644 --- a/ui-tui/src/app/turnController.ts +++ b/ui-tui/src/app/turnController.ts @@ -1040,6 +1040,7 @@ class TurnController { } const base: SubagentProgress = existing ?? { + delegationId: p.delegation_id, depth: p.depth ?? 0, goal: p.goal, id, @@ -1071,6 +1072,7 @@ class TurnController { ...base, apiCalls: p.api_calls ?? base.apiCalls, costUsd: p.cost_usd ?? base.costUsd, + delegationId: p.delegation_id ?? base.delegationId, depth: p.depth ?? base.depth, filesRead: p.files_read ?? base.filesRead, filesWritten: p.files_written ?? base.filesWritten, diff --git a/ui-tui/src/app/useConfigSync.ts b/ui-tui/src/app/useConfigSync.ts index 32e5b4f462..2f3f31dca3 100644 --- a/ui-tui/src/app/useConfigSync.ts +++ b/ui-tui/src/app/useConfigSync.ts @@ -253,10 +253,11 @@ const _pasteCollapseCharsFromConfig = (cfg: ConfigFullResponse | null): number = export async function hydrateFullConfig( gw: GatewayClient, setBell: (v: boolean) => void, - setVoiceRecordKey?: (v: ParsedVoiceRecordKey) => void + setVoiceRecordKey?: (v: ParsedVoiceRecordKey) => void, + setBellOnPrompt?: (v: boolean) => void ): Promise { const cfg = await quietRpc(gw, 'config.get', { key: 'full' }) - applyDisplay(cfg, setBell, setVoiceRecordKey) + applyDisplay(cfg, setBell, setVoiceRecordKey, setBellOnPrompt) return cfg } @@ -264,13 +265,16 @@ export async function hydrateFullConfig( export const applyDisplay = ( cfg: ConfigFullResponse | null, setBell: (v: boolean) => void, - setVoiceRecordKey?: (v: ParsedVoiceRecordKey) => void + setVoiceRecordKey?: (v: ParsedVoiceRecordKey) => void, + setBellOnPrompt?: (v: boolean) => void ) => { const d = cfg?.config?.display ?? {} const approvals = cfg?.config?.approvals setBell(!!d.bell_on_complete) + setBellOnPrompt?.(!!d.bell_on_prompt) + applyConfiguredTuiTheme(d.tui_theme) // Only push the voice record key when the RPC actually returned a @@ -314,6 +318,7 @@ export const applyDisplay = ( export function useConfigSync({ gw, setBellOnComplete, + setBellOnPrompt, setVoiceEnabled, setVoiceRecordKey, sid @@ -339,8 +344,8 @@ export function useConfigSync({ // mcp_rev) look like an MCP change and fire a needless reload.mcp. mcpRevRef.current.accepted = String(r?.mcp_rev ?? '') }) - void hydrateFullConfig(gw, setBellOnComplete, setVoiceRecordKey) - }, [gw, setBellOnComplete, setVoiceEnabled, setVoiceRecordKey, sid]) + void hydrateFullConfig(gw, setBellOnComplete, setVoiceRecordKey, setBellOnPrompt) + }, [gw, setBellOnComplete, setBellOnPrompt, setVoiceEnabled, setVoiceRecordKey, sid]) useEffect(() => { if (!sid) { @@ -387,17 +392,18 @@ export function useConfigSync({ ) } - void hydrateFullConfig(gw, setBellOnComplete, setVoiceRecordKey) + void hydrateFullConfig(gw, setBellOnComplete, setVoiceRecordKey, setBellOnPrompt) }) }, MTIME_POLL_MS) return () => clearInterval(id) - }, [gw, setBellOnComplete, setVoiceRecordKey, sid]) + }, [gw, setBellOnComplete, setBellOnPrompt, setVoiceRecordKey, sid]) } export interface UseConfigSyncOptions { gw: GatewayClient setBellOnComplete: (v: boolean) => void + setBellOnPrompt?: (v: boolean) => void setVoiceEnabled: (v: boolean) => void setVoiceRecordKey?: (v: ParsedVoiceRecordKey) => void sid: null | string diff --git a/ui-tui/src/app/useMainApp.ts b/ui-tui/src/app/useMainApp.ts index 4a794fc29b..7d57382644 100644 --- a/ui-tui/src/app/useMainApp.ts +++ b/ui-tui/src/app/useMainApp.ts @@ -207,6 +207,7 @@ export function useMainApp(gw: GatewayClient) { // Bumped by the gateway `reaction` event (core-detected affection). const goodVibesTick = useStore($goodVibesTick) const [bellOnComplete, setBellOnComplete] = useState(false) + const [bellOnPrompt, setBellOnPrompt] = useState(false) const ui = useStore($uiState) const overlay = useStore($overlayState) @@ -578,7 +579,7 @@ export function useMainApp(gw: GatewayClient) { } }, [ui.busy, turnStartedAt]) - useConfigSync({ gw, setBellOnComplete, setVoiceEnabled, setVoiceRecordKey, sid: ui.sid }) + useConfigSync({ gw, setBellOnComplete, setBellOnPrompt, setVoiceEnabled, setVoiceRecordKey, sid: ui.sid }) useBatteryPoll(gw) useEffect(() => { @@ -857,7 +858,7 @@ export function useMainApp(gw: GatewayClient) { setCatalog }, submission: { submitLiteralRef, submitRef }, - system: { bellOnComplete, stdout, sys }, + system: { bellOnComplete, bellOnPrompt, stdout, sys }, transcript: { appendMessage, panel, setHistoryItems }, voice: { setProcessing: setVoiceProcessing, @@ -869,6 +870,7 @@ export function useMainApp(gw: GatewayClient) { [ appendMessage, bellOnComplete, + bellOnPrompt, composerActions.setInput, gateway, panel, diff --git a/ui-tui/src/app/useSubmission.ts b/ui-tui/src/app/useSubmission.ts index 0a7ef55bad..57ce9f60f9 100644 --- a/ui-tui/src/app/useSubmission.ts +++ b/ui-tui/src/app/useSubmission.ts @@ -40,6 +40,24 @@ export const prepareSubmission = (display: string, tokens: ComposerToken[]) => ( text: expandTokens(tokens)(display) }) +/** + * Split a slash submission into the two things it has to be at once. + * + * A slash command's argument is ordinary user text, so a collapsed paste in it + * must resolve BEFORE the command runs — otherwise `/pr-triage [[ … [412 lines] + * … ]]` hands the skill the label and the agent faithfully reports that the + * paste is truncated. The transcript still shows the compact form, because a + * 412-line paste inlined into the scrollback is exactly what collapsing it was + * for. + * + * Image tokens stay as labels: the gateway already holds those files in + * `attached_images` and splices them in at submit. + */ +export const prepareSlashSubmission = (display: string, tokens: ComposerToken[]) => ({ + command: expandPasteTokens(tokens)(display), + display +}) + export const shouldInterpolateSubmission = (display: string) => hasInterpolation(display) export function useSubmission(opts: UseSubmissionOptions) { @@ -245,22 +263,23 @@ export function useSubmission(opts: UseSubmissionOptions) { const submissionTokens = [...composerRefs.tokensRef.current] const submission = prepareSubmission(full, submissionTokens) const toHistory = submission.text - const queuePayload = expandPasteTokens(submissionTokens)(full) if (looksLikeSlashCommand(full)) { - appendMessage({ kind: 'slash', role: 'system', text: full }) + const slash = prepareSlashSubmission(full, submissionTokens) + + appendMessage({ kind: 'slash', role: 'system', text: slash.display }) composerActions.pushHistory(toHistory) const parsed = parseSlashCommand(full) const queued = - parsed.name === 'queue' || parsed.name === 'q' ? queueItemFromSlash(full, queuePayload) : undefined + parsed.name === 'queue' || parsed.name === 'q' ? queueItemFromSlash(slash.display, slash.command) : undefined if (queued) { composerActions.enqueue(queued.text, queued.display) sys(`queued: "${queued.display.slice(0, 50)}${queued.display.length > 50 ? '…' : ''}"`) } else { - slashRef.current(full) + slashRef.current(slash.command) } composerActions.clearIn() diff --git a/ui-tui/src/components/thinking.tsx b/ui-tui/src/components/thinking.tsx index d3225bee6a..da484a0fa6 100644 --- a/ui-tui/src/components/thinking.tsx +++ b/ui-tui/src/components/thinking.tsx @@ -332,7 +332,16 @@ function SubagentAccordion({ ? 'warn' : 'dim' - const prefix = item.taskCount > 1 ? `[${item.index + 1}/${item.taskCount}] ` : '' + // `[6a66 3/9]` when the gateway tags the batch; `[3/9]` on older gateways. + const batchTag = item.delegationId?.split('_').at(-1)?.slice(0, 4) + + const prefix = + item.taskCount > 1 + ? `[${batchTag ? `${batchTag} ` : ''}${item.index + 1}/${item.taskCount}] ` + : batchTag + ? `[${batchTag}] ` + : '' + const goalLabel = item.goal || `Subagent ${item.index + 1}` const title = `${prefix}${open ? goalLabel : compactPreview(goalLabel, 60)}` const summary = compactPreview((item.summary || '').replace(/\s+/g, ' ').trim(), 72) diff --git a/ui-tui/src/domain/slash.ts b/ui-tui/src/domain/slash.ts index 1f1b12847d..3b24d1e7a3 100644 --- a/ui-tui/src/domain/slash.ts +++ b/ui-tui/src/domain/slash.ts @@ -47,10 +47,17 @@ export const inlineSlashTrigger = (text: string): { query: string; start: number return { query, start: text.length - query.length - 1 } } -export const parseSlashCommand = (cmd: string) => { - const [name = '', ...rest] = cmd.slice(1).split(/\s+/) +// Only the separator between the command name and its argument is whitespace +// the parser owns. Everything after it is the user's text and survives +// verbatim: splitting the whole line on `\s+` and rejoining with a space +// flattened every pasted diff, log, or PR thread into one run-on line before +// the command ever saw it. +const SLASH_PARTS_RE = /^(\S*)\s*([\s\S]*)$/ - return { arg: rest.join(' '), cmd, name: name.toLowerCase() } +export const parseSlashCommand = (cmd: string) => { + const [, name = '', arg = ''] = SLASH_PARTS_RE.exec(cmd.slice(1)) ?? [] + + return { arg, cmd, name: name.toLowerCase() } } /** diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index bc08637c14..b2c2fd5955 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -79,6 +79,7 @@ export type CommandDispatchResponse = export interface ConfigDisplayConfig { battery?: boolean bell_on_complete?: boolean + bell_on_prompt?: boolean busy_input_mode?: string details_mode?: string /** Focus view (/focus) — display-only reduced-output mode. */ @@ -543,6 +544,9 @@ export interface RollbackRestoreResponse { export interface SubagentEventPayload { api_calls?: number cost_usd?: number + /** Batch (delegation) id this subagent belongs to — distinguishes + * interleaved `[n/N]` progress from concurrent or nested fan-outs. */ + delegation_id?: string depth?: number duration_seconds?: number files_read?: string[] diff --git a/ui-tui/src/types.ts b/ui-tui/src/types.ts index 1803402bb5..1b016da56f 100644 --- a/ui-tui/src/types.ts +++ b/ui-tui/src/types.ts @@ -25,6 +25,9 @@ export type SubagentStatus = 'completed' | 'error' | 'failed' | 'interrupted' | export interface SubagentProgress { apiCalls?: number costUsd?: number + /** Batch (delegation) id — tags `[n/N]` rows so concurrent/nested fan-outs + * are distinguishable. Absent on older gateways. */ + delegationId?: string depth: number durationSeconds?: number filesRead?: string[] diff --git a/uv.lock b/uv.lock index cfe76c92b5..428e43ea1a 100644 --- a/uv.lock +++ b/uv.lock @@ -2097,8 +2097,8 @@ requires-dist = [ { name = "microsoft-teams-apps", marker = "extra == 'teams'", specifier = "==2.0.13.4" }, { name = "mistralai", marker = "extra == 'mistral'", specifier = "==2.4.8" }, { name = "modal", marker = "extra == 'modal'", specifier = "==1.3.4" }, - { name = "nemo-relay", marker = "(platform_machine == 'aarch64' and 'android' not in platform_release and sys_platform == 'linux') or (platform_machine == 'x86_64' and 'android' not in platform_release and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')", specifier = ">=0.7.1,<0.8" }, - { name = "numpy", marker = "extra == 'audio-io'", specifier = "==2.4.3" }, + { name = "nemo-relay", marker = "(platform_machine == 'aarch64' and 'android' not in platform_release and sys_platform == 'linux') or (platform_machine == 'x86_64' and 'android' not in platform_release and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'ARM64' and sys_platform == 'win32')", specifier = ">=0.8.3,<0.9" }, + { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "numpy", marker = "extra == 'wake'", specifier = "==2.4.3" }, { name = "onnxruntime", marker = "(platform_machine != 'x86_64' and extra == 'wake') or (sys_platform != 'darwin' and extra == 'wake')", specifier = "==1.27.0" }, @@ -2973,17 +2973,17 @@ wheels = [ [[package]] name = "nemo-relay" -version = "0.7.2" +version = "0.8.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/81/a7a545ac3a2f8c670d261c89df599aa8fbf49d8be45fd1f52efb36b489eb/nemo_relay-0.7.2.tar.gz", hash = "sha256:828d9f6c7d7e4e42276bb7192bd44202c761e0c76fa4943d84e051b5a99028e5", size = 1295616, upload-time = "2026-08-08T01:54:00.953Z" } +sdist = { url = "https://files.pythonhosted.org/packages/03/73/ac90ccb08faca19b2c8470bdd4d5b9bae89fc5edfde8dae72ee7b1a2d8df/nemo_relay-0.8.3.tar.gz", hash = "sha256:3670c0689f0709354068a1460131e6f01ea44cd7c2b2186930dd20b23cd079ff", size = 1615748, upload-time = "2026-09-02T03:32:45.828Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/cd/f50440257f01bc5ab3d668331c90e06cf4edcc84dc7dc582d322ad05b622/nemo_relay-0.7.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:e7c7977f0903793cc34c5542bf2b2e44d107def8a5ae9f1b28f06dd61ddec4ed", size = 9246341, upload-time = "2026-08-08T01:53:19.832Z" }, - { url = "https://files.pythonhosted.org/packages/ed/9f/4041446dd134218799a34b5b5fad3a62d3e1d0a6c322ba2ca4b896ba1393/nemo_relay-0.7.2-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ae77c1f3d58eabda264e82ffaca54548df80caede7dd6af8cbd8f72b4a82ed", size = 8454070, upload-time = "2026-08-08T01:53:22.524Z" }, - { url = "https://files.pythonhosted.org/packages/11/83/90230c2e9fae1aee39f768d4a9ef57e9f2716bcaed1a5923cce8b526c66b/nemo_relay-0.7.2-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ce7103aec546766649c182619d16aa6ad07439e4d0ebd16d95c5004afb3e56a", size = 8954377, upload-time = "2026-08-08T01:53:25.267Z" }, - { url = "https://files.pythonhosted.org/packages/71/e7/463fa461d0801146fec6a00cbc02e8961b30089d65ba170f9dfa9e6e3dcd/nemo_relay-0.7.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e7d0c2629ade7313aaed71d2272dca96a2fafad0248d0f87cf40a7720b252a0", size = 10322132, upload-time = "2026-08-08T01:53:27.991Z" }, - { url = "https://files.pythonhosted.org/packages/32/8c/e20ec9c52bd1edd953157aaf24d0d9f9ab8afcbf108fc2356f398e252da8/nemo_relay-0.7.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b841c92395d7686c7f233036008294b9d362af1ec5123ab0babbfd11cbb04054", size = 10704141, upload-time = "2026-08-08T01:53:30.453Z" }, - { url = "https://files.pythonhosted.org/packages/5a/c1/92a73961ea759b433b1f897b225662d499123cb962b48dc8ece19f610a09/nemo_relay-0.7.2-cp311-abi3-win_amd64.whl", hash = "sha256:0cdcc5e09d6d62d5c1d385dc62c9233eb714a25f36a09da81e5b9731e3c67903", size = 8803938, upload-time = "2026-08-08T01:53:33.437Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ec/2de114dab437431173988b9b11f46e8d377e12d57e1b4903258f3e03c2df/nemo_relay-0.7.2-cp311-abi3-win_arm64.whl", hash = "sha256:ca5f66e617311f836a10d96f120f3f32a99b4267d65048453b31951de3419a9d", size = 8438997, upload-time = "2026-08-08T01:53:36.12Z" }, + { url = "https://files.pythonhosted.org/packages/2a/6a/199c2061358684550780bfc3184e588cea6533ec697ae6fa2f36f8881412/nemo_relay-0.8.3-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:2b0a59b8a95d6ed9de099e471318af672b51426a89c2019766ac889b1e23f1d4", size = 10200865, upload-time = "2026-09-02T03:32:15.373Z" }, + { url = "https://files.pythonhosted.org/packages/df/90/3486d10c2003cda3bd4ce98d34e40f283faf87dcb1292cddca356a86c6be/nemo_relay-0.8.3-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c84a93ad0d4bea9a7bf67917874cd4eee24694764456a5f94c9609b1df42904", size = 9199160, upload-time = "2026-09-02T03:32:17.467Z" }, + { url = "https://files.pythonhosted.org/packages/18/00/15705b941df64443e50c49139140dd84603f23125ab48f9c122942911439/nemo_relay-0.8.3-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10b08a939678d8f54ad02e5fb086454ffad15f269eae4854e9597956dc96b176", size = 9760299, upload-time = "2026-09-02T03:32:19.255Z" }, + { url = "https://files.pythonhosted.org/packages/00/d3/2dac6a938713cfb6f59d8f3e82e991a1635a666f25f8d7fed8a3fa9c56b7/nemo_relay-0.8.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c2fba57f39100068244f8fcaffbdd801a0a8ccc7b9dbee330bb087f3970d8ccc", size = 11287368, upload-time = "2026-09-02T03:32:21.359Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/f61379d244139306c3832082d446ad6167f50108969b3e8720e62e1b2ae5/nemo_relay-0.8.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ae6804050b309a12dae5a7afc8c85b29b601c2c4a4257d365a998b774a2e7d16", size = 11717569, upload-time = "2026-09-02T03:32:23.321Z" }, + { url = "https://files.pythonhosted.org/packages/db/96/333f62b4176450154a8908f619e3323b70b43c3b733a0a0752ba3b9232b1/nemo_relay-0.8.3-cp311-abi3-win_amd64.whl", hash = "sha256:b1f006e50e44967821f3a77a2e0001eed8154db01168abcc0681891f9c339fed", size = 9656294, upload-time = "2026-09-02T03:32:25.904Z" }, + { url = "https://files.pythonhosted.org/packages/fe/bd/37ab6038111d3cdec1858e584ef9e8fdfe0db07aa6073d66d1303b18d791/nemo_relay-0.8.3-cp311-abi3-win_arm64.whl", hash = "sha256:0d3db4f0a9d1f909acbf659612bfb9859e304b53cd425ef8e0f60ca851e85173", size = 9235572, upload-time = "2026-09-02T03:32:28.042Z" }, ] [[package]] diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index f02841275b..783d3b6bc5 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -1948,6 +1948,9 @@ export interface SessionInfo { output_tokens: number; preview: string | null; parent_session_id?: string | null; + /** Owning profile stamped by the list/detail endpoints (the store the row + * was read from). Absent on search-endpoint rows, which carry no stamp. */ + profile?: string; } export interface SessionLatestDescendantResponse { diff --git a/web/src/lib/cron-job.test.ts b/web/src/lib/cron-job.test.ts index 172f7f939e..25420d0a3a 100644 --- a/web/src/lib/cron-job.test.ts +++ b/web/src/lib/cron-job.test.ts @@ -4,6 +4,7 @@ import { buildCronJobPayload, cronJobHasExecutionContent, cronJobFormFromJob, + cronLastResult, splitCronList, type CronJobFormState, } from "./cron-job"; @@ -153,3 +154,49 @@ describe("cronJobFormFromJob", () => { }); }); }); + +describe("cronLastResult", () => { + it("renders nothing for a job that never ran", () => { + expect(cronLastResult({ last_status: null })).toBeNull(); + expect(cronLastResult({ last_status: "" })).toBeNull(); + }); + + it("is green for ok with no detail", () => { + expect(cronLastResult({ last_status: "ok", last_error: null })).toEqual({ + status: "ok", + tone: "success", + detail: null, + }); + }); + + it("is amber for delivery_failed and explains it from last_delivery_error", () => { + // The agent run succeeded (last_error is null for these runs); the reason + // lives in last_delivery_error. Must never render as green or as "unknown". + expect( + cronLastResult({ + last_status: "delivery_failed", + last_error: null, + last_delivery_error: "telegram: 502 Bad Gateway", + }), + ).toEqual({ + status: "delivery_failed", + tone: "warning", + detail: "telegram: 502 Bad Gateway", + }); + }); + + it("is red for error and any unrecognised literal", () => { + expect(cronLastResult({ last_status: "error", last_error: "boom" })).toEqual({ + status: "error", + tone: "destructive", + detail: "boom", + }); + expect(cronLastResult({ last_status: "something_new" })?.tone).toBe("destructive"); + }); + + it("is amber for blocked_config (preflight refused to burn a run)", () => { + expect( + cronLastResult({ last_status: "blocked_config", last_error: "missing API key" }), + ).toEqual({ status: "blocked_config", tone: "warning", detail: "missing API key" }); + }); +}); diff --git a/web/src/lib/cron-job.ts b/web/src/lib/cron-job.ts index ab8e834582..de4fa9fc47 100644 --- a/web/src/lib/cron-job.ts +++ b/web/src/lib/cron-job.ts @@ -102,3 +102,39 @@ export function cronJobFormFromJob(job: CronJob): CronJobFormState { workdir: asString(job.workdir), }; } + +/** How a job's `last_status` should render. The scheduler writes a small, + * closed set of literals; every literal maps to an explicit tone here so a + * new status can never fall through to a neutral "unknown"-looking badge. + * In particular `delivery_failed` (agent run succeeded, output never reached + * the target) is amber, not green and not the same red as a run error, and + * its detail lives in `last_delivery_error` (last_error is null for it). */ +export type CronLastResultTone = "success" | "warning" | "destructive"; + +export interface CronLastResult { + status: string; + tone: CronLastResultTone; + /** Human detail to show next to the badge; null when nothing to add. */ + detail: string | null; +} + +const CRON_LAST_RESULT_TONE: Record = { + ok: "success", + delivery_failed: "warning", + blocked_config: "warning", + error: "destructive", +}; + +export function cronLastResult( + job: Pick, +): CronLastResult | null { + const status = asString(job.last_status).trim(); + if (!status) return null; + const tone = CRON_LAST_RESULT_TONE[status] ?? "destructive"; + if (status === "ok") return { status, tone, detail: null }; + const detail = + status === "delivery_failed" + ? asString(job.last_delivery_error).trim() || asString(job.last_error).trim() + : asString(job.last_error).trim() || asString(job.last_delivery_error).trim(); + return { status, tone, detail: detail || null }; +} diff --git a/web/src/pages/CronPage.tsx b/web/src/pages/CronPage.tsx index b501d5675f..a29a192dc2 100644 --- a/web/src/pages/CronPage.tsx +++ b/web/src/pages/CronPage.tsx @@ -22,6 +22,7 @@ import { buildCronJobPayload, cronJobHasExecutionContent, cronJobFormFromJob, + cronLastResult, type CronJobFormState, } from "@/lib/cron-job"; import { DeleteConfirmDialog } from "@/components/DeleteConfirmDialog"; @@ -1100,6 +1101,7 @@ export default function CronPage() { const toolsets = Array.isArray(job.enabled_toolsets) ? job.enabled_toolsets.filter(Boolean) : []; + const lastResult = cronLastResult(job); return ( @@ -1112,6 +1114,15 @@ export default function CronPage() { {state} + {lastResult && lastResult.status !== "ok" && ( + + {lastResult.status} + + )} {profileLabel(profile)} {deliver && deliver !== "local" && ( {deliver} diff --git a/web/src/pages/SessionsPage.test.tsx b/web/src/pages/SessionsPage.test.tsx new file mode 100644 index 0000000000..a621835544 --- /dev/null +++ b/web/src/pages/SessionsPage.test.tsx @@ -0,0 +1,154 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { MemoryRouter } from "react-router"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const apiMocks = vi.hoisted(() => ({ + getSessions: vi.fn(), + getSessionMessages: vi.fn(), + getEmptySessionsCount: vi.fn(), + getStatus: vi.fn(), + searchSessions: vi.fn(), + importSessions: vi.fn(), + exportSessionUrl: vi.fn(), + renameSession: vi.fn(), + pruneSessions: vi.fn(), + deleteSession: vi.fn(), + deleteEmptySessions: vi.fn(), + bulkDeleteSessions: vi.fn(), + getProfiles: vi.fn(), + getActiveProfile: vi.fn(), + getSessionStats: vi.fn(), +})); + +vi.mock("@/lib/api", () => ({ + api: apiMocks, + // ProfileProvider mirrors its selection into the api module. + setManagementProfile: vi.fn(), + getManagementProfile: vi.fn(() => ""), +})); +vi.mock("@/components/PlatformsCard", () => ({ PlatformsCard: () => null })); +vi.mock("@/components/Markdown", () => ({ Markdown: () => null })); + +let container: HTMLDivElement; +let root: Root; +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +async function waitFor(cond: () => boolean, timeoutMs = 5000) { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > timeoutMs) throw new Error("waitFor: condition never became true"); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + }); + } +} + +function click(el: Element | null) { + if (!el) throw new Error("element not rendered"); + el.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); +} + +const button = (label: string) => document.querySelector(`button[aria-label="${label}"]`); + +async function renderSessionsPage(rows: Record[]) { + // Page list uses limit 20; the overview tab's recent-cards fetch uses 50 — + // keep the overview empty so the list view (with row actions) renders. + apiMocks.getSessions.mockImplementation(async (limit: number) => ({ + sessions: limit >= 50 ? [] : rows, + total: limit >= 50 ? 0 : rows.length, + limit, + offset: 0, + })); + const [{ default: SessionsPage }, { I18nProvider }, { SystemActionsProvider }, { ProfileProvider }, { PageHeaderProvider }] = + await Promise.all([ + import("./SessionsPage"), + import("@/i18n"), + import("@/contexts/SystemActions"), + import("@/contexts/ProfileProvider"), + import("@/contexts/PageHeaderProvider"), + ]); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + await act(async () => + root.render( + + + + + + + + + + + , + ), + ); + await waitFor(() => Boolean(button("Delete session"))); +} + +beforeEach(() => { + for (const fn of Object.values(apiMocks)) fn.mockReset(); + apiMocks.getStatus.mockResolvedValue({}); + apiMocks.getEmptySessionsCount.mockResolvedValue({ count: 0 }); + apiMocks.getProfiles.mockResolvedValue({ profiles: [] }); + // active === current keeps the management profile "" — the precondition + // under which an unstamped request hits the process's own store. + apiMocks.getActiveProfile.mockResolvedValue({ current: "default", active: "default" }); + apiMocks.getSessionStats.mockResolvedValue({ by_source: {} }); + apiMocks.getSessionMessages.mockResolvedValue({ messages: [] }); + apiMocks.deleteSession.mockResolvedValue({ ok: true }); + apiMocks.renameSession.mockResolvedValue({ ok: true, title: "Renamed" }); + apiMocks.exportSessionUrl.mockReturnValue("/api/sessions/x/export"); + vi.stubGlobal("fetch", vi.fn(async () => ({ ok: false, status: 500 }))); + vi.stubGlobal("ResizeObserver", class { disconnect() {} observe() {} unobserve() {} }); + // gsap ticks through rAF; a synchronous callback recurses to death. + vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => setTimeout(() => cb(0), 0) as unknown as number); + vi.stubGlobal("cancelAnimationFrame", (id: number) => clearTimeout(id)); + vi.stubGlobal("matchMedia", () => ({ addEventListener() {}, matches: false, media: "", removeEventListener() {} })); + sessionStorage.clear(); +}); + +afterEach(async () => { + await act(async () => root?.unmount()); + container?.remove(); + vi.unstubAllGlobals(); +}); + +describe("SessionsPage per-row profile routing (#99387)", () => { + it("sends every per-row request to the row's owning profile, not the management default", async () => { + await renderSessionsPage([ + { id: "sid-guanli", profile: "guanli", source: "cli", model: null, title: "Managed", started_at: 1, ended_at: null, + last_active: 1, is_active: false, message_count: 2, tool_call_count: 0, input_tokens: 1, output_tokens: 1, preview: "hi" }, + ]); + + // expand → transcript read + await act(async () => click(button("Delete session")!.closest("div.cursor-pointer"))); + await waitFor(() => apiMocks.getSessionMessages.mock.calls.length > 0); + expect(apiMocks.getSessionMessages).toHaveBeenCalledWith("sid-guanli", "guanli"); + + await act(async () => click(button("Export session"))); + expect(apiMocks.exportSessionUrl).toHaveBeenCalledWith("sid-guanli", "guanli"); + + await act(async () => click(button("Rename session"))); + const input = document.querySelector('input[placeholder="Session title"]'); + if (!input) throw new Error("rename input not rendered"); + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!.call(input, "Renamed"); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + await act(async () => click(button("Save title"))); + expect(apiMocks.renameSession).toHaveBeenCalledWith("sid-guanli", "Renamed", "guanli"); + + await act(async () => click(button("Delete session"))); + await waitFor(() => Boolean(document.querySelector('[role="alertdialog"]'))); + const confirm = Array.from(document.querySelectorAll('[role="alertdialog"] button')).find( + (b) => b.textContent?.trim() === "Delete", + ); + await act(async () => click(confirm ?? null)); + expect(apiMocks.deleteSession).toHaveBeenCalledWith("sid-guanli", "guanli"); + }); +}); diff --git a/web/src/pages/SessionsPage.tsx b/web/src/pages/SessionsPage.tsx index 09140f9e41..8115cd5742 100644 --- a/web/src/pages/SessionsPage.tsx +++ b/web/src/pages/SessionsPage.tsx @@ -487,7 +487,7 @@ function SessionRow({ if (!isExpanded || messages !== null) return; let cancelled = false; api - .getSessionMessages(session.id) + .getSessionMessages(session.id, session.profile) .then((resp) => { if (!cancelled) setMessages(resp.messages); }) @@ -497,7 +497,7 @@ function SessionRow({ return () => { cancelled = true; }; - }, [isExpanded, session.id, messages]); + }, [isExpanded, session.id, session.profile, messages]); const sourceKey = session.source?.split(":")[0]; const sourceInfo = (session.source @@ -1274,11 +1274,22 @@ export default function SessionsPage() { }; }, [search, sessionQueryOptions]); + // The profile a listed row was read from — the store that owns it. Every + // per-row request (delete, rename, export, messages) must go there, not to + // the global management profile, which lags the row (it stays "" while the + // sticky active profile equals the dashboard process's own, so the request + // hits the process store — a delete then "succeeds" as already_absent). + // Search rows carry no stamp: undefined falls back to the management profile. + const rowProfile = useCallback( + (id: string) => sessions.find((s) => s.id === id)?.profile, + [sessions], + ); + const sessionDelete = useConfirmDelete({ onDelete: useCallback( async (id: string) => { try { - await api.deleteSession(id); + await api.deleteSession(id, rowProfile(id)); setSessions((prev) => prev.filter((s) => s.id !== id)); setTotal((prev) => prev - 1); if (expandedId === id) setExpandedId(null); @@ -1304,6 +1315,7 @@ export default function SessionsPage() { [ expandedId, refreshEmptyCount, + rowProfile, showToast, loadStats, t.sessions.sessionDeleted, @@ -1373,7 +1385,13 @@ export default function SessionsPage() { } setDeletingSelected(true); try { - const resp = await api.bulkDeleteSessions(ids); + // The selection comes from one listed page, so its rows share one + // owning profile; a mixed selection falls back to the management profile. + const owners = new Set(ids.map(rowProfile)); + const resp = await api.bulkDeleteSessions( + ids, + owners.size === 1 ? [...owners][0] : undefined, + ); showToast( t.sessions.selectedSessionsDeleted.replace( "{count}", @@ -1404,6 +1422,7 @@ export default function SessionsPage() { loadSessions, page, refreshEmptyCount, + rowProfile, selectedIds, showToast, t.sessions.failedToDeleteSelected, @@ -1449,7 +1468,7 @@ export default function SessionsPage() { const handleRename = useCallback( async (id: string, title: string) => { try { - await api.renameSession(id, title); + await api.renameSession(id, title, rowProfile(id)); setSessions((prev) => prev.map((s) => (s.id === id ? { ...s, title } : s)), ); @@ -1462,13 +1481,13 @@ export default function SessionsPage() { showToast("Failed to rename session", "error"); } }, - [showToast, loadStats], + [rowProfile, showToast, loadStats], ); const handleExport = useCallback( async (id: string) => { try { - const res = await fetch(api.exportSessionUrl(id), { + const res = await fetch(api.exportSessionUrl(id, rowProfile(id)), { credentials: "include", headers: { "X-Hermes-Session-Token": @@ -1488,7 +1507,7 @@ export default function SessionsPage() { showToast("Failed to export session", "error"); } }, - [showToast], + [rowProfile, showToast], ); const handlePrune = useCallback(async () => { diff --git a/website/docs/developer-guide/adding-platform-adapters.md b/website/docs/developer-guide/adding-platform-adapters.md index 5df1baba48..40e7ec04e6 100644 --- a/website/docs/developer-guide/adding-platform-adapters.md +++ b/website/docs/developer-guide/adding-platform-adapters.md @@ -637,7 +637,7 @@ Three touchpoints: Six touchpoints: -1. **`_create_adapter()`** — Add an `elif platform == Platform.NEWPLAT:` branch +1. **`_instantiate_adapter()`** — Add an `elif platform == Platform.NEWPLAT:` branch. The `_create_adapter()` wrapper binds every successful adapter to its gateway runner. 2. **`_is_user_authorized()` allowed_users map** — `Platform.NEWPLAT: "NEWPLAT_ALLOWED_USERS"` 3. **`_is_user_authorized()` allow_all map** — `Platform.NEWPLAT: "NEWPLAT_ALLOW_ALL_USERS"` 4. **Early env check `_any_allowlist` tuple** — Add `"NEWPLAT_ALLOWED_USERS"` diff --git a/website/docs/developer-guide/context-compression-and-caching.md b/website/docs/developer-guide/context-compression-and-caching.md index 2234289af4..223b802797 100644 --- a/website/docs/developer-guide/context-compression-and-caching.md +++ b/website/docs/developer-guide/context-compression-and-caching.md @@ -73,6 +73,21 @@ Located in `agent/context_compressor.py`. This is the **primary compression system** that runs inside the agent's tool loop with access to accurate, API-reported token counts. +#### Failure cooldown and provider-proven overflow + +A failed or stalled summary attempt arms a per-session **failure cooldown** +(escalating 60s → 300s → 900s, persisted in `state.db`). While it is armed, +ordinary threshold-triggered compaction is deferred so a broken summary backend +does not re-fire every turn. Two paths run a real attempt anyway: + +- Manual `/compress` (`force=True`) — clears the cooldown and retries. +- **Provider-proven overflow** — when the provider itself rejects the request + with a context-length error, the recovery pass ignores the cooldown for one + bounded attempt (`max_compression_attempts`) without clearing it. Deferring + here would wedge the session: every turn would bounce off the provider and + the next failure would extend the ladder (#100661). If that attempt fails, + the cooldown is recorded normally. + ## Configuration diff --git a/website/docs/developer-guide/cron-internals.md b/website/docs/developer-guide/cron-internals.md index 427692eb92..968af066cd 100644 --- a/website/docs/developer-guide/cron-internals.md +++ b/website/docs/developer-guide/cron-internals.md @@ -63,6 +63,20 @@ Jobs are stored in `~/.hermes/cron/jobs.json` with atomic write semantics (write } ``` +### `last_status` literals + +`last_status` is a closed set written only by `cron.jobs.mark_job_run`. Every +renderer (`hermes cron list`/`doctor`, the `cronjob` tool, the web dashboard +badge, the Desktop routine inspector) maps each literal explicitly — a consumer +must never test `== "ok"` for "the user got their result": + +| Literal | Meaning | Detail field | +|---------|---------|--------------| +| `ok` | Agent run succeeded and (if targeted) delivery was confirmed | — | +| `error` | Agent run failed | `last_error` | +| `delivery_failed` | Agent run succeeded, but the output never reached its target | `last_delivery_error` (`last_error` is `null`) | +| `blocked_config` | Pre-dispatch validation refused to burn a run | `last_error` | + ### Job Lifecycle States | State | Meaning | diff --git a/website/docs/developer-guide/gateway-internals.md b/website/docs/developer-guide/gateway-internals.md index 30c8cb9e1f..a96c3d930d 100644 --- a/website/docs/developer-guide/gateway-internals.md +++ b/website/docs/developer-guide/gateway-internals.md @@ -191,6 +191,8 @@ Adapters implement a common interface: Adapters that connect with unique credentials call `acquire_scoped_lock()` in `connect()` and `release_scoped_lock()` in `disconnect()`. This prevents two profiles from using the same bot token simultaneously. +A lock conflict is emitted as `{scope}_lock` with `retryable=True` so a **mid-run** reconnect can recover once the other holder exits. At **startup**, though, a live foreign holder is a configuration conflict: `gateway/restart.py::is_global_startup_conflict()` recognizes the `*_lock` / `lock_conflict` code families and the startup router parks the platform `fatal` instead of retry-queueing it. With nothing else connected the gateway exits `78` (`EX_CONFIG`, `gateway_state=startup_failed`) so the supervisor stops restarting it; alongside a genuinely transient peer failure the gateway stays alive and only the peer retries. + ## Delivery Path Outgoing deliveries (`gateway/delivery.py`) handle: diff --git a/website/docs/developer-guide/image-gen-provider-plugin.md b/website/docs/developer-guide/image-gen-provider-plugin.md index 44b5090295..a42aa3c974 100644 --- a/website/docs/developer-guide/image-gen-provider-plugin.md +++ b/website/docs/developer-guide/image-gen-provider-plugin.md @@ -6,7 +6,7 @@ description: "How to build an image-generation backend plugin for Hermes Agent" # Building an Image Generation Provider Plugin -Image-gen provider plugins register a backend that services every `image_generate` tool call — DALL·E, gpt-image, Grok, Flux, Imagen, Stable Diffusion, fal, Replicate, a local ComfyUI rig, anything. Built-in providers (OpenAI, OpenAI-Codex, xAI, FAL, Krea, DeepInfra, OpenRouter) all ship as plugins. You can add a new one, or override a bundled one, by dropping a directory into `plugins/image_gen//`. +Image-gen provider plugins register a backend that services every `image_generate` tool call — DALL·E, gpt-image, Grok, Flux, Imagen, Stable Diffusion, fal, Replicate, a local ComfyUI rig, anything. Built-in providers (OpenAI, OpenAI-Codex, xAI, FAL, Krea, DeepInfra, OpenRouter, Meta Model API) all ship as plugins. You can add a new one, or override a bundled one, by dropping a directory into `plugins/image_gen//`. :::tip Image-gen is one of several **backend plugins** Hermes supports. The others (with more specialized ABCs) are [Memory Provider Plugins](/developer-guide/memory-provider-plugin), [Context Engine Plugins](/developer-guide/context-engine-plugin), and [Model Provider Plugins](/developer-guide/model-provider-plugin). General tool/hook/CLI plugins live in [Build a Hermes Plugin](/developer-guide/plugins). diff --git a/website/docs/developer-guide/model-provider-plugin.md b/website/docs/developer-guide/model-provider-plugin.md index 5127107fa3..df11914f3f 100644 --- a/website/docs/developer-guide/model-provider-plugin.md +++ b/website/docs/developer-guide/model-provider-plugin.md @@ -18,7 +18,8 @@ Model provider plugins are the third kind of **provider plugin**. The others are 1. **Bundled plugins** — `/plugins/model-providers//` — ship with Hermes 2. **User plugins** — `$HERMES_HOME/plugins/model-providers//` — drop in any directory; no restart required for subsequent sessions -3. **Legacy single-file** — `/providers/.py` — back-compat for out-of-tree editable installs +3. **Installed plugins** — `$HERMES_HOME/plugins//` (where `hermes plugins install owner/repo` clones) — imported only when `plugin.yaml` declares `kind: model-provider`; every other kind there belongs to the general PluginManager +4. **Legacy single-file** — `/providers/.py` — back-compat for out-of-tree editable installs **User plugins override bundled plugins of the same name** because `register_provider()` is last-writer-wins. Drop a `$HERMES_HOME/plugins/model-providers/gmi/` directory to replace the built-in GMI profile without touching the repo. @@ -141,8 +142,31 @@ class AcmeProfile(ProviderProfile): Bearer auth. Override for: custom auth (Anthropic), no REST endpoint (Bedrock → None), or public/unauthenticated catalogs (OpenRouter).""" return super().fetch_models(api_key=api_key, base_url=base_url, timeout=timeout) + + def create_client(self, **client_kwargs): + """Supply your own client object instead of the shared openai.OpenAI. + Default returns None (= use the standard client). Override when the + wire protocol is not OpenAI-over-HTTP — e.g. an ACP subprocess shim. + client_kwargs is what the core would have passed to openai.OpenAI + (api_key, base_url, command, args, timeouts, headers…); accept **kwargs + and pick what you need. A raise is logged and falls back to the + standard client.""" + return None ``` +## External-process (ACP) providers + +An agent CLI driven over stdio is not an HTTP endpoint. Set `auth_type="external_process"`, describe how to launch the binary, and supply the client with `create_client`. No core edits are needed — `hermes -m `, `/model`, credential resolution, runtime resolution and the auxiliary client (compression, vision) all key on `auth_type`, not on the provider name. `plugins/model-providers/copilot-acp/` is the in-tree example. + +| Field | Purpose | +|---|---| +| `process_command` | Default binary, e.g. `"copilot"` | +| `process_args` | Default argv tail, e.g. `("--acp", "--stdio")` | +| `process_command_env_vars` | Env vars that override the binary, checked in order | +| `process_args_env_var` | Env var that overrides argv (shlex-split) | + +The client your `create_client` returns receives `command` and `args` in `client_kwargs`. If it is already complete and async-safe, declare `HERMES_SKIP_TRANSPORT_WRAP = True` / `HERMES_SKIP_ASYNC_WRAP = True` as class attributes so the auxiliary client does not re-dispatch it through an HTTP wire adapter. + ## Hook reference examples Look at these bundled plugins for idioms: @@ -198,7 +222,7 @@ Set `profile.api_mode` to match the default your provider ships — it acts as a | `oauth_external` | User signs in elsewhere, tokens land in `auth.json` | Anthropic OAuth, MiniMax OAuth, Qwen Portal, Nous Portal | | `copilot` | GitHub Copilot token refresh cycle | `copilot` plugin only | | `aws_sdk` | AWS SDK credential chain (IAM role, profile, env) | `bedrock` plugin only | -| `external_process` | Auth handled by a subprocess the agent spawns | `copilot-acp` plugin only | +| `external_process` | Auth handled by a subprocess the agent spawns (see [External-process providers](#external-process-acp-providers)) | `copilot-acp` plugin, out-of-tree ACP plugins | `auth_type` gates which codepaths treat your provider as a "simple api-key provider" — if it's not `api_key`, the PluginManager still records the manifest but Hermes' CLI-level automation (doctor checks, `--provider` flag, setup wizard delegation) may skip over it. diff --git a/website/docs/developer-guide/session-storage.md b/website/docs/developer-guide/session-storage.md index 0ff701d7f3..4627b0ffa6 100644 --- a/website/docs/developer-guide/session-storage.md +++ b/website/docs/developer-guide/session-storage.md @@ -21,9 +21,15 @@ Source file: `hermes_state.py` ├── gateway_routing — Gateway routing metadata ├── compression_locks — Cross-process compression locking ├── async_delegations — Async delegation bookkeeping +├── delivery_obligations — Gateway outbox (owed replies); created lazily by gateway/delivery_ledger.py └── schema_version — Single-row table tracking migration state ``` +`hermes sessions recover` copies the row-bearing tables above into the +recovered database (FTS indexes and `schema_version` are regenerated), including +the lazily-created `delivery_obligations` ledger when the source has one — its +row count is verified like `sessions`/`messages`. + Key design decisions: - **WAL mode** for concurrent readers + one writer (gateway multi-platform) - **FTS5 virtual table** for fast text search across all session messages @@ -163,6 +169,8 @@ The `schema_version` table stores a single integer. Simple column additions are | 20 | Per-model usage attribution — seed `session_model_usage` rows from historical per-session aggregate totals | | 22 | Task-dimension usage attribution — rebuild `session_model_usage` so the `task` column participates in the PRIMARY KEY | | 23 | FTS storage redesign — external-content FTS tables replacing the v11 inline-mode copies (opt-in transition for existing DBs) | +| 29 | Cron sessions leave the trigram (substring/CJK) index; `messages_fts_trigram_src` view + triggers filter on `sessions.source`, one-time rebuild purges historical rows | +| 30 | Delegate-child (subagent) sessions leave the trigram index too — `source='subagent'` or the `$._delegate_from` marker (`FTS_TRIGRAM_SESSION_SQL`). Rows stay in `messages` and the standard `messages_fts` word index, so `session_search` still finds them; only the ~2.6× trigram shadow tables shrink. Same one-time rebuild as v29 | Versions not listed above were declarative column additions handled by `_reconcile_columns()` (version bump only, no data migration). diff --git a/website/docs/getting-started/updating.md b/website/docs/getting-started/updating.md index 5ae1f14787..8c20bc1558 100644 --- a/website/docs/getting-started/updating.md +++ b/website/docs/getting-started/updating.md @@ -29,7 +29,8 @@ When you run `hermes update`, the following steps occur: 3. **Post-pull syntax validation + auto-rollback** — after the pull, Hermes compiles the nine critical files every `hermes` invocation imports at startup. If any fails to parse (e.g. an orphan merge-conflict marker, an accidentally truncated file), Hermes runs `git reset --hard ` to roll the install back so your shell stays bootable. Re-run `hermes update` once the upstream fix lands. 4. **Dependency install** — runs `uv pip install -e ".[all]"` to pick up new or changed dependencies 5. **Config migration** — detects new config options added since your version and prompts you to set them -6. **Gateway auto-restart** — running gateways are refreshed after the update completes so the new code takes effect immediately. Service-managed gateways (systemd on Linux, launchd on macOS) are restarted through the service manager. Manual gateways are relaunched automatically when Hermes can map the running PID back to a profile. Manually-launched `hermes serve` / `hermes dashboard` backends (for example a network-bound serve powering a remote Desktop) are handled the same way: each backend records its bind address in the install's spawn ledger at startup, so the update stops it before the code swap and relaunches it afterward on the **same host and port** — a remote Desktop pointed at that endpoint reconnects instead of stranding. Backends owned by a running Desktop app are left to the app's own respawn. +6. **Desktop rebuild (stage-and-swap)** — if the Hermes Desktop app was built from this checkout, it is rebuilt so the GUI matches the new code. The rebuild packs into a temporary staging directory next to `apps/desktop/release/`, verifies the staged app, and only then renames it over the previous build. A rebuild that fails at any point — corrupt Electron download, missing dependency, disk full — leaves the previous app untouched and launchable; the update reports `⚠ Update partially complete` and `hermes desktop` retries the rebuild. +7. **Gateway auto-restart** — running gateways are refreshed after the update completes so the new code takes effect immediately. Service-managed gateways (systemd on Linux, launchd on macOS) are restarted through the service manager. Manual gateways are relaunched automatically when Hermes can map the running PID back to a profile. Manually-launched `hermes serve` / `hermes dashboard` backends (for example a network-bound serve powering a remote Desktop) are handled the same way: each backend records its bind address in the install's spawn ledger at startup, so the update stops it before the code swap and relaunches it afterward on the **same host and port** — a remote Desktop pointed at that endpoint reconnects instead of stranding. Backends owned by a running Desktop app are left to the app's own respawn. ### Updating against a non-default branch: `--branch` diff --git a/website/docs/guides/automate-with-cron.md b/website/docs/guides/automate-with-cron.md index 20bb490207..c73d9b39ea 100644 --- a/website/docs/guides/automate-with-cron.md +++ b/website/docs/guides/automate-with-cron.md @@ -74,6 +74,10 @@ Set up the cron job: For cron monitoring jobs, instruct the agent to respond with only `[SILENT]` when nothing changed. Cron delivery treats `[SILENT]` as the quiet marker, so you only get notified when something actually happens — no spam on quiet hours. ::: +:::tip Keeping failure notices out of shared channels +`[SILENT]` only applies to successful runs — when a job hard-fails, the engine posts a `⚠️ Cron 'X' failed…` notice to the job's delivery target. For jobs that deliver into busy shared channels, set `--failure-deliver local` to suppress those notices entirely (run state stays visible in `hermes cron list` and run history), or point failures at an ops channel with `--failure-deliver slack:C_OPS`. Same grammar as `--deliver`; omit it and failures follow `--deliver` as before. +::: + --- ## Pattern 2: Weekly Report diff --git a/website/docs/integrations/nous-portal.md b/website/docs/integrations/nous-portal.md index ffcbe49409..b574213352 100644 --- a/website/docs/integrations/nous-portal.md +++ b/website/docs/integrations/nous-portal.md @@ -237,6 +237,13 @@ The OAuth refresh token is stored separately at `~/.hermes/auth.json` (not in `c Hermes mints a short-lived JWT from your stored Portal refresh token on each inference call rather than reusing a long-lived API key. The token lifecycle is fully automatic — refresh, mint, retry on transient 401 — and you never see it. +Long-running gateway and dashboard processes also run a background keepalive that refreshes the token before it expires, so idle agents don't pay a 401 round-trip on their first request of each credential lifetime. The keepalive derives its tick from the lifetime the Portal actually issued (several ticks per lifetime), bounded above by: + +```yaml +nous: + keepalive_interval_seconds: 900 # upper bound on the tick; 0 disables the keepalive +``` + If the Portal invalidates the refresh token (password change, manual revoke, session expiry), the invalid refresh token is **quarantined locally** so Hermes stops replaying it and you don't see a stream of identical 401s. The next call surfaces a clear "re-authentication required" message. Run `hermes auth add nous` to log in again; the quarantine clears on the next successful login. ## Troubleshooting diff --git a/website/docs/integrations/providers.md b/website/docs/integrations/providers.md index 0d4569accd..d9de727acd 100644 --- a/website/docs/integrations/providers.md +++ b/website/docs/integrations/providers.md @@ -36,8 +36,8 @@ You need at least one way to connect to an LLM. Use `hermes model` to switch pro | **xAI (Grok) — Responses API** | `XAI_API_KEY` in `~/.hermes/.env` (provider: `xai`) | | **xAI Grok OAuth (SuperGrok)** | `hermes model` → "xAI Grok OAuth (SuperGrok / Premium+)" — browser login, no API key. See [guide](../guides/xai-grok-oauth.md) | | **Qwen Cloud (Alibaba DashScope)** | `DASHSCOPE_API_KEY` in `~/.hermes/.env` (provider: `alibaba`; mainland-China endpoint: `alibaba-cn`) | -| **Alibaba Cloud (Coding Plan)** | `ALIBABA_CODING_PLAN_API_KEY` (falls back to `DASHSCOPE_API_KEY`) (provider: `alibaba-coding-plan`, alias: `alibaba_coding`; mainland-China endpoint: `alibaba-coding-plan-cn`) — separate billing SKU, different endpoint | -| **Alibaba Cloud (Token Plan)** | `ALIBABA_TOKEN_PLAN_API_KEY` in `~/.hermes/.env` (provider: `alibaba-token-plan`; mainland-China endpoint: `alibaba-token-plan-cn`) — Model Studio flat-token tier | +| **Alibaba Cloud (Coding Plan)** | `ALIBABA_CODING_PLAN_API_KEY` (falls back to `DASHSCOPE_API_KEY`) (provider: `alibaba-coding-plan`, alias: `alibaba_coding`; mainland-China endpoint: `alibaba-coding-plan-cn` with `ALIBABA_CODING_PLAN_CN_API_KEY`, falling back to the shared keys) — separate billing SKU, different endpoint | +| **Alibaba Cloud (Token Plan)** | `ALIBABA_TOKEN_PLAN_API_KEY` in `~/.hermes/.env` (provider: `alibaba-token-plan`; mainland-China endpoint: `alibaba-token-plan-cn` with `ALIBABA_TOKEN_PLAN_CN_API_KEY`, falling back to the shared key) — Model Studio flat-token tier | | **Kilo Code** | `KILOCODE_API_KEY` in `~/.hermes/.env` (provider: `kilocode`) | | **Xiaomi MiMo** | `XIAOMI_API_KEY` in `~/.hermes/.env` (provider: `xiaomi`, aliases: `mimo`, `xiaomi-mimo`) | | **Tencent TokenHub** | `TOKENHUB_API_KEY` in `~/.hermes/.env` (provider: `tencent-tokenhub`, aliases: `tencent`, `tokenhub`, `tencentmaas`) | @@ -61,6 +61,8 @@ You need at least one way to connect to an LLM. Use `hermes model` to switch pro | **LM Studio** | `hermes model` → "LM Studio" (provider: `lmstudio`, optional `LM_API_KEY`) | | **Custom Endpoint** | `hermes model` → choose "Custom endpoint" (saved in `config.yaml`) | +All three OpenCode providers send an opaque, per-conversation `x-opencode-session` header on every request (main turns on every transport plus auxiliary calls such as compression and titles). OpenCode uses it to pin a conversation to one backend so its prompt cache stays warm; the value is derived from the Hermes session id and carries no personal data. + For the official API-key path, see the dedicated [Google Gemini guide](/guides/google-gemini). :::tip Model key alias @@ -327,7 +329,7 @@ model: Base URLs can be overridden with `NOVITA_BASE_URL`, `GLM_BASE_URL`, `KIMI_BASE_URL`, `MINIMAX_BASE_URL`, `MINIMAX_CN_BASE_URL`, `DASHSCOPE_BASE_URL`, `XIAOMI_BASE_URL`, `GMI_BASE_URL`, `META_BASE_URL`, or `TOKENHUB_BASE_URL` environment variables. :::note Meta contributor tier -`muse-spark-1.2-contributor` is Meta's discounted tier — Meta may train on your prompts and completions, so [interactive model selection asks for confirmation](../user-guide/configuring-models.md) before using it. Use `muse-spark-1.2` (standard pricing, no training) for confidential work. +`muse-spark-1.2-contributor` and `muse-spark-1.3-contributor` are Meta's contributor tiers — Meta may train on your prompts and completions, so [interactive model selection asks for confirmation](../user-guide/configuring-models.md) before using either. For current pricing and rate limits, see [Meta Model API pricing and rate limits](https://dev.meta.ai/docs/pricing-rate-limits/). Use the standard `muse-spark-1.2` / `muse-spark-1.3` (no training) for confidential work. ::: :::note Z.AI Endpoint Auto-Detection @@ -503,6 +505,8 @@ hermes chat --provider alibaba_coding --model qwen3-coder-plus `alibaba_coding` uses the same `DASHSCOPE_API_KEY` your `alibaba` entry already uses — no separate key needed, just a different routing target. Before this provider was registered, users who set `provider: alibaba_coding` in `config.yaml` silently fell through to OpenRouter routing. +For the mainland-China endpoint (`alibaba-coding-plan-cn`, `https://coding.dashscope.aliyuncs.com/v1`) set `ALIBABA_CODING_PLAN_CN_API_KEY`. The CN provider still falls back to `ALIBABA_CODING_PLAN_API_KEY` / `DASHSCOPE_API_KEY`, but with only the shared key set the `/model` picker lists just the international row — set the CN key (or `provider: alibaba-coding-plan-cn` in `config.yaml`) to surface the CN one. The same applies to `alibaba-token-plan-cn` with `ALIBABA_TOKEN_PLAN_CN_API_KEY`. + ### MiniMax (OAuth) MiniMax-M2.7 via browser OAuth login — no API key needed. Pick **MiniMax (OAuth)** in `hermes model`, sign in through the browser, and Hermes persists the access + refresh tokens. Uses the Anthropic Messages-compatible endpoint (`/anthropic`) under the hood. diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index 8760b5b796..6cad7bcebf 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -1018,6 +1018,21 @@ Restore a previously created Hermes backup into your Hermes home directory. All Stop the gateway before importing to avoid conflicts with running processes. ::: +### SQLite databases + +`.db` members (`state.db`, `kanban.db`, `response_store.db`, …) are not published with a rename like ordinary files. Renaming would replace the file's inode while a gateway, dashboard, or WebUI process still holds the old one open: that process would keep reading pre-import pages and keep writing sessions nobody else can see, and those sessions would simply be absent from the database everyone opens next — with nothing logged. Instead the imported pages are written **into the existing database file**, the same way `/snapshot restore` does it, so every open connection converges on the imported data. + +If the live database cannot be replaced safely — the page copy failed *and* another process still holds the file open — the import leaves that database untouched and lists it under `Warnings (N files skipped)`. Stop the holding processes and re-run. + +Importing an older backup over newer work is still allowed, but it is no longer silent. When the imported `state.db` holds fewer messages than the one it replaced, the summary reports it: + +``` + ⚠ Session data replaced by older backup contents: + state.db: 12 session(s) / 8912 message(s) -> 3 / 24 + Anything recorded after the backup was taken is not in it. + Recover from a newer backup or snapshot: hermes snapshot list +``` + ### Examples ```bash hermes import ~/hermes-backup-20260423.zip # Prompts before overwriting existing config diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 4c87009a64..bd1b8e0ba2 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -83,10 +83,12 @@ Hermes reads environment variables from the process environment and, for user-ma | `DASHSCOPE_API_KEY` | Qwen Cloud (Alibaba DashScope) API key for Qwen models ([modelstudio.console.alibabacloud.com](https://modelstudio.console.alibabacloud.com/)) | | `DASHSCOPE_BASE_URL` | Custom DashScope base URL (default: `https://dashscope-intl.aliyuncs.com/compatible-mode/v1`; use `https://dashscope.aliyuncs.com/compatible-mode/v1` for mainland-China region) | | `DASHSCOPE_CN_BASE_URL` | Override the `alibaba-cn` mainland-China DashScope base URL | -| `ALIBABA_CODING_PLAN_API_KEY` | Qwen Coding Plan API key (`alibaba-coding-plan` / `alibaba-coding-plan-cn` providers) | +| `ALIBABA_CODING_PLAN_API_KEY` | Qwen Coding Plan API key (`alibaba-coding-plan`; also a fallback for `alibaba-coding-plan-cn`) | +| `ALIBABA_CODING_PLAN_CN_API_KEY` | Qwen Coding Plan API key for the mainland-China `alibaba-coding-plan-cn` provider (checked before the shared key, so only the CN row lights up) | | `ALIBABA_CODING_PLAN_BASE_URL` | Override the Qwen Coding Plan base URL (international) | | `ALIBABA_CODING_PLAN_CN_BASE_URL` | Override the Qwen Coding Plan base URL (mainland China) | -| `ALIBABA_TOKEN_PLAN_API_KEY` | Alibaba Model Studio Token Plan API key (`alibaba-token-plan` / `alibaba-token-plan-cn` providers) | +| `ALIBABA_TOKEN_PLAN_API_KEY` | Alibaba Model Studio Token Plan API key (`alibaba-token-plan`; also a fallback for `alibaba-token-plan-cn`) | +| `ALIBABA_TOKEN_PLAN_CN_API_KEY` | Token Plan API key for the mainland-China `alibaba-token-plan-cn` provider (checked before the shared key) | | `ALIBABA_TOKEN_PLAN_BASE_URL` | Override the Token Plan base URL (international) | | `ALIBABA_TOKEN_PLAN_CN_BASE_URL` | Override the Token Plan base URL (mainland China) | | `DEEPSEEK_API_KEY` | DeepSeek API key for direct DeepSeek access ([platform.deepseek.com](https://platform.deepseek.com/api_keys)) | diff --git a/website/docs/reference/faq.md b/website/docs/reference/faq.md index 19bd27e4d4..d9f04d7d27 100644 --- a/website/docs/reference/faq.md +++ b/website/docs/reference/faq.md @@ -319,6 +319,8 @@ If this happens on the first long conversation, Hermes may have the wrong contex Look at the CLI startup line — it shows the detected context length (e.g., `📊 Context limit: 128000 tokens`). You can also check with `/usage` during a session. +**Local servers (llama.cpp, Ollama) that go silent instead of erroring:** when a provider rejects a request as too large, Hermes compacts the conversation and rebuilds the request. Hermes re-measures the *complete* rebuilt request (system prompt + tool schemas + messages) before retrying, and runs further bounded compaction passes if it is still over the threshold. If the request still cannot fit, the turn ends with `Context length exceeded: compression could not reduce the rebuilt request below the safe threshold` rather than sending an oversized request that llama.cpp would silently truncate (`stop processing: n_tokens = 65535, truncated = 1` in the server log). If you hit that message, the fix is almost always the configured `context_length` above: make it match the server's actual `-c` / `--ctx-size`. + To fix context detection, set it explicitly: ```yaml diff --git a/website/docs/reference/model-catalog.md b/website/docs/reference/model-catalog.md index 4769a720c8..b26a1399f0 100644 --- a/website/docs/reference/model-catalog.md +++ b/website/docs/reference/model-catalog.md @@ -59,6 +59,7 @@ Field notes: | When | What happens | |---|---| | `/model` or `hermes model` | Fetches if disk cache is stale, else uses cache | +| Gateway running | Background refresh every `ttl_minutes` (default 20), so the picker never lags the published manifest by more than one window | | Disk cache fresh (< TTL) | No network hit | | Network failure with cache | Silent fallback to cache, one log line | | Network failure, no cache | Silent fallback to in-repo snapshot | @@ -72,11 +73,11 @@ Cache location: `~/.hermes/cache/model_catalog.json`. model_catalog: enabled: true url: https://hermes-agent.nousresearch.com/docs/api/model-catalog.json - ttl_hours: 1 + ttl_minutes: 20 providers: {} ``` -Set `enabled: false` to disable remote fetch entirely and always use the in-repo snapshot. +Set `enabled: false` to disable remote fetch entirely and always use the in-repo snapshot (this also disables the gateway's background refresh). `ttl_minutes` sets both the cache lifetime and the gateway refresh cadence; the legacy `ttl_hours` key is still honoured if you set it explicitly. ### Per-provider override URLs diff --git a/website/docs/reference/slash-commands.md b/website/docs/reference/slash-commands.md index 41dd223b7f..0bcdcb52cb 100644 --- a/website/docs/reference/slash-commands.md +++ b/website/docs/reference/slash-commands.md @@ -76,12 +76,12 @@ Type `/` in the CLI to open the autocomplete menu. Built-in commands are case-in | Command | Description | |---------|-------------| | `/config` | Show current configuration | -| `/model [model-name]` | Show or change the current model. Supports: `/model claude-sonnet-4`, `/model provider:model` (switch providers), `/model custom:model` (custom endpoint), `/model custom:name:model` (named custom provider), `/model custom` (auto-detect from endpoint), and user-defined aliases (`/model fav`, `/model grok` — see [Custom model aliases](#custom-model-aliases)). Flags: `--global` persists the change to config.yaml; `--session` forces session-only; `--once` applies to the next turn only; `--refresh` re-fetches the provider's model list; `--provider ` switches backend (session-only unless `--global`). A plain `/model ` is session-only unless `model.persist_switch_by_default: true` is set. **Interactive picker:** running `/model` with no arguments opens the provider→model picker; on the model list you can **type to fuzzy-filter** the models (e.g. type `grok` to narrow to matching models), Backspace to trim the filter, Esc to clear it (or close the picker). Selection always resolves to one concrete model — the filter only narrows the list, it never guesses. **Note:** `/model` can only switch between already-configured providers. To add a new provider, exit the session and run `hermes model` from your terminal. **Cost note:** switching models mid-conversation resets the prompt cache — the cache key includes the model, so your next turn re-reads the entire conversation at full input price instead of the ~75%-discounted cached rate. Expected and unavoidable, but worth knowing on long sessions. | +| `/model [model-name]` | Show or change the current model. Supports: `/model claude-sonnet-4`, `/model provider:model` (switch providers), `/model custom:model` (custom endpoint), `/model custom:name:model` (named custom provider), `/model custom` (auto-detect from endpoint), and user-defined aliases (`/model fav`, `/model grok` — see [Custom model aliases](#custom-model-aliases)). Flags: `--global` persists the change to config.yaml; `--session` forces session-only; `--once` applies to the next turn only; `--refresh` re-fetches the provider's model list; `--provider ` switches backend (session-only unless `--global`). A plain `/model ` is session-only unless `model.persist_switch_by_default: true` is set — except when no `model.default`/`model.provider` is configured yet, in which case the first pick persists so the profile gets a real default. The same rule governs the desktop composer picker. **Interactive picker:** running `/model` with no arguments opens the provider→model picker; on the model list you can **type to fuzzy-filter** the models (e.g. type `grok` to narrow to matching models), Backspace to trim the filter, Esc to clear it (or close the picker). Selection always resolves to one concrete model — the filter only narrows the list, it never guesses. **Note:** `/model` can only switch between already-configured providers. To add a new provider, exit the session and run `hermes model` from your terminal. **Cost note:** switching models mid-conversation resets the prompt cache — the cache key includes the model, so your next turn re-reads the entire conversation at full input price instead of the ~75%-discounted cached rate. Expected and unavoidable, but worth knowing on long sessions. | | `/codex-runtime [auto\|codex_app_server\|on\|off]` | Toggle the optional [Codex app-server runtime](../user-guide/features/codex-app-server-runtime) for OpenAI/Codex models. `auto` (default) uses Hermes' standard chat completions; `codex_app_server` hands turns to a `codex app-server` subprocess for native shell, apply_patch, ChatGPT subscription auth, and migrated Codex plugins. Effective on next session. | | `/personality` | Set a predefined personality. `/personality none` (or `default` / `neutral`) clears the overlay and returns to base behavior. | | `/verbose` | Cycle tool progress display: off → new → all → verbose. Can be [enabled for messaging](#notes) via config. | | `/focus [on\|off\|status]` | Toggle **focus view** — a display-only reduced-output mode showing just your prompt and the final response. Composes with `/verbose`: turning it on snaps tool progress to `off` and remembers your previous mode, and `/focus off` restores it. Each turn ends with a dim recovery line (`⋯ 7 tool lines hidden · /focus off to show`) and a persistent `◉ focus` badge sits in the status bar so you always know you're in the reduced view. Nothing is sent differently to the model — detail is hidden, never discarded. | -| `/fast [normal\|fast\|status]` | Toggle fast mode — OpenAI Priority Processing / Anthropic Fast Mode. Options: `normal`, `fast`, `status`. | +| `/fast [normal\|fast\|auto\|cold\|status]` | Fast mode — OpenAI Priority Processing / Anthropic Fast Mode. `fast` = every request; `auto` = only requests in the first `agent.fast_auto_seconds` (default 60s) of each turn; `cold` = that same window on the first turn of a session only. Default `normal` (off). See [Fast mode](../user-guide/configuration.md#fast-mode). | | `/reasoning [level\|show\|hide\|full\|clamp] [--global]` | Manage reasoning effort and display. Levels include `none` / `minimal` / `low` / `medium` / `high` / `xhigh` / `max` / `ultra`. `show` / `hide` (or `on` / `off`) toggle reasoning display; `full` and `clamp` adjust how reasoning is shown. `--global` persists effort to config. | | `/skin` | Show or change the display skin/theme | | `/export [profile] [-o out.tar.gz]` | **CLI only.** Pack a profile into a shareable `.tar.gz` — skills, memory, persona, crons, plugins, settings, and (from the desktop) themes and layout. Credentials (`auth.json`, `.env`) are stripped. Defaults to the active profile and `.tar.gz` in the current directory. Same archive as `hermes profile export`; for a versioned, updatable share use a [profile distribution](../user-guide/profile-distributions.md) instead. | @@ -115,7 +115,7 @@ Type `/` in the CLI to open the autocomplete menu. Built-in commands are case-in | `/blueprint [name] [slot=value ...]` (alias: `/bp`) | Set up an automation from a blueprint template. Bare `/blueprint` lists the catalog; `/blueprint ` starts a guided slot-filling flow on the next agent turn; `/blueprint slot=value ...` creates the job directly. | | `/curator` | Background skill maintenance — `status`, `run`, `pin`, `archive`. See [Curator](/user-guide/features/curator). | | `/kanban ` | Drive the multi-profile, multi-project collaboration board without leaving chat. Full `hermes kanban` surface is available: `/kanban list`, `/kanban show t_abc`, `/kanban create "title" --assignee X`, `/kanban comment t_abc "text"`, `/kanban unblock t_abc`, `/kanban dispatch`, etc. Multi-board support included: `/kanban boards list`, `/kanban boards create `, `/kanban boards switch `, `/kanban --board `. See [Kanban slash command](/user-guide/features/kanban#kanban-slash-command). | -| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config.yaml | +| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config.yaml and re-probe tool availability (credentials/daemons that appeared mid-session) | | `/reload-skills` (alias: `/reload_skills`) | Re-scan `~/.hermes/skills/` for newly installed or removed skills | | `/reload` | Reload `.env` variables into the running session (picks up new API keys without restarting) | | `/plugins` | List installed plugins and their status | @@ -246,7 +246,7 @@ The messaging gateway supports the following built-in commands inside Telegram, | `/model [provider:model]` | Show or change the model. Supports provider switches (`/model zai:glm-5`), custom endpoints (`/model custom:model`), named custom providers (`/model custom:local:qwen`), auto-detect (`/model custom`), and user-defined aliases (`/model fav`, `/model grok` — see [Custom model aliases](#custom-model-aliases)). Use `--global` to persist the change to config.yaml. **Note:** `/model` can only switch between already-configured providers. To add a new provider or set up API keys, use `hermes model` from your terminal (outside the chat session). **Cost note:** a mid-session model switch resets the prompt cache (the cache key includes the model), so the next message re-reads the whole conversation at full input price. | | `/codex-runtime [auto\|codex_app_server\|on\|off]` | Toggle the optional [Codex app-server runtime](../user-guide/features/codex-app-server-runtime). Persists to `model.openai_runtime` in config.yaml and evicts the cached agent so the next message picks up the new runtime. Effective on next session. | | `/personality [name]` | Set a personality overlay for the session. `/personality none` (or `default` / `neutral`) clears it. | -| `/fast [normal\|fast\|status]` | Toggle fast mode — OpenAI Priority Processing / Anthropic Fast Mode. | +| `/fast [normal\|fast\|auto\|cold\|status]` | Fast mode — OpenAI Priority Processing / Anthropic Fast Mode. `auto`/`cold` open a bounded fast window per turn / per session. | | `/retry` | Retry the last message. | | `/undo` | Remove the last exchange. | | `/sethome` (alias: `/set-home`) | Mark the current chat as the platform home channel for deliveries. | @@ -291,7 +291,7 @@ The messaging gateway supports the following built-in commands inside Telegram, | `/skills [pending\|approve\|reject\|diff\|approval]` | Review pending **skill** writes staged by the write-approval gate (`skills.write_approval`). Shows a one-line gist per staged write; `/skills diff ` is truncated for chat — read the full diff on the CLI or in `~/.hermes/pending/skills/.json`. Only appears when the gate is on (or staged writes remain); search/install stay CLI-only. | | `/kanban ` | Drive the multi-profile, multi-project collaboration board from chat — identical argument surface to the CLI. Bypasses the running-agent guard, so `/kanban unblock t_abc`, `/kanban comment t_abc "…"`, `/kanban list --mine`, `/kanban boards switch `, etc. work mid-turn. `/kanban create …` auto-subscribes the originating chat to the new task's terminal events. See [Kanban slash command](/user-guide/features/kanban#kanban-slash-command). | | `/platform [name]` | Operate a running gateway platform right from chat. `/platform list` shows every adapter and its state (running, paused-by-breaker, manually-paused); `/platform pause ` stops dispatching new messages to that adapter without unloading it; `/platform resume ` re-enables it and clears a tripped circuit breaker once the upstream is healthy. | -| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config. | +| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config and re-probe tool availability. | | `/verbose` | Cycle tool progress display. **Off by default on messaging** — enable with `display.tool_progress_command: true` in `config.yaml`. | | `/yolo` | Toggle YOLO mode — skip all dangerous command approval prompts. | | `/commands [page]` | Browse all commands and skills (paginated). | diff --git a/website/docs/user-guide/bot-mode.md b/website/docs/user-guide/bot-mode.md index 0e50502a3f..b529b7daa0 100644 --- a/website/docs/user-guide/bot-mode.md +++ b/website/docs/user-guide/bot-mode.md @@ -17,7 +17,7 @@ There is no new primitive to learn: a Bot **is** a Hermes profile — isolated c The roster shows one row per agent profile: avatar, latest-message preview, and timestamp. -- **Click a Bot** to land in its chat — every Bot has a canonical, persistent **Bot Chat** conversation that is created (and pinned) the moment the Bot is born. +- **Click a Bot** to land in its chat — every Bot has a canonical, persistent **Bot Chat** conversation that is created (and pinned) the moment the Bot is born. A row click always opens that Bot Chat (the same conversation the row previews), even when you have other tabs open for the Bot; those tabs stay open beside it. In the tab strip the Bot Chat is captioned with the Bot's name, so two open Bots are told apart at a glance. - **Active now** — a presence strip above the roster shows every Bot currently working: the gateway-busy profile plus any Bot that wrote within the last 90 seconds. Each chip opens that Bot's chat. The strip never reorders the roster and disappears when the fleet is idle. - **Search** filters the roster as you type. - **Hide a Bot** — right-click a row → **Hide Bot** to take a Bot you don't use out of the roster and the Active-now strip. Hiding is display-only: @mentions still resolve, group-chat memberships are untouched, and routines keep running. Once at least one Bot is hidden, an **eye toggle** appears in the pane header — click it to reveal hidden Bots dimmed in place, then right-click → **Unhide Bot** to bring one back. Hidden Bots never toast, but they accumulate unread activity silently and the eye badges a dot so you know something happened. Hidden state is saved in the Bot's profile metadata, so it follows the Bot to every desktop connected to that backend. @@ -26,6 +26,17 @@ The roster shows one row per agent profile: avatar, latest-message preview, and Typing `/new` (or `/reset`) inside a Bot's canonical chat would fork the relationship into a scratch session — the one thing Bot Mode promises never happens. The composer reroutes it to `/compact` instead: fresh working context, same conversation. Regular sessions on the same profile keep full `/new` freedom. ::: +### Organize bots into sections + +Sections are folders you make yourself — **Clients**, **Team**, whatever fits — as a second axis beside the automatic per-gateway grouping. With no sections created the roster is the plain list it always was. + +- **Create one** from the pane's **+** menu → **New section**, or right-click a Bot → **Move to section** → **New section…** (that files the Bot into it as you create it). +- **File a Bot** by dragging its row onto a section — the target highlights while you hover, and **Esc** cancels the drag — or right-click → **Move to section** and pick one. **Remove from section** puts it back in **Unassigned**. +- **Rename, reorder, or delete** a section from its heading's right-click menu (or the **⋯** that appears on hover); double-click a heading to rename. Headings fold like the gateway headings do. +- **Deleting a section never deletes Bots** — they return to **Unassigned**, and the toast offers **Undo**. No confirmation is asked. + +Membership is stored in each Bot's profile metadata (`ui_meta`), so a Bot's section follows it to every desktop connected to that backend. When the roster shows more than one gateway, sections nest inside each gateway's bucket. + ## Creating a Bot Hit **New Agent** in the roster. The quick path is three fields — **Name**, **Title**, **Description** — and the Bot exists in seconds, introducing itself as the first message of its new Bot Chat. diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index b6654defea..95668e8f15 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -136,6 +136,8 @@ delegation: Multiple references in a single value work: `url: "${HOST}:${PORT}"`. If a referenced variable is not set, the placeholder is kept verbatim (`${UNDEFINED_VAR}` stays as-is) and a warning is logged. Bare `$VAR` is not expanded. +Under a [multiplexed multi-profile gateway](/user-guide/multi-profile-gateways), references in a profile's `config.yaml` resolve against **that profile's** `.env` (its secret scope), not the shared process environment — a `${MATRIX_ACCESS_TOKEN}` in profile B stays unresolved unless B defines the variable itself. Single-profile runs are unchanged. + Cursor-style SecretRef syntax is also accepted: `${env:VAR_NAME}` resolves exactly like `${VAR_NAME}` (the `env:` prefix is stripped), so MCP or provider snippets copied from Cursor / Claude configs work unchanged in both `config.yaml` and the `mcp_servers` block. Other SecretRef sources (`${file:...}`, `${vault:...}`, `${bitwarden:...}`) are **not** resolved inline — external secret backends inject their values into the environment at startup via the `secrets:` block, so reference them as `${env:NAME}` instead; unknown prefixes warn once and stay verbatim. For AI provider setup (OpenRouter, Anthropic, Copilot, custom endpoints, self-hosted LLMs, fallback models, etc.), see [AI Providers](/integrations/providers). @@ -747,6 +749,12 @@ Set a positive integer to pin a fixed cap instead of the dynamic behavior: context_file_max_chars: 25000 ``` +Each context file read is also bounded by `context_file_read_timeout` (seconds, default `5.0`). A file that takes longer to read — typically on a network-backed filesystem such as iCloud Drive, OneDrive or NFS — is skipped with a warning so the rest of the system prompt still loads: + +```yaml +context_file_read_timeout: 5.0 +``` + ## File Read Safety Controls how much content a single `read_file` call can return. Reads that exceed the limit are rejected with an error telling the agent to use `offset` and `limit` for a smaller range. This prevents a single read of a minified JS bundle or large data file from flooding the context window. @@ -909,7 +917,7 @@ Older configs with `compression.summary_model`, `compression.summary_provider`, `hygiene_total_ceiling_seconds` (default `600`) bounds the total wait even while tokens are still moving, so a degenerate trickle stream can't hold a turn hostage indefinitely. It is clamped to at least `hygiene_timeout_seconds`. -`hygiene_max_turn_hold_seconds` (default `10`) is the gateway's **turn-hold budget** — the maximum wall-clock the incoming message is held waiting on hygiene compression before the gateway stops waiting and proceeds on the uncompressed transcript. It exists because `hygiene_total_ceiling_seconds` alone can leave the wire silent for far longer than a chat transport's idle-timeout: a summary model that keeps streaming tokens keeps resetting the inactivity slice, so without a turn-hold budget the wait can stretch toward the ceiling while zero bytes reach the user — Telegram (and similar transports) then drop the connection and the turn appears frozen. Capping the turn's wait at this budget (well under the typical ~30s transport idle-timeout) guarantees the message is answered promptly; the compression worker keeps running detached and its commit is fenced (`CompressionCommitFence`), so when it eventually finishes it cannot overwrite the turns appended after the wait was abandoned. Raise it if your summary model routinely needs longer and your transport tolerates it; lower it for snappier recovery on very slow backends. +`hygiene_max_turn_hold_seconds` (default `10`) is the gateway's **turn-hold budget** — the maximum wall-clock the incoming message is held waiting on hygiene compression before the gateway stops waiting and proceeds on the uncompressed transcript. It exists because `hygiene_total_ceiling_seconds` alone can leave the wire silent for far longer than a chat transport's idle-timeout: a summary model that keeps streaming tokens keeps resetting the inactivity slice, so without a turn-hold budget the wait can stretch toward the ceiling while zero bytes reach the user — Telegram (and similar transports) then drop the connection and the turn appears frozen. Capping the turn's wait at this budget (well under the typical ~30s transport idle-timeout) guarantees the message is answered promptly. **The compression is not lost when the budget expires**: the worker keeps running detached and — when its commit is watermark-fenced (the normal case with a session DB) — it keeps its commit admission, so the finished summary is adopted at the next safe boundary and turns appended after the wait was abandoned survive verbatim as concurrent tail. This matters especially for **thinking/reasoning summary models** (DeepSeek, QwQ, etc.) whose reasoning phase alone can exceed the budget: their summaries land one turn late instead of never. If the commit cannot be safely fenced, the late result is discarded (`CompressionCommitFence`) and it cannot overwrite newer turns. Raise the budget if you'd rather have compression apply within the same turn and your transport tolerates the wait; lower it for snappier recovery on very slow backends. `hygiene_failure_cooldown_seconds` controls that per-session cooldown after a hygiene compression timeout or abort. During the cooldown, the gateway skips repeated hygiene attempts for the same oversized session so every incoming message does not block on the same broken auxiliary backend. `/compress`, `/reset`, or a healthy later turn can still recover the session. @@ -917,7 +925,7 @@ The value is the **first rung** of an escalating ladder, not a fixed interval: c `context_timeout_seconds` (default `120`) is the same **inactivity budget** for in-agent `compress_context` — the conversation loop, preflight compaction, and manual `/compress` — so a hung summary model cannot stall a session indefinitely. Streamed summary tokens extend the wait; only a silent worker is cut off. On timeout Hermes retries the summary once against the first entry of `auxiliary.compression.fallback_chain` (using that entry's own `timeout` when it declares one) — a stalled route never raises, so the auxiliary client's own fallback handling cannot see it. Only if that attempt also fails, or no fallback chain is configured, does Hermes skip compaction, keep the existing messages, and warn the user. Set to `0` to disable. Gateway session hygiene keeps its own `hygiene_timeout_seconds` path and is not double-wrapped. -`context_total_ceiling_seconds` (default `600`) bounds the in-agent **pre-commit** wait (summary / stream phase) even while tokens are still moving. It is clamped to at least `context_timeout_seconds`. The exact guarantee: **the summary phase is bounded by this ceiling; the commit phase is logged and surfaced if it exceeds it.** Once the worker has entered the compression commit fence and SessionDB mutation is in flight, the commit is never abandoned mid-flight — that would risk transcript divergence — but the wait is no longer silent: if the commit runs past the ceiling, Hermes logs the overrun (WARNING, escalating to ERROR on repeat), sends a one-shot warning through the user-visible warning channel, and keeps waiting in bounded increments until the commit completes. +`context_total_ceiling_seconds` (default `600`) bounds the in-agent **pre-commit** wait (summary / stream phase) even while tokens are still moving. It is clamped to at least `context_timeout_seconds`. The exact guarantee: **the summary phase is bounded by this ceiling; the commit phase is logged and surfaced if it exceeds it.** Once the worker has entered the compression commit fence and SessionDB mutation is in flight, the commit is never abandoned mid-flight — that would risk transcript divergence — but the wait is no longer silent: if the commit runs past the ceiling, Hermes logs the overrun (WARNING, escalating to ERROR on repeat), sends a one-shot warning through the user-visible warning channel, and keeps waiting in bounded increments until the commit completes. When the ceiling expires during the summary phase, the summary model's stream is closed at that same instant on every auxiliary wire (chat.completions, Codex Responses, Anthropic Messages) — an abandoned summary is not billed to completion on a connection nobody is waiting for, and its session lease is freed for the next attempt. `protect_first_n` controls how many **non-system** head messages are pinned across every compaction. Default `3` — the opening user/assistant exchange survives every summarizer pass so the original goal stays visible. On long-running rolling-compaction sessions where the opening turn is no longer relevant, set `protect_first_n: 0` to pin nothing but the system prompt + summary + tail. The system prompt itself is always preserved regardless of this setting. @@ -1152,7 +1160,18 @@ The **stale stream detection** kills connections that receive SSE keep-alive pin The **stale non-stream detection** kills non-streaming calls that produce no response for too long. By default Hermes disables this on local endpoints to avoid false positives during long prefills. If you explicitly set `providers..stale_timeout_seconds`, `providers..models..stale_timeout_seconds`, or `HERMES_API_CALL_STALE_TIMEOUT`, that explicit value is honored even on local endpoints. -This budget bounds every non-streaming call, including the ones cron jobs and delegated subagents run inline. A provider that accepts a request and then goes silent — connection held open, no bytes, no error — is aborted at the stale timeout and retried, rather than hanging until the much longer socket read timeout (or, for an unattended cron run, until something external kills the process). +This budget bounds every non-streaming call. A provider that accepts a request and then goes silent — connection held open, no bytes, no error — is aborted at the stale timeout and retried, rather than hanging until the much longer socket read timeout (or, for an unattended cron run, until something external kills the process). + +Cron jobs and delegated subagents stream too. They run the request inline on their own thread (the interrupt worker other sessions use wedges inside the gateway's nested thread pools), but the wire request is still `stream: true`, so the **stale stream detection** budget above governs them — every token counts as liveness, so a reasoning model that thinks for minutes is not mistaken for a hung provider, and edge proxies that kill silent connections keep seeing bytes. + +### Disabling API streaming + +`model.streaming: false` forces non-streaming requests for the whole session — parent and subagents alike. It is an escape hatch for self-hosted OpenAI-compatible servers whose *streaming* tool-call path is broken (for example vLLM with `--tool-call-parser qwen3_xml` plus a reasoning parser can leak tool-call markup into plain text and return zero `tool_calls`, so delegated tasks silently no-op). Default is `true`; leave it unless you hit that class of bug, since non-streaming calls lose the liveness properties described above. This is separate from `display.streaming`, which only controls token rendering in the terminal. + +```yaml +model: + streaming: false +``` ## Context Pressure Warnings @@ -1690,6 +1709,27 @@ There is no `hermes config set` support for `reasoning_overrides` keys — edit The override applies automatically everywhere: CLI startup, messaging gateway, Desktop/TUI, cron jobs, `/model` mid-session switches, and fallback model activation. +## Fast Mode + +Fast mode asks the provider for faster output at a premium price: OpenAI [Priority Processing](https://openai.com/api-priority-processing/) (`service_tier: priority`), xAI Priority Processing on Grok 4.6, and Anthropic [Fast Mode](https://platform.claude.com/docs/en/build-with-claude/fast-mode) (`speed: fast`, Opus 4.8 / Opus 5 only). It is **off by default**. + +```yaml +agent: + service_tier: "" # "" / normal | fast | auto | cold + fast_auto_seconds: 60 # window for auto / cold +``` + +| Mode | When fast params are sent | Use it for | +|------|---------------------------|------------| +| `normal` (default, `""`) | Never | Cheapest; standard latency | +| `fast` | Every request | Long interactive sessions where you always want speed | +| `auto` | Requests in the first `fast_auto_seconds` of **every** turn | Snappy first reply; long tool loops fall back to standard pricing | +| `cold` | Same window, but only on the **first turn** of a session (no prior history) | Fast onboarding reply, standard pricing afterwards | + +`/fast normal|fast|auto|cold` switches the mode for the session; add `--global` to persist to `config.yaml`. `/fast` alone shows the current mode. + +**Cost note:** both providers bill fast requests at a multiplier on standard rates (Anthropic: $10 / $50 per MTok in/out on Opus 4.8 and Opus 5), stacking with prompt-cache pricing. `auto`/`cold` bound that premium to the window only. Fast params are only sent to the first-party endpoint that supports them (`api.openai.com` / Codex subscription, `api.anthropic.com`, `api.x.ai`); OpenRouter, Nous Portal, Copilot, Azure, Bedrock, and custom `base_url` routes never receive them in any mode. Only the per-request parameter changes between requests — the system prompt, tools, and messages stay byte-identical, so the prompt cache survives the window boundary. + ## Tool-Use Enforcement Some models occasionally describe intended actions as text instead of making tool calls ("I would run the tests..." instead of actually calling the terminal). Tool-use enforcement injects system prompt guidance that steers the model back to actually calling tools. @@ -1701,7 +1741,7 @@ agent: | Value | Behavior | |-------|----------| -| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `gemini`, `gemma`, `grok`, `glm`, `qwen`, `deepseek`. Disabled for all others (e.g. Claude). | +| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `gemini`, `gemma`, `grok`, `glm`, `qwen`, `deepseek`, `muse`. Disabled for all others (e.g. Claude). | | `true` | Always enabled, regardless of model. Useful if you notice your current model describing actions instead of performing them. | | `false` | Always disabled, regardless of model. | | `["gpt", "codex", "qwen", "llama"]` | Enabled only when the model name contains one of the listed substrings (case-insensitive). | @@ -1736,7 +1776,7 @@ agent: | Value | Behavior | |-------|----------| -| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `grok`, `deepseek`, `kimi`, `qwen`, `glm`, `minimax`, `mimo`, `mistral`. | +| `"auto"` (default) | Enabled for models matching: `gpt`, `codex`, `grok`, `deepseek`, `kimi`, `qwen`, `glm`, `minimax`, `mimo`, `mistral`, `muse`. | | `true` | Always enabled, regardless of model. | | `false` | Always disabled, regardless of model. | | `["deepseek", "my-custom-model"]` | Enabled only when the model name contains one of the listed substrings (case-insensitive). | @@ -1754,14 +1794,15 @@ The gate is independent of `tool_use_enforcement` — either can be on without t ## Tool-Loop Guardrails -Hermes detects when the agent is stuck in an unproductive tool-calling loop — the same tool call failing repeatedly, the same tool failing over and over, or an idempotent call returning the same result with no progress. By default it injects a **warning** into the tool result so the model self-corrects; it does not hard-stop, since a person watching the CLI/TUI can intervene. +Hermes detects when the agent is stuck in an unproductive tool-calling loop — the same tool call failing repeatedly, the same tool failing over and over, or an idempotent call returning the same result with no progress. By default it injects a **warning** into the tool result so the model self-corrects. Interactive CLI, TUI, Desktop, and ACP sessions remain warning-only because a person can intervene; unattended gateway and cron sessions enable hard stops by default. -For unattended gateway / server deployments, enable hard stops so a stuck agent is circuit-broken instead of burning the iteration budget: +The platform-aware default can be disabled for an unattended deployment, or hard stops can be explicitly enabled on every platform: ```yaml tool_loop_guardrails: warnings_enabled: true # inject warnings into tool results (default: true) hard_stop_enabled: false # also BLOCK the call past the hard-stop threshold (default: false) + non_interactive_hard_stop_enabled: true # default hard stops for gateway/cron warn_after: exact_failure: 2 # identical failing call repeated N times same_tool_failure: 3 # same tool failing N times (different args) @@ -1775,7 +1816,13 @@ tool_loop_guardrails: max_subagents: 50 # max subagents spawned per turn (0 = unlimited) ``` -`hard_stop_enabled` defaults to `false` because interactive sessions have a human in the loop. In unattended deployments (gateway, cron, kanban workers) set it to `true` so repeated failures are blocked rather than only warned. See also [Docker / unattended deployments](docker.md). +`hard_stop_enabled` explicitly enables hard stops on every platform. When it remains `false`, `non_interactive_hard_stop_enabled` still enables them for unattended gateway/cron-style platforms while preserving warning-only behavior for CLI, TUI, Desktop, ACP, subagents, and `api_server` runs (supervised task loops with a live parent or client). Set `non_interactive_hard_stop_enabled: false` to opt an unattended deployment out. See also [Docker / unattended deployments](docker.md). + +Hard stops are designed to catch **replays** — the same call, unchanged, with nothing happening in between — not legitimate iteration: + +- **Edit → re-run is never a loop.** Any successful mutating call (`write_file`, `patch`, a green `terminal`/`execute_code`, a browser action, a job/message/cron mutation) marks progress for every failing call still being counted. The next identical retry (re-running a red test after a fix, re-snapshotting after a click) starts a fresh streak instead of accumulating toward a block. +- **Distinct red commands are diagnosis, not a loop.** For tools whose non-zero exit is ordinary output (`terminal`, `execute_code`, process pollers, `browser_navigate`, `web_extract`) the `same_tool_failure` threshold only warns and never halts. Only an exact-args replay with no intervening change, or an identical-result streak, can stop them. +- **A halt ends the turn, not the session.** The agent replies with which guardrail fired and why; replying "continue" resumes with fresh per-turn counters. ### Per-turn runaway-loop caps @@ -1787,7 +1834,7 @@ This mirrors Claude Code's per-session WebSearch and subagent caps (v2.1.212), w ### Runtime anti-stall guards -Complementing the failure-based guardrails above, `agent.stall_guards` (default `true`) enables two conservative runtime guards against wasted turns. First, an **identical-call loop breaker**: when the same tool is called 3+ consecutive times with identical arguments *and* returns an identical result, a short one-line notice is appended to that tool result telling the model not to repeat the call — it never blocks the call, and legitimately-repeatable pollers (`process`, `*_get_result`, `*_poll`) are exempt. Second, a **continue-intent recovery**: when the model ends a turn with no tool calls but its short reply trails off announcing an action ("Let me now update the file…"), Hermes re-prompts it to act via the same bounded continuation mechanism used for intent-ack recovery (max 2 re-prompts per turn). Both are cache-safe (notices are added at result construction, never retroactively) and can be disabled together: +Complementing the failure-based guardrails above, `agent.stall_guards` (default `true`) enables two conservative runtime guards against wasted turns. First, an **identical-call loop breaker**: when the same tool is called 3+ consecutive times with identical arguments *and* returns an identical result, a short one-line notice is appended to that tool result telling the model not to repeat the call — in warning-only sessions it never blocks the call, and legitimately-repeatable pollers (`process`, `*_get_result`, `*_poll`) are exempt. When hard stops are active (explicit `hard_stop_enabled`, or an unattended gateway/cron platform), the same streak also becomes a hard stop once it reaches `hard_stop_after.idempotent_no_progress` consecutive identical calls — for **any** tool, not just the read-only ones the `idempotent_no_progress` guardrail tracks — so a model replaying the same successful `terminal` or `skill_view` call is halted instead of running out the iteration budget (`identical_call_streak_halt`). Second, a **continue-intent recovery**: when the model ends a turn with no tool calls but its short reply trails off announcing an action ("Let me now update the file…"), Hermes re-prompts it to act via the same bounded continuation mechanism used for intent-ack recovery (max 2 re-prompts per turn). Both are cache-safe (notices are added at result construction, never retroactively) and can be disabled together: ```yaml agent: @@ -1870,6 +1917,11 @@ display: cli_multiline_shortcuts: true # CLI: Ctrl+J, \ + Enter, and supported Shift+Enter insert newlines (false = legacy c-j submit fallback) resume_display: full # full (show previous messages on resume) | minimal (one-liner only) bell_on_complete: false # Play terminal bell when agent finishes (great for long tasks) + bell_on_prompt: false # Play terminal bell when a blocking prompt opens (clarify, approval, sudo password, secret capture) — works over SSH + # Both bell flags also emit an OSC 9 desktop notification (Ghostty, iTerm2, Kitty, WezTerm raise an OS + # notification; other terminals ignore it) and, inside Warp (TERM_PROGRAM=WarpTerminal with the CLI-agent + # protocol advertised), a warp://cli-agent OSC 777 event (`stop` on completion, `permission_request` on + # blocking prompts) so Warp's tab status and notification mailbox track Hermes. No extra keys needed. show_reasoning: true # Show model reasoning/thinking above each response (default: true; toggle with /reasoning show|hide) streaming: false # Stream tokens to terminal as they arrive (real-time output) show_cost: false # Show estimated $ cost in the CLI status bar @@ -2053,6 +2105,8 @@ display: tool_progress: 'off' # quiet in shared Slack workspace ``` +From the CLI, use the canonical path — `hermes config set display.platforms.telegram.streaming false`. The shorthand `hermes config set platforms.telegram.streaming false` is accepted too: because per-platform *display* settings (`streaming`, `show_reasoning`, `tool_progress`, …) are only ever read from `display.platforms`, `config set`/`get`/`unset` redirect that shorthand to the canonical key and print a note. Connection keys under the top-level `platforms.` block (`token`, `enabled`, `reply_to_mode`, `extra`) are not redirected. + Platforms without an override fall back to the global `tool_progress` value. Valid platform keys: `telegram`, `discord`, `slack`, `signal`, `whatsapp`, `matrix`, `mattermost`, `email`, `sms`, `homeassistant`, `dingtalk`, `feishu`, `wecom`, `weixin`, `bluebubbles`, `qqbot`. The legacy `display.tool_progress_overrides` key still loads for backward compatibility but is deprecated and migrated into `display.platforms` on first load. Signal is listed as a valid platform key because the setting can be saved per platform, but the current Signal adapter cannot edit sent messages and does not render tool-progress bubbles. Keep Signal `tool_progress` set to `off`; use the CLI or an editing-capable messaging platform if you need to watch each tool call live. diff --git a/website/docs/user-guide/configuring-models.md b/website/docs/user-guide/configuring-models.md index e67e85d5c3..0456c391e4 100644 --- a/website/docs/user-guide/configuring-models.md +++ b/website/docs/user-guide/configuring-models.md @@ -57,7 +57,7 @@ Prompt caches are keyed to the model serving the request, so any mid-conversatio ### Unattended data-training tiers -Models such as `muse-spark-1.2-contributor` are discounted because the vendor may train on your prompts and completions. Interactive model selection always shows a confirmation prompt. Non-interactive startup paths such as Kanban workers and cron agents fail closed because they cannot ask that question. +Models with a `-contributor` suffix (e.g. `muse-spark-1.2-contributor`, `muse-spark-1.3-contributor`) are discounted because the vendor may train on your prompts and completions. Interactive model selection always shows a confirmation prompt. Non-interactive startup paths such as Kanban workers and cron agents fail closed because they cannot ask that question. If training on the unattended workload's data is acceptable, record a persistent acknowledgement: diff --git a/website/docs/user-guide/desktop.md b/website/docs/user-guide/desktop.md index 09cd4b3f11..f569daebfc 100644 --- a/website/docs/user-guide/desktop.md +++ b/website/docs/user-guide/desktop.md @@ -44,7 +44,7 @@ The center of the app. You get: - **The same conversation history** as every other Hermes surface — sessions started here resume in the CLI/TUI and vice versa. - **Drag-and-drop files** anywhere in the chat area to attach them to your next message. - **A right-hand preview rail** — render web pages, files, and tool outputs side by side while you keep chatting. -- **Comment mode in the in-app browser** — click **Annotate** in the preview browser bar, then click any element (or drag a box) on the live page and type a note; each saved comment stays as a numbered pin on the page. Saving a pin never sends a turn — when you're done, **Add N comments** attaches a cropped screenshot per pin and a short prompt naming each comment to the composer, and you still hit send yourself. Pin numbers hold steady if you delete one, and switching chats clears the stack. +- **Comment mode in the in-app browser** — click **Annotate** in the preview browser bar, then click any element (or drag a box) on the live page and type a note; each saved comment stays as a numbered pin on the page. Saving a pin never sends a turn — when you're done, **Add N comments** attaches a cropped screenshot per pin and a short prompt naming each comment to the composer, and you still hit send yourself. Each element comment carries its CSS selector, its markup, and the computed styles that matter for layout, so the agent can find the element in your source instead of guessing from the picture. Password and hidden field values, and any attribute that looks like a key or token, are redacted on the page before the markup leaves it. Larger batches arrive grouped by which part of the page each comment sits in, so twenty-odd comments become a handful of pieces of work rather than one task each — and because the groups are separate DOM subtrees they usually touch separate files, which is what makes handing them to parallel workers safe. Pin numbers hold steady if you delete one, and switching chats clears the stack. - **Composer history and queue editing** — press the up/down arrow keys in an empty composer to recall and reuse previous prompts, and edit messages you've queued up before they're sent. Pressing Stop (or Esc) while turns are queued pauses the queue and expands it above the composer; resume it from there, or send, edit, and delete individual entries. - **A conversation timeline rail** — long chats get a slim rail of markers along the edge of the transcript, one per prompt. Hover it to pop open the list of prompts, click one to jump straight to that point in the conversation. (It appears once the chat has a handful of turns.) - **Find in page** — press **Cmd/Ctrl+F** to open a find bar that searches the rendered chat transcript. Enter / Shift+Enter (or Cmd/Ctrl+G / Cmd/Ctrl+Shift+G while the bar is open) step through matches; Esc closes it. @@ -55,7 +55,8 @@ The bar along the bottom of the chat shows live session state and exposes quick - **Per-session YOLO toggle** — flip YOLO on or off for just this session (matching the TUI). YOLO bypasses the dangerous-command approval prompts, so know what you're turning off — see [Security → YOLO Mode](./security.md#yolo-mode). - **Context-usage meter** — a live "% full" meter of the session's context window. Click it to open the **Context Usage** popover with a token breakdown by category (system prompt, tool definitions, skills, memory, rules, MCP, subagent definitions, and the conversation itself) so you can see exactly what's eating the window before compression kicks in. -- **Customizable items** — right-click the status bar (**Show in status bar**) to choose what appears: the context meter, workspace, model, approvals, turn/session timers, terminal, Command Center, backend version, and more — or hide the bar entirely (**Cmd/Ctrl+Shift+S** toggles it). +- **Cache hit rate and tokens per second** — off by default; turn them on from the right-click menu. Cache hit rate is the share of this session's prompt tokens served from the provider's prompt cache (cached tokens cost less, so higher is cheaper — you can watch a session get cheaper as it warms up). Tokens per second is output throughput averaged over the last 10 model calls. Both update live during a turn. +- **Customizable items** — right-click the status bar (**Show in status bar**) to choose what appears: the context meter, cache hit rate, tokens per second, workspace, model, approvals, turn/session timers, terminal, Command Center, backend version, and more — or hide the bar entirely (**Cmd/Ctrl+Shift+S** toggles it). Chatting against a Hermes instance on another machine instead of the bundled local backend? See [Connecting to a remote backend](#connecting-to-a-remote-backend) below — and for the full picture of how the remote-hosted dashboard connection works (the auth gate, the `/api/ws` chat socket, and WebSocket close-code triage), see [Web Dashboard → Connecting Hermes Desktop to a remote backend](./features/web-dashboard.md#connecting-hermes-desktop-to-a-remote-backend). @@ -80,7 +81,7 @@ Changing any of these values invalidates only that profile's disk-discovery cach The model picker lives in the **composer**, just left of the microphone. Click it to switch the model, reasoning effort, and fast mode from one dropdown. -- **The composer picker is sticky UI state and never touches your default.** It's remembered locally (per device) and **follows** across new chats and restarts instead of snapping back to the default — pick a model once and the next `Cmd/Ctrl+N` opens on it. With a live chat, switching models scopes the change to that **current chat**; either way the selection rides along when the session is created/switched and is **never** written to the profile default. (Switching [profiles](#sessions--profiles) reseeds to that profile's own default.) +- **The composer picker is sticky UI state and never touches your default.** It's remembered locally (per device) and **follows** across new chats and restarts instead of snapping back to the default — pick a model once and the next `Cmd/Ctrl+N` opens on it. With a live chat, switching models scopes the change to that **current chat**; either way the selection rides along when the session is created/switched and is **never** written to the profile default — with one exception: on a fresh profile that has no `model.default`/`model.provider` configured yet, the first pick is persisted so the app has a real default instead of falling through to a stray API-key env var on restart. Persistence follows the same rule as `/model` (`model.persist_switch_by_default`); use **Settings → Model** to change the default deliberately. (Switching [profiles](#sessions--profiles) reseeds to that profile's own default.) - **Set the default in Settings → Model.** That "main" model is your **per-profile global default** — it's what new chats, crons, subagents, and auxiliary tasks start from, and it's the only place that writes it. Each [profile](#sessions--profiles) keeps its own default. - **Per-model effort/fast presets.** Each model remembers its own reasoning effort and fast-mode choice in the desktop app, re-applied to the session whenever you pick that model. These presets are a desktop convenience and don't change crons or subagents. - **Mid-chat switches reset the prompt cache.** Switching the model inside a live chat means the next message re-reads the whole conversation at full input price (provider prompt caches are keyed to the model). Fine occasionally; on a long chat, a fresh chat on the new model is often cheaper than bouncing back and forth. @@ -162,6 +163,7 @@ Manage providers, models, tools, and credentials from a real UI instead of editi - **xAI Grok OAuth** — Grok is a first-class OAuth provider in the launcher; sign in through the browser flow like the other OAuth providers. - **Tool-backend installs from the GUI** — run a tool backend's post-setup install steps directly from the app instead of dropping to a terminal. - **Terminal font picker** — choose an installed font in **Settings → Appearance**. Nerd Fonts such as `MesloLGS NF` render Powerlevel10k separators and icons in both interactive and agent terminals; the setting is saved per profile. +- **Reopen Last Chat on Launch** — by default the app picks up where you left off on cold start. Turn it off in **Settings → Appearance** (or set `display.resume_last_session: false` in `config.yaml`) to always begin with a fresh chat. Deep links and explicit destinations are never overridden either way. - **Auxiliary-model warning** — if you switch the main model to a new provider while auxiliary tasks (titling, summarization, and similar helpers) are still pinned to another provider, the app warns you so you don't unknowingly split work across two providers. - **VS Code Marketplace themes** — beyond the built-in theme presets, the appearance settings include a live VS Code Marketplace search: pick any color theme and the app downloads, converts, and installs it as a desktop theme. The same importer is available from the command palette (*Install theme*), and imported themes can be removed again from the appearance settings. - **Keep computer awake** — **Settings → Advanced → Keep computer awake** stops the machine from sleeping so long or overnight agent runs keep going (the display can still dim). This is a per-computer setting. diff --git a/website/docs/user-guide/docker.md b/website/docs/user-guide/docker.md index da737a4b17..dbdb77cb21 100644 --- a/website/docs/user-guide/docker.md +++ b/website/docs/user-guide/docker.md @@ -71,14 +71,11 @@ See the [Where the logs go](#where-the-logs-go) section below for the full routi ::: :::note Tool-loop hard stops for unattended gateways -The `tool_loop_guardrails.hard_stop_enabled` setting defaults to `false`, which is reasonable for interactive CLI and TUI sessions where a person can see repeated tool-call warnings. In unattended gateway or server deployments, warnings alone may not stop an agent that gets stuck in a repeated tool-call loop. Operators who want circuit-breaker behavior should explicitly enable hard stops in the profile's `config.yaml`: +Unattended gateway and cron sessions enable tool-loop hard stops by default through `non_interactive_hard_stop_enabled`. Interactive CLI, TUI, Desktop, and ACP sessions remain warning-only. To opt an unattended deployment out in the profile's `config.yaml`: ```yaml tool_loop_guardrails: - hard_stop_enabled: true - hard_stop_after: - exact_failure: 5 - idempotent_no_progress: 5 + non_interactive_hard_stop_enabled: false ``` ::: diff --git a/website/docs/user-guide/features/api-server.md b/website/docs/user-guide/features/api-server.md index 47c0373613..36b6e1c7c0 100644 --- a/website/docs/user-guide/features/api-server.md +++ b/website/docs/user-guide/features/api-server.md @@ -477,8 +477,9 @@ When the agent delegates work to background subagents, the stream also carries `subagent.start` and `subagent.complete` lifecycle events, so clients can observe delegation outcomes — including timeouts and failures — instead of the run going silent while a child works. The `subagent.complete` payload carries -the child's status, summary, duration, token/cost figures, and a -`child_session_id` for correlation; free-text fields pass forced secret +the child's status, summary, duration, token/cost figures, a +`child_session_id` for correlation, and the `delegation_id` of the batch it +belongs to (so concurrent or nested fan-outs stay distinguishable); free-text fields pass forced secret redaction before leaving the process. Per-tool child events (`subagent.tool`, progress ticks) are intentionally **not** forwarded — they are high-volume UI noise; use the per-child live transcript files for @@ -628,6 +629,10 @@ to the routed profile**: - Unprefixed routes and `/p/default/...` keep using the default profile's key. - A named profile with no `API_SERVER_KEY` of its own fails closed — its prefix is unreachable until you set one. +- Runs are per-profile scoped: `/v1/runs/{run_id}` and its `events`, `stop`, + `steer`, and `approval` routes only answer for the profile that created + the run (including runs started via `/api/sessions/{id}/chat/stream`); + another profile's run id returns `404`, never `403`. :::warning Breaking change (July 2026) Before this fix, a valid default-profile key was accepted on any diff --git a/website/docs/user-guide/features/browser.md b/website/docs/user-guide/features/browser.md index 1b2df2a39a..97b5e4ca4a 100644 --- a/website/docs/user-guide/features/browser.md +++ b/website/docs/user-guide/features/browser.md @@ -455,7 +455,7 @@ AGENT_BROWSER_ENGINE=lightpanda The engine works with both browser drivers: -- **Browser Use mode (the default).** Hermes launches `lightpanda serve --host 127.0.0.1 --port ` itself — one process per `browser_exec` session name (or per task) — and points the Browser Use CLI at it. No Chromium, Playwright or Node.js is needed. The process is reaped after `browser.inactivity_timeout`, on exit, and by the orphan sweep if Hermes crashes. Lightpanda has no graphical renderer, so `capture_screenshot()` is unavailable and the tool description tells the model to work text-first; it also holds one page per session, so the model is told to call `new_tab()` once and `goto_url()` afterwards (tracked upstream in [lightpanda-io/browser#1962](https://github.com/lightpanda-io/browser/issues/1962)). +- **Browser Use mode (the default).** Hermes launches `lightpanda serve --host 127.0.0.1 --port ` itself — one process per `browser_exec` session name (or per task) — and points the Browser Use CLI at it. No Chromium, Playwright or Node.js is needed. The process is reaped after `browser.inactivity_timeout`, on exit, and by the orphan sweep if Hermes crashes. All of these processes share one on-disk HTTP cache at `$HERMES_HOME/cache/browser-use/lightpanda/http-cache`, so repeat visits skip re-downloading assets. Hermes passes the cache flag only when the installed Lightpanda supports it (0.3.x+); older binaries simply run without a cache. To clear it, stop your Lightpanda sessions first, then delete that directory. Lightpanda has no graphical renderer, so `capture_screenshot()` is unavailable and the tool description tells the model to work text-first; it also holds one page per session, so the model is told to call `new_tab()` once and `goto_url()` afterwards (tracked upstream in [lightpanda-io/browser#1962](https://github.com/lightpanda-io/browser/issues/1962)). - **Built-in browser tools** (`/browser use off`). Hermes drives Lightpanda through `agent-browser --engine lightpanda` over CDP, the same way it drives local Chrome, with **automatic Chrome fallback**: Lightpanda handles the actions it supports (navigate, snapshot, click, type, scroll, back, press, eval) and Hermes transparently retries on Chrome for anything it doesn't. Screenshots and `browser_vision` are routed straight to Chrome. **When the engine is ignored.** `browser.engine` is the lowest-precedence browser setting: a cloud provider (including the Nous subscription browser — and on never-configured setups, any `BROWSERBASE_API_KEY` / `BROWSER_USE_API_KEY` in `~/.hermes/.env` auto-selects one), Camofox, a `browser.cdp_url` / `/browser connect` override, or `browser.use_real_profile` all take precedence. Picking Lightpanda in `hermes tools` writes `cloud_provider: local` for you; `/browser status` and `hermes doctor` report when the engine is configured but shadowed, and by what. diff --git a/website/docs/user-guide/features/computer-use.md b/website/docs/user-guide/features/computer-use.md index d1d2983c3f..1148e2949a 100644 --- a/website/docs/user-guide/features/computer-use.md +++ b/website/docs/user-guide/features/computer-use.md @@ -430,6 +430,17 @@ computer_use: capability_manifest: "" # capability manifest path, required for bounded ``` +On Linux, native Wayland support remains an explicit opt-in. Hermes passes the +opt-in to every cua-driver process, including gateway sessions, only when that +process also has `WAYLAND_DISPLAY`: + +```yaml +computer_use: + native_wayland: true +``` + +Restart a running gateway after changing this setting. + Override the driver binary path (tests / CI / local builds): ``` diff --git a/website/docs/user-guide/features/context-files.md b/website/docs/user-guide/features/context-files.md index b5c628213d..2906c4f780 100644 --- a/website/docs/user-guide/features/context-files.md +++ b/website/docs/user-guide/features/context-files.md @@ -190,6 +190,7 @@ This scanner protects against common injection patterns, but it's not a substitu | Limit | Value | |-------|-------| | Max chars per file | `context_file_max_chars` when set; otherwise dynamic (scales with model context window, floor 20,000, ceiling 500,000) | +| Read timeout per file | `context_file_read_timeout` (default 5 seconds); a file that takes longer to read — e.g. on iCloud Drive, OneDrive or NFS — is skipped with a warning | | Head truncation ratio | 70% | | Tail truncation ratio | 20% | | Truncation marker | 10% (shows char counts and suggests using file tools) | diff --git a/website/docs/user-guide/features/credential-pools.md b/website/docs/user-guide/features/credential-pools.md index f66f2a624a..05c0e988b6 100644 --- a/website/docs/user-guide/features/credential-pools.md +++ b/website/docs/user-guide/features/credential-pools.md @@ -208,6 +208,8 @@ This means subagents benefit from the same rate-limit resilience as the parent, The credential pool uses a threading lock for all state mutations (`select()`, `mark_exhausted_and_rotate()`, `try_refresh_current()`, `mark_used()`). This ensures safe concurrent access when the gateway handles multiple chat sessions simultaneously. +Across processes (many subagents, a gateway plus a CLI, cron jobs), OAuth refreshes are serialized through a file lock on `auth.json`. When one shared OAuth grant expires under many concurrent processes, exactly one process performs the refresh; the others detect that the on-disk token no longer matches the one that failed and adopt it instead of rotating the single-use refresh token again. A process that loses the lock race keeps its entry healthy and retries — lock contention is never recorded as a credential failure. + ## Architecture For the full data flow diagram, see [`docs/credential-pool-flow.excalidraw`](https://excalidraw.com/#json=2Ycqhqpi6f12E_3ITyiwh,c7u9jSt5BwrmiVzHGbm87g) in the repository. diff --git a/website/docs/user-guide/features/cron.md b/website/docs/user-guide/features/cron.md index 410237b6f5..e9a24600c1 100644 --- a/website/docs/user-guide/features/cron.md +++ b/website/docs/user-guide/features/cron.md @@ -440,6 +440,19 @@ When scheduling jobs, you specify where the output goes: The agent's final response is automatically delivered to the configured `deliver:` target — the agent does not send messages itself, so there is nothing to call in the cron prompt. +### Delivery failures are a distinct status + +Execution and delivery are tracked separately. When the agent run succeeds but +the output never reaches the target (platform 5xx, rate limit, stale session, +adapter returned no positive evidence of a send), the job records +`last_status: delivery_failed` — never a plain `ok` — with the reason in +`last_delivery_error`. `hermes cron list` shows it in yellow as +`delivery_failed: `, `hermes cron doctor` reports it as a delivery +issue, and a manual `cronjob run` reports `success: false` with the delivery +error. A delivery failure does not count toward the job's `failure_streak` +(the agent did its job); the next fully successful run returns the status to +`ok`. + ### Bot Chat delivery (`bot-chat`) `bot-chat` delivers the output **into a profile's canonical "Bot Chat" session as a real message**. Unlike every other target — where the recipient is a human reading a channel — the recipient here is the bot itself: it receives the output as an incoming message, acts on anything that needs action, and responds in its chat. Use it when scheduled output should be *processed*, not just posted. @@ -489,6 +502,43 @@ cron: wrap_response: false ``` +### Push notifications (`cron.delivery.notify`) + +Cron output is a *final* delivery, not a progress message, so by default it is +sent with the platform's notification flag set — on Telegram this means the +brief triggers a push even when the adapter's notification mode is `important` +(which otherwise sends with `disable_notification=true`, and users report the +silent brief as "never delivered"). To restore silent deliveries: + +```yaml +# ~/.hermes/config.yaml +cron: + delivery: + notify: false # default: true +``` + +The flag rides both the text send and any media attachments, so a run never +pushes for one and stays silent for the other. + +### Delivery confirmation and the `UNVERIFIED` state + +A live-adapter delivery is logged as delivered only on positive evidence from +the adapter: an explicit `success` that is not a filtered drop +(`delivered: false`), plus a `message_id` or `raw_response`. A result carrying +`success` but neither piece of evidence — the shape Slack, Matrix and +Mattermost adapters return — is still accepted (it is not proof of failure), +but the run is recorded on the job as `last_delivery_unverified` and surfaces +in `hermes cron list`: + +``` +⚠ Delivery UNVERIFIED: adapter acked slack:C0123456 without message_id/raw_response +``` + +and in `hermes cron doctor` as `last delivery unverified (...)`. The marker is +cleared by the next run that delivers with evidence. An empty payload (no text +and no media) is never handed to an adapter; it fails closed and is reported in +`last_delivery_error` instead of being logged as delivered. + ### Continuable jobs (reply to a cron delivery) By default a cron delivery is fire-and-forget: the message is sent, but it does diff --git a/website/docs/user-guide/features/goals.md b/website/docs/user-guide/features/goals.md index b4a9f31585..d187dd9bef 100644 --- a/website/docs/user-guide/features/goals.md +++ b/website/docs/user-guide/features/goals.md @@ -49,7 +49,7 @@ What you'll see: 1. **Goal accepted** — `⊙ Goal set (20-turn budget): ` 2. **Turn 1 runs** — Hermes starts working as if you'd sent the goal as a normal message. -3. **Judge runs** — after the turn, the judge model decides `done` or `continue`. +3. **Judge runs** — after the turn, the judge model decides `done`, `continue`, or `blocked`. 4. **Loop fires if needed** — if `continue`, you'll see `↻ Continuing toward goal (1/20): ` and Hermes takes the next step automatically. 5. **Terminates** — eventually you see either `✓ Goal achieved: ` or `⏸ Goal paused — N/20 turns used`. @@ -140,7 +140,7 @@ A completion contract makes the judge stricter, but the judge is still an LLM re How it works, each turn: 1. **Gates run before the judge.** If any gate fails, the judge is *not called* — a red gate is deterministic evidence the goal isn't done. The gate's exit code and output tail (last ~3 KB) become the continuation prompt, so the agent iterates against the actual failure instead of a vibe. -2. **All gates pass → normal judging.** The LLM judge then decides done/continue/wait exactly as before. +2. **All gates pass → normal judging.** The LLM judge then decides done/blocked/continue/wait exactly as before. 3. **Unchanged workspace → no re-run.** If a gate failed and nothing changed in the workspace since (tracked via a git fingerprint of HEAD + working-tree status), the gate is not re-run — the recorded failure is replayed and the attempt count advances. A stuck agent can't burn wall-clock re-running an identical red suite. Outside a git repo, gates simply always re-run. 4. **Retries are bounded.** Each gate defaults to 3 retries and a 5-minute timeout. When a gate exhausts its retries the goal auto-pauses (like the turn budget) with a message telling you to fix it manually, remove the gate, or `/goal resume`. @@ -179,9 +179,9 @@ After every turn, Hermes calls an auxiliary model with: - The standing goal text - The agent's most recent final response (last ~4 KB of text) -- A system prompt telling the judge to reply with strict one-line JSON: `{"verdict": "done" | "continue" | "wait", "reason": ""}` (wait verdicts add `wait_on_session` / `wait_on_pid` / `wait_for_seconds`; the legacy `{"done": , "reason": "..."}` shape is still accepted) +- A system prompt telling the judge to reply with strict one-line JSON: `{"verdict": "done" | "blocked" | "continue" | "wait", "reason": ""}` (wait verdicts add `wait_on_session` / `wait_on_pid` / `wait_for_seconds`; the legacy `{"done": , "reason": "..."}` shape is still accepted) -The judge is deliberately conservative: it marks a goal `done` only when the response **explicitly** confirms the goal is complete, when the final deliverable is clearly produced, or when the goal is unachievable/blocked (treated as DONE with a block reason so we don't burn budget on impossible tasks). +The judge is deliberately conservative: it marks a goal `done` only when the response **explicitly** confirms the goal is complete, when the final deliverable is clearly produced. A goal the agent explains is **unachievable** (impossible, out of scope, needs user input) gets a `blocked` verdict instead — never `done`: the goal **pauses** with the judge's reason (`🚫 Goal judged unachievable — paused`), so you can re-scope it with `/goal ` or override with `/goal resume` rather than burning budget or having an impossible task waved through as complete. ### Fail-open semantics diff --git a/website/docs/user-guide/features/hooks.md b/website/docs/user-guide/features/hooks.md index 8ef9e7b180..e30b3b0d39 100644 --- a/website/docs/user-guide/features/hooks.md +++ b/website/docs/user-guide/features/hooks.md @@ -1876,11 +1876,12 @@ Secrets: prefer `secret_env` (the name of an environment variable, typically set ### Wire format -Each firing POSTs a JSON body with the same top-level shape as shell hooks' stdin, plus delivery metadata: +Each firing POSTs a JSON body with the same top-level shape as shell hooks' stdin, plus delivery metadata. `profile` names the Hermes profile that emitted the event (`"default"` outside profiles), so receivers behind a multiplexed gateway can tell profiles apart: ```json { "hook_event_name": "on_session_end", + "profile": "default", "tool_name": null, "tool_input": null, "session_id": "sess_abc123", diff --git a/website/docs/user-guide/features/image-generation.md b/website/docs/user-guide/features/image-generation.md index 33c4abc747..f9ad545524 100644 --- a/website/docs/user-guide/features/image-generation.md +++ b/website/docs/user-guide/features/image-generation.md @@ -104,6 +104,28 @@ image_gen: The `fal-ai/gpt-image-1.5` and `fal-ai/gpt-image-2` request quality is pinned to `medium` (~$0.034–$0.06/image at 1024×1024). We don't expose the `low` / `high` tiers as a user-facing option so that Nous Portal billing stays predictable across all users — the cost spread between tiers is 3–22×. If you want a cheaper option, pick Klein 9B or Z-Image Turbo; if you want higher quality, use Nano Banana Pro or Recraft V4 Pro. +### Meta Model API: Muse Image + +With `image_gen.provider: meta-ai`, images are generated through the +[Meta Model API](https://api.meta.ai) (`https://api.meta.ai/v1`), the same +OpenAI-compatible endpoint that serves the Muse Spark chat models. It is the +image-gen companion to the bundled `meta-ai` chat provider. + +| Model | Speed | Strengths | Price | +|---|---|---|---| +| `muse-image-1.0` *(default)* | ~10s | Meta Model API image generation | $0.01/image | + +```yaml +image_gen: + provider: meta-ai + model: muse-image-1.0 +``` + +Auth reuses the same env vars as the Meta chat provider — `MODEL_API_KEY` +(Meta's documented name), with `META_API_KEY` / `META_MODEL_API_KEY` accepted +as aliases. Set `META_BASE_URL` to point at a proxy or alternate host. Text-to-image +only for now; responses are saved to `$HERMES_HOME/cache/images/`. + ## Usage The agent-facing schema is intentionally minimal — the model picks up whatever you've configured: diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.md index fccda51c1b..084a36f3f8 100644 --- a/website/docs/user-guide/features/kanban.md +++ b/website/docs/user-guide/features/kanban.md @@ -512,7 +512,7 @@ def register(ctx): ### Goal-mode cards (`--goal`) -By default each worker gets **one shot** at its card — do the work, call `kanban_complete`/`kanban_block`, exit. Pass `--goal` (CLI) or `goal_mode=True` (the `kanban_create` tool / dashboard) to instead run that worker in a **goal loop**, the same Ralph-style engine behind the `/goal` slash command: after every turn an auxiliary judge checks the worker's output against the card's title + body (treated as the acceptance criteria), and if the work isn't done — and the turn budget remains — the worker keeps going **in the same session** until the judge agrees, the worker terminates the task itself, or the budget runs out (which **blocks** the card for human review rather than exiting silently). +By default each worker gets **one shot** at its card — do the work, call `kanban_complete`/`kanban_block`, exit. Pass `--goal` (CLI) or `goal_mode=True` (the `kanban_create` tool / dashboard) to instead run that worker in a **goal loop**, the same Ralph-style engine behind the `/goal` slash command: after every turn an auxiliary judge checks the worker's output against the card's title + body (treated as the acceptance criteria), and if the work isn't done — and the turn budget remains — the worker keeps going **in the same session** until the judge agrees, the worker terminates the task itself, or the budget runs out (which **blocks** the card for human review rather than exiting silently). If the judge rules the goal **unachievable** as written, the card is blocked immediately with the judge's reason — an impossible card is never marked done, and `kanban complete` / `kanban request-review` on such a card are rejected with a pointer to `kanban block` or re-scoping. ```bash hermes kanban create "Translate the docs site to French" \ @@ -614,7 +614,7 @@ Config knobs (all under `kanban:` in `~/.hermes/config.yaml`): | `orchestrator_profile` | `""` | Profile assigned to the root/orchestration task after decomposition. Empty = fall back to active default profile. | | `default_assignee` | `""` | Where a child task lands when the LLM picks an unknown profile. Empty = fall back to active default. | | `auto_subscribe_on_create` | `true` | When `kanban_create` runs inside a persistent gateway/TUI session, terminal events resume that originating agent with a synthetic status turn. Set to `false` for passive completion or to require explicit `kanban_notify-subscribe` calls. Independent of `auto_decompose`. | -| `done_sub_retention_days` | `30` | Notify subscriptions survive `done` (reopen-safe) and are removed on `archived`. The notifier GC purges subscriptions whose task has been `done` with no new events for this many days, bounding sub-table growth on boards that never archive. `0` disables the sweep. | +| `done_sub_retention_days` | `30` | Notify subscriptions survive `done` (reopen-safe) and are removed on `archived`. The notifier GC purges subscriptions whose task has been `done` or `blocked` with no new events for this many days, bounding sub-table growth on boards that never archive. `0` disables the sweep. | And the two auxiliary LLM slots: @@ -887,7 +887,7 @@ bot> ✓ t_9fc1a3 completed by transcriber transcribed 42 minutes, saved to podcast/2026-05-04.md ``` -Subscriptions survive a task reaching `done` — completion is reversible (a reviewer or controller can reopen a done task), so the origin session keeps getting notified through reopen cycles. They auto-remove on `archived` (the irreversible end state). On boards that never archive, a GC sweep purges subscriptions for tasks that have sat in `done` with no new activity for `kanban.done_sub_retention_days` days (default 30; set 0 to disable), so stale rows don't accumulate forever. If you script a create with `--json` (machine output) the auto-subscribe is skipped — the assumption is that scripted callers want to manage subscriptions explicitly via `/kanban notify-subscribe`. +Subscriptions survive a task reaching `done` — completion is reversible (a reviewer or controller can reopen a done task), so the origin session keeps getting notified through reopen cycles. They auto-remove on `archived` (the irreversible end state). On boards that never archive, a GC sweep purges subscriptions for tasks that have sat in `done` or `blocked` with no new activity for `kanban.done_sub_retention_days` days (default 30; set 0 to disable), so stale rows don't accumulate forever. If you script a create with `--json` (machine output) the auto-subscribe is skipped — the assumption is that scripted callers want to manage subscriptions explicitly via `/kanban notify-subscribe`. A chat-originated auto-subscribe is created in `notify+wake` mode: on a terminal event the destination agent both receives the passive message **and** takes a real turn, so it can read the board context and reply in its own voice. See [Delivery modes](#delivery-modes) below. diff --git a/website/docs/user-guide/features/loops.md b/website/docs/user-guide/features/loops.md index dfd0acb589..ad52195c3e 100644 --- a/website/docs/user-guide/features/loops.md +++ b/website/docs/user-guide/features/loops.md @@ -61,7 +61,7 @@ A loop ends when any of these fires: |---|---| | The agent decides it's done | The wakeup prompt teaches the agent to end its reply with `LOOP_COMPLETE` on its own line when the task is finished or moot. | | A run cap | `--times N` — stop after N wakeups. | -| An evidence-based condition | `--until ` — after each wakeup, the same auxiliary judge that powers `/goal` checks the reply against your condition (fail-open: a broken judge never wedges the loop). | +| An evidence-based condition | `--until ` — after each wakeup, the same auxiliary judge that powers `/goal` checks the reply against your condition. If the judge rules the condition unachievable, the loop **pauses** with the reason instead of re-firing until the tick budget (fail-open: a broken judge never wedges the loop). | | You | `/loop stop` (or `/loop pause` to keep it around). | | The backstop budget | `loops.max_ticks` (default 100) pauses the loop so an unattended session can't burn tokens forever. `0` = unlimited. | diff --git a/website/docs/user-guide/features/lsp.md b/website/docs/user-guide/features/lsp.md index 8f5830f479..c34ac0e9c0 100644 --- a/website/docs/user-guide/features/lsp.md +++ b/website/docs/user-guide/features/lsp.md @@ -237,6 +237,12 @@ respawned automatically on the next relevant file operation. Set `idle_timeout: 0` to disable reaping and hold every server's index warm for the life of the process. +Servers that support multi-root workspaces (currently pyright) run as a +**single process** per Hermes process: the first Python project spawns +it, and every further project root — for example sibling git worktrees +edited by parallel subagents — is attached to that same server as an +additional workspace folder instead of starting another copy. + ## Disabling Set `lsp.enabled: false` in `config.yaml` to disable the entire diff --git a/website/docs/user-guide/features/mcp.md b/website/docs/user-guide/features/mcp.md index 3d2e82a85f..a3fe5f0802 100644 --- a/website/docs/user-guide/features/mcp.md +++ b/website/docs/user-guide/features/mcp.md @@ -644,7 +644,7 @@ If you change MCP config, use: /reload-mcp ``` -This reloads MCP servers from config and refreshes the available tool list. For runtime tool changes pushed by the server itself, see [Dynamic Tool Discovery](#dynamic-tool-discovery) above. +This reloads MCP servers from config and refreshes the available tool list. It is also the explicit way to re-probe availability-gated tools (Docker, `HASS_TOKEN`, OAuth…): a session's tool set is otherwise frozen, so a credential or daemon that appears mid-session is only picked up on `/reload-mcp`, `/new`, or context compaction. For runtime tool changes pushed by the server itself, see [Dynamic Tool Discovery](#dynamic-tool-discovery) above. ### Toolsets diff --git a/website/docs/user-guide/features/tool-search.md b/website/docs/user-guide/features/tool-search.md index 8264594632..f64e59caf1 100644 --- a/website/docs/user-guide/features/tool-search.md +++ b/website/docs/user-guide/features/tool-search.md @@ -156,6 +156,12 @@ to any progressive-disclosure design, not specific to this implementation: result enters the conversation history (so it does get cached on subsequent turns) but it never benefits from the system-prompt cache prefix. +- **No provider-native validation for deferred schemas.** `tool_describe` + lets the model read a deferred tool's schema, but the provider still sees + only the generic `tool_call.arguments` object. Hermes therefore coerces and + validates the underlying arguments locally before dispatch; the concrete + tool or MCP server remains responsible for schemas Hermes cannot safely + validate, such as malformed schemas or external references. - **Model-quality dependence.** Tool Search assumes the model can write a reasonable search query for the tool it wants. Smaller models do this less well; the published Anthropic numbers (49% → 74% on Opus 4 with diff --git a/website/docs/user-guide/features/tts.md b/website/docs/user-guide/features/tts.md index 3fbfce34b6..e2ae021a81 100644 --- a/website/docs/user-guide/features/tts.md +++ b/website/docs/user-guide/features/tts.md @@ -256,6 +256,19 @@ tts: **Advanced knobs** (`tts.piper.length_scale` / `noise_scale` / `noise_w_scale` / `volume` / `normalize_audio`, `use_cuda`) correspond 1:1 to Piper's `SynthesisConfig`. They're ignored on older `piper-tts` versions. +### Warm-up and unload via speech toggles (local engines) + +Local engines (Piper, KittenTTS) load their model lazily, so without help the *first* spoken reply after you turn speech on pays the whole model load — and on a fresh install the voice download — as silence before the first word. Hermes treats the speech-output toggles as the signal that TTS is about to be needed: + +- **Desktop** — turning on **Read replies aloud**, or starting a **voice conversation**, pre-loads the configured engine in the background right away. Turning both off again unloads the resident model (a Piper voice is tens of MB; KittenTTS up to ~80MB) so it isn't parked in RAM for nothing. +- **CLI / TUI** — `/voice tts` (and `/voice on` when `voice.auto_tts` is set) do the same; `/voice off` releases. + +Each toggle holds a *lease* on the engine; the model is only unloaded when the last lease across surfaces is released, so switching off read-aloud in one Desktop window never pulls the voice out from under a conversation running in another. For cloud providers there is no model to hold — the toggle only makes sure a lazily-installed SDK (edge-tts, ElevenLabs, Mistral) is present. Warm-up is best-effort: if the engine can't load, the toggle still succeeds and the first reply falls back to loading on demand as before. + +The Desktop calls `POST /api/audio/tts-lease` with `{"lease": "", "active": true|false}`; other frontends can use the same endpoint. + +The same lease also reaches user-declared providers, so a self-hosted TTS server can preload and unload its model on the toggles: a [command provider](#custom-command-providers) runs its optional `warm_command` / `release_command`, and a [Python plugin provider](#python-plugin-providers) gets `warm()` / `release()`. + ### Custom command providers If a TTS engine you want isn't natively supported (VoxCPM, MLX-Kokoro, XTTS CLI, a voice-cloning script, anything else that exposes a CLI), you can wire it in as a **command-type provider** without writing any Python. Hermes writes the input text to a temp UTF-8 file, runs your shell command, and reads the audio file the command produced. @@ -348,6 +361,7 @@ Use `{{` and `}}` for literal braces. | `voice_compatible` | `false` | When `true`, Hermes converts MP3/WAV output to Opus/OGG via ffmpeg so Telegram renders a voice bubble. | | `max_text_length` | `5000` | Maximum input characters per command invocation; longer text is split into ordered chunks. | | `voice` / `model` | empty | Passed to the command as placeholder values only. | +| `warm_command` / `release_command` | unset | Shell commands run when a surface toggles speech output on / when the last lease across surfaces is released — e.g. `curl -s localhost:5002/load?model={model}` to preload a local TTS server, and its `unload` counterpart. Best-effort and non-blocking: run in the background with the same `timeout`, `env_passthrough` and `{voice}` / `{model}` / `{speed}` placeholders as `command`; output is discarded and failures are only logged at debug. | #### Behavior notes @@ -437,6 +451,7 @@ Override these on your provider class for richer integration: - `get_setup_schema()` → return `{name, badge, tag, env_vars: [{key, prompt, url}]}` to power the picker row in `hermes tools` / `hermes setup`. Without this, the plugin still works but its row in the picker is minimal. - `stream(text, *, voice, model, format, **extra)` → iterator yielding audio bytes for streaming delivery (default raises `NotImplementedError`). - `voice_compatible` property → set `True` if your output is Opus-compatible and the gateway should deliver it as a voice bubble (default `False` = regular audio attachment). +- `warm()` / `release()` → called when a surface toggles speech output on / when the last lease across surfaces is released, while your provider is the configured `tts.provider` — preload or unload a local model server here. Both default to no-ops; exceptions are logged at debug and never fail the toggle. See `agent/tts_provider.py` for the full ABC including docstrings. diff --git a/website/docs/user-guide/messaging/email.md b/website/docs/user-guide/messaging/email.md index eabde5da49..71f932d6e9 100644 --- a/website/docs/user-guide/messaging/email.md +++ b/website/docs/user-guide/messaging/email.md @@ -48,6 +48,31 @@ Most email providers support IMAP/SMTP. Check your provider's documentation for: - SMTP host and port (usually port 587 with STARTTLS) - Whether app passwords are required +### Proton Mail Bridge / local relays + +Proton Mail Bridge (and similar local relays such as a self-hosted MTA) listen on +loopback with **STARTTLS** and a self-signed certificate, so the defaults +(implicit TLS on IMAP 993, verified certificates) won't connect. Override the +transport in `~/.hermes/config.yaml`: + +```yaml +platforms: + email: + enabled: true + extra: + imap_host: 127.0.0.1 + imap_security: starttls # tls (default) | starttls | plain + imap_tls_verify: false # Bridge uses a self-signed cert + smtp_host: 127.0.0.1 + smtp_security: starttls # default: tls on port 465, starttls otherwise + smtp_tls_verify: false +``` + +and set `EMAIL_IMAP_PORT=1143` / `EMAIL_SMTP_PORT=1025` alongside your Bridge +credentials in `~/.hermes/.env`. Unknown `*_security` values log a warning and +fall back to the secure default. Only disable `*_tls_verify` for loopback hosts — +Hermes logs a warning when verification is off for any other host. + --- ## Step 1: Configure Hermes diff --git a/website/docs/user-guide/messaging/google_chat.md b/website/docs/user-guide/messaging/google_chat.md index e613331a4d..e47e5a495a 100644 --- a/website/docs/user-guide/messaging/google_chat.md +++ b/website/docs/user-guide/messaging/google_chat.md @@ -166,6 +166,15 @@ GOOGLE_CHAT_MAX_BYTES=16777216 # 16 MiB — cap on in-flight me The project ID also falls back to `GOOGLE_CLOUD_PROJECT`, and the SA path falls back to `GOOGLE_APPLICATION_CREDENTIALS` — use whichever convention you prefer. +Under a [multi-profile gateway](../multi-profile-gateways.md), every +`GOOGLE_CHAT_*` setting is read from the routed profile's own `.env`; a +secondary profile never inherits the default profile's project, subscription, +or service account. If a profile has no SA configured while the process +environment carries one for another profile, the adapter refuses to fall back +to Application Default Credentials (which would authenticate as that other +profile) and logs an explicit error instead — put +`GOOGLE_CHAT_SERVICE_ACCOUNT_JSON` in that profile's `.env`. + Install the Google Chat adapter dependencies through its maintained installer. It applies the same pinned security floors used by the runtime checks: diff --git a/website/docs/user-guide/messaging/index.md b/website/docs/user-guide/messaging/index.md index a0aaf6b5e4..72fc4288eb 100644 --- a/website/docs/user-guide/messaging/index.md +++ b/website/docs/user-guide/messaging/index.md @@ -673,6 +673,54 @@ Once the gateway is running, use the `/platform` slash command from any connecte See also the broader status summary command [`/platforms`](../../reference/slash-commands.md#info). +### Disabling a platform whose credentials are still in `.env` + +`platforms..enabled: false` in `~/.hermes/config.yaml` is authoritative. +Credentials for that platform left in the environment (`TELEGRAM_BOT_TOKEN`, +`WEIXIN_TOKEN`, `HASS_TOKEN`, `EMAIL_*`, `TWILIO_ACCOUNT_SID`, ...) are still +wired into the platform's config so send-only tooling keeps working, but they +no longer start the adapter: + +```yaml title="~/.hermes/config.yaml" +platforms: + weixin: + enabled: false # wins over WEIXIN_TOKEN in .env +``` + +Earlier releases let the mere presence of credentials re-enable twelve +platforms (Weixin, WhatsApp Cloud, Home Assistant, Email, SMS, DingTalk, Feishu, +WeCom, WeCom callback, BlueBubbles, QQ Bot, Yuanbao) regardless of that key. If +you relied on that, the gateway now logs one WARNING per affected platform at +startup so it does not just go dark: + +``` +Platform 'weixin' is explicitly disabled by platforms.weixin.enabled: false in config.yaml, +so the credentials found in the environment (WEIXIN_TOKEN, WEIXIN_ACCOUNT_ID) will NOT start +its adapter. Environment credentials no longer override an explicit disable. Remove the key +or set platforms.weixin.enabled: true to turn it back on. +``` + +Omitting the `enabled` key entirely keeps the env-only behaviour: credentials +present → adapter starts. + +### Ignoring an inherited proxy (`gateway.trust_env`) + +By default every platform adapter honors `HTTP_PROXY` / `HTTPS_PROXY` / +`NO_PROXY` (and `SSL_CERT_FILE`) from the gateway's environment, and +auto-detects the macOS system proxy. A gateway started by a Windows Scheduled +Task or a service manager can inherit a proxy the interactive shell never +sees — a local Clash/V2Ray listener that isn't running yet — and log +`Cannot connect to host 127.0.0.1:7890` on every poll. Turn the inherited +proxy off for all adapters at once: + +```yaml title="~/.hermes/config.yaml" +gateway: + trust_env: false +``` + +Explicit per-platform proxy variables (`DISCORD_PROXY`, `TELEGRAM_PROXY`, +`MATRIX_PROXY`, ...) are still honored. Restart the gateway after changing it. + ### Automatic circuit breaker Each adapter is wrapped in a circuit breaker. Repeated retryable failures (network blips, rate-limit replies, 5xx upstream responses, websocket disconnects) cause the breaker to trip — the adapter is auto-paused, an operator notification is sent to the home channel of another live platform when one is configured, and a structured log line is emitted. diff --git a/website/docs/user-guide/messaging/slack.md b/website/docs/user-guide/messaging/slack.md index 4bc66f5a30..6a0a396571 100644 --- a/website/docs/user-guide/messaging/slack.md +++ b/website/docs/user-guide/messaging/slack.md @@ -472,6 +472,7 @@ platforms: | `platforms.slack.extra.suggested_prompts` | `[]` | Up to four `{title, message}` prompts for Agent/Assistant DM entry points; accepts either a list or `{title, prompts}`. | | `platforms.slack.extra.assistant_thread_titles` | `true` | When `true`, names Agent/Assistant DM threads from the first user message. | | `platforms.slack.extra.allow_bots` | `"none"` | Controls messages from other Slack bots: `"none"` ignores them, `"mentions"` accepts a bot message only when **that message itself** @mentions Hermes, and `"all"` accepts all of them. Use `"mentions"` for the safest bot-to-bot collaboration mode. See [Accepting messages from other bots](#accepting-messages-from-other-bots-allow_bots). | +| `platforms.slack.extra.api_human_users` | `[]` | Slack user IDs whose **Web-API (user-token) posts count as human**. Such posts carry the posting `app_id` and no `client_msg_id`, so by default they are dropped as app traffic; allowlist your own front-end's users here instead of `allow_bots: all`. See [Treating your own app's user-token posts as human](#treating-your-own-apps-user-token-posts-as-human-api_human_users). | | `platforms.slack.extra.cron_continuable_surface` | `"thread"` | Delivery surface for [continuable cron jobs](../features/cron.md#flat-in-channel-continuation-slack). `"thread"` opens a dedicated thread per delivery (default); `"in_channel"` delivers flat into the channel timeline. Pair `in_channel` with `reply_in_thread: false` (and `require_mention: false`) so a plain channel reply continues the job. | The equivalent environment variable is `SLACK_ALLOW_BOTS=none|mentions|all`. @@ -701,6 +702,38 @@ How `mentions` mode gates: For strict multi-bot deployments, pair with `require_mention: true` and `strict_mention: true` — see the smoke-check profile below. +### Treating your own app's user-token posts as human (`api_human_users`) + +A message posted through the Web API with a **user token** (`xoxp-`) is +authored by a real person, but it arrives with the posting `app_id` and no +`client_msg_id` — the same signature Hermes uses to recognise app posts — so it +is dropped as bot traffic. This blocks a common pattern: a custom front-end (an +internal dashboard, a mobile shell, a kiosk) that sends messages to Hermes *as* +the logged-in user. + +`allow_bots: all` would let those posts through, but it opens the door to every +bot in the channel and weakens the loop protections. Instead, allowlist just +the people who use your front-end: + +```yaml +platforms: + slack: + extra: + api_human_users: ["U0AAAAAAA", "U0BBBBBBB"] +``` + +The equivalent environment variable is `SLACK_API_HUMAN_USERS` (comma-separated). + +Scope and safety: + +- The allowlist is **users only**. There is deliberately no app-ID variant: a + modern bot token (`xoxb-`) posts with the same `user` + `app_id` shape, so + trusting an app would also admit its own bot posts and defeat the loop guard. +- Events carrying `bot_id` or `subtype: bot_message`, or no `user` at all, are + always treated as bot posts regardless of the allowlist. +- The rest of the pipeline is unchanged: mention gating, `allowed_channels`, + and `SLACK_ALLOWED_USERS` still apply to the (now human) sender. + ### Reaction Triggers (`reaction_triggers`) By default, emoji reactions are acknowledged and dropped — a 👍 on a bot diff --git a/website/docs/user-guide/messaging/telegram.md b/website/docs/user-guide/messaging/telegram.md index cd651d7df5..2becfab4ca 100644 --- a/website/docs/user-guide/messaging/telegram.md +++ b/website/docs/user-guide/messaging/telegram.md @@ -848,29 +848,31 @@ Shows the current topic's binding: session title, session ID, and hints for `/ne ### Under the hood -- Activation persists to `telegram_dm_topic_mode(chat_id, user_id, enabled, ...)` in `state.db` -- Each topic binding persists to `telegram_dm_topic_bindings(chat_id, thread_id, session_id, ...)` with `ON DELETE CASCADE` on `session_id` — pruning a session automatically clears its topic binding -- The topic-mode SQLite migration is **opt-in**: it runs on the first `/topic` call, never on gateway startup. Until a user runs `/topic` in this profile, `state.db` is unchanged -- Each inbound DM message looks up its `(chat_id, thread_id)` binding. If present, the lookup routes the message to the bound session via `SessionStore.switch_session()` so the session-key-to-session-id mapping stays consistent on disk +- Activation persists to `telegram_dm_topic_mode(profile_name, chat_id, user_id, enabled, ...)` in `state.db`. Primary key is `(profile_name, chat_id)` so multiplexed / profile-routed bots sharing one `state.db` do not clobber each other when the same Telegram user DMs multiple bots (private `chat_id` is the user id and is identical across bots). +- Each topic binding persists to `telegram_dm_topic_bindings(profile_name, chat_id, thread_id, session_id, ...)` with PK `(profile_name, chat_id, thread_id)` and `ON DELETE CASCADE` on `session_id` — pruning a session automatically clears its topic binding +- The topic-mode SQLite migration is **opt-in**: it runs on the first `/topic` call, never on gateway startup. Until a user runs `/topic` in this profile, `state.db` is unchanged. Schema v3 adds `profile_name`; legacy rows migrate into the `default` namespace only +- Each inbound DM message looks up its `(profile_name, chat_id, thread_id)` binding using the **routed** profile (`source.profile`, not the process-global active profile). If present, the lookup routes the message to the bound session via `SessionStore.switch_session()` so the session-key-to-session-id mapping stays consistent on disk - `/new` inside a topic rewrites the binding row to point at the new session ID, so the next message stays on the fresh session - Topics declared in `extra.dm_topics` are **never auto-renamed** — the operator-chosen name is preserved even when multi-session mode is enabled - Set `extra.disable_topic_auto_rename: true` to turn off auto-rename for **all** topics in the chat (ad-hoc topics created via Threaded Mode included) - The General (pinned top) topic in a forum-enabled DM is treated as the root lobby, regardless of whether Telegram delivers its messages with `message_thread_id=1` or with no thread_id -- Root-lobby reminders are rate-limited to one message per 30 seconds per chat — a user who forgets topic mode is on and types ten prompts in the root won't get ten replies -- BotFather setup screenshots are rate-limited to one send per 5 minutes per chat — repeated `/topic` attempts while Threads Settings are still disabled won't re-upload the same image +- Root-lobby reminders are rate-limited to one message per 30 seconds per **(profile, chat)** — a user who forgets topic mode is on and types ten prompts in the root won't get ten replies, and two multiplexed profiles sharing a chat id do not suppress each other's reminders +- BotFather setup screenshots are rate-limited to one send per 5 minutes per **(profile, chat)** — repeated `/topic` attempts while Threads Settings are still disabled won't re-upload the same image - `/bg ` started inside a topic delivers its result back to the same topic; background sessions don't trigger auto-rename of the owning topic - `/topic` itself is gated by the bot's user authorization check — unauthorized DMs get a refusal instead of activation ### Disabling multi-session mode -Send `/topic off` in the root DM. Hermes flips the row off, clears the chat's `(thread_id → session_id)` bindings, and the root DM reverts to a normal Hermes chat. Existing topics in Telegram aren't deleted — they just stop being gated as independent sessions. Re-run `/topic` later to turn it back on. +Send `/topic off` in the root DM. Hermes flips the row off for **this profile's** namespace, clears that profile's `(thread_id → session_id)` bindings for the chat, and the root DM reverts to a normal Hermes chat. Existing topics in Telegram aren't deleted — they just stop being gated as independent sessions. Re-run `/topic` later to turn it back on. -If you need to clean up by hand (e.g. a bulk reset across many chats), remove the rows directly: +If you need to clean up by hand (e.g. a bulk reset across many chats), scope rows by `profile_name` (use `default` for single-profile installs): ```bash sqlite3 ~/.hermes/state.db \ - "UPDATE telegram_dm_topic_mode SET enabled = 0 WHERE chat_id = ''; \ - DELETE FROM telegram_dm_topic_bindings WHERE chat_id = '';" + "UPDATE telegram_dm_topic_mode SET enabled = 0 + WHERE profile_name = 'default' AND chat_id = ''; + DELETE FROM telegram_dm_topic_bindings + WHERE profile_name = 'default' AND chat_id = '';" ``` ### Downgrading Hermes diff --git a/website/docs/user-guide/multi-profile-gateways.md b/website/docs/user-guide/multi-profile-gateways.md index 7feddd069a..9825f4d047 100644 --- a/website/docs/user-guide/multi-profile-gateways.md +++ b/website/docs/user-guide/multi-profile-gateways.md @@ -157,7 +157,7 @@ configure them only on the default profile. Port-binding platforms covered by this rule: `webhook`, `api_server`, `msgraph_webhook`, `feishu`, `wecom_callback`, `bluebubbles`, `sms`, -`whatsapp_cloud`, `line`. Configure any of these **only on the default profile**; +`whatsapp_cloud`, `line`, `teams`. Configure any of these **only on the default profile**; every profile is reachable through its `/p//` prefix. Authentication follows the profile named in the URL. Unprefixed endpoints keep @@ -213,7 +213,13 @@ keep working. Per-profile `.env` credential isolation is preserved and, if anything, stricter: a profile's keys are resolved from its own scope and are never unioned into a shared environment (this also means subprocesses like MCP servers and -Kanban workers only ever see their own profile's secrets). Kanban, +Kanban workers only ever see their own profile's secrets). Terminal settings +(`terminal.backend`, `terminal.cwd`, `terminal.docker_volumes`, +`terminal.docker_shared_container_key`, SSH targets, …) are likewise resolved +per profile on every routed turn: a profile that omits a terminal key gets the +documented default, never the launch profile's value, and a profile whose +`config.yaml`/`.env` cannot be parsed has terminal execution refused rather than +run under another profile's sandbox policy. Kanban, profile-scoped skills/memory/SOUL, and model routing all behave per-profile exactly as they do with separate gateways. @@ -272,6 +278,12 @@ gateway: platform: telegram chat_id: "-1001234567890" profile: tg-profile + + # A WhatsApp DM — write the phone number; JID and LID forms also match + - name: owner-whatsapp + platform: whatsapp + chat_id: "15551234567" + profile: owner ``` Routes are matched most-specific-first (`thread_id` > `chat_id` > `guild_id`), @@ -281,6 +293,18 @@ no route stay on the default/active profile. The routed profile gets the full per-profile isolation described above (config, skills, memory, credentials, session namespace). Routing works on every platform adapter, not just Discord. +On WhatsApp and WhatsApp Cloud, a `chat_id` route matches across user-identity +forms: a bare phone number (`15551234567`), a JID +(`15551234567@s.whatsapp.net`), and a LID (`…@lid`) all refer to the same +person once the bridge has paired them (the same canonicalization session keys +and adapter allowlists already use). You can put the phone number in +`profile_routes` and inbound DMs still match whether WhatsApp delivers a JID or +a LID. Without a LID mapping yet, the number form still matches a JID (the +suffix is stripped) but cannot resolve an unknown LID — that inbound falls +through to the default profile until the mapping appears. Group chats +(`…@g.us`) are not sender identities and still match exactly. Telegram numeric +ids are unchanged. + `profile_routes` requires `gateway.multiplex_profiles: true`; with multiplexing off the routes are ignored. If an explicit route matches but its target profile is not installed or is outside `multiplex_profile_allowlist`, @@ -288,6 +312,12 @@ the gateway rejects that ingress and logs the route and target. It does not run the default profile. Traffic that matches no route keeps the historical default-profile behavior. +Cron jobs owned by a routed profile deliver through the shared bot too, but +only to targets an enabled route with a `chat_id`/`thread_id` maps to that +profile — a routed profile's job targeting an unrouted chat (or a chat routed +to another profile) is never sent through the shared bot. Guild-only routes do +not qualify a cron target; add a `chat_id` route for the delivery channel. + ## Start, stop, or restart all gateways at once The CLI ships with single-profile lifecycle commands. To act across every diff --git a/website/docs/user-guide/profiles.md b/website/docs/user-guide/profiles.md index ae4ad7055f..ca3349defd 100644 --- a/website/docs/user-guide/profiles.md +++ b/website/docs/user-guide/profiles.md @@ -64,6 +64,10 @@ hermes profile create backup --clone-all Copies **everything** — config, API keys, personality, all memories, skills, cron jobs, plugins. A complete working snapshot. Per-profile history is excluded (session history, `state.db`, `backups/`, `state-snapshots/`, `checkpoints/`) — these belong to the source profile and can reach tens of GB. For a full backup including history, use `hermes profile export` or `hermes backup` instead. +:::note OAuth logins are shared, not copied +Anthropic (Claude Pro/Max), OpenAI Codex, and xAI OAuth logins use **single-use refresh tokens** — a copy of one is not a second credential, it is the same credential with two owners, and the first profile to refresh it revokes it for every other copy. `--clone-all` (and the dashboard's credential mirroring) therefore drops those OAuth rows from the clone. The new profile keeps reading the login from the root `~/.hermes/auth.json`, and a token refresh performed inside any profile is written back to root, so all profiles stay signed in. Static API keys are copied as usual. To give a profile its own separate OAuth login, run `hermes -p auth add ` inside it. +::: + ### Clone from a specific profile ```bash diff --git a/website/docs/user-guide/sessions.md b/website/docs/user-guide/sessions.md index af18e5c5ba..67fa40e093 100644 --- a/website/docs/user-guide/sessions.md +++ b/website/docs/user-guide/sessions.md @@ -869,24 +869,69 @@ Key tables in `state.db`: - Gateway sessions auto-reset based on the configured reset policy - Before reset, the agent saves memories and skills from the expiring session -- Opt-in auto-pruning: when `sessions.auto_prune` is `true`, ended sessions inactive for `sessions.retention_days` (default 90) are pruned at CLI/gateway startup -- After a prune that actually removed rows, `state.db` is `VACUUM`ed to reclaim disk space when at least `sessions.min_vacuum_interval_days` (default 30) have elapsed since the last successful `VACUUM` (SQLite does not shrink the file on plain DELETE) +- Auto-pruning (**on by default** since #54189): when `sessions.auto_prune` is `true`, ended sessions inactive for `sessions.retention_days` (default 90) are pruned at CLI/gateway/cron startup +- After a prune that actually removed rows, `state.db` is `VACUUM`ed to reclaim disk space only when **both** gates pass: at least `sessions.min_vacuum_interval_days` (default 30) have elapsed since the last successful `VACUUM`, **and** more than 25% of the file's pages are reclaimable (`PRAGMA freelist_count / page_count`). A dense database never pays for a full rewrite to reclaim a few MB (SQLite does not shrink the file on plain DELETE) - Pruning runs at most once per `sessions.min_interval_hours` (default 24); the last-run timestamp is tracked inside `state.db` itself so it's shared across every Hermes process in the same `HERMES_HOME` -Default is **off** — session history is valuable for `session_search` recall, and silently deleting it could surprise users. Enable in `~/.hermes/config.yaml`: +Without pruning, `state.db` grows without bound — multi-GB files within weeks were reported on gateway + cron installs. If you would rather keep every ended session forever (the pre-#54189 behavior), turn it off in `~/.hermes/config.yaml`: ```yaml sessions: - auto_prune: true # opt in — default is false + auto_prune: false # default is true — set false to keep all history retention_days: 90 # keep ended sessions active within this window vacuum_after_prune: true # reclaim disk space after a pruning sweep min_vacuum_interval_days: 30 # don't rewrite the DB more often than this min_interval_hours: 24 # don't re-run the sweep more often than this ``` -Active sessions are never auto-pruned, regardless of age. Ended sessions are -aged from their latest message, so a long-lived conversation used recently is -not deleted merely because it began before the retention window. +Existing installs that already set any of these keys explicitly keep their +values; only unset keys pick up the new defaults. + +Only **ended** sessions are ever deleted. Active sessions are never auto-pruned, +regardless of age. Ended sessions are aged from their latest message, so a +long-lived conversation used recently is not deleted merely because it began +before the retention window. + +**Stale open sessions from automation.** Some producers — cron jobs, kanban +workers, subagents, one-shot CLI runs — can die without ever marking their +session ended, and pruning only deletes *ended* rows. To keep those from +accumulating forever, each auto-prune pass also *closes* open sessions from +those state-owned sources (`cli`, `cron`, `kanban`, `acp`, `api_server`, +`subagent`, `tool`) whose last activity is older than `retention_days` +(`end_reason: startup_orphan_reap`). Closing is non-destructive — the +session stays resumable — and the row is aged from its close, so it is only +deleted by a *later* pass after a further full retention window. Messaging +platform sessions (Telegram, Discord, …), TUI/desktop sessions, pinned +sessions, and sessions with a live turn or compression in progress are +never closed by this sweep. + +### Oversized-Transcript Guards + +Two limits stop a runaway transcript from being loaded into memory all at once +(both default to `20000` active messages; `0` disables the guard): + +```yaml +sessions: + max_resume_messages: 20000 # interactive resume (CLI / TUI / Desktop) + max_export_messages: 20000 # one-shot in-memory export of a single session +``` + +`max_resume_messages` bounds **what the resume actually loads**, not the whole +history of the conversation: + +- A plain interactive resume (CLI `--resume`, the TUI) materializes the full + compression lineage — every compacted segment plus the live tip — so it is + bounded across the lineage. +- Desktop's cold resume pages the transcript over REST and only holds the live + tip segment in memory, so it is bounded by the tip alone. A long-lived chat + that has been compacted many times (dozens of segments, tens of thousands of + archived rows behind a small tip) is exactly what compression is meant to + produce and opens normally; its footer message count reflects the stored + lineage, not the live prompt. + +When a resume is refused the client receives error code `4130` with the count +and the scope it was measured against (`across its lineage` or +`in its tip segment`). `hermes sessions export` still works for such sessions. ### Manual Cleanup diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/adding-platform-adapters.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/adding-platform-adapters.md index 43bd0b49fe..b52870e848 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/adding-platform-adapters.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/adding-platform-adapters.md @@ -537,9 +537,9 @@ await self.handle_message(event) ### 4. Gateway Runner（`gateway/run.py`） -五个接触点： +六个接触点： -1. **`_create_adapter()`** — 添加 `elif platform == Platform.NEWPLAT:` 分支 +1. **`_instantiate_adapter()`** — 添加 `elif platform == Platform.NEWPLAT:` 分支。`_create_adapter()` 包装器会将每个成功创建的适配器绑定到其网关运行器。 2. **`_is_user_authorized()` allowed_users 映射** — `Platform.NEWPLAT: "NEWPLAT_ALLOWED_USERS"` 3. **`_is_user_authorized()` allow_all 映射** — `Platform.NEWPLAT: "NEWPLAT_ALLOW_ALL_USERS"` 4. **早期环境检查 `_any_allowlist` 元组** — 添加 `"NEWPLAT_ALLOWED_USERS"` diff --git a/website/static/api/model-catalog.json b/website/static/api/model-catalog.json index df049f7a88..e96aeb87bb 100644 --- a/website/static/api/model-catalog.json +++ b/website/static/api/model-catalog.json @@ -1,6 +1,6 @@ { "version": 1, - "updated_at": "2026-09-01T18:20:04Z", + "updated_at": "2026-09-03T07:08:39Z", "metadata": { "source": "hermes-agent repo", "docs": "https://hermes-agent.nousresearch.com/docs/reference/model-catalog" @@ -84,6 +84,10 @@ "id": "google/gemini-3.1-pro-preview", "description": "" }, + { + "id": "google/gemini-3.8-flash", + "description": "" + }, { "id": "google/gemini-3.7-flash", "description": "" @@ -161,6 +165,18 @@ "id": "meta/muse-spark-1.2", "description": "" }, + { + "id": "meta/muse-spark-1.2-contributor", + "description": "" + }, + { + "id": "meta/muse-spark-1.3", + "description": "" + }, + { + "id": "meta/muse-spark-1.3-contributor", + "description": "" + }, { "id": "sakana/fugu-ultra", "description": "" @@ -261,6 +277,9 @@ { "id": "google/gemini-3.1-pro-preview" }, + { + "id": "google/gemini-3.8-flash" + }, { "id": "google/gemini-3.7-flash" },