9d7921cd6e
Use the canonical version identity reader instead of importing its deleted predecessor. Handle update metadata flags before mutation. Restore terminal environment imports and make platform-parameterized TUI tests host-independent.
825 lines
38 KiB
Python
825 lines
38 KiB
Python
"""Base class for all Hermes execution environment backends.
|
|
|
|
Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
|
|
A session snapshot (env vars, functions, aliases) is captured once at init and
|
|
re-sourced before each command. CWD persists via in-band stdout markers (remote)
|
|
or a temp file (local). Cohesive pieces live in sibling modules (``base_output``,
|
|
``base_session_env``, ``base_wait``, ``path_utils``).
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shlex
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from abc import ABC, abstractmethod
|
|
from pathlib import Path
|
|
from typing import Callable, IO, Iterable, Protocol
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from tools.interrupt import consume_yield, is_interrupted, is_thread_interrupted
|
|
from tools.environments.base_output import (
|
|
ProcessHandle, _finalize_wait_result, _new_output_collector, _start_drain_thread,
|
|
)
|
|
from tools.environments.base_session_env import (
|
|
_SHELL_ENV_NAME_RE, _SNAP_TMP_SUFFIX, _cwd_marker, _snapshot_bootstrap_script, _split_cwd_marker,
|
|
_wrap_command_script,
|
|
)
|
|
from tools.environments.base_wait import _WaitTrace
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Opt-in debug tracing for the interrupt/activity/poll machinery
|
|
# (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs.
|
|
_DEBUG_INTERRUPT = bool(os.getenv("HERMES_DEBUG_INTERRUPT"))
|
|
|
|
# Extra seconds the ``run_bounded_sync`` backstop waits past the inner ``_wait_for_process``
|
|
# deadline: the inner loop returns partial output + 124; the outer bound only fires when that
|
|
# loop never returns. Keep small so a healthy timeout still comes from the inner path.
|
|
# The inner poll loop is what returns partial output + returncode 124; this outer bound only exists for when
|
|
# that loop itself never returns (family A of #94285: a blocked wait that silently disables asyncio timers).
|
|
_EXECUTE_WAIT_BOUND_GRACE_S = 2.0
|
|
|
|
if _DEBUG_INTERRUPT:
|
|
# quiet_mode forces the `tools` logger to ERROR on CLI startup, which would
|
|
# swallow every trace; force this logger back to INFO in the opt-in case.
|
|
logger.setLevel(logging.INFO)
|
|
|
|
# Thread-local activity callback: the agent sets it before a tool call so
|
|
# long-running _wait_for_process loops can report liveness to the gateway.
|
|
_activity_callback_local = threading.local()
|
|
|
|
|
|
class FileFetchError(RuntimeError):
|
|
"""A file could not be extracted from the backend filesystem."""
|
|
|
|
|
|
# Pulling a large artifact over a slow exec channel can outlast the per-command terminal timeout.
|
|
_FETCH_TIMEOUT_SECONDS = 300
|
|
|
|
|
|
class EnvironmentConnectionError(RuntimeError):
|
|
"""Infrastructure/connection-class failure of a terminal backend (SSH host down, Docker
|
|
daemon not running, remote sync on a dead link) — never a command that merely exited
|
|
nonzero. Subclassing RuntimeError keeps every ``except RuntimeError`` catcher working.
|
|
``terminal_tool`` turns this into a structured ``status: "degraded"`` result; the failed
|
|
backend is never cached, so a later call retries from scratch."""
|
|
|
|
def __init__(self, reason: str, *, retry_hint: str = ""):
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
self.retry_hint = retry_hint or (
|
|
"This is an infrastructure failure, not a command failure. "
|
|
"Verify the backend is reachable (network, service running, "
|
|
"credentials), then retry the same command — recovery is "
|
|
"automatic once the backend is back.")
|
|
|
|
|
|
def set_activity_callback(cb: Callable[[str], None] | None) -> None:
|
|
"""Register a callback that _wait_for_process fires periodically."""
|
|
_activity_callback_local.callback = cb
|
|
|
|
|
|
def get_activity_callback() -> Callable[[str], None] | None:
|
|
"""Thread-local activity callback; capture it before handing work to another thread.
|
|
|
|
Public accessor for callers outside this module that need to capture the calling thread's callback
|
|
before handing work to another thread (the callback is thread-local, so a freshly spawned thread cannot
|
|
read it back) — e.g. the manual cron-run heartbeat (#76502).
|
|
"""
|
|
return getattr(_activity_callback_local, "callback", None)
|
|
|
|
|
|
def touch_activity_if_due(state: dict, label: str) -> None:
|
|
"""Fire the activity callback at most once every ``state['interval']`` (default 10 s).
|
|
*state* holds ``last_touch``/``start`` monotonic timestamps. Swallows all exceptions."""
|
|
now = time.monotonic()
|
|
if now - state["last_touch"] < state.get("interval", 10.0):
|
|
return
|
|
state["last_touch"] = now
|
|
try:
|
|
cb = get_activity_callback()
|
|
if cb:
|
|
cb(f"{label} ({int(now - state['start'])}s elapsed)")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def get_sandbox_dir() -> Path:
|
|
"""Host-side root for all sandbox storage (Docker workspaces, Singularity
|
|
overlays/SIF cache). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``."""
|
|
custom = os.getenv("TERMINAL_SANDBOX_DIR")
|
|
p = Path(custom) if custom else get_hermes_home() / "sandboxes"
|
|
p.mkdir(parents=True, exist_ok=True)
|
|
return p
|
|
|
|
|
|
def _load_json_store(path: Path) -> dict:
|
|
"""Load a JSON file as a dict, returning ``{}`` on any error."""
|
|
if path.exists():
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except Exception:
|
|
pass
|
|
return {}
|
|
|
|
|
|
def _save_json_store(path: Path, data: dict) -> None:
|
|
"""Write *data* as pretty-printed JSON to *path*."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
|
|
|
|
def _file_mtime_key(host_path: str) -> tuple[float, int] | None:
|
|
"""Return ``(mtime, size)`` for cache comparison, or ``None`` if unreadable."""
|
|
try:
|
|
st = Path(host_path).stat()
|
|
return (st.st_mtime, st.st_size)
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ProcessHandle protocol
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class ProcessHandle(Protocol):
|
|
"""Duck type that every backend's _run_bash() must return.
|
|
|
|
subprocess.Popen satisfies this natively. SDK backends (Modal, Daytona)
|
|
return _ThreadedProcessHandle which adapts their blocking calls.
|
|
"""
|
|
|
|
def poll(self) -> int | None: ...
|
|
def kill(self) -> None: ...
|
|
def wait(self, timeout: float | None = None) -> int: ...
|
|
|
|
@property
|
|
def stdout(self) -> IO[str] | None: ...
|
|
|
|
@property
|
|
def returncode(self) -> int | None: ...
|
|
|
|
|
|
class _ThreadedProcessHandle:
|
|
"""Adapter for SDK backends (Modal, Daytona) that have no real subprocess.
|
|
|
|
Wraps a blocking ``exec_fn() -> (output_str, exit_code)`` in a background
|
|
thread and exposes a ProcessHandle-compatible interface. An optional
|
|
``cancel_fn`` is invoked on ``kill()`` for backend-specific cancellation
|
|
(e.g. Modal sandbox.terminate, Daytona sandbox.stop).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
exec_fn: Callable[[], tuple[str, int]],
|
|
cancel_fn: Callable[[], None] | None = None,
|
|
):
|
|
self._cancel_fn = cancel_fn
|
|
self._done = threading.Event()
|
|
self._returncode: int | None = None
|
|
self._error: Exception | None = None
|
|
|
|
# Pipe for stdout — drain thread in _wait_for_process reads the read end.
|
|
read_fd, write_fd = os.pipe()
|
|
self._stdout = os.fdopen(read_fd, "r", encoding="utf-8", errors="replace") # windows-footgun: ok (pipe is BOM-free)
|
|
self._write_fd = write_fd
|
|
|
|
def _worker():
|
|
try:
|
|
output, exit_code = exec_fn()
|
|
self._returncode = exit_code
|
|
# Write output into the pipe so drain thread picks it up.
|
|
try:
|
|
os.write(self._write_fd, output.encode("utf-8", errors="replace"))
|
|
except OSError:
|
|
pass
|
|
except Exception as exc:
|
|
self._error = exc
|
|
self._returncode = 1
|
|
finally:
|
|
try:
|
|
os.close(self._write_fd)
|
|
except OSError:
|
|
pass
|
|
self._done.set()
|
|
|
|
t = threading.Thread(target=_worker, daemon=True)
|
|
t.start()
|
|
|
|
@property
|
|
def stdout(self):
|
|
return self._stdout
|
|
|
|
@property
|
|
def returncode(self) -> int | None:
|
|
return self._returncode
|
|
|
|
def poll(self) -> int | None:
|
|
return self._returncode if self._done.is_set() else None
|
|
|
|
def kill(self):
|
|
if self._cancel_fn:
|
|
try:
|
|
self._cancel_fn()
|
|
except Exception:
|
|
pass
|
|
|
|
def wait(self, timeout: float | None = None) -> int:
|
|
self._done.wait(timeout=timeout)
|
|
return self._returncode
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CWD marker for remote backends
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _cwd_marker(session_id: str) -> str:
|
|
return f"__HERMES_CWD_{session_id}__"
|
|
|
|
|
|
# Per-session variables that the gateway bridges freshly onto every command's
|
|
# process environment (via tools/environments/local._inject_session_context_env,
|
|
# reading gateway.session_context._VAR_MAP). They must NEVER be persisted into
|
|
# the shared bash session snapshot: a single long-lived backend serves many
|
|
# concurrent sessions (the messaging gateway, TUI, desktop/web dashboard all
|
|
# collapse the terminal to one "default" environment), so ``export -p`` dumping
|
|
# the FIRST session's HERMES_SESSION_ID into the snapshot makes every LATER
|
|
# session ``source`` that stale value and see a FOREIGN session's identity —
|
|
# overriding the correct per-command Popen env (issue: cross-session
|
|
# HERMES_SESSION_ID leak via the shared snapshot). Stripping them from the
|
|
# snapshot is safe because they are re-injected on every command; a snapshot
|
|
# should only carry the user's own shell state (PATH, functions, exports they
|
|
# set), not Hermes' per-turn session identity.
|
|
#
|
|
# Kept in sync with gateway.session_context._VAR_MAP: every bridged name starts
|
|
# with one of these prefixes (or is HERMES_UI_SESSION_ID). Used by unit tests
|
|
# as the Python-side contract for the exclusion set; the dump path unsets by
|
|
# name/prefix instead of grepping declare lines (see below / issue #71296).
|
|
_SNAPSHOT_EXCLUDED_ENV_REGEX = (
|
|
"^declare -x (HERMES_SESSION_|HERMES_UI_SESSION_ID|HERMES_CRON_AUTO_DELIVER_|"
|
|
"HERMES_CRON_SESSION|HERMES_BROWSER_CONTROL_)"
|
|
)
|
|
_SHELL_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
|
|
|
|
def _export_dump_excluding_session_vars(
|
|
tmp_path: str,
|
|
excluded_names: Iterable[str] = (),
|
|
) -> str:
|
|
"""Return a shell snippet that dumps ``export -p`` to *tmp_path* minus the
|
|
per-session bridged vars (see ``_SNAPSHOT_EXCLUDED_ENV_REGEX``) and any
|
|
additional names supplied by the caller.
|
|
|
|
Unset the bridged vars in a subshell *before* ``export -p``. A line-based
|
|
``grep -vE`` filter is unsafe: bash 3.2 prints a value containing a newline
|
|
as a multi-line ``declare -x NAME="…`` block, so only the opener matches the
|
|
regex and continuation lines (e.g. ``curl … | bash #`` smuggled into a
|
|
Matrix room/display name via ``HERMES_SESSION_CHAT_NAME``) land in the
|
|
snapshot and execute on the next ``source`` (issue #71296). Unsetting first
|
|
means ``export -p`` never emits those vars — including any continuation
|
|
lines. ``|| true`` keeps the success contract for callers that chain on it.
|
|
|
|
The dump MUST be wrapped in a brace group with the redirection applied to
|
|
the group. *tmp_path* is typically a shell-variable expansion (a
|
|
mktemp-allocated per-writer temp name); a redirection attached to a
|
|
pipeline segment would expand it inside that segment's subshell,
|
|
potentially inconsistently with the parent that expands the follow-up
|
|
``mv``. The brace-group redirect is expanded in the current shell,
|
|
keeping both expansions consistent.
|
|
"""
|
|
# ${!PREFIX*} is bash 3.2+ name-prefix expansion; empty matches are fine
|
|
# because ``unset`` with only missing names is ignored under 2>/dev/null.
|
|
# Quote caller-provided names so malformed configuration can never become
|
|
# shell syntax. Valid environment names remain unquoted by shlex.quote().
|
|
safe_names = {
|
|
name for name in excluded_names
|
|
if isinstance(name, str) and name
|
|
}
|
|
extra_unset = " ".join(shlex.quote(name) for name in sorted(safe_names))
|
|
if extra_unset:
|
|
extra_unset = f" {extra_unset}"
|
|
return (
|
|
"{ ( "
|
|
"unset ${!HERMES_SESSION_*} ${!HERMES_CRON_AUTO_DELIVER_*} "
|
|
"${!HERMES_BROWSER_CONTROL_*} "
|
|
# AI_AGENT / HERMES_AGENT are per-command attribution markers
|
|
# (re-exported by every _wrap_command with outer-harness-preserving
|
|
# ${VAR:-default} semantics). Persisting them into the snapshot
|
|
# would make the FIRST command's value override a later outer
|
|
# harness value arriving via the process env, exactly like the
|
|
# session-var leak this dump already guards against.
|
|
"AI_AGENT HERMES_AGENT "
|
|
f"HERMES_UI_SESSION_ID{extra_unset} 2>/dev/null; "
|
|
"export -p; "
|
|
") || true; } "
|
|
f"> {tmp_path}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BaseEnvironment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class BaseEnvironment(ABC):
|
|
"""Common interface and unified execution flow for all Hermes backends. Subclasses
|
|
implement ``_run_bash()`` and ``cleanup()``; the base provides ``execute()`` with
|
|
snapshot sourcing, CWD tracking, interrupt handling and timeout enforcement."""
|
|
|
|
# Subclasses that embed stdin as a heredoc (Modal, Daytona) set this.
|
|
_stdin_mode: str = "pipe" # "pipe" or "heredoc"
|
|
|
|
# True only when commands execute on the SAME host as the Hermes process
|
|
# (LocalEnvironment); controller-host facts then describe the execution target.
|
|
is_local: bool = False
|
|
|
|
# Snapshot creation timeout (override for slow cold-starts).
|
|
_snapshot_timeout: int = 30
|
|
|
|
# Local and Docker override this because they resolve allowlisted values
|
|
# through the active profile scope; other backends keep plain snapshots.
|
|
_profile_scoped_passthrough: bool = False
|
|
|
|
def get_temp_dir(self) -> str:
|
|
"""Return the backend temp directory used for session artifacts.
|
|
|
|
Most sandboxed backends use ``/tmp`` inside the target environment.
|
|
LocalEnvironment overrides this on hosts where ``/tmp`` may be missing
|
|
and ``TMPDIR`` is the portable writable location.
|
|
"""
|
|
return "/tmp"
|
|
|
|
def __init__(self, cwd: str, timeout: int, env: dict = None):
|
|
self.cwd = cwd
|
|
self.timeout = timeout
|
|
self.env = env or {}
|
|
|
|
self._session_id = uuid.uuid4().hex[:12]
|
|
temp_dir = self.get_temp_dir().rstrip("/") or "/"
|
|
self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
|
|
self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
|
|
self._cwd_marker = _cwd_marker(self._session_id)
|
|
self._snapshot_ready = False
|
|
self._snapshot_passthrough_names: set[str] = set()
|
|
# True when login bash is unusable (e.g. broken Git-for-Windows startup)
|
|
# so execute() must fall back to non-login ``bash -c``, not ``bash -l``.
|
|
self._prefer_nonlogin = False
|
|
|
|
# --- Abstract methods ---
|
|
def _run_bash(
|
|
self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None,
|
|
) -> ProcessHandle:
|
|
"""Spawn a bash process to run *cmd_string*; every backend overrides this."""
|
|
raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
|
|
|
|
@abstractmethod
|
|
def cleanup(self):
|
|
"""Release backend resources (container, instance, connection)."""
|
|
...
|
|
|
|
# --- File extraction (backend -> host) ---
|
|
def fetch_file(self, remote_path: str, local_dest: Path, *, max_bytes: int) -> None:
|
|
"""Copy a regular file out of the backend filesystem to ``local_dest`` on the host.
|
|
|
|
One transport for every backend: base64 over the exec channel, bounded INSIDE the sandbox
|
|
(``head -c max+1`` so /dev/zero can't stream unbounded data into host memory — the same
|
|
shape ``tools.image_source`` uses to read sandbox images). The payload is fenced between
|
|
unique markers so login-shell noise in the merged stdout/stderr can't corrupt the decode.
|
|
Raises :class:`FileFetchError` on a missing/unreadable/oversized file.
|
|
"""
|
|
import base64
|
|
import binascii
|
|
marker = f"__HERMES_FETCH_{uuid.uuid4().hex[:12]}__"
|
|
quoted = shlex.quote(remote_path)
|
|
# ``[ -f ]`` follows symlinks, so a link to a denied host file is judged by the CALLER on
|
|
# ``readlink -f`` output before any bytes move.
|
|
result = self.execute(
|
|
f"[ -f {quoted} ] && echo {marker} && head -c {max_bytes + 1} < {quoted} | base64 && echo {marker}",
|
|
timeout=_FETCH_TIMEOUT_SECONDS, rewrite_compound_background=False)
|
|
output = result.get("output") or ""
|
|
first, last = output.find(marker), output.rfind(marker)
|
|
if int(result.get("returncode") or 0) != 0 or first == -1 or last <= first:
|
|
raise FileFetchError(f"could not read {remote_path!r} in the sandbox (missing, not a regular file, or unreadable)")
|
|
try:
|
|
data = base64.b64decode("".join(output[first + len(marker):last].split()), validate=True)
|
|
except (ValueError, binascii.Error) as exc:
|
|
raise FileFetchError(f"transfer of {remote_path!r} was corrupted in transit: {exc}") from exc
|
|
if len(data) > max_bytes:
|
|
raise FileFetchError(f"{remote_path!r} exceeds the {max_bytes // (1024 * 1024)} MB delivery limit")
|
|
Path(local_dest).write_bytes(data)
|
|
|
|
def fetch_realpath(self, remote_path: str) -> str | None:
|
|
"""``readlink -f`` inside the backend, or None when it cannot be resolved."""
|
|
result = self.execute(f"readlink -f {shlex.quote(remote_path)} 2>/dev/null", rewrite_compound_background=False)
|
|
if int(result.get("returncode") or 0) != 0:
|
|
return None
|
|
return next((ln.strip() for ln in reversed((result.get("output") or "").splitlines()) if ln.strip().startswith("/")), None)
|
|
|
|
# --- Session snapshot (init_session) ---
|
|
def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]:
|
|
"""Return backend-specific names that must not persist in snapshots."""
|
|
return ()
|
|
|
|
def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]:
|
|
"""Profile-scoped names that must not persist in the snapshot. Monotonic for the
|
|
environment lifetime: an allowlist can be cleared after a value was captured, and
|
|
retaining the exclusion keeps that old value from leaking to a later profile."""
|
|
if not self._profile_scoped_passthrough:
|
|
return ()
|
|
try:
|
|
from agent.secret_scope import is_multiplex_active
|
|
if is_multiplex_active():
|
|
from tools.env_passthrough import get_all_passthrough
|
|
names = (*get_all_passthrough(), *self._additional_profile_scoped_passthrough_names())
|
|
self._snapshot_passthrough_names.update(
|
|
name for name in names if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name))
|
|
except Exception:
|
|
logger.debug("Could not refresh profile-scoped snapshot exclusions", exc_info=True)
|
|
return tuple(sorted(self._snapshot_passthrough_names))
|
|
|
|
def _snapshot_script_kwargs(self, cwd: str) -> dict:
|
|
"""Quoting inputs shared by the bootstrap and per-command wrapper scripts.
|
|
``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote) let the Windows
|
|
subclass rewrite ``C:\\...`` to ``/c/...`` so ``cd`` resolves and MSYS doesn't choke."""
|
|
return dict(
|
|
quoted_cwd=self._quote_cwd_for_cd(cwd),
|
|
quoted_snap=self._quote_shell_path(self._snapshot_path),
|
|
snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX),
|
|
cwd_marker=self._cwd_marker)
|
|
|
|
def init_session(self):
|
|
"""Capture the login shell environment into the snapshot file (once, after construction).
|
|
On success ``_snapshot_ready`` is set so commands source the snapshot instead of running
|
|
under ``bash -l``. On failure, fall back to ``bash -l`` per command — unless a non-login
|
|
probe shows login bash itself is dead, in which case prefer ``bash -c``."""
|
|
bootstrap = _snapshot_bootstrap_script(
|
|
excluded_names=self._snapshot_excluded_passthrough_names(), **self._snapshot_script_kwargs(self.cwd))
|
|
try:
|
|
proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
|
|
result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
|
|
if int(result.get("returncode") or 0) != 0:
|
|
raise RuntimeError(f"snapshot bootstrap failed with exit code {result.get('returncode')}")
|
|
self._snapshot_ready = True
|
|
self._update_cwd(result)
|
|
logger.info("Session snapshot created (session=%s, cwd=%s)", self._session_id, self.cwd)
|
|
except Exception as exc:
|
|
self._snapshot_ready = False
|
|
self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc))
|
|
if self._prefer_nonlogin:
|
|
logger.warning(
|
|
"init_session failed (session=%s): %s — "
|
|
"login bash unusable; falling back to non-login bash -c",
|
|
self._session_id, exc)
|
|
else:
|
|
logger.warning(
|
|
"init_session failed (session=%s): %s — falling back to bash -l per command",
|
|
self._session_id, detail)
|
|
|
|
def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]:
|
|
"""Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``."""
|
|
probe_timeout = min(15, self._snapshot_timeout)
|
|
try:
|
|
probe = self._run_bash("true", login=False, timeout=probe_timeout)
|
|
probe_result = self._wait_for_process(probe, timeout=probe_timeout)
|
|
prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0
|
|
if not prefer_nonlogin:
|
|
detail = (probe_result.get("stdout") or detail).strip() or detail
|
|
return prefer_nonlogin, detail
|
|
except Exception as probe_exc:
|
|
return False, f"{detail}; non-login probe: {probe_exc}"
|
|
|
|
# --- Command wrapping ---
|
|
@staticmethod
|
|
def _quote_cwd_for_cd(cwd: str) -> str:
|
|
"""Quote a ``cd`` target while preserving ``~`` expansion (``~/...``
|
|
goes through ``$HOME`` so suffixes with spaces stay one word)."""
|
|
if cwd == "~":
|
|
return cwd
|
|
if cwd == "~/":
|
|
return "$HOME"
|
|
if cwd.startswith("~/"):
|
|
return f"$HOME/{shlex.quote(cwd[2:])}"
|
|
return shlex.quote(cwd)
|
|
|
|
def _quote_shell_path(self, path: str) -> str:
|
|
"""Quote *path* for a bash script. LocalEnvironment overrides this to
|
|
rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX."""
|
|
return shlex.quote(path)
|
|
|
|
def _wrap_command(self, command: str, cwd: str) -> str:
|
|
"""Full bash script: source snapshot, cd, run, re-dump env, emit CWD markers."""
|
|
return _wrap_command_script(
|
|
command,
|
|
passthrough_names=self._snapshot_excluded_passthrough_names(),
|
|
snapshot_ready=self._snapshot_ready,
|
|
**self._snapshot_script_kwargs(cwd))
|
|
|
|
@staticmethod
|
|
def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
|
|
"""Append stdin_data as a shell heredoc to the command string (SDK backends)."""
|
|
delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
|
|
return f"{command} << '{delimiter}'\n{stdin_data}\n{delimiter}"
|
|
|
|
# --- Process lifecycle ---
|
|
def _wait_for_process(
|
|
self, proc: ProcessHandle, timeout: int = 120, *,
|
|
bounded_capture: bool = False, watch_interrupt_tid: int | None = None,
|
|
output=None, yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
|
|
"""Poll-based wait with interrupt checking and stdout draining (shared, not overridden).
|
|
``yield_handler(proc, output_so_far)``: when the tool thread is asked to yield
|
|
(``tools.interrupt.request_yield`` — a user message arrived mid-command), the drain
|
|
thread is stopped, the still-running process is handed to the handler and its dict
|
|
is returned as the result; the process is NOT killed.
|
|
``bounded_capture=True`` (foreground terminal-tool path only) retains at most
|
|
``tool_output.max_bytes`` in a head/tail window so a verbose subprocess cannot OOM the
|
|
process; the default keeps full fidelity for internal consumers. Fires the activity
|
|
callback every 10s so the gateway's inactivity timeout doesn't kill long commands.
|
|
``watch_interrupt_tid`` is the tool-worker thread that submitted this wait: ``execute()``
|
|
may move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still interrupts the
|
|
original tid, so both bits are honored. ``KeyboardInterrupt``/``SystemExit`` mid-poll
|
|
kills the process first — the local backend spawns into its own process group, so an
|
|
unkilled child would be orphaned.
|
|
|
|
The default (False) preserves full-fidelity capture for internal consumers — file-operation ``cat``
|
|
reads feeding the patch engine, code-execution RPC reads, log reads — where truncation would corrupt
|
|
data. See #64435.
|
|
"""
|
|
if output is None:
|
|
output = _new_output_collector(proc, bounded_capture)
|
|
drain_stop = threading.Event() if yield_handler is not None else None
|
|
drain_thread = _start_drain_thread(proc, output, drain_stop)
|
|
_now = time.monotonic()
|
|
deadline = _now + timeout
|
|
_activity_state = {"last_touch": _now, "start": _now}
|
|
trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger)
|
|
trace.enter()
|
|
|
|
def _kill_and_join():
|
|
self._kill_process(proc)
|
|
drain_thread.join(timeout=2)
|
|
|
|
try:
|
|
# Adaptive poll: start at 5ms so fast commands return in ~6ms, back
|
|
# off exponentially toward 200ms so long builds don't pay poll CPU.
|
|
_poll_sleep = 0.005
|
|
while proc.poll() is None:
|
|
trace.iterations += 1
|
|
if is_interrupted() or is_thread_interrupted(watch_interrupt_tid):
|
|
trace.interrupted()
|
|
_kill_and_join()
|
|
return self._finalize_wait_result(output, output.render(suffix="\n[Command interrupted]"), 130)
|
|
if yield_handler is not None and consume_yield(watch_interrupt_tid):
|
|
drain_stop.set()
|
|
drain_thread.join(timeout=1)
|
|
try:
|
|
handed = yield_handler(proc, output.render())
|
|
except Exception:
|
|
logger.warning("yield-to-background handoff failed; continuing to wait", exc_info=True)
|
|
handed = None
|
|
if handed is not None:
|
|
output.close_spill()
|
|
return handed
|
|
drain_stop.clear()
|
|
drain_thread = _start_drain_thread(proc, output, drain_stop)
|
|
if time.monotonic() > deadline:
|
|
trace.timed_out()
|
|
_kill_and_join()
|
|
rendered = output.render(suffix=f"\n[Command timed out after {timeout}s]")
|
|
if output.total_chars == 0:
|
|
rendered = rendered.lstrip()
|
|
return self._finalize_wait_result(output, rendered, 124)
|
|
touch_activity_if_due(_activity_state, "terminal command running")
|
|
trace.heartbeat()
|
|
time.sleep(_poll_sleep)
|
|
if _poll_sleep < 0.2:
|
|
_poll_sleep = min(_poll_sleep * 1.5, 0.2)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
trace.exception_exit()
|
|
try:
|
|
_kill_and_join()
|
|
except Exception:
|
|
pass # cleanup is best-effort
|
|
raise
|
|
|
|
# The drain thread exits promptly after bash does (~300ms idle check);
|
|
# a long join here would itself indicate a bug in the drain loop.
|
|
drain_thread.join(timeout=2)
|
|
try:
|
|
proc.stdout.close()
|
|
except Exception:
|
|
pass
|
|
trace.natural_exit(proc.returncode)
|
|
|
|
# Join the stdin writer before reading its error list: a child that exits without
|
|
# reading stdin can otherwise race ahead of a recorded encode failure. The timeout
|
|
# is a pure safety net (write raises BrokenPipeError once the pipe closes).
|
|
stdin_thread = getattr(proc, "_hermes_stdin_thread", None)
|
|
if stdin_thread is not None:
|
|
stdin_thread.join(timeout=5)
|
|
rendered = output.render()
|
|
result = self._finalize_wait_result(output, rendered, proc.returncode)
|
|
if stdin_errors := getattr(proc, "_hermes_stdin_errors", None):
|
|
result["stdin_error"] = err = str(stdin_errors[0])
|
|
result["output"] = rendered + f"\n[stdin write failed: {err}]"
|
|
return result
|
|
|
|
_finalize_wait_result = staticmethod(_finalize_wait_result)
|
|
|
|
def _kill_process(self, proc: ProcessHandle):
|
|
"""Terminate a process. Subclasses may override for process-group kill."""
|
|
try:
|
|
proc.kill()
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
pass
|
|
|
|
# --- CWD extraction ---
|
|
def _update_cwd(self, result: dict):
|
|
"""Extract CWD from command output. Override for local file-based read."""
|
|
self._extract_cwd_from_output(result)
|
|
|
|
def _extract_cwd_from_output(self, result: dict):
|
|
"""Parse the ``__HERMES_CWD_{session}__`` marker from ``result["output"]``, update
|
|
``self.cwd`` and strip the marker line. ``result["cwd_observed"]``/``["cwd"]`` are set
|
|
only when THIS command emitted a marker: a killed/timed-out command emits none and
|
|
``self.cwd`` keeps the previous value. The environment is shared across sessions, so
|
|
concurrent callers must read ``result["cwd"]`` rather than ``self.cwd``."""
|
|
split = _split_cwd_marker(result.get("output", ""), self._cwd_marker)
|
|
if split is None:
|
|
return
|
|
cwd_path, cleaned = split
|
|
if cwd_path:
|
|
self.cwd = cwd_path
|
|
result["cwd_observed"] = True
|
|
result["cwd"] = cwd_path
|
|
result["output"] = cleaned
|
|
|
|
# --- Hooks ---
|
|
def _before_execute(self) -> None:
|
|
"""Hook before each command. Remote backends (SSH, Modal, Daytona)
|
|
trigger their FileSyncManager here; bind-mount backends and Local don't."""
|
|
pass
|
|
|
|
# --- Unified execute() ---
|
|
def execute(
|
|
self,
|
|
command: str,
|
|
cwd: str = "",
|
|
*,
|
|
timeout: int | None = None,
|
|
stdin_data: str | None = None,
|
|
rewrite_compound_background: bool = True,
|
|
bounded_capture: bool = False,
|
|
yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
|
|
"""Execute a command, return {"output": str, "returncode": int}. ``bounded_capture=True``
|
|
caps retention at ``tool_output.max_bytes`` WHILE draining; only the foreground terminal
|
|
tool may set it — internal full-fidelity consumers (file-op ``cat`` reads feeding the
|
|
patch engine, RPC reads, log reads) MUST leave it False or data is corrupted. The wait is
|
|
bounded by ``agent.deadline.run_bounded_sync`` so a wedged poll loop cannot hang past
|
|
``timeout`` and silently disable every asyncio timer in the process.
|
|
|
|
``bounded_capture=True`` caps stdout/stderr retention at ``tool_output.max_bytes`` WHILE the stream
|
|
is drained (head/tail window) instead of holding the full output in memory (#64435).
|
|
See #94285.
|
|
"""
|
|
self._before_execute()
|
|
|
|
exec_command, sudo_stdin = self._prepare_command(command)
|
|
# Guard against the `A && B &` subshell-wait trap by default; callers
|
|
# that already produce shell-safe wrappers (spawn_via_env) pass False.
|
|
if rewrite_compound_background:
|
|
from tools.terminal_tool_sudo import _rewrite_compound_background
|
|
exec_command = _rewrite_compound_background(exec_command)
|
|
effective_timeout = timeout or self.timeout
|
|
effective_cwd = cwd or self.cwd
|
|
|
|
# Merge sudo stdin with caller stdin.
|
|
effective_stdin = sudo_stdin + (stdin_data or "") if sudo_stdin is not None else stdin_data
|
|
if effective_stdin and self._stdin_mode == "heredoc":
|
|
exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
|
|
effective_stdin = None
|
|
|
|
wrapped = self._wrap_command(exec_command, effective_cwd)
|
|
|
|
# Login shell if the snapshot failed (so the user's profile still
|
|
# loads), unless login itself is broken — then non-login is the only path.
|
|
login = not self._snapshot_ready and not self._prefer_nonlogin
|
|
|
|
parent_tid = threading.current_thread().ident
|
|
# The activity callback is thread-local and the wait runs on the
|
|
# deadline worker, so copy it across or long commands look idle.
|
|
parent_activity_cb = get_activity_callback()
|
|
proc_holder: list = []
|
|
output_holder: list = []
|
|
|
|
def _spawn_and_wait() -> dict:
|
|
if parent_activity_cb is not None:
|
|
set_activity_callback(parent_activity_cb)
|
|
spawned = self._run_bash(wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin)
|
|
proc_holder.append(spawned)
|
|
output = _new_output_collector(spawned, bounded_capture)
|
|
output_holder.append(output)
|
|
return self._wait_for_process(
|
|
spawned, timeout=effective_timeout, bounded_capture=bounded_capture,
|
|
watch_interrupt_tid=parent_tid, output=output,
|
|
**({"yield_handler": yield_handler} if yield_handler is not None else {}))
|
|
|
|
def _on_timeout() -> None:
|
|
if proc_holder:
|
|
self._kill_spawned_tree(proc_holder[0])
|
|
|
|
# Hard wall-clock backstop: ``_wait_for_process`` polls to ``effective_timeout`` on the
|
|
# tool thread; if that is the event-loop thread, or the wait never returns (Windows
|
|
# pipe/poll hang), every asyncio timer is silently disabled. ``run_bounded_sync`` drives
|
|
# expiry from a daemon worker + ``Event.wait`` so a blocked loop cannot disable it; the
|
|
# grace lets the inner loop return the partial-output 124 path.
|
|
# See #94285.
|
|
from agent.deadline import run_bounded_sync
|
|
|
|
try:
|
|
bound_s = float(effective_timeout)
|
|
except (TypeError, ValueError):
|
|
bound_s = 120.0 # a non-numeric timeout must not disable the backstop
|
|
bound_s += _EXECUTE_WAIT_BOUND_GRACE_S
|
|
|
|
try:
|
|
bounded = run_bounded_sync(
|
|
_spawn_and_wait, bound_s, label=f"terminal.wait:{type(self).__name__}", on_timeout=_on_timeout)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
_on_timeout()
|
|
raise
|
|
|
|
if bounded.timed_out:
|
|
suffix = f"\n[Command timed out after {effective_timeout}s]"
|
|
if output_holder:
|
|
collector = output_holder[0]
|
|
result = self._finalize_wait_result(collector, collector.render(suffix=suffix).lstrip("\n"), 124)
|
|
else:
|
|
result = {"output": suffix.lstrip(), "returncode": 124}
|
|
else:
|
|
result = bounded.value
|
|
self._update_cwd(result)
|
|
return result
|
|
|
|
def _kill_spawned_tree(self, spawned) -> None:
|
|
"""Best-effort kill of a wedged spawned process and its tree (backstop path)."""
|
|
try:
|
|
self._kill_process(spawned)
|
|
except Exception:
|
|
logger.debug("terminal wait-bound kill_process failed", exc_info=True)
|
|
pid = getattr(spawned, "pid", None)
|
|
if not pid:
|
|
return
|
|
try:
|
|
from agent.deadline import kill_process_tree
|
|
kill_process_tree(int(pid))
|
|
except Exception:
|
|
logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True)
|
|
|
|
# --- Shared helpers ---
|
|
def __del__(self):
|
|
try:
|
|
self.cleanup()
|
|
except Exception:
|
|
pass
|
|
|
|
def _prepare_command(self, command: str) -> tuple[str, str | None]:
|
|
"""Transform sudo commands if SUDO_PASSWORD is available."""
|
|
from tools.terminal_tool_sudo import _transform_sudo_command
|
|
return _transform_sudo_command(command)
|
|
|
|
|
|
# ---- BEGIN PLUGIN-COMPAT (revert-scheduled; see COMPAT_MANIFEST.md) ----
|
|
# Names external plugins imported from this module before the Sep 2026 decomposition.
|
|
# Internal code MUST NOT use these (scripts/check_compat_pointers.py fails CI if it does).
|
|
# The whole block is removed by reverting the commit that added it.
|
|
from typing import IO # noqa: F401,E402
|
|
from typing import Protocol # noqa: F401,E402
|
|
import codecs # noqa: F401,E402
|
|
from collections import deque # noqa: F401,E402
|
|
import re # noqa: F401,E402
|
|
import select # noqa: F401,E402
|
|
import subprocess # noqa: F401,E402
|
|
|
|
|
|
_PLUGIN_COMPAT_LAZY = {
|
|
'sanitize_task_id_for_path': ('tools.environments.path_utils', 'sanitize_task_id_for_path'),
|
|
'windows_hide_flags': ('hermes_cli._subprocess_compat', 'windows_hide_flags'),
|
|
}
|
|
|
|
|
|
def __getattr__(name): # PEP 562 — lazy so no import cycles
|
|
target = _PLUGIN_COMPAT_LAZY.get(name)
|
|
if target is None:
|
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
import importlib
|
|
from hermes_cli.plugin_compat import warn_once
|
|
warn_once(__name__, name, *target)
|
|
return getattr(importlib.import_module(target[0]), target[1])
|
|
# ---- END PLUGIN-COMPAT ----
|