670 lines
28 KiB
Python
670 lines
28 KiB
Python
"""Base class for all Hermes execution environment backends.
|
|
|
|
Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
|
|
A session snapshot (env vars, functions, aliases) is captured once at init and
|
|
re-sourced before each command. CWD persists via in-band stdout markers (remote)
|
|
or a temp file (local).
|
|
|
|
Cohesive pieces live in sibling modules and are re-exported here so
|
|
``from tools.environments.base import X`` / ``patch("tools.environments.base.X")``
|
|
keep working: ``base_output`` (collector, ProcessHandle, stdin/drain plumbing),
|
|
``base_session_env`` (snapshot/wrapper shell scripting), ``base_wait`` (tracing).
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import shlex
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from abc import ABC, abstractmethod
|
|
from pathlib import Path
|
|
from typing import Callable, Iterable
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from tools.interrupt import is_interrupted, is_thread_interrupted
|
|
from tools.environments.base_output import ( # noqa: F401
|
|
ProcessHandle,
|
|
_BoundedOutputCollector,
|
|
_ThreadedProcessHandle,
|
|
_UNBOUNDED_CAPTURE_CHARS,
|
|
_finalize_wait_result,
|
|
_new_output_collector,
|
|
_pipe_stdin,
|
|
_popen_bash,
|
|
_start_drain_thread,
|
|
)
|
|
from tools.environments.base_session_env import ( # noqa: F401
|
|
_SHELL_ENV_NAME_RE,
|
|
_SNAP_TMP,
|
|
_SNAP_TMP_SUFFIX,
|
|
_SNAPSHOT_EXCLUDED_ENV_REGEX,
|
|
_cwd_marker,
|
|
_export_dump_excluding_session_vars,
|
|
_snapshot_bootstrap_script,
|
|
_split_cwd_marker,
|
|
_wrap_command_script,
|
|
)
|
|
from tools.environments.base_wait import _WaitTrace
|
|
from tools.environments.path_utils import ( # noqa: F401
|
|
_SANDBOX_DIR_HASH_LEN,
|
|
_SANDBOX_DIR_MAX_LEN,
|
|
_SANDBOX_DIR_UNSAFE_RE,
|
|
sanitize_task_id_for_path,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Opt-in debug tracing for the interrupt/activity/poll machinery
|
|
# (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs.
|
|
_DEBUG_INTERRUPT = bool(os.getenv("HERMES_DEBUG_INTERRUPT"))
|
|
|
|
# Extra seconds the ``run_bounded_sync`` backstop waits past the inner
|
|
# ``_wait_for_process`` deadline. The inner poll loop returns partial output +
|
|
# returncode 124; the outer bound only fires when that loop itself never
|
|
# returns (a blocked wait that silently disables asyncio timers). Keep it
|
|
# small so a healthy timeout still comes from the inner path.
|
|
_EXECUTE_WAIT_BOUND_GRACE_S = 2.0
|
|
|
|
if _DEBUG_INTERRUPT:
|
|
# quiet_mode forces the `tools` logger to ERROR on CLI startup, which would
|
|
# swallow every trace; force this logger back to INFO in the opt-in case.
|
|
logger.setLevel(logging.INFO)
|
|
|
|
# Thread-local activity callback: the agent sets it before a tool call so
|
|
# long-running _wait_for_process loops can report liveness to the gateway.
|
|
_activity_callback_local = threading.local()
|
|
|
|
|
|
class EnvironmentConnectionError(RuntimeError):
|
|
"""Infrastructure/connection-class failure of a terminal backend.
|
|
|
|
Raised when the backend itself is unreachable (SSH host down, Docker daemon
|
|
not running, remote file sync failing on a dead link) — never for a command
|
|
that merely exited nonzero. Subclassing RuntimeError keeps every existing
|
|
``except RuntimeError`` catcher working. ``terminal_tool`` turns this into a
|
|
structured ``status: "degraded"`` result (config ``terminal.degraded_mode``);
|
|
the failed backend is never cached, so a later call retries from scratch.
|
|
"""
|
|
|
|
def __init__(self, reason: str, *, retry_hint: str = ""):
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
self.retry_hint = retry_hint or (
|
|
"This is an infrastructure failure, not a command failure. "
|
|
"Verify the backend is reachable (network, service running, "
|
|
"credentials), then retry the same command — recovery is "
|
|
"automatic once the backend is back."
|
|
)
|
|
|
|
|
|
def set_activity_callback(cb: Callable[[str], None] | None) -> None:
|
|
"""Register a callback that _wait_for_process fires periodically."""
|
|
_activity_callback_local.callback = cb
|
|
|
|
|
|
def get_activity_callback() -> Callable[[str], None] | None:
|
|
"""Return the thread-local activity callback (see ``set_activity_callback``).
|
|
|
|
For callers that must capture the calling thread's callback before handing
|
|
work to another thread — a freshly spawned thread cannot read it back.
|
|
"""
|
|
return getattr(_activity_callback_local, "callback", None)
|
|
|
|
|
|
def touch_activity_if_due(state: dict, label: str) -> None:
|
|
"""Fire the activity callback at most once every ``state['interval']`` seconds.
|
|
|
|
*state* must contain ``last_touch`` and ``start`` (monotonic timestamps);
|
|
optional ``interval`` overrides the default 10 s cadence. Swallows all
|
|
exceptions so callers don't need their own try/except.
|
|
"""
|
|
now = time.monotonic()
|
|
interval = state.get("interval", 10.0)
|
|
if now - state["last_touch"] < interval:
|
|
return
|
|
state["last_touch"] = now
|
|
try:
|
|
cb = get_activity_callback()
|
|
if cb:
|
|
elapsed = int(now - state["start"])
|
|
cb(f"{label} ({elapsed}s elapsed)")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def get_sandbox_dir() -> Path:
|
|
"""Host-side root for all sandbox storage (Docker workspaces, Singularity
|
|
overlays/SIF cache). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``."""
|
|
custom = os.getenv("TERMINAL_SANDBOX_DIR")
|
|
p = Path(custom) if custom else get_hermes_home() / "sandboxes"
|
|
p.mkdir(parents=True, exist_ok=True)
|
|
return p
|
|
|
|
|
|
def _load_json_store(path: Path) -> dict:
|
|
"""Load a JSON file as a dict, returning ``{}`` on any error."""
|
|
if path.exists():
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
pass
|
|
return {}
|
|
|
|
|
|
def _save_json_store(path: Path, data: dict) -> None:
|
|
"""Write *data* as pretty-printed JSON to *path*."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
|
|
|
|
def _file_mtime_key(host_path: str) -> tuple[float, int] | None:
|
|
"""Return ``(mtime, size)`` for cache comparison, or ``None`` if unreadable."""
|
|
try:
|
|
st = Path(host_path).stat()
|
|
return (st.st_mtime, st.st_size)
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BaseEnvironment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class BaseEnvironment(ABC):
|
|
"""Common interface and unified execution flow for all Hermes backends.
|
|
|
|
Subclasses implement ``_run_bash()`` and ``cleanup()``. The base class
|
|
provides ``execute()`` with session snapshot sourcing, CWD tracking,
|
|
interrupt handling, and timeout enforcement.
|
|
"""
|
|
|
|
# Subclasses that embed stdin as a heredoc (Modal, Daytona) set this.
|
|
_stdin_mode: str = "pipe" # "pipe" or "heredoc"
|
|
|
|
# True only when commands execute on the SAME host as the Hermes process
|
|
# (LocalEnvironment). Controller-host facts (sys.platform, Path.home())
|
|
# describe the execution target only when this is True.
|
|
is_local: bool = False
|
|
|
|
# Snapshot creation timeout (override for slow cold-starts).
|
|
_snapshot_timeout: int = 30
|
|
|
|
# Local and Docker override this because they resolve allowlisted values
|
|
# through the active profile scope; other backends keep plain snapshots.
|
|
_profile_scoped_passthrough: bool = False
|
|
|
|
def get_temp_dir(self) -> str:
|
|
"""Backend temp directory for session artifacts (``/tmp`` in sandboxes;
|
|
LocalEnvironment overrides for Termux where only ``TMPDIR`` is writable)."""
|
|
return "/tmp"
|
|
|
|
def __init__(self, cwd: str, timeout: int, env: dict = None):
|
|
self.cwd = cwd
|
|
self.timeout = timeout
|
|
self.env = env or {}
|
|
|
|
self._session_id = uuid.uuid4().hex[:12]
|
|
temp_dir = self.get_temp_dir().rstrip("/") or "/"
|
|
self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
|
|
self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
|
|
self._cwd_marker = _cwd_marker(self._session_id)
|
|
self._snapshot_ready = False
|
|
self._snapshot_passthrough_names: set[str] = set()
|
|
# True when login bash is unusable (e.g. broken Git-for-Windows startup)
|
|
# so execute() must fall back to non-login ``bash -c``, not ``bash -l``.
|
|
self._prefer_nonlogin = False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Abstract methods
|
|
# ------------------------------------------------------------------
|
|
|
|
def _run_bash(
|
|
self,
|
|
cmd_string: str,
|
|
*,
|
|
login: bool = False,
|
|
timeout: int = 120,
|
|
stdin_data: str | None = None,
|
|
) -> ProcessHandle:
|
|
"""Spawn a bash process to run *cmd_string*; every backend overrides this."""
|
|
raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
|
|
|
|
@abstractmethod
|
|
def cleanup(self):
|
|
"""Release backend resources (container, instance, connection)."""
|
|
...
|
|
|
|
# ------------------------------------------------------------------
|
|
# Session snapshot (init_session)
|
|
# ------------------------------------------------------------------
|
|
|
|
def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]:
|
|
"""Return backend-specific names that must not persist in snapshots."""
|
|
return ()
|
|
|
|
def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]:
|
|
"""Profile-scoped names that must not persist in the snapshot.
|
|
|
|
Monotonic for the environment lifetime: an allowlist can be cleared
|
|
after a value was captured, and retaining the exclusion keeps that old
|
|
value from leaking to a later profile through the shared snapshot.
|
|
"""
|
|
if not self._profile_scoped_passthrough:
|
|
return ()
|
|
try:
|
|
from agent.secret_scope import is_multiplex_active
|
|
if is_multiplex_active():
|
|
from tools.env_passthrough import get_all_passthrough
|
|
names = (
|
|
*get_all_passthrough(),
|
|
*self._additional_profile_scoped_passthrough_names(),
|
|
)
|
|
self._snapshot_passthrough_names.update(
|
|
name
|
|
for name in names
|
|
if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name)
|
|
)
|
|
except Exception:
|
|
logger.debug(
|
|
"Could not refresh profile-scoped snapshot exclusions",
|
|
exc_info=True,
|
|
)
|
|
return tuple(sorted(self._snapshot_passthrough_names))
|
|
|
|
def init_session(self):
|
|
"""Capture the login shell environment into the snapshot file.
|
|
|
|
Called once after construction. On success ``_snapshot_ready`` is set so
|
|
commands source the snapshot instead of running under ``bash -l``. On
|
|
failure, fall back to ``bash -l`` per command — unless a non-login probe
|
|
shows login bash itself is dead, in which case prefer ``bash -c``.
|
|
"""
|
|
bootstrap = _snapshot_bootstrap_script(
|
|
# ``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote)
|
|
# let the Windows subclass rewrite ``C:\\...`` to ``/c/...`` so the
|
|
# bootstrap ``cd`` resolves and MSYS doesn't choke on drive paths.
|
|
quoted_cwd=self._quote_cwd_for_cd(self.cwd),
|
|
quoted_snap=self._quote_shell_path(self._snapshot_path),
|
|
snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX),
|
|
excluded_names=self._snapshot_excluded_passthrough_names(),
|
|
cwd_marker=self._cwd_marker,
|
|
)
|
|
try:
|
|
proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
|
|
result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
|
|
if int(result.get("returncode") or 0) != 0:
|
|
raise RuntimeError(
|
|
f"snapshot bootstrap failed with exit code {result.get('returncode')}"
|
|
)
|
|
self._snapshot_ready = True
|
|
self._update_cwd(result)
|
|
logger.info(
|
|
"Session snapshot created (session=%s, cwd=%s)",
|
|
self._session_id,
|
|
self.cwd,
|
|
)
|
|
except Exception as exc:
|
|
self._snapshot_ready = False
|
|
self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc))
|
|
if self._prefer_nonlogin:
|
|
logger.warning(
|
|
"init_session failed (session=%s): %s — "
|
|
"login bash unusable; falling back to non-login bash -c",
|
|
self._session_id,
|
|
exc,
|
|
)
|
|
else:
|
|
logger.warning(
|
|
"init_session failed (session=%s): %s — "
|
|
"falling back to bash -l per command",
|
|
self._session_id,
|
|
detail,
|
|
)
|
|
|
|
def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]:
|
|
"""Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``."""
|
|
probe_timeout = min(15, self._snapshot_timeout)
|
|
try:
|
|
probe = self._run_bash("true", login=False, timeout=probe_timeout)
|
|
probe_result = self._wait_for_process(probe, timeout=probe_timeout)
|
|
prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0
|
|
if not prefer_nonlogin:
|
|
detail = (probe_result.get("stdout") or detail).strip() or detail
|
|
return prefer_nonlogin, detail
|
|
except Exception as probe_exc:
|
|
return False, f"{detail}; non-login probe: {probe_exc}"
|
|
|
|
# ------------------------------------------------------------------
|
|
# Command wrapping
|
|
# ------------------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _quote_cwd_for_cd(cwd: str) -> str:
|
|
"""Quote a ``cd`` target while preserving ``~`` expansion (``~/...``
|
|
goes through ``$HOME`` so suffixes with spaces stay one word)."""
|
|
if cwd == "~":
|
|
return cwd
|
|
if cwd == "~/":
|
|
return "$HOME"
|
|
if cwd.startswith("~/"):
|
|
return f"$HOME/{shlex.quote(cwd[2:])}"
|
|
return shlex.quote(cwd)
|
|
|
|
def _quote_shell_path(self, path: str) -> str:
|
|
"""Quote *path* for a bash script. LocalEnvironment overrides this to
|
|
rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX."""
|
|
return shlex.quote(path)
|
|
|
|
def _wrap_command(self, command: str, cwd: str) -> str:
|
|
"""Build the full bash script that sources snapshot, cd's, runs command,
|
|
re-dumps env vars, and emits CWD markers (see ``_wrap_command_script``)."""
|
|
return _wrap_command_script(
|
|
command,
|
|
quoted_cwd=self._quote_cwd_for_cd(cwd),
|
|
quoted_snap=self._quote_shell_path(self._snapshot_path),
|
|
snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX),
|
|
passthrough_names=self._snapshot_excluded_passthrough_names(),
|
|
snapshot_ready=self._snapshot_ready,
|
|
cwd_marker=self._cwd_marker,
|
|
)
|
|
|
|
@staticmethod
|
|
def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
|
|
"""Append stdin_data as a shell heredoc to the command string (SDK backends)."""
|
|
delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
|
|
return f"{command} << '{delimiter}'\n{stdin_data}\n{delimiter}"
|
|
|
|
# ------------------------------------------------------------------
|
|
# Process lifecycle
|
|
# ------------------------------------------------------------------
|
|
|
|
def _wait_for_process(
|
|
self,
|
|
proc: ProcessHandle,
|
|
timeout: int = 120,
|
|
*,
|
|
bounded_capture: bool = False,
|
|
watch_interrupt_tid: int | None = None,
|
|
) -> dict:
|
|
"""Poll-based wait with interrupt checking and stdout draining.
|
|
|
|
Shared across all backends — not overridden. ``bounded_capture=True``
|
|
(foreground terminal-tool path only) retains at most
|
|
``tool_output.max_bytes`` in a head/tail window so a verbose subprocess
|
|
cannot OOM the process; the default keeps full fidelity for internal
|
|
consumers (file-op ``cat`` reads, RPC reads) where truncation corrupts
|
|
data. Fires the activity callback every 10s so the gateway's
|
|
inactivity timeout doesn't kill long commands. ``watch_interrupt_tid``
|
|
is the tool-worker thread that submitted this wait: ``execute()`` may
|
|
move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still
|
|
interrupts the original tid, so both bits are honored. A
|
|
``KeyboardInterrupt``/``SystemExit`` mid-poll kills the process first —
|
|
the local backend spawns into its own process group (``os.setsid``), so
|
|
an unkilled child would be orphaned to PPID=1 and keep running.
|
|
"""
|
|
output = _new_output_collector(proc, bounded_capture)
|
|
drain_thread = _start_drain_thread(proc, output)
|
|
deadline = time.monotonic() + timeout
|
|
_now = time.monotonic()
|
|
_activity_state = {"last_touch": _now, "start": _now}
|
|
trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger)
|
|
trace.enter()
|
|
|
|
def _kill_and_join():
|
|
self._kill_process(proc)
|
|
drain_thread.join(timeout=2)
|
|
|
|
try:
|
|
# Adaptive poll: start at 5ms so fast commands return in ~6ms, back
|
|
# off exponentially toward 200ms so long builds don't pay poll CPU.
|
|
_poll_sleep = 0.005
|
|
while proc.poll() is None:
|
|
trace.iterations += 1
|
|
if is_interrupted() or is_thread_interrupted(watch_interrupt_tid):
|
|
trace.interrupted()
|
|
_kill_and_join()
|
|
return self._finalize_wait_result(
|
|
output, output.render(suffix="\n[Command interrupted]"), 130
|
|
)
|
|
if time.monotonic() > deadline:
|
|
trace.timed_out()
|
|
_kill_and_join()
|
|
timeout_msg = f"\n[Command timed out after {timeout}s]"
|
|
rendered = output.render(suffix=timeout_msg)
|
|
if output.total_chars == 0:
|
|
rendered = rendered.lstrip()
|
|
return self._finalize_wait_result(output, rendered, 124)
|
|
touch_activity_if_due(_activity_state, "terminal command running")
|
|
trace.heartbeat()
|
|
time.sleep(_poll_sleep)
|
|
if _poll_sleep < 0.2:
|
|
_poll_sleep = min(_poll_sleep * 1.5, 0.2)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
trace.exception_exit()
|
|
try:
|
|
_kill_and_join()
|
|
except Exception:
|
|
pass # cleanup is best-effort
|
|
raise
|
|
|
|
# The drain thread exits promptly after bash does (~300ms idle check);
|
|
# a long join here would itself indicate a bug in the drain loop.
|
|
drain_thread.join(timeout=2)
|
|
try:
|
|
proc.stdout.close()
|
|
except Exception:
|
|
pass
|
|
trace.natural_exit(proc.returncode)
|
|
|
|
# Join the stdin writer before reading its error list: a child that
|
|
# exits without reading stdin can otherwise race ahead of a recorded
|
|
# encode failure. The timeout is a pure safety net (write raises
|
|
# BrokenPipeError once the pipe closes).
|
|
stdin_thread = getattr(proc, "_hermes_stdin_thread", None)
|
|
if stdin_thread is not None:
|
|
stdin_thread.join(timeout=5)
|
|
rendered = output.render()
|
|
result = self._finalize_wait_result(output, rendered, proc.returncode)
|
|
stdin_errors = getattr(proc, "_hermes_stdin_errors", None)
|
|
if stdin_errors:
|
|
err = str(stdin_errors[0])
|
|
result["stdin_error"] = err
|
|
result["output"] = rendered + f"\n[stdin write failed: {err}]"
|
|
return result
|
|
|
|
_finalize_wait_result = staticmethod(_finalize_wait_result)
|
|
|
|
def _kill_process(self, proc: ProcessHandle):
|
|
"""Terminate a process. Subclasses may override for process-group kill."""
|
|
try:
|
|
proc.kill()
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
pass
|
|
|
|
# ------------------------------------------------------------------
|
|
# CWD extraction
|
|
# ------------------------------------------------------------------
|
|
|
|
def _update_cwd(self, result: dict):
|
|
"""Extract CWD from command output. Override for local file-based read."""
|
|
self._extract_cwd_from_output(result)
|
|
|
|
def _extract_cwd_from_output(self, result: dict):
|
|
"""Parse the ``__HERMES_CWD_{session}__`` marker from ``result["output"]``.
|
|
|
|
Updates ``self.cwd`` and strips the marker line. Sets
|
|
``result["cwd_observed"]`` (and ``result["cwd"]``) only when THIS command
|
|
emitted a marker: the wrapper prints it after the command returns, so a
|
|
killed/timed-out command emits none and ``self.cwd`` keeps the previous
|
|
value. The environment is shared across sessions, so callers must not
|
|
attribute an unobserved cwd to this session, and concurrent callers
|
|
must read ``result["cwd"]`` rather than ``self.cwd``.
|
|
"""
|
|
split = _split_cwd_marker(result.get("output", ""), self._cwd_marker)
|
|
if split is None:
|
|
return
|
|
cwd_path, cleaned = split
|
|
if cwd_path:
|
|
self.cwd = cwd_path
|
|
result["cwd_observed"] = True
|
|
result["cwd"] = cwd_path
|
|
result["output"] = cleaned
|
|
|
|
# ------------------------------------------------------------------
|
|
# Hooks
|
|
# ------------------------------------------------------------------
|
|
|
|
def _before_execute(self) -> None:
|
|
"""Hook before each command. Remote backends (SSH, Modal, Daytona)
|
|
trigger their FileSyncManager here; bind-mount backends and Local don't."""
|
|
pass
|
|
|
|
# ------------------------------------------------------------------
|
|
# Unified execute()
|
|
# ------------------------------------------------------------------
|
|
|
|
def execute(
|
|
self,
|
|
command: str,
|
|
cwd: str = "",
|
|
*,
|
|
timeout: int | None = None,
|
|
stdin_data: str | None = None,
|
|
rewrite_compound_background: bool = True,
|
|
bounded_capture: bool = False,
|
|
) -> dict:
|
|
"""Execute a command, return {"output": str, "returncode": int}.
|
|
|
|
``bounded_capture=True`` caps retention at ``tool_output.max_bytes``
|
|
WHILE draining; only the foreground terminal tool may set it — internal
|
|
full-fidelity consumers (file-op ``cat`` reads feeding the patch
|
|
engine, RPC reads, log reads) MUST leave it False or data is corrupted.
|
|
The wait is bounded by ``agent.deadline.run_bounded_sync`` so a wedged
|
|
poll loop cannot hang past ``timeout`` and silently disable every
|
|
asyncio timer in the process.
|
|
"""
|
|
self._before_execute()
|
|
|
|
exec_command, sudo_stdin = self._prepare_command(command)
|
|
# Guard against the `A && B &` subshell-wait trap by default; callers
|
|
# that already produce shell-safe wrappers (spawn_via_env) pass False.
|
|
if rewrite_compound_background:
|
|
from tools.terminal_tool import _rewrite_compound_background
|
|
exec_command = _rewrite_compound_background(exec_command)
|
|
effective_timeout = timeout or self.timeout
|
|
effective_cwd = cwd or self.cwd
|
|
|
|
# Merge sudo stdin with caller stdin.
|
|
if sudo_stdin is not None:
|
|
effective_stdin = sudo_stdin + (stdin_data or "")
|
|
else:
|
|
effective_stdin = stdin_data
|
|
|
|
if effective_stdin and self._stdin_mode == "heredoc":
|
|
exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
|
|
effective_stdin = None
|
|
|
|
wrapped = self._wrap_command(exec_command, effective_cwd)
|
|
|
|
# Login shell if the snapshot failed (so the user's profile still
|
|
# loads), unless login itself is broken — then non-login is the only path.
|
|
login = not self._snapshot_ready and not self._prefer_nonlogin
|
|
|
|
parent_tid = threading.current_thread().ident
|
|
# The activity callback is thread-local and the wait runs on the
|
|
# deadline worker, so copy it across or long commands look idle.
|
|
parent_activity_cb = get_activity_callback()
|
|
proc_holder: list = []
|
|
|
|
def _spawn_and_wait() -> dict:
|
|
if parent_activity_cb is not None:
|
|
set_activity_callback(parent_activity_cb)
|
|
spawned = self._run_bash(
|
|
wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin
|
|
)
|
|
proc_holder.append(spawned)
|
|
return self._wait_for_process(
|
|
spawned,
|
|
timeout=effective_timeout,
|
|
bounded_capture=bounded_capture,
|
|
watch_interrupt_tid=parent_tid,
|
|
)
|
|
|
|
def _on_timeout() -> None:
|
|
if not proc_holder:
|
|
return
|
|
self._kill_spawned_tree(proc_holder[0])
|
|
|
|
# Hard wall-clock backstop: ``_wait_for_process`` polls to
|
|
# ``effective_timeout`` on the tool thread; if that is the event-loop
|
|
# thread, or the wait never returns (Windows pipe/poll hang), every
|
|
# asyncio timer is silently disabled. ``run_bounded_sync`` drives expiry
|
|
# from a daemon worker + ``Event.wait`` so a blocked loop cannot disable
|
|
# it; the grace lets the inner loop return the partial-output 124 path.
|
|
from agent.deadline import run_bounded_sync
|
|
|
|
try:
|
|
bound_s = float(effective_timeout) + _EXECUTE_WAIT_BOUND_GRACE_S
|
|
except (TypeError, ValueError):
|
|
# A non-numeric timeout must not disable the backstop; use the 120s default.
|
|
bound_s = 120.0 + _EXECUTE_WAIT_BOUND_GRACE_S
|
|
|
|
try:
|
|
bounded = run_bounded_sync(
|
|
_spawn_and_wait,
|
|
bound_s,
|
|
label=f"terminal.wait:{type(self).__name__}",
|
|
on_timeout=_on_timeout,
|
|
)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
_on_timeout()
|
|
raise
|
|
|
|
if bounded.timed_out:
|
|
timeout_msg = f"\n[Command timed out after {effective_timeout}s]"
|
|
result = {"output": timeout_msg.lstrip(), "returncode": 124}
|
|
else:
|
|
result = bounded.value
|
|
self._update_cwd(result)
|
|
|
|
return result
|
|
|
|
def _kill_spawned_tree(self, spawned) -> None:
|
|
"""Best-effort kill of a wedged spawned process and its tree (backstop path)."""
|
|
try:
|
|
self._kill_process(spawned)
|
|
except Exception:
|
|
logger.debug("terminal wait-bound kill_process failed", exc_info=True)
|
|
pid = getattr(spawned, "pid", None)
|
|
if not pid:
|
|
return
|
|
try:
|
|
from agent.deadline import kill_process_tree
|
|
|
|
kill_process_tree(int(pid))
|
|
except Exception:
|
|
logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Shared helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def stop(self):
|
|
"""Alias for cleanup (compat with older callers)."""
|
|
self.cleanup()
|
|
|
|
def __del__(self):
|
|
try:
|
|
self.cleanup()
|
|
except Exception:
|
|
pass
|
|
|
|
def _prepare_command(self, command: str) -> tuple[str, str | None]:
|
|
"""Transform sudo commands if SUDO_PASSWORD is available."""
|
|
from tools.terminal_tool import _transform_sudo_command
|
|
|
|
return _transform_sudo_command(command)
|