407 lines
18 KiB
Python
407 lines
18 KiB
Python
"""Shared file sync manager for remote execution backends.
|
|
|
|
Tracks local file changes via mtime+size, detects deletions, and syncs to
|
|
remote environments transactionally. Used by SSH, Modal, and Daytona.
|
|
Docker and Singularity use bind mounts (live host FS view) and don't need this.
|
|
"""
|
|
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import posixpath
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import tarfile
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
|
|
try:
|
|
import fcntl
|
|
except ImportError:
|
|
fcntl = None # Windows — file locking skipped
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from tools.environments.base import _file_mtime_key
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Tests patch these module-level aliases instead of ``time.sleep`` /
|
|
# ``time.monotonic``: patching attributes on the shared ``time`` module object
|
|
# leaks into unrelated threads under xdist and inflates retry call counts.
|
|
_sleep = time.sleep
|
|
_monotonic = time.monotonic
|
|
|
|
_SYNC_INTERVAL_SECONDS = 5.0
|
|
_FORCE_SYNC_ENV = "HERMES_FORCE_FILE_SYNC"
|
|
|
|
# Transport callbacks provided by each backend
|
|
UploadFn = Callable[[str, str], None] # (host_path, remote_path) -> raises on failure
|
|
BulkUploadFn = Callable[[list[tuple[str, str]]], None] # [(host_path, remote_path), ...] -> raises on failure
|
|
BulkDownloadFn = Callable[[Path], None] # (dest_tar_path) -> writes tar archive, raises on failure
|
|
DeleteFn = Callable[[list[str]], None] # (remote_paths) -> raises on failure
|
|
GetFilesFn = Callable[[], list[tuple[str, str]]] # () -> [(host_path, remote_path), ...]
|
|
|
|
_SYNC_BACK_MAX_RETRIES = 3
|
|
_SYNC_BACK_BACKOFF = (2, 4, 8) # seconds between retries
|
|
_SYNC_BACK_MAX_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB — refuse to extract larger tars
|
|
|
|
|
|
def iter_sync_files(container_base: str = "/root/.hermes") -> list[tuple[str, str]]:
|
|
"""Enumerate all (host_path, remote_path) pairs to sync to a remote. Credential paths are
|
|
remapped from the hardcoded /root/.hermes to *container_base* (remote home may differ)."""
|
|
# Late import: credential_files pulls in agent modules (circular at module level).
|
|
from tools.credential_files import get_credential_file_mounts, iter_cache_files, iter_skills_files
|
|
|
|
files = [
|
|
(entry["host_path"], entry["container_path"].replace("/root/.hermes", container_base, 1))
|
|
for entry in get_credential_file_mounts()]
|
|
files += [
|
|
(entry["host_path"], entry["container_path"])
|
|
for entry in (*iter_skills_files(container_base=container_base),
|
|
*iter_cache_files(container_base=container_base))]
|
|
return files
|
|
|
|
|
|
def _resolve_host_path_str(host_path: str) -> str:
|
|
"""Canonical string form of a host path (``resolve()`` falling back to ``expanduser()``)."""
|
|
try:
|
|
return str(Path(host_path).expanduser().resolve())
|
|
except OSError:
|
|
return str(Path(host_path).expanduser())
|
|
|
|
|
|
def _credential_host_paths() -> set[str]:
|
|
"""Return credential files that are upload-only for remote sandboxes."""
|
|
try:
|
|
from tools.credential_files import get_credential_file_mounts
|
|
mounts = get_credential_file_mounts()
|
|
except Exception:
|
|
return set()
|
|
return {
|
|
_resolve_host_path_str(entry["host_path"])
|
|
for entry in mounts if isinstance(entry, dict) and entry.get("host_path")}
|
|
|
|
|
|
def quoted_rm_command(remote_paths: list[str]) -> str:
|
|
"""Build a shell ``rm -f`` command for a batch of remote paths."""
|
|
return "rm -f " + " ".join(shlex.quote(p) for p in remote_paths)
|
|
|
|
|
|
def quoted_mkdir_command(dirs: list[str]) -> str:
|
|
"""Build a shell ``mkdir -p`` command for a batch of directories."""
|
|
return "mkdir -p " + " ".join(shlex.quote(d) for d in dirs)
|
|
|
|
|
|
def unique_parent_dirs(files: list[tuple[str, str]]) -> list[str]:
|
|
"""Extract sorted unique parent directories from (host, remote) pairs."""
|
|
return sorted({posixpath.dirname(remote) for _, remote in files})
|
|
|
|
|
|
def _sha256_file(path: str) -> str:
|
|
"""Return hex SHA-256 digest of a file."""
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(65536), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
class FileSyncManager:
|
|
"""Tracks local file changes and syncs to a remote environment. Backends supply transport
|
|
callbacks (upload, delete) and a file-source callable; the manager handles mtime-based
|
|
change detection, deletion tracking, rate limiting, and transactional state."""
|
|
|
|
def __init__(
|
|
self,
|
|
get_files_fn: GetFilesFn,
|
|
upload_fn: UploadFn,
|
|
delete_fn: DeleteFn,
|
|
sync_interval: float = _SYNC_INTERVAL_SECONDS,
|
|
bulk_upload_fn: BulkUploadFn | None = None,
|
|
bulk_download_fn: BulkDownloadFn | None = None):
|
|
self._get_files_fn = get_files_fn
|
|
self._upload_fn = upload_fn
|
|
self._bulk_upload_fn = bulk_upload_fn
|
|
self._bulk_download_fn = bulk_download_fn
|
|
self._delete_fn = delete_fn
|
|
self._transaction_lock = threading.Lock()
|
|
self._synced_files: dict[str, tuple[float, int]] = {} # remote_path -> (mtime, size)
|
|
self._pushed_hashes: dict[str, str] = {} # remote_path -> sha256 hex digest
|
|
self._upload_only_host_paths: set[str] = set()
|
|
self._last_sync_time: float = 0.0 # monotonic; 0 ensures first sync runs
|
|
self._sync_interval = sync_interval
|
|
|
|
def sync(self, *, force: bool = False) -> None:
|
|
"""Run a sync cycle: upload changed files, delete removed files. Rate-limited to once
|
|
per ``sync_interval`` unless *force* or ``HERMES_FORCE_FILE_SYNC=1``. Transactional:
|
|
state is committed only if ALL operations succeed; on failure it rolls back so the
|
|
next cycle retries everything."""
|
|
with self._transaction_lock:
|
|
self._sync_transaction(force=force)
|
|
|
|
def _sync_transaction(self, *, force: bool = False) -> None:
|
|
"""Execute one sync cycle while holding the per-manager lock."""
|
|
if (
|
|
not force
|
|
and not os.environ.get(_FORCE_SYNC_ENV)
|
|
and _monotonic() - self._last_sync_time < self._sync_interval):
|
|
return
|
|
|
|
current_files = self._get_files_fn()
|
|
self._upload_only_host_paths.update(_credential_host_paths())
|
|
to_upload, new_files, to_delete = self._plan_sync(current_files)
|
|
|
|
if not to_upload and not to_delete:
|
|
self._last_sync_time = _monotonic()
|
|
return
|
|
|
|
prev_files = dict(self._synced_files)
|
|
prev_hashes = dict(self._pushed_hashes)
|
|
try:
|
|
# Hash and upload the same bytes: the original may be saved while
|
|
# the transport is reading it or waiting for remote acknowledgement.
|
|
with tempfile.TemporaryDirectory(prefix="hermes-sync-push-") as staging:
|
|
staged_files = []
|
|
pushed_hashes = {}
|
|
for index, (host_path, remote_path) in enumerate(to_upload):
|
|
staged_path = os.path.join(staging, str(index))
|
|
shutil.copy2(host_path, staged_path)
|
|
pushed_hashes[remote_path] = _sha256_file(staged_path)
|
|
staged_files.append((staged_path, remote_path))
|
|
self._push(staged_files, to_delete)
|
|
# Commit (all succeeded).
|
|
self._pushed_hashes.update(pushed_hashes)
|
|
for p in to_delete:
|
|
new_files.pop(p, None)
|
|
self._pushed_hashes.pop(p, None)
|
|
self._synced_files = new_files
|
|
self._last_sync_time = _monotonic()
|
|
except Exception as exc:
|
|
self._synced_files = prev_files
|
|
self._pushed_hashes = prev_hashes
|
|
# Do NOT advance _last_sync_time: bumping the rate-limit clock on failure would
|
|
# suppress the retry for up to _sync_interval, contradicting the retry contract.
|
|
logger.warning("file_sync: sync failed, rolled back state: %s", exc)
|
|
|
|
def _plan_sync(
|
|
self, current_files: list[tuple[str, str]]
|
|
) -> tuple[list[tuple[str, str]], dict[str, tuple[float, int]], list[str]]:
|
|
"""Diff *current_files* against synced state -> ``(to_upload, new_synced_state, to_delete)``."""
|
|
to_upload: list[tuple[str, str]] = []
|
|
new_files = dict(self._synced_files)
|
|
for host_path, remote_path in current_files:
|
|
file_key = _file_mtime_key(host_path)
|
|
if file_key is None or self._synced_files.get(remote_path) == file_key:
|
|
continue
|
|
to_upload.append((host_path, remote_path))
|
|
new_files[remote_path] = file_key
|
|
current_remote_paths = {remote for _, remote in current_files}
|
|
to_delete = [p for p in self._synced_files if p not in current_remote_paths]
|
|
return to_upload, new_files, to_delete
|
|
|
|
def _push(self, to_upload: list[tuple[str, str]], to_delete: list[str]) -> None:
|
|
"""Run the transport calls for one cycle (bulk upload when available)."""
|
|
if to_upload:
|
|
logger.debug("file_sync: uploading %d file(s)", len(to_upload))
|
|
if to_delete:
|
|
logger.debug("file_sync: deleting %d stale remote file(s)", len(to_delete))
|
|
if to_upload and self._bulk_upload_fn is not None:
|
|
self._bulk_upload_fn(to_upload)
|
|
logger.debug("file_sync: bulk-uploaded %d file(s)", len(to_upload))
|
|
else:
|
|
for host_path, remote_path in to_upload:
|
|
self._upload_fn(host_path, remote_path)
|
|
logger.debug("file_sync: uploaded %s -> %s", host_path, remote_path)
|
|
if to_delete:
|
|
self._delete_fn(to_delete)
|
|
logger.debug("file_sync: deleted %s", to_delete)
|
|
|
|
# --- Sync-back: pull remote changes to host on teardown ---
|
|
def sync_back(self, hermes_home: Path | None = None) -> None:
|
|
"""Pull remote changes back to the host: download the remote ``.hermes/`` as a tar and
|
|
apply only files whose SHA-256 differs from what was pushed. SIGINT is deferred until
|
|
complete; concurrent gateway sandboxes are serialized via a file lock."""
|
|
with self._transaction_lock:
|
|
self._sync_back_transaction(hermes_home=hermes_home)
|
|
|
|
def _sync_back_transaction(self, hermes_home: Path | None = None) -> None:
|
|
"""Execute sync-back (with retries) against a stable snapshot of manager state."""
|
|
if self._bulk_download_fn is None:
|
|
return
|
|
|
|
# Nothing was ever committed (initial push failed or never ran): skip
|
|
# to avoid retry storms against an uninitialized remote .hermes/.
|
|
if not self._pushed_hashes and not self._synced_files:
|
|
logger.debug("sync_back: no prior push state — skipping")
|
|
return
|
|
|
|
lock_path = (hermes_home or get_hermes_home()) / ".sync.lock"
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
last_exc: Exception | None = None
|
|
for attempt in range(_SYNC_BACK_MAX_RETRIES):
|
|
try:
|
|
self._sync_back_once(lock_path)
|
|
return
|
|
except Exception as exc:
|
|
last_exc = exc
|
|
if attempt < _SYNC_BACK_MAX_RETRIES - 1:
|
|
delay = _SYNC_BACK_BACKOFF[attempt]
|
|
logger.warning("sync_back: attempt %d failed (%s), retrying in %ds", attempt + 1, exc, delay)
|
|
_sleep(delay)
|
|
|
|
logger.warning("sync_back: all %d attempts failed: %s", _SYNC_BACK_MAX_RETRIES, last_exc)
|
|
|
|
def _sync_back_once(self, lock_path: Path) -> None:
|
|
"""Single sync-back attempt with SIGINT protection and file lock."""
|
|
# signal.signal() only works from the main thread; gateway cleanup()
|
|
# may run from a worker thread — skip SIGINT deferral there.
|
|
on_main_thread = threading.current_thread() is threading.main_thread()
|
|
|
|
deferred_sigint: list[object] = []
|
|
original_handler = None
|
|
if on_main_thread:
|
|
original_handler = signal.getsignal(signal.SIGINT)
|
|
|
|
def _defer_sigint(signum, frame):
|
|
deferred_sigint.append((signum, frame))
|
|
logger.debug("sync_back: SIGINT deferred until sync completes")
|
|
|
|
signal.signal(signal.SIGINT, _defer_sigint)
|
|
try:
|
|
self._sync_back_locked(lock_path)
|
|
finally:
|
|
if on_main_thread and original_handler is not None:
|
|
signal.signal(signal.SIGINT, original_handler)
|
|
if deferred_sigint:
|
|
# Re-deliver the deferred Ctrl+C to the restored handler. ``os.kill(os.getpid(),
|
|
# SIGINT)`` is NOT graceful on Windows (routes to TerminateProcess, hard-killing
|
|
# the CLI); ``raise_signal`` invokes the handler everywhere.
|
|
signal.raise_signal(signal.SIGINT)
|
|
|
|
def _sync_back_locked(self, lock_path: Path) -> None:
|
|
"""Sync-back under file lock (serializes concurrent gateways)."""
|
|
if fcntl is None:
|
|
# Windows: no flock — run without serialization
|
|
self._sync_back_impl()
|
|
return
|
|
lock_fd = open(lock_path, "w", encoding="utf-8")
|
|
try:
|
|
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
|
self._sync_back_impl()
|
|
finally:
|
|
try:
|
|
fcntl.flock(lock_fd, fcntl.LOCK_UN)
|
|
except (OSError, IOError):
|
|
pass
|
|
lock_fd.close()
|
|
|
|
def _sync_back_impl(self) -> None:
|
|
"""Download, diff, and apply remote changes to host."""
|
|
if self._bulk_download_fn is None:
|
|
raise RuntimeError("_sync_back_impl called without bulk_download_fn")
|
|
|
|
# Cache file mapping once to avoid O(n*m) from repeated iteration
|
|
try:
|
|
file_mapping = list(self._get_files_fn())
|
|
except Exception:
|
|
file_mapping = []
|
|
|
|
# mkstemp + close: NamedTemporaryFile keeps an exclusive handle on Windows, so the
|
|
# backend's open(dest, "wb") / write_bytes on the same path raised PermissionError.
|
|
fd, tar_path = tempfile.mkstemp(suffix=".tar")
|
|
os.close(fd)
|
|
try:
|
|
self._bulk_download_fn(Path(tar_path))
|
|
|
|
# A misbehaving sandbox could produce an arbitrarily large tar.
|
|
try:
|
|
tar_size = os.path.getsize(tar_path)
|
|
except OSError:
|
|
tar_size = 0
|
|
if tar_size > _SYNC_BACK_MAX_BYTES:
|
|
logger.warning(
|
|
"sync_back: remote tar is %d bytes (cap %d) — skipping extraction",
|
|
tar_size, _SYNC_BACK_MAX_BYTES)
|
|
return
|
|
|
|
with tempfile.TemporaryDirectory(prefix="hermes-sync-back-") as staging:
|
|
with tarfile.open(tar_path) as tar:
|
|
tar.extractall(staging, filter="data")
|
|
|
|
upload_only = self._upload_only_host_paths | _credential_host_paths()
|
|
applied = 0
|
|
for dirpath, _dirnames, filenames in os.walk(staging):
|
|
for fname in filenames:
|
|
staged_file = os.path.join(dirpath, fname)
|
|
# Remote keys are POSIX; relpath uses host separators (backslashes on Windows).
|
|
remote_path = "/" + Path(os.path.relpath(staged_file, staging)).as_posix()
|
|
applied += self._apply_staged_file(staged_file, remote_path, file_mapping, upload_only)
|
|
|
|
if applied:
|
|
logger.info("sync_back: applied %d changed file(s)", applied)
|
|
else:
|
|
logger.debug("sync_back: no remote changes detected")
|
|
finally:
|
|
try:
|
|
os.unlink(tar_path)
|
|
except OSError:
|
|
pass
|
|
|
|
def _apply_staged_file(
|
|
self, staged_file: str, remote_path: str, file_mapping: list[tuple[str, str]], upload_only_host_paths: set[str],
|
|
) -> int:
|
|
"""Copy one extracted remote file onto the host if it changed since push. Returns 1 if
|
|
applied, 0 if skipped (unchanged, unmapped, or an upload-only credential). A host file
|
|
modified since push is overwritten with the remote version (last-write-wins) with a warning."""
|
|
pushed_hash = self._pushed_hashes.get(remote_path)
|
|
if pushed_hash is not None and _sha256_file(staged_file) == pushed_hash:
|
|
return 0 # unchanged from push
|
|
|
|
host_path = self._resolve_host_path(remote_path, file_mapping)
|
|
if host_path is None:
|
|
host_path = self._infer_host_path(remote_path, file_mapping, upload_only_host_paths=upload_only_host_paths)
|
|
if host_path is None:
|
|
logger.debug("sync_back: skipping %s (no host mapping)", remote_path)
|
|
return 0
|
|
|
|
if self._is_upload_only_host_path(host_path, upload_only_host_paths):
|
|
logger.debug("sync_back: skipping upload-only credential file %s", remote_path)
|
|
return 0
|
|
|
|
if pushed_hash is not None and os.path.exists(host_path) and _sha256_file(host_path) != pushed_hash:
|
|
logger.warning(
|
|
"sync_back: conflict on %s — host modified "
|
|
"since push, remote also changed. Applying remote version (last-write-wins).",
|
|
remote_path)
|
|
|
|
os.makedirs(os.path.dirname(host_path), exist_ok=True)
|
|
shutil.copy2(staged_file, host_path)
|
|
return 1
|
|
|
|
def _resolve_host_path(self, remote_path: str, file_mapping: list[tuple[str, str]] | None = None) -> str | None:
|
|
"""Find the host path for a known remote path from the file mapping."""
|
|
return next((host for host, remote in file_mapping or [] if remote == remote_path), None)
|
|
|
|
def _infer_host_path(self, remote_path: str, file_mapping: list[tuple[str, str]] | None = None, *,
|
|
upload_only_host_paths: set[str] | None = None) -> str | None:
|
|
"""Infer a host path for a new remote file by matching path prefixes: an existing
|
|
remote->host pair whose parent directory prefixes *remote_path* gets the same
|
|
substitution (``/root/.hermes/skills/b.md`` -> ``~/.hermes/skills/b.md``)."""
|
|
upload_only_host_paths = upload_only_host_paths or set()
|
|
for host, remote in file_mapping or []:
|
|
if self._is_upload_only_host_path(host, upload_only_host_paths):
|
|
continue
|
|
remote_dir = posixpath.dirname(remote) # remote paths are POSIX even on a Windows host
|
|
if remote_path.startswith(remote_dir + "/"):
|
|
return str(Path(host).parent / remote_path[len(remote_dir) + 1:])
|
|
return None
|
|
|
|
@staticmethod
|
|
def _is_upload_only_host_path(host_path: str, upload_only_host_paths: set[str]) -> bool:
|
|
return _resolve_host_path_str(host_path) in upload_only_host_paths
|