From 57307300ddd501c6780c8d9fe0a144a2cf9f60d3 Mon Sep 17 00:00:00 2001 From: ethernet Date: Wed, 23 Sep 2026 11:37:45 -0400 Subject: [PATCH] feat(update-test): clone and rename-swap instead of tarring the whole home (macOS/Linux) pre tarred the entire HERMES_HOME and post extracted it over an emptied home: two full copies of a tree that is mostly node_modules and venvs. pre now takes a `hermes backup` of the user's data, then clones every top-level entry of HERMES_HOME except cache/ (and the Electron userData) into the backup dir: copy-on-write where the filesystem can (clonefile on APFS, reflink on btrfs/xfs), a plain copy elsewhere. post moves the live entries aside and the clones in, by rename only, so the home directory itself never moves (it may be a mountpoint or symlink target) and cache/ stays where it is. It then deletes the moved-aside post-update trees; hermes-backup.zip is never deleted. --- .../update-test/hermes-update-rehearsal.sh | 207 +++++++++++++----- scripts/update-test/smoke-test.sh | 41 +++- 2 files changed, 182 insertions(+), 66 deletions(-) diff --git a/scripts/update-test/hermes-update-rehearsal.sh b/scripts/update-test/hermes-update-rehearsal.sh index 3cf4a33476..b5cbe68e66 100755 --- a/scripts/update-test/hermes-update-rehearsal.sh +++ b/scripts/update-test/hermes-update-rehearsal.sh @@ -2,16 +2,18 @@ # hermes-update-rehearsal.sh — for an EXISTING Hermes install. # # Two steps: -# pre back up your ENTIRE HERMES_HOME and the desktop app's Electron -# userData, then point the install's update source at a custom repo + -# ref so `hermes update` pulls it. Prints what to do next. -# post wipe both trees and put the backup back exactly as it was. +# pre take a `hermes backup` of your data, clone HERMES_HOME and the desktop +# app's Electron userData into the backup dir, then point the install's +# update source at a custom repo + ref so `hermes update` pulls it. +# Prints what to do next. +# post swap the clones back in, so both trees are exactly as they were. # # Plus `status`, which only prints. This script never judges your install: it # reports what it did and stops. Whether the update worked is for you to see. # -# The backup is one plain tar per tree with nothing filtered out, and `post` -# restores those tars over empty directories, so every file comes back as it was. +# On APFS (macOS) and btrfs/xfs the clone is copy-on-write: no file data is +# copied. Elsewhere it is a plain copy. post swaps by rename, so it is instant +# when the backup dir is on the same disk. HERMES_HOME/cache is left as it is. # # ./hermes-update-rehearsal.sh pre --source --ref # # ... run `hermes update`, use Hermes, test whatever you need ... @@ -23,7 +25,7 @@ # --backup-root DIR where the backup lives (default ~/hermes-update-rehearsal) # --yes post: skip the confirmation # -# Requires: tar and git. `pre` needs network access to --source. +# Requires: git. `pre` needs network access to --source. set -euo pipefail @@ -45,12 +47,12 @@ die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } step() { printf '\n=== %s ===\n' "$*"; } usage() { - if [ -n "$SELF" ] && [ -r "$SELF" ]; then sed -n '2,26p' "$SELF"; exit 0; fi + if [ -n "$SELF" ] && [ -r "$SELF" ]; then sed -n '2,28p' "$SELF"; exit 0; fi cat <<'EOF' hermes-update-rehearsal.sh -- run against an EXISTING Hermes install. - pre back up everything, then point the update source at a custom repo+ref - post wipe both trees and restore the backup exactly as it was + pre back up your data, clone both trees, point the update source at a custom repo+ref + post swap the clones back in, exactly as they were status print what is prepared (read-only; nothing is touched) Options: @@ -86,7 +88,6 @@ done [ -n "$SUBCMD" ] || usage command -v git >/dev/null 2>&1 || die "git is required" -command -v tar >/dev/null 2>&1 || die "tar is required" SNAP="" @@ -95,7 +96,7 @@ SNAP="" # --------------------------------------------------------------------------- # Under git-bash/cygwin, POSIX paths are not translated for native tools -# (git.exe, tar.exe), so normalise them. On macOS cygpath is absent: no-op. +# (git.exe), so normalise them. On macOS cygpath is absent: no-op. native_path() { if command -v cygpath >/dev/null 2>&1; then cygpath -m "$1" 2>/dev/null || printf '%s' "$1" @@ -143,8 +144,9 @@ load_snapshot() { SNAP="$(latest_backup_root)" || die "no backup found under $BACKUP_ROOT — run 'pre' first" local recorded_path="$SNAP/hermes-home.txt" [ -f "$recorded_path" ] || die "$SNAP is not a rehearsal backup (no hermes-home.txt)" + [ -d "$SNAP/home" ] || die "$SNAP has no clone of HERMES_HOME (made by an older version of this script, or already restored)" # Plain text, not JSON: this string is compared byte-for-byte to decide whether - # the backup belongs to the home post is about to wipe. + # the backup belongs to the home post is about to restore. local recorded current recorded="$(tr -d '\r' < "$recorded_path")" current="$(printf '%s' "$HERMES_HOME" | tr '\\' '/')" @@ -152,6 +154,74 @@ load_snapshot() { || die "that backup belongs to HERMES_HOME=$recorded, not $HERMES_HOME; pass --backup-root to pick the right one" } +# --------------------------------------------------------------------------- +# cloning +# --------------------------------------------------------------------------- + +# Pick how to clone into $SNAP: copy-on-write where the filesystem can, a plain +# copy elsewhere. Probed on a real file, since support is per filesystem. +CLONE_MODE="" +detect_clone_mode() { + local probe="$SNAP/.clone-probe" + printf 'x' > "$probe" + # GNU cp also accepts -c (a deprecated --preserve=context), so only ask + # macOS's cp for clonefile. + if [ "$(uname -s)" = Darwin ] && cp -c "$probe" "$probe.c" 2>/dev/null; then + CLONE_MODE="clonefile" # macOS APFS + elif cp --reflink=always "$probe" "$probe.c" 2>/dev/null; then + CLONE_MODE="reflink" # btrfs, xfs, bcachefs + elif cp --reflink=auto "$probe" "$probe.c" 2>/dev/null; then + CLONE_MODE="copy-gnu" # GNU cp, no CoW here (ext4, ...) + else + CLONE_MODE="copy" # BSD cp without clonefile (non-APFS mac) + fi + rm -f "$probe" "$probe.c" +} + +clone_entry() { + case "$CLONE_MODE" in + clonefile) cp -cpR "$1" "$2/" ;; + reflink|copy-gnu) cp -a --reflink=auto "$1" "$2/" ;; + *) cp -pR "$1" "$2/" ;; + esac +} + +# Clone every top-level entry of $1 into $2, except one named $3 (may be empty). +clone_tree() { + local src="$1" dst="$2" skip="$3" e + mkdir -p "$dst" + while IFS= read -r -d '' e; do + [ -n "$skip" ] && [ "$(basename "$e")" = "$skip" ] && continue + clone_entry "$e" "$dst" || return 1 + done < <(find "$src" -mindepth 1 -maxdepth 1 -print0) +} + +# post: move $live's top-level entries (except $skip) into $aside, then the +# clone's entries into $live. Renames only, so the live directory itself never +# moves -- it may be a mountpoint or a symlink target. +swap_tree() { + local live="$1" clone="$2" aside="$3" skip="$4" e + mkdir -p "$aside" "$live" + while IFS= read -r -d '' e; do + [ -n "$skip" ] && [ "$(basename "$e")" = "$skip" ] && continue + mv "$e" "$aside/" || return 1 + done < <(find "$live" -mindepth 1 -maxdepth 1 -print0) + while IFS= read -r -d '' e; do + mv "$e" "$live/" || return 1 + done < <(find "$clone" -mindepth 1 -maxdepth 1 -print0) + rmdir "$clone" +} + +# The install's own hermes: pre-branch installs keep it in the venv. +resolve_hermes_exe() { + local c + for c in "$INSTALL_DIR/venv/bin/hermes" "$INSTALL_DIR/.hermes/bin/hermes" \ + "$INSTALL_DIR/venv/Scripts/hermes.exe" "$HERMES_HOME/bin/hermes"; do + [ -x "$c" ] && { printf '%s' "$c"; return 0; } + done + return 1 +} + # --------------------------------------------------------------------------- # pre: back up, then point the install at the rehearsal source # --------------------------------------------------------------------------- @@ -165,6 +235,8 @@ cmd_pre() { say "backup to $BACKUP_ROOT" [ -d "$HERMES_HOME" ] || die "no HERMES_HOME at $HERMES_HOME" [ -d "$INSTALL_DIR/.git" ] || die "no git checkout at $INSTALL_DIR — this tool covers source installs" + local hermes_exe + hermes_exe="$(resolve_hermes_exe)" || die "no hermes executable found in $INSTALL_DIR (venv/bin, .hermes/bin) or $HERMES_HOME/bin" step "before we start (nothing here is a pass/fail, just read it)" if command -v pgrep >/dev/null 2>&1; then @@ -187,32 +259,39 @@ cmd_pre() { [ ! -e "$SNAP" ] || die "backup dir already exists: $SNAP" mkdir -p "$SNAP" - step "backing up your entire HERMES_HOME" - local started elapsed tar_pid + step "backing up your data (hermes backup)" + # Config, keys, sessions, memories, skills, with SQLite copied consistently: + # a second, independent copy of what matters most. post never deletes it. + local started code=0 out started="$SECONDS" - # No excludes: checkout, venv, PM store and node_modules come too, so `post` - # is a true rollback rather than a re-download. SQLite sidecars travel WITH - # their db on purpose (a raw copy of db+wal+shm is consistent). - # No -z: the backup root is typically the same internal disk, so the size - # saving buys nothing and gzip costs ~5x the wall time on a checkout-sized - # tree (72s vs 14s measured on an M1). bsdtar (macOS) and GNU tar differ on - # progress options, so poll the growing archive instead — portable, and it - # doubles as a liveness signal. - tar -cf "$SNAP/hermes-home.tar" -C "$HERMES_HOME" . & - tar_pid=$! - while kill -0 "$tar_pid" 2>/dev/null; do - printf '\r %s written... ' "$(du -h "$SNAP/hermes-home.tar" 2>/dev/null | cut -f1)" - sleep 2 - done - printf '\r \r' - wait "$tar_pid" || die "backup failed (tar)" - elapsed=$((SECONDS - started)) - ok "hermes-home.tar ($(du -h "$SNAP/hermes-home.tar" | cut -f1), ${elapsed}s)" + out="$(HERMES_HOME="$HERMES_HOME" "$hermes_exe" backup -o "$SNAP/hermes-backup.zip" 2>&1)" || code=$? + if [ "$code" = 1 ] && [ -f "$SNAP/hermes-backup.zip" ]; then + printf ' %s\n' "$out" + warn "hermes backup finished INCOMPLETE ($((SECONDS - started))s): the files listed above are not in $SNAP/hermes-backup.zip" + elif [ "$code" != 0 ] || [ ! -f "$SNAP/hermes-backup.zip" ]; then + printf ' %s\n' "$out" + die "hermes backup failed (exit $code) — nothing else was done" + else + ok "hermes-backup.zip ($(du -h "$SNAP/hermes-backup.zip" | cut -f1), $((SECONDS - started))s)" + fi - step "backing up the desktop app's data" + step "cloning HERMES_HOME and the desktop app's data" + detect_clone_mode + case "$CLONE_MODE" in + clonefile|reflink) ok "copy-on-write clones ($CLONE_MODE): no file data is copied" ;; + *) ok "this filesystem cannot clone, so this is a plain copy" ;; + esac + started="$SECONDS" + # Everything but cache/: checkout, venv, PM store and node_modules come too, + # so post is a true rollback rather than a re-download. SQLite sidecars travel + # WITH their db on purpose (a raw copy of db+wal+shm is consistent). + clone_tree "$HERMES_HOME" "$SNAP/home" cache || die "cloning HERMES_HOME failed" + ok "HERMES_HOME -> $SNAP/home ($((SECONDS - started))s, cache/ left out)" + local userdata_existed=false if [ -d "$USERDATA_DIR" ]; then - tar -cf "$SNAP/electron-userdata.tar" -C "$USERDATA_DIR" . || die "userData backup failed" - ok "electron-userdata.tar ($(du -h "$SNAP/electron-userdata.tar" | cut -f1))" + userdata_existed=true + clone_tree "$USERDATA_DIR" "$SNAP/userdata" "" || die "cloning the desktop app's data failed" + ok "desktop data -> $SNAP/userdata" else warn "no Electron userData at $USERDATA_DIR (desktop app not installed?)" fi @@ -221,12 +300,14 @@ cmd_pre() { printf '%s\n' "$HERMES_HOME" > "$SNAP/hermes-home.txt" cat > "$SNAP/manifest.json" < "$SNAP/target-sha" ok "official repo URL now resolves to the rehearsal copy" @@ -309,6 +389,8 @@ cmd_status() { else say "prepared no" fi + say "clone $([ -d "$SNAP/home" ] && echo "present ($(sed -n 's/.*"clone_mode": "\(.*\)",/\1/p' "$SNAP/manifest.json" 2>/dev/null | head -1))" || echo 'none (restored already?)')" + say "data backup $([ -f "$SNAP/hermes-backup.zip" ] && echo hermes-backup.zip || echo none)" say "marker $([ -f "$HERMES_HOME/.skip_upstream_prompt" ] && echo present || echo absent)" local n=0 while IFS= read -r _; do n=$((n + 1)); done \ @@ -320,7 +402,7 @@ cmd_status() { } # --------------------------------------------------------------------------- -# post: wipe both trees, then put the backup back +# post: swap the clones back in # --------------------------------------------------------------------------- confirm() { @@ -359,10 +441,12 @@ remove_stale_global_redirect() { cmd_post() { resolve_paths load_snapshot - step "this will delete and restore:" - say " $HERMES_HOME (all of it, including the checkout)" + local userdata_existed + userdata_existed="$(sed -n 's/.*"userdata_existed": \([a-z]*\).*/\1/p' "$SNAP/manifest.json" | head -1)" + step "this will restore:" + say " $HERMES_HOME (everything except cache/, including the checkout)" say " $USERDATA_DIR" - say " from $SNAP" + say " to how they were at $SNAP" confirm "Put everything back from $SNAP?" step "stopping Hermes" @@ -370,31 +454,40 @@ cmd_post() { pkill -f "hermes gateway" 2>/dev/null || true ok "asked Hermes to stop (if anything was running)" - step "clearing both trees" - rm -rf "$HERMES_HOME"; ok "removed $HERMES_HOME" - rm -rf "$USERDATA_DIR"; ok "removed $USERDATA_DIR" + local aside="$SNAP/replaced" + [ ! -e "$aside" ] || die "$aside already exists (an interrupted post?) — nothing was changed; move it away and run post again" step "restoring your HERMES_HOME" - mkdir -p "$HERMES_HOME" - tar -xf "$SNAP/hermes-home.tar" -C "$HERMES_HOME" || die "restore failed — your backup is intact at $SNAP" - ok "restored" + swap_tree "$HERMES_HOME" "$SNAP/home" "$aside/home" cache \ + || die "restore stopped part-way: the post-update files are in $aside/home, the rest of the clone in $SNAP/home" + ok "restored (cache/ left as it was)" step "restoring the desktop app's data" - if [ -f "$SNAP/electron-userdata.tar" ]; then - mkdir -p "$USERDATA_DIR" - tar -xf "$SNAP/electron-userdata.tar" -C "$USERDATA_DIR" || die "userData restore failed (backup intact at $SNAP)" + if [ "$userdata_existed" = true ]; then + swap_tree "$USERDATA_DIR" "$SNAP/userdata" "$aside/userdata" "" \ + || die "restore stopped part-way: the post-update files are in $aside/userdata, the rest of the clone in $SNAP/userdata" ok "restored" + elif [ -e "$USERDATA_DIR" ]; then + # There was no desktop data at the snapshot: exact means none now either. + mkdir -p "$aside" + mv "$USERDATA_DIR" "$aside/userdata" + ok "removed $USERDATA_DIR (it did not exist before)" else - warn "there was no desktop app data to restore" + ok "nothing to restore (there was no desktop data before)" fi + step "cleaning up" + # Only the whole trees the update left behind. hermes-backup.zip stays. + rm -rf "$aside" + ok "deleted the post-update trees ($aside)" + remove_stale_global_redirect step "done" say "Your HERMES_HOME and the desktop app's data are back exactly as they were." say "Open the desktop app once and run 'hermes doctor' to confirm." - say "Nothing was judged or changed by this script; the backup at $SNAP is" - say "yours to keep or delete." + say "Nothing was judged or changed by this script; the backup at $SNAP" + say "(including hermes-backup.zip) is yours to keep or delete." } case "$SUBCMD" in diff --git a/scripts/update-test/smoke-test.sh b/scripts/update-test/smoke-test.sh index 61e3314892..06d951fc45 100755 --- a/scripts/update-test/smoke-test.sh +++ b/scripts/update-test/smoke-test.sh @@ -17,7 +17,6 @@ export HOME="$ROOT/home"; mkdir -p "$HOME" export GIT_CONFIG_GLOBAL="$ROOT/gitconfig-test"; : > "$GIT_CONFIG_GLOBAL" export HERMES_HOME="$ROOT/home/.hermes" export HERMES_DESKTOP_USER_DATA_DIR="$ROOT/electron-user-data" -if tar --help 2>&1 | grep -q -- '--force-local'; then export TAR_OPTIONS=--force-local; fi H="$HERMES_HOME"; INSTALL="$H/hermes-agent" @@ -37,6 +36,8 @@ printf 'console.log(1)\n' > "$H/photon/sidecar/index.mjs" printf '{"name":"sidecar"}\n' > "$H/photon/sidecar/package.json" printf '{"lockfileVersion":3}\n' > "$H/photon/sidecar/package-lock.json" printf '{"lockfileVersion":3}\n' > "$H/photon/sidecar/node_modules/.package-lock.json" +mkdir -p "$H/cache/scratch" +printf 'throwaway\n' > "$H/cache/scratch/keep.txt" python - "$H/state.db" <<'PY' import sqlite3, sys @@ -57,7 +58,13 @@ git -C "$INSTALL" -c commit.gpgsign=false commit -qm initial git -C "$INSTALL" remote add origin https://github.com/NousResearch/hermes-agent.git HEAD_SHA="$(git -C "$INSTALL" rev-parse HEAD)" mkdir -p "$INSTALL/.hermes/bin" "$INSTALL/.hermes-runtime/python" -printf '#!/bin/sh\necho hermes 0.0.0\n' > "$INSTALL/.hermes/bin/hermes"; chmod +x "$INSTALL/.hermes/bin/hermes" +# A fake launcher that understands `backup -o `: pre calls it for the data backup. +cat > "$INSTALL/.hermes/bin/hermes" <<'SH' +#!/bin/sh +if [ "$1" = backup ] && [ "$2" = -o ]; then printf 'fake-zip\n' > "$3"; exit 0; fi +echo hermes 0.0.0 +SH +chmod +x "$INSTALL/.hermes/bin/hermes" printf 'big' > "$INSTALL/.hermes-runtime/python/interpreter.bin" mkdir -p "$HOME/.local/bin" ln -sfn "$INSTALL/.hermes/bin/hermes" "$HOME/.local/bin/hermes" @@ -88,15 +95,16 @@ cp -a "$HERMES_DESKTOP_USER_DATA_DIR/." "$ROOT/pristine/userdata/" "${RUN[@]}" pre --source "$INSTALL" --ref main --backup-root "$BACKUPS" > "$ROOT/pre.log" 2>&1 check $? "pre exits 0" SNAP="$(ls -1d "$BACKUPS"/*/ | head -1)"; SNAP="${SNAP%/}" -for f in hermes-home.tar electron-userdata.tar manifest.json hermes-home.txt target-sha; do +for f in hermes-backup.zip home manifest.json hermes-home.txt target-sha; do [ -e "$SNAP/$f" ]; check $? "backup artifact $f" done -TARLIST="$(tar -tf "$SNAP/hermes-home.tar" 2>&1 || true)" -grep -qE '^\./hermes-agent/\.git/config$' <<< "$TARLIST"; check $? "whole home: checkout .git in the tar" -grep -qE '^\./hermes-agent/\.hermes-runtime/python/interpreter\.bin$' <<< "$TARLIST"; check $? "whole home: PM store in the tar" -grep -qE '^\./config\.yaml$' <<< "$TARLIST"; check $? "whole home: config.yaml in the tar" -UDLIST="$(tar -tf "$SNAP/electron-userdata.tar" 2>&1 || true)" -grep -qE '^\./Cache/data\.bin$' <<< "$UDLIST"; check $? "whole userData: nothing filtered out of the tar" +[ -f "$SNAP/home/hermes-agent/.git/config" ]; check $? "whole home: checkout .git in the clone" +[ -f "$SNAP/home/hermes-agent/.hermes-runtime/python/interpreter.bin" ]; check $? "whole home: PM store in the clone" +[ -f "$SNAP/home/config.yaml" ]; check $? "whole home: config.yaml in the clone" +[ ! -e "$SNAP/home/cache" ]; check $? "cache/ left out of the clone" +[ -f "$SNAP/userdata/Cache/data.bin" ]; check $? "whole userData: nothing filtered out of the clone" +grep -q 'clone_mode' "$SNAP/manifest.json"; check $? "manifest records the clone mode" +echo " clone mode: $(sed -n 's/.*"clone_mode": "\(.*\)",/\1/p' "$SNAP/manifest.json")" echo echo "--- pre points the install at the rehearsal copy ---" @@ -115,6 +123,17 @@ grep -q 'nothing has been updated yet' "$ROOT/pre.log"; check $? "pre says it di echo echo "--- status (read-only) ---" check_out "$HEAD_SHA" "status reports what it prepared" "${RUN[@]}" status --backup-root "$BACKUPS" +check_out "clone *present" "status reports the clone present" "${RUN[@]}" status --backup-root "$BACKUPS" + +echo +echo "--- simulate an update: modify, add and delete in both trees ---" +printf 'timezone: changed-by-update\n' > "$H/config.yaml" +rm "$H/memories/note.md" +printf 'print(2)\n' > "$INSTALL/added_by_update.py" +mkdir -p "$H/photon/sidecar/node_modules/newdep"; printf 'x' > "$H/photon/sidecar/node_modules/newdep/index.js" +printf '{"window":{"changed":true}}\n' > "$HERMES_DESKTOP_USER_DATA_DIR/Preferences" +printf '{}\n' > "$HERMES_DESKTOP_USER_DATA_DIR/new-after-update.json" +! diff -r --no-dereference "$ROOT/pristine/home" "$H" >/dev/null 2>&1; check $? "the simulated update changed HERMES_HOME" echo echo "--- post ---" @@ -133,6 +152,10 @@ check $? "post exits 0" [ "$(git -C "$INSTALL" config --local --get-regexp 'insteadOf' 2>/dev/null | wc -l | tr -d ' ')" = 0 ]; check $? "no stale insteadOf left in the checkout" [ ! -f "$H/.skip_upstream_prompt" ]; check $? "upstream-prompt marker removed" git -C "$INSTALL" remote get-url origin | grep -q 'NousResearch'; check $? "origin resolves officially again" +[ -f "$H/cache/scratch/keep.txt" ]; check $? "cache/ left in place" +[ -f "$SNAP/hermes-backup.zip" ]; check $? "post keeps hermes-backup.zip" +[ ! -e "$SNAP/home" ] && [ ! -e "$SNAP/userdata" ]; check $? "post consumed the clones" +[ ! -e "$SNAP/replaced" ]; check $? "post deleted the post-update trees" if [ "$SYMLINKS_OK" = 1 ]; then [ -L "$HOME/.local/bin/hermes" ]; check $? "shim outside the two trees left untouched" else