hermes-agent/apps/bootstrap-installer/src-tauri/src/update.rs

//! Update orchestration.
//!
//! Driven when the installer is launched as `Hermes-Setup.exe --update` (see
//! `AppMode` in lib.rs). The desktop app hands off to us — it exits, then we:
//!
//! Application output locks protect replacement. Python owns dependency
//! generations, product compilation, and gateway draining/restart. Only a
//! pre-PM runtime gets the historical extra rebuild/retry. Published launchers
//! bind every command to the installation; user-bin migration verifies identity
//! through the existing `--version` surface before selecting an older launcher.

use std::collections::VecDeque;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};

use anyhow::{anyhow, Result};
use tauri::{AppHandle, Emitter};
use tokio::process::Command;

use crate::events::{BootstrapEvent, LogStream, StageInfo, StageState};
use crate::marker::{
    should_heal_self_marker_refusal, AcquireError, MarkerOwner, UpdateMarkerGuard,
};
use crate::powershell::{pump_child, DRAIN_GRACE};

/// `hermes update` exit code meaning "another hermes process is holding the
/// venv shim open / dirty precondition" — see _cmd_update_impl in
/// hermes_cli/main.py (sys.exit(2)). We surface a targeted message for this.
pub(crate) const UPDATE_EXIT_CONCURRENT: i32 = 2;

/// How long to wait for the old desktop process to release files under the
/// install tree before refusing the handoff.
const DESKTOP_EXIT_WAIT: Duration = Duration::from_secs(20);
const DESKTOP_EXIT_POLL: Duration = Duration::from_millis(500);

/// Guards against concurrent update runs. The frontend kicks `startUpdate()`
/// from a mount effect, which can fire more than once (React strict-mode
/// double-invokes effects in dev; a window reload or stray re-init can do it
/// in prod). Two `run_update` tasks racing on `git stash` corrupt the working
/// tree — one stashes the changes the other then can't find. Exactly one task
/// may hold this flag at a time.
static UPDATE_RUNNING: AtomicBool = AtomicBool::new(false);

/// Frontend → Rust: kick off the update flow. Mirrors `start_bootstrap`'s
/// fire-and-forget shape; progress arrives on the `bootstrap` event channel.
#[tauri::command]
pub async fn start_update(app: AppHandle) -> Result<(), String> {
    // Re-entrancy guard (see UPDATE_RUNNING). compare_exchange lets exactly one
    // caller flip false→true; any concurrent caller no-ops instead of spawning
    // a second racing update.
    if UPDATE_RUNNING
        .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
        .is_err()
    {
        // Already running: re-emit the manifest so a duplicate startUpdate()
        // call (which resets the frontend store) can recover its stage list.
        let target_app = if cfg!(target_os = "macos") {
            target_app_from_args(std::env::args().skip(1))
        } else {
            None
        };
        emit(
            &app,
            BootstrapEvent::Manifest {
                stages: update_stages(target_app.is_some()),
                protocol_version: None,
            },
        );
        return Ok(());
    }
    tokio::spawn(async move {
        if let Err(err) = run_update(app.clone()).await {
            // run_update already emits a Failed event on the paths that matter;
            // this catches anything that escaped. Emit defensively.
            emit(
                &app,
                BootstrapEvent::Failed {
                    stage: None,
                    error: format!("{err:#}"),
                },
            );
        }
        UPDATE_RUNNING.store(false, Ordering::SeqCst);
    });
    Ok(())
}

/// The refusal shown when a live update already holds the marker. pid 0 is
/// a claim still being published (see `marker::inspect_marker_locked`), or,
/// with `held`, a dead update whose process still holds the checkout (R6).
fn busy_update_message(owner: &MarkerOwner) -> String {
    let wait = "Wait for it to finish, or close the window or dashboard tab that \
                started it, then try again.";
    let mins = owner.age_secs / 60;
    let secs = owner.age_secs % 60;
    let elapsed = if mins > 0 {
        format!("{mins}m {secs}s")
    } else {
        format!("{secs}s")
    };
    if owner.held {
        return format!(
            "Another Hermes update is already running (started {elapsed} ago; its owner \
             exited but a process it started still holds the checkout). {wait}"
        );
    }
    if owner.pid == 0 {
        return format!("Another Hermes update is starting right now. {wait}");
    }
    format!(
        "Another Hermes update is already running (PID {}, started {elapsed} ago). {wait}",
        owner.pid
    )
}

async fn run_update(app: AppHandle) -> Result<()> {
    let hermes_home = crate::paths::hermes_home();
    let install_root = hermes_home.join("hermes-agent");

    // Mutual exclusion (#50238): publish an "update in progress" marker for the
    // entire duration of this update. A desktop instance the user relaunches
    // mid-update consults this before spawning its own local backend — without
    // it, that backend re-locks the venv shim, our `force_kill_other_hermes`
    // straggler-cleanup kills it, and the relaunch/kill cycle loops. The guard
    // removes the marker on every exit path (incl. early returns / panics).
    //
    // The same marker is the cross-process update lock (hermes_cli/
    // update_lock.py claims it too), so a live foreign owner means another
    // updater — most often a dashboard-spawned `hermes update` — is already
    // mutating this checkout. Refuse instead of running a second one over it.
    let _update_marker = match UpdateMarkerGuard::acquire(
        crate::paths::update_in_progress_marker(),
        &install_root,
    ) {
        Ok(guard) => guard,
        Err(err) => {
            let msg = match err {
                AcquireError::Busy(owner) => busy_update_message(&owner),
                AcquireError::Unwritable(msg) => msg,
            };
            emit(
                &app,
                BootstrapEvent::Failed {
                    stage: None,
                    error: msg.clone(),
                },
            );
            return Err(anyhow!(msg));
        }
    };

    let update_branch = update_branch_from_args(std::env::args().skip(1))
        .or_else(|| option_env_string("BUILD_PIN_BRANCH"))
        .unwrap_or_else(|| "main".to_string());
    let target_app = if cfg!(target_os = "macos") {
        target_app_from_args(std::env::args().skip(1))
    } else {
        None
    };

    let legacy_install = !install_root.join("pm").is_dir();
    let hermes = resolve_hermes(&install_root).await.ok_or_else(|| {
        let msg = format!(
            "Could not find the hermes CLI under {}. Is Hermes installed? \
             Re-run the installer to repair the install.",
            install_root.display()
        );
        emit(
            &app,
            BootstrapEvent::Failed {
                stage: None,
                error: msg.clone(),
            },
        );
        anyhow!(msg)
    })?;

    // Synthetic manifest so the existing progress UI renders our stages.
    emit(
        &app,
        BootstrapEvent::Manifest {
            stages: update_stages(target_app.is_some()),
            protocol_version: None,
        },
    );

    // ---- stage 1: wait for the old desktop to die ------------------------
    // Windows process teardown is asynchronous; the old app must release the
    // packaged payload before it can be replaced. Python readers are unrelated.
    let started = Instant::now();
    emit_stage(&app, "handoff", StageState::Running, None, None);
    wait_for_install_locks_free(&install_root, &app, "handoff").await?;
    emit_stage(
        &app,
        "handoff",
        StageState::Succeeded,
        Some(started.elapsed().as_millis() as u64),
        None,
    );

    // ---- stage 2: hermes update -----------------------------------------
    // Pass --branch so `hermes update` targets the branch this installer was
    // built/pinned against (BUILD_PIN_BRANCH), NOT its built-in default of
    // `main`. The install was a detached-HEAD checkout of a specific commit;
    // without --branch, `hermes update` switches the checkout to `main` (a
    // divergent branch that may not even have the desktop CLI command), then
    // reports "already up to date" against the wrong branch. The desktop
    // detected the update against this same branch, so we must update against
    // it too.
    emit_log(
        &app,
        Some("update"),
        LogStream::Stdout,
        &format!("[update] updating against branch {update_branch}"),
    );
    let child_env = update_child_env(&install_root);
    let mut update_args: Vec<String> =
        vec!["update".into(), "--yes".into(), "--gateway".into()];
    // Only historical in-place updaters need the old shim bypass.
    if legacy_install { update_args.push("--force".into()); }
    update_args.push("--branch".into());
    update_args.push(update_branch);

    emit_stage(&app, "update", StageState::Running, None, None);
    let started = Instant::now();
    let mut update = run_streamed(
        &app,
        &hermes,
        &update_args,
        &install_root,
        &child_env,
        Some("update"),
    )
    .await?;

    // Retry-once for the update-boundary crash. `hermes update` lazily imports
    // the FRESHLY PULLED modules, but the dependency-install step still runs the
    // already-in-memory pre-pull code for one invocation. A release that changed
    // an updater-path contract across that boundary (e.g. #39780's `_UvResult`,
    // whose `__iter__` injected a bool into the argv and crashed Windows
    // `list2cmdline` with `TypeError: sequence item 1: expected str instance,
    // bool found`, fixed in #39820) therefore kills the FIRST update on the
    // parked population — even though the fix is already on disk by then. A
    // second `hermes update` runs clean because the now-current module is loaded
    // from the start. Rather than make the parked user click Update twice (and
    // stare at a scary crash first), retry once automatically. Skip the retry
    // for the concurrent-instance guard (exit 2) — that's a "close Hermes" state
    // a retry can't fix.
    if legacy_install && !matches!(update.exit_code, Some(0) | Some(UPDATE_EXIT_CONCURRENT)) {
        emit_log(
            &app,
            Some("update"),
            LogStream::Stdout,
            "[update] first update attempt failed; retrying once (the fix it just \
             pulled loads on the second run)…",
        );
        update = run_streamed(
            &app,
            &hermes,
            &update_args,
            &install_root,
            &child_env,
            Some("update"),
        )
        .await?;
    }

    // Self-owned-marker heal (#75788). Exit 2 means the child refused over a
    // live update marker with a foreign owner. When that "foreign" owner is
    // THIS process, the child simply failed to recognize the handoff — a
    // checkout predating the HERMES_UPDATE_HANDOFF_PID env fix (8c76fe19f)
    // and the ancestor-pid fallback runs its pre-pull update_lock.py, reads
    // our marker, and exits 2 every time. The refusal loop is unbreakable
    // from the user's side because the update being refused is the one that
    // ships the fix. The marker exists to serialize updates and this process
    // IS the update: drop our claim and retry once with the marker absent.
    // The guard re-removes on Drop (idempotent), and the desktop is already
    // gone at this point, so nothing races the brief marker-free window.
    if legacy_install && should_heal_self_marker_refusal(
        update.exit_code,
        &crate::paths::update_in_progress_marker(),
        &install_root,
    ) {
        emit_log(
            &app,
            Some("update"),
            LogStream::Stdout,
            "[update] child refused over this updater's own marker (stale \
             checkout without handoff recognition); clearing the claim and \
             retrying once…",
        );
        _update_marker.complete();
        update = run_streamed(
            &app,
            &hermes,
            &update_args,
            &install_root,
            &child_env,
            Some("update"),
        )
        .await?;
    }
    let update_ms = started.elapsed().as_millis() as u64;

    match update.exit_code {
        Some(0) => {
            emit_stage(&app, "update", StageState::Succeeded, Some(update_ms), None);
        }
        Some(code) if code == UPDATE_EXIT_CONCURRENT => {
            let msg = concurrent_update_message(&update.stdout_tail);
            emit_stage(
                &app,
                "update",
                StageState::Failed,
                Some(update_ms),
                Some(msg.clone()),
            );
            emit(
                &app,
                BootstrapEvent::Failed {
                    stage: Some("update".into()),
                    error: msg.clone(),
                },
            );
            return Err(anyhow!(msg));
        }
        other => {
            let msg = format!(
                "hermes update failed (exit {:?}). See {} for details.",
                other,
                crate::paths::hermes_home()
                    .join("logs")
                    .join("update.log")
                    .display()
            );
            emit_stage(
                &app,
                "update",
                StageState::Failed,
                Some(update_ms),
                Some(msg.clone()),
            );
            emit(
                &app,
                BootstrapEvent::Failed {
                    stage: Some("update".into()),
                    error: msg.clone(),
                },
            );
            return Err(anyhow!(msg));
        }
    }

    // Older updaters did not own desktop compilation. Current PM update
    // composes all products and propagates failures; never build them twice.
    if legacy_install {
        emit_stage(&app, "rebuild", StageState::Running, None, None);
        let started = Instant::now();
        let rebuild_args: Vec<String> = vec!["desktop".into(), "--build-only".into()];
        let mut rebuild = run_streamed(&app, &hermes, &rebuild_args, &install_root, &child_env, Some("rebuild")).await?;

        // Retry-once: the first `--build-only` can return nonzero on a still-settling
        // post-update tree or a network-blocked Electron fetch that our self-heal
        // repaired mid-run. A second attempt then builds clean off the healed dist
        // (the content-hash stamp makes it a near-no-op when the first actually
        // succeeded). Without this the updater bails here and never reaches the
        // relaunch below — the app updates but doesn't restart. Matches the
        // retry-once `hermes update` already does above, and `hermes update`'s own
        // desktop rebuild in cmd_update.
        if rebuild_needs_retry(rebuild.exit_code) {
            emit_log(
                &app,
                Some("rebuild"),
                LogStream::Stdout,
                "[rebuild] first desktop rebuild failed; retrying once (a self-healed \
                 Electron download builds clean on the second run)…",
            );
            rebuild = run_streamed(
                &app,
                &hermes,
                &rebuild_args,
                &install_root,
                &child_env,
                Some("rebuild"),
            )
            .await?;
        }
        let rebuild_ms = started.elapsed().as_millis() as u64;

        if rebuild.exit_code != Some(0) {
            let msg = format!(
                "Rebuilding the desktop app failed (exit {:?}). The update was \
                 applied but the app could not be rebuilt; run `hermes desktop` \
                 from a terminal to see the error.",
                rebuild.exit_code
            );
            emit_stage(
                &app,
                "rebuild",
                StageState::Failed,
                Some(rebuild_ms),
                Some(msg.clone()),
            );
            emit(
                &app,
                BootstrapEvent::Failed {
                    stage: Some("rebuild".into()),
                    error: msg.clone(),
                },
            );
            return Err(anyhow!(msg));
        }
        emit_stage(&app, "rebuild", StageState::Succeeded, Some(rebuild_ms), None);
    } else {
        emit_stage(&app, "rebuild", StageState::Succeeded, Some(0), None);
    }

    let launch_target = if let Some(target_app) = target_app {
        let started = Instant::now();
        emit_stage(&app, "install", StageState::Running, None, None);
        match install_macos_app_update(&app, &install_root, &target_app).await {
            Ok(installed_app) => {
                emit_stage(
                    &app,
                    "install",
                    StageState::Succeeded,
                    Some(started.elapsed().as_millis() as u64),
                    None,
                );
                Some(installed_app)
            }
            Err(err) => {
                let msg = format!("{err:#}");
                emit_stage(
                    &app,
                    "install",
                    StageState::Failed,
                    Some(started.elapsed().as_millis() as u64),
                    Some(msg.clone()),
                );
                emit(
                    &app,
                    BootstrapEvent::Failed {
                        stage: Some("install".into()),
                        error: msg.clone(),
                    },
                );
                return Err(anyhow!(msg));
            }
        }
    } else {
        None
    };

    // ---- done: signal complete, then launch the fresh desktop ------------
    emit(
        &app,
        BootstrapEvent::Complete {
            install_root: install_root.to_string_lossy().into_owned(),
            marker: None,
        },
    );
    // Every install-tree mutation is finished. Release the lock BEFORE the
    // relaunch: this process can stay wedged in its native event loop even
    // after a successful app.exit(), and a live pid on a fresh marker would
    // make a completed update look active — blocking desktop startup and
    // every other updater for as long as this process lives (v2 markers
    // have no age ceiling).
    _update_marker.complete();

    if let Some(target_app) = launch_target {
        if let Err(err) = launch_macos_app_and_exit(&app, &target_app).await {
            emit_log(
                &app,
                None,
                LogStream::Stderr,
                &format!("[update] could not auto-launch desktop: {err}. Launch Hermes manually."),
            );
        }
    } else if let Err(err) =
        crate::bootstrap::launch_hermes_desktop(app.clone(), install_root.to_string_lossy().into_owned()).await
    {
        // Launch failed: don't hard-fail the update (it succeeded); surface a
        // log line so the success screen can still tell the user to launch
        // manually.
        emit_log(
            &app,
            None,
            LogStream::Stdout,
            &format!("[update] could not auto-launch desktop: {err}. Launch Hermes manually."),
        );
    }

    // The launch helpers normally request exit themselves, but their failure
    // paths must still close a successful updater. A native event loop can
    // ignore that graceful request, so arm a process-exit fallback now that
    // all update state and the marker have been settled.
    exit_after_success(&app);
    Ok(())
}

/// Ask the app to exit, with a hard `process::exit` fallback for a native
/// event loop that ignores the graceful request. Without it a finished updater
/// can linger as a live pid forever.
fn exit_after_success(app: &AppHandle) {
    std::thread::spawn(|| {
        std::thread::sleep(std::time::Duration::from_secs(3));
        tracing::warn!("graceful updater exit timed out; forcing process exit");
        std::process::exit(0);
    });
    app.exit(0);
}

/// Wait for the application payload being replaced, never PM dependency readers.
pub(crate) async fn wait_for_install_locks_free(install_root: &Path, app: &AppHandle, stage: &str) -> Result<()> {
    let lock_targets = install_lock_probe_paths(install_root);
    let deadline = Instant::now() + DESKTOP_EXIT_WAIT;
    emit_log(app, Some(stage), LogStream::Stdout, "[handoff] waiting for Hermes to exit…");
    loop {
        let locked = locked_paths(&lock_targets);
        if locked.is_empty() {
            return Ok(());
        }
        if Instant::now() >= deadline {
            return Err(anyhow!("Desktop application files are still locked: {}. Close the other Hermes window and retry.", format_locked_paths(&locked)));
        }
        tokio::time::sleep(DESKTOP_EXIT_POLL).await;
    }
}

fn install_lock_probe_paths(install_root: &Path) -> Vec<PathBuf> {
    desktop_app_payload_paths(install_root)
}

fn desktop_app_payload_paths(install_root: &Path) -> Vec<PathBuf> {
    let release = install_root.join("apps").join("desktop").join("release");
    if cfg!(target_os = "windows") {
        vec![
            release.join("win-unpacked").join("resources").join("app.asar"),
            release.join("win-arm64-unpacked").join("resources").join("app.asar"),
        ]
    } else if cfg!(target_os = "macos") {
        vec![
            release.join("mac").join("Hermes.app").join("Contents").join("Resources").join("app.asar"),
            release.join("mac-arm64").join("Hermes.app").join("Contents").join("Resources").join("app.asar"),
        ]
    } else {
        // x64 builds land in `linux-unpacked`, ARM64 in `linux-arm64-unpacked` (#94703).
        vec![
            release.join("linux-unpacked").join("resources").join("app.asar"),
            release.join("linux-arm64-unpacked").join("resources").join("app.asar"),
        ]
    }
}

fn locked_paths(paths: &[PathBuf]) -> Vec<PathBuf> {
    paths.iter().filter(|p| is_locked(p)).cloned().collect()
}

fn format_locked_paths(paths: &[PathBuf]) -> String {
    paths.iter().map(|p| p.display().to_string()).collect::<Vec<_>>().join(", ")
}


/// Best-effort lock probe: try to open the file for read+write. On Windows an
/// exclusively-held running .exe refuses the open with a sharing violation.
/// On Unix this almost always succeeds (no mandatory locking), which is fine —
/// application-output contention is a Windows-only problem.
fn is_locked(path: &Path) -> bool {
    if !path.exists() {
        return false;
    }
    match std::fs::OpenOptions::new().read(true).write(true).open(path) {
        Ok(_) => false,
        Err(_) => true,
    }
}

/// Whether the `desktop --build-only` rebuild should be retried once. Any
/// non-success exit qualifies: the common cause is a transient first-attempt
/// failure (still-settling tree / self-healed Electron download) that a clean
/// second run resolves.
fn rebuild_needs_retry(exit_code: Option<i32>) -> bool {
    exit_code != Some(0)
}

/// Lines of child stdout kept for the exit-2 diagnosis. Every refusal block
/// is a handful of lines printed right before `sys.exit(2)`.
const STDOUT_TAIL_LINES: usize = 40;

/// User-facing message for an exit-2 refusal from `hermes update`.
///
/// Exit 2 has several causes (another update holds the marker, a live
/// hermes.exe or venv holder, a self-mapped `.pyd` after the code swap), and
/// the child always prints the specific one as a block starting with `✗`
/// just before exiting. Show that block — it names the real holder/PID — and
/// fall back to the generic "still running" text only when none was captured
/// (#78135).
fn concurrent_update_message(stdout_tail: &[String]) -> String {
    match stdout_tail.iter().rposition(|l| l.trim_start().starts_with('✗')) {
        Some(start) => stdout_tail[start..].join("\n").trim().to_string(),
        None => "Hermes is still running. Close all Hermes windows and try \
                 the update again."
            .to_string(),
    }
}

/// Spawn `hermes <args>` from `cwd`, stream stdout/stderr as Log events on the
/// bootstrap channel, and return the exit code. Mirrors powershell::run_script
/// but for an arbitrary command (no install.ps1 -File wrapping).
async fn run_streamed(
    app: &AppHandle,
    program: &Path,
    args: &[String],
    cwd: &Path,
    envs: &[(String, OsString)],
    stage: Option<&str>,
) -> Result<CmdResult> {
    let mut stdout_tail: VecDeque<String> = VecDeque::with_capacity(STDOUT_TAIL_LINES);
    let current = resolve_hermes(cwd).await.ok_or_else(|| anyhow!("Installation launcher missing under {}", cwd.display()))?;
    let mut command: Vec<String> = vec![current.to_string_lossy().into_owned()];
    if current.starts_with(cwd.join(".hermes").join("bin")) {
        let mut query = Command::new(&current);
        query.arg("--print-runtime-command").current_dir(cwd);
        #[cfg(windows)]
        query.creation_flags(0x0800_0000);
        for (key, value) in envs { query.env(key, value); }
        let output = query.output().await?;
        if !output.status.success() { return Err(anyhow!("Installation launcher could not resolve its runtime: {}", current.display())); }
        command = serde_json::from_slice(&output.stdout)?;
        if command.len() < 2 || command.iter().any(|part| part.is_empty()) {
            return Err(anyhow!("Installation launcher returned invalid runtime command"));
        }
    }
    let mut cmd = Command::new(&command[0]);
    cmd.args(&command[1..]).args(args)
        .current_dir(cwd)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    for (key, value) in envs {
        cmd.env(key, value);
    }

    #[cfg(target_os = "windows")]
    {
        use std::os::windows::process::CommandExt;
        // CREATE_NO_WINDOW = 0x08000000 — no flashing console behind the GUI.
        cmd.creation_flags(0x0800_0000);
    }

    let mut child = cmd
        .spawn()
        .map_err(|e| anyhow!("spawning {} {:?}: {e}", program.display(), args))?;

    // Same non-UTF-8-safe decode path as powershell::run_script (#67193), and
    // the same rule about pipe EOF: `hermes update` is precisely the shape that
    // leaves resident descendants holding an inherited stdout handle, and every
    // stage this drives sits downstream of the read.
    let stage_owned = stage.map(|s| s.to_string());
    let outcome = pump_child(
        &mut child,
        |l| {
            emit_log(app, stage_owned.as_deref(), LogStream::Stdout, l);
            if stdout_tail.len() == STDOUT_TAIL_LINES {
                stdout_tail.pop_front();
            }
            stdout_tail.push_back(l.to_string());
        },
        |l| emit_log(app, stage_owned.as_deref(), LogStream::Stderr, l),
        &mut None,
        DRAIN_GRACE,
    )
    .await
    .map_err(|e| anyhow!("streaming {} {:?}: {e}", program.display(), args))?;

    if outcome.abandoned {
        let note = format!(
            "{} exited but a surviving descendant still holds its stdout/stderr; \
             gave up on the last {}s of output (#90455)",
            program.display(),
            DRAIN_GRACE.as_secs()
        );
        tracing::warn!("{note}");
        emit_log(app, stage_owned.as_deref(), LogStream::Stderr, &note);
    }

    Ok(CmdResult {
        exit_code: outcome.exit_code,
        stdout_tail: stdout_tail.into(),
    })
}

struct CmdResult {
    exit_code: Option<i32>,
    /// Last [`STDOUT_TAIL_LINES`] stdout lines; see [`concurrent_update_message`].
    stdout_tail: Vec<String>,
}

/// Resolve only a launcher owned by this installation, never PATH.
async fn resolve_hermes(install_root: &Path) -> Option<PathBuf> {
    let names: &[&str] = if cfg!(target_os = "windows") { &["hermes.exe", "hermes.cmd"] } else { &["hermes"] };
    for name in names {
        let launcher = install_root.join(".hermes").join("bin").join(name);
        if launcher.is_file() { return Some(launcher); }
    }
    // Earlier PM publication lived in user-bin only. The CLI's existing
    // version surface proves which source tree that command belongs to.
    if install_root.join("hermes_cli/_launchers.py").is_file() {
        let mut directories = vec![crate::paths::hermes_home().join("bin")];
        if let Some(home) = dirs::home_dir() { directories.push(home.join(".local/bin")); }
        if let Some(parent) = install_root.parent() { directories.push(parent.join("bin")); }
        if let Some(local) = std::env::var_os("LOCALAPPDATA") { directories.push(PathBuf::from(local).join("hermes/bin")); }
        for directory in directories {
            for name in names {
                let candidate = directory.join(name);
                if candidate.is_file() && launcher_targets_installation(&candidate, install_root).await {
                    return Some(candidate);
                }
            }
        }
    }
    // Required transition for pre-PM releases. A broken PM publication must
    // fail closed rather than select an unrelated interpreter or checkout.
    if !install_root.join("pm").is_dir() {
        let legacy = install_root.join("venv")
            .join(if cfg!(target_os = "windows") { "Scripts" } else { "bin" })
            .join(names[0]);
        if legacy.is_file() { return Some(legacy); }
    }
    None
}

async fn launcher_targets_installation(launcher: &Path, root: &Path) -> bool {
    let mut command = Command::new(launcher);
    command.arg("--version").current_dir(root).kill_on_drop(true);
    #[cfg(windows)]
    command.creation_flags(0x0800_0000);
    let Ok(Ok(output)) = tokio::time::timeout(Duration::from_secs(15), command.output()).await else { return false; };
    if !output.status.success() { return false; }
    let stdout = String::from_utf8_lossy(&output.stdout);
    let Some(reported) = stdout.lines().find_map(|line| line.strip_prefix("Install directory: ")) else { return false; };
    match (std::fs::canonicalize(reported.trim()), std::fs::canonicalize(root)) {
        (Ok(actual), Ok(expected)) => actual == expected,
        _ => false,
    }
}

fn update_child_env(install_root: &Path) -> Vec<(String, OsString)> {
    let hermes_home = crate::paths::hermes_home();
    let mut envs = vec![(
        "HERMES_HOME".to_string(),
        hermes_home.as_os_str().to_os_string(),
    )];
    // `hermes update` is a Python CLI writing to a pipe here, so CPython
    // block-buffers its stdout: nothing reaches run_streamed (and the live
    // log UI) until 8 KB accumulate or the process exits. Long quiet steps —
    // the pre-update backup can zip multi-GB archives for minutes — render as
    // a frozen stage, and users cancel a healthy update. Force line-by-line
    // output instead.
    envs.push(("PYTHONUNBUFFERED".to_string(), OsString::from("1")));
    // We hold the update-in-progress marker for this whole run, and the
    // `hermes update` child claims that SAME lock (hermes_cli/update_lock.py).
    // Name our pid so the child recognizes the live holder as its own
    // orchestrator and runs under our claim — without this every GUI update
    // refuses its parent's marker with exit 2 ("Hermes is still running")
    // and no number of retries can ever succeed. Keep the variable name in
    // sync with HANDOFF_PID_ENV in hermes_cli/update_lock.py.
    envs.push((
        "HERMES_UPDATE_HANDOFF_PID".to_string(),
        OsString::from(std::process::id().to_string()),
    ));
    envs.push(("HERMES_INSTALL_ROOT".to_string(), install_root.as_os_str().to_os_string()));
    envs
}

fn update_branch_from_args<I, S>(args: I) -> Option<String>
where
    I: IntoIterator<Item = S>,
    S: AsRef<str>,
{
    arg_value_from_args(args, "--branch")
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty())
}

fn target_app_from_args<I, S>(args: I) -> Option<PathBuf>
where
    I: IntoIterator<Item = S>,
    S: AsRef<str>,
{
    arg_value_from_args(args, "--target-app")
        .map(PathBuf::from)
        .filter(|p| p.extension().and_then(|e| e.to_str()) == Some("app"))
}

fn arg_value_from_args<I, S>(args: I, name: &str) -> Option<String>
where
    I: IntoIterator<Item = S>,
    S: AsRef<str>,
{
    let mut iter = args.into_iter().map(|s| s.as_ref().to_string()).peekable();
    while let Some(arg) = iter.next() {
        if arg == name {
            return iter.next();
        }
        if let Some(value) = arg.strip_prefix(&format!("{name}=")) {
            return Some(value.to_string());
        }
    }
    None
}

#[cfg(target_os = "macos")]
async fn install_macos_app_update(
    app: &AppHandle,
    install_root: &Path,
    target_app: &Path,
) -> Result<PathBuf> {
    if target_app.extension().and_then(|e| e.to_str()) != Some("app") {
        return Err(anyhow!(
            "refusing to install update into non-app path: {}",
            target_app.display()
        ));
    }

    let rebuilt_app = crate::bootstrap::resolve_hermes_desktop_app(install_root).ok_or_else(|| {
        anyhow!(
            "desktop rebuild succeeded but no Hermes.app was found under {}",
            install_root.join("apps").join("desktop").join("release").display()
        )
    })?;

    let same = match (rebuilt_app.canonicalize(), target_app.canonicalize()) {
        (Ok(a), Ok(b)) => a == b,
        _ => rebuilt_app == target_app,
    };
    if same {
        emit_log(
            app,
            Some("install"),
            LogStream::Stdout,
            &format!(
                "[update] rebuilt app is already the launch target: {}",
                target_app.display()
            ),
        );
        return Ok(target_app.to_path_buf());
    }

    emit_log(
        app,
        Some("install"),
        LogStream::Stdout,
        &format!(
            "[update] installing rebuilt app {} -> {}",
            rebuilt_app.display(),
            target_app.display()
        ),
    );

    if let Some(parent) = target_app.parent() {
        tokio::fs::create_dir_all(parent).await?;
    }
    let tmp = PathBuf::from(format!("{}.hermes-update-new", target_app.display()));
    let old = PathBuf::from(format!("{}.hermes-update-old", target_app.display()));
    remove_dir_if_exists(&tmp).await;
    remove_dir_if_exists(&old).await;

    let ditto = Command::new("/usr/bin/ditto")
        .arg(&rebuilt_app)
        .arg(&tmp)
        .current_dir(crate::paths::hermes_home())
        .status()
        .await
        .map_err(|e| anyhow!("running ditto: {e}"))?;
    if !ditto.success() {
        return Err(anyhow!(
            "ditto failed while copying updated app into {}",
            tmp.display()
        ));
    }

    // Atomic-as-possible swap with rollback. Extracted so the invariant
    // (target is never left deleted-with-no-replacement) can be unit-tested
    // without ditto / a real .app bundle.
    swap_in_new_bundle(&tmp, target_app, &old).await?;

    let _ = Command::new("/usr/bin/xattr")
        .arg("-dr")
        .arg("com.apple.quarantine")
        .arg(target_app)
        .current_dir(crate::paths::hermes_home())
        .status()
        .await;

    Ok(target_app.to_path_buf())
}

#[cfg(not(target_os = "macos"))]
async fn install_macos_app_update(
    _app: &AppHandle,
    _install_root: &Path,
    target_app: &Path,
) -> Result<PathBuf> {
    Ok(target_app.to_path_buf())
}

/// Move a freshly-staged bundle (`tmp`) into place at `target`, parking any
/// existing bundle at `old` so the move can succeed (macOS `rename` won't
/// overwrite a non-empty directory).
///
/// Invariant: on ANY failure path, `target` is left pointing at a working
/// bundle — either the original (rolled back from `old`) or untouched — and we
/// never delete the running app with no replacement in place. The staged `tmp`
/// copy is cleaned up on failure.
async fn swap_in_new_bundle(tmp: &Path, target: &Path, old: &Path) -> Result<()> {
    let moved_old = if target.exists() {
        if let Err(err) = tokio::fs::rename(target, old).await {
            // Could not move the existing app aside. Leave it untouched and
            // bail — a failed update must not brick the install.
            remove_dir_if_exists(tmp).await;
            return Err(anyhow!(
                "could not move existing app aside at {} (leaving it in place): {err}",
                target.display()
            ));
        }
        true
    } else {
        false
    };
    if let Err(err) = tokio::fs::rename(tmp, target).await {
        // Restore the original app from the backup so the user keeps a working
        // install, and clean up the staged copy.
        if moved_old {
            let _ = tokio::fs::rename(old, target).await;
        }
        remove_dir_if_exists(tmp).await;
        return Err(anyhow!("installing updated app at {}: {err}", target.display()));
    }
    remove_dir_if_exists(old).await;
    Ok(())
}

async fn remove_dir_if_exists(path: &Path) {
    if path.exists() {
        let _ = tokio::fs::remove_dir_all(path).await;
    }
}

#[cfg(target_os = "macos")]
async fn launch_macos_app_and_exit(app: &AppHandle, target_app: &Path) -> Result<()> {
    crate::bootstrap::open_macos_app_detached(target_app)
        .map_err(|e| anyhow!("launching {}: {e}", target_app.display()))?;
    tokio::time::sleep(std::time::Duration::from_millis(150)).await;
    app.exit(0);
    Ok(())
}

#[cfg(not(target_os = "macos"))]
async fn launch_macos_app_and_exit(_app: &AppHandle, _target_app: &Path) -> Result<()> {
    Ok(())
}

// ---------------------------------------------------------------------------
// Event helpers — keep emit shape identical to bootstrap.rs so the UI is reused
// ---------------------------------------------------------------------------

fn stage_info(name: &str, title: &str) -> StageInfo {
    StageInfo {
        name: name.to_string(),
        title: title.to_string(),
        category: "update".to_string(),
        needs_user_input: false,
    }
}

/// The synthetic update manifest. Mirrors the real operations `run_update`
/// performs so the progress UI shows them as discrete steps (with the live log
/// underneath) instead of one monolithic bar. `include_install` adds the macOS
/// app-swap stage. Both the happy path and the re-entrancy guard build the
/// manifest here so the two can never drift apart.
fn update_stages(include_install: bool) -> Vec<StageInfo> {
    let mut stages = vec![
        stage_info("handoff", "Preparing to update"),
        stage_info("update", "Downloading the latest version"),
        stage_info("rebuild", "Rebuilding the desktop app"),
    ];
    if include_install {
        stages.push(stage_info("install", "Installing the update"));
    }
    stages
}

// option_env! only accepts string literals, so the build-time pins are read
// by their literal names here. Mirrors bootstrap.rs's helper of the same name
// (kept local rather than shared because option_env! can't be parameterized).
fn option_env_string(key: &str) -> Option<String> {
    let val = match key {
        "BUILD_PIN_COMMIT" => option_env!("BUILD_PIN_COMMIT"),
        "BUILD_PIN_BRANCH" => option_env!("BUILD_PIN_BRANCH"),
        _ => None,
    };
    val.map(|s| s.to_string())
}

fn emit(app: &AppHandle, event: BootstrapEvent) {
    // Same UI boundary as bootstrap.rs's emit_event: the update flow's log
    // lines also reach the plain-text Live output pane (#112675).
    let event = event.sanitized_for_ui();
    if let Err(e) = app.emit(BootstrapEvent::CHANNEL, &event) {
        tracing::warn!(?e, "failed to emit update event");
    }
}

fn emit_stage(
    app: &AppHandle,
    name: &str,
    state: StageState,
    duration_ms: Option<u64>,
    error: Option<String>,
) {
    tracing::info!(stage = %name, ?state, ?duration_ms, ?error, "update stage");
    emit(
        app,
        BootstrapEvent::Stage {
            name: name.to_string(),
            state,
            duration_ms,
            result: None,
            error,
        },
    );
}

fn emit_log(app: &AppHandle, stage: Option<&str>, stream: LogStream, line: &str) {
    match stage {
        Some(s) => tracing::info!(target: "bootstrap.log", stage = %s, "{line}"),
        None => tracing::info!(target: "bootstrap.log", "{line}"),
    }
    emit(
        app,
        BootstrapEvent::Log {
            stage: stage.map(|s| s.to_string()),
            line: line.to_string(),
            stream,
        },
    );
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn launcher_resolution_is_installation_bound() {
        let root = unique_tmp_dir("launcher");
        let legacy = root.join("venv").join(if cfg!(windows) { "Scripts" } else { "bin" })
            .join(if cfg!(windows) { "hermes.exe" } else { "hermes" });
        std::fs::create_dir_all(legacy.parent().unwrap()).unwrap();
        std::fs::write(&legacy, "old").unwrap();
        assert_eq!(resolve_hermes(&root).await, Some(legacy));
        std::fs::create_dir(root.join("pm")).unwrap();
        assert_eq!(resolve_hermes(&root).await, None, "PM must never fall back to the old venv");
        let launcher = root.join(".hermes/bin").join(if cfg!(windows) { "hermes.cmd" } else { "hermes" });
        std::fs::create_dir_all(launcher.parent().unwrap()).unwrap();
        std::fs::write(&launcher, "new").unwrap();
        assert_eq!(resolve_hermes(&root).await, Some(launcher));
        std::fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn missing_file_is_not_locked() {
        assert!(!is_locked(Path::new("/nonexistent/does/not/exist/xyz")));
    }

    #[test]
    fn update_child_env_forces_unbuffered_python() {
        let envs = update_child_env(Path::new("/x/hermes-agent"));
        assert!(
            envs.iter()
                .any(|(k, v)| k == "PYTHONUNBUFFERED" && v.to_str() == Some("1")),
            "update children must run unbuffered so long steps stream to the live log"
        );
    }

    #[test]
    fn update_child_env_names_our_pid_for_the_lock_handoff() {
        let envs = update_child_env(Path::new("/x/hermes-agent"));
        assert!(
            envs.iter().any(|(k, v)| k == "HERMES_UPDATE_HANDOFF_PID"
                && v.to_str() == Some(std::process::id().to_string().as_str())),
            "the hermes update child claims the same marker we hold; without our pid \
             it refuses its own parent's lock and every GUI update dead-ends on exit 2"
        );
    }

    #[test]
    fn lock_probe_paths_include_desktop_app_payload() {
        let root = Path::new("/x/hermes-agent");
        let probes = install_lock_probe_paths(root);

        assert!(probes.iter().all(|p| p.starts_with(root.join("apps/desktop/release"))),
                "only replaced application outputs belong to the update lock set");
        assert!(
            // Windows/Linux payloads live under `resources/`, the macOS bundle
            // under `Contents/Resources/` — Path::ends_with is case-sensitive.
            probes.iter().any(|p| {
                p.ends_with(Path::new("resources/app.asar"))
                    || p.ends_with(Path::new("Resources/app.asar"))
            }),
            "packaged app.asar must be probed so repair/re-clone waits for the old desktop to exit"
        );
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn lock_probe_paths_cover_arm64_linux_build() {
        let root = Path::new("/x/hermes-agent");
        let probes = install_lock_probe_paths(root);

        for dir in ["linux-unpacked", "linux-arm64-unpacked"] {
            let asar = root.join("apps/desktop/release").join(dir).join("resources/app.asar");
            assert!(probes.contains(&asar), "{dir} payload must be probed (#94703)");
        }
    }

    #[test]
    fn locked_paths_ignores_missing_payloads() {
        let root = Path::new("/nonexistent/hermes-agent");
        let probes = install_lock_probe_paths(root);

        assert!(locked_paths(&probes).is_empty());
    }

    fn lines(text: &str) -> Vec<String> {
        text.lines().map(str::to_string).collect()
    }

    #[test]
    fn concurrent_update_message_shows_the_childs_refusal_block() {
        // Exit 2 after the code swap: fetch/pull noise precedes the refusal,
        // and only the refusal (which names the real holder) is the message.
        let tail = lines(
            "→ Fetching updates...\n\
             ✓ Updated to 6b2c23ae42\n\
             ✗ Another Hermes update is already running (started 3m 42s ago, process 65285).\n\
             \n  Wait for it to finish, then run `hermes update` again.\n",
        );
        assert_eq!(
            concurrent_update_message(&tail),
            "✗ Another Hermes update is already running (started 3m 42s ago, process 65285).\n\
             \n  Wait for it to finish, then run `hermes update` again."
        );
    }

    #[test]
    fn concurrent_update_message_falls_back_without_a_refusal_block() {
        let generic = "Hermes is still running. Close all Hermes windows and try the update again.";
        assert_eq!(concurrent_update_message(&[]), generic);
        assert_eq!(concurrent_update_message(&lines("→ Fetching updates...\n")), generic);
    }

    #[test]
    fn parses_update_branch_from_space_or_equals_args() {
        assert_eq!(
            update_branch_from_args(["--update", "--branch", "bb/test"]),
            Some("bb/test".to_string())
        );
        assert_eq!(
            update_branch_from_args(["--update", "--branch=main"]),
            Some("main".to_string())
        );
        assert_eq!(update_branch_from_args(["--update"]), None);
    }

    #[test]
    fn update_manifest_leads_with_handoff_and_gates_install() {
        let base = update_stages(false);
        assert_eq!(
            base.first().map(|s| s.name.as_str()),
            Some("handoff"),
            "the lock-wait must surface as the first visible step"
        );
        assert!(
            base.iter().any(|s| s.name == "update") && base.iter().any(|s| s.name == "rebuild"),
            "update + rebuild remain distinct stages"
        );
        assert!(
            base.iter().all(|s| s.name != "install"),
            "no app-swap stage unless an install target was passed"
        );

        let with_install = update_stages(true);
        assert_eq!(
            with_install.last().map(|s| s.name.as_str()),
            Some("install"),
            "the macOS app-swap is the final stage when present"
        );
        assert_eq!(
            with_install.len(),
            base.len() + 1,
            "include_install adds exactly one stage"
        );
    }

    #[test]
    fn rebuild_retries_only_on_failure() {
        assert!(!rebuild_needs_retry(Some(0)), "a clean rebuild must not retry");
        assert!(rebuild_needs_retry(Some(1)), "a failed rebuild retries once");
        assert!(
            rebuild_needs_retry(None),
            "a killed/signalled rebuild (no exit code) retries once"
        );
    }

    #[test]
    fn parses_only_app_targets() {
        assert_eq!(
            target_app_from_args(["--update", "--target-app", "/Applications/Hermes.app"]),
            Some(PathBuf::from("/Applications/Hermes.app"))
        );
        assert_eq!(target_app_from_args(["--target-app", "/tmp/not-an-app"]), None);
    }

    // Helpers for the swap tests: make a throwaway dir tree we can rename.
    fn unique_tmp_dir(tag: &str) -> PathBuf {
        let base = std::env::temp_dir().join(format!(
            "hermes-swap-test-{tag}-{}-{}",
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        std::fs::create_dir_all(&base).unwrap();
        base
    }

    fn write_marker(dir: &Path, contents: &str) {
        std::fs::create_dir_all(dir).unwrap();
        std::fs::write(dir.join("marker.txt"), contents).unwrap();
    }

    #[tokio::test]
    async fn swap_installs_new_bundle_and_cleans_up() {
        let base = unique_tmp_dir("ok");
        let target = base.join("Hermes.app");
        let tmp = base.join("Hermes.app.hermes-update-new");
        let old = base.join("Hermes.app.hermes-update-old");
        write_marker(&target, "OLD");
        write_marker(&tmp, "NEW");

        swap_in_new_bundle(&tmp, &target, &old).await.unwrap();

        // New bundle is now at target; staging + backup dirs are gone.
        assert_eq!(
            std::fs::read_to_string(target.join("marker.txt")).unwrap(),
            "NEW"
        );
        assert!(!tmp.exists(), "staged copy should be cleaned up");
        assert!(!old.exists(), "backup should be cleaned up on success");
        let _ = std::fs::remove_dir_all(&base);
    }

    #[tokio::test]
    async fn swap_failure_never_leaves_target_missing() {
        // Regression guard for the catastrophic path: the move-aside of the
        // existing app fails AND the staged bundle can't be installed. The
        // buggy version deleted `target` when move-aside failed and then
        // skipped rollback, bricking the install. The fixed version must leave
        // the original app intact on disk.
        //
        // Trigger both failures deterministically:
        //  - `old` is a NON-EMPTY dir  -> rename(target, old) fails
        //  - `tmp` does not exist       -> rename(tmp, target) fails
        let base = unique_tmp_dir("fail");
        let target = base.join("Hermes.app");
        let tmp = base.join("Hermes.app.hermes-update-new"); // intentionally absent
        let old = base.join("Hermes.app.hermes-update-old");
        write_marker(&target, "OLD");
        write_marker(&old, "OCCUPIED"); // non-empty => rename(target,old) fails

        let result = swap_in_new_bundle(&tmp, &target, &old).await;

        assert!(result.is_err(), "swap should fail when neither move can complete");
        assert!(target.exists(), "original app must NOT be deleted on failure");
        assert_eq!(
            std::fs::read_to_string(target.join("marker.txt")).unwrap(),
            "OLD",
            "original app contents must be intact after a failed swap"
        );
        let _ = std::fs::remove_dir_all(&base);
    }

    #[tokio::test]
    async fn swap_rolls_back_when_install_step_fails() {
        // Move-aside succeeds but installing the staged bundle fails (tmp
        // absent). The original must be rolled back from `old` to `target`.
        let base = unique_tmp_dir("rollback");
        let target = base.join("Hermes.app");
        let tmp = base.join("Hermes.app.hermes-update-new"); // absent
        let old = base.join("Hermes.app.hermes-update-old");
        write_marker(&target, "OLD");

        let result = swap_in_new_bundle(&tmp, &target, &old).await;

        assert!(result.is_err());
        assert!(target.exists(), "original must be restored after failed install");
        assert_eq!(
            std::fs::read_to_string(target.join("marker.txt")).unwrap(),
            "OLD"
        );
        assert!(!old.exists(), "backup should be rolled back, not left behind");
        let _ = std::fs::remove_dir_all(&base);
    }
}