hermes-agent/apps/desktop/electron/managed-ssh-update.test.ts

import assert from 'node:assert/strict'
import { exec as execCallback, spawn } from 'node:child_process'
import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'

import { test } from 'vitest'

import {
  buildPosixManagedUpdateLaunch,
  buildRemoteUpdateObservationCommand,
  buildWindowsManagedUpdateLaunch,
  fenceManagedSshBootstrapPublication,
  joinManagedUpdatesForApply,
  ManagedConnectionUpdateGate,
  managedSshDrainBlocker,
  managedSshRecoveryDisposition,
  managedSshRecoveryScopes,
  managedSshScopeRole,
  managedSshTokenPersistencePlan,
  managedSshUpdateAllRow,
  MAX_MANAGED_SSH_RECOVERY_ATTEMPTS,
  parseRemoteUpdateObservation,
  RECEIPT_GRACE_MS,
  recoverManagedSshScopes,
  refusedManagedSshUpdate,
  runManagedSshUpdate,
  waitForManagedRemoteClearance,
  waitForManagedRemoteUpdate,
  waitForManagedSshBootstrapFence,
  waitForManagedUpdateOperations
} from './managed-ssh-update'
import { createBootstrapCoordinator } from './ssh-bootstrap-coordinator'

const CORRELATION = '12345678-1234-4678-9234-567812345678'
const exec = promisify(execCallback)

function observation(over: Record<string, unknown> = {}) {
  return JSON.stringify({
    marker: 'absent',
    launchIntent: 'absent',
    exitCode: null,
    receipt: null,
    coordinatorReady: null,
    ...over
  })
}

test('ManagedConnectionUpdateGate blocks new dials but admits the exact restoring transaction', () => {
  const gate = new ManagedConnectionUpdateGate()

  assert.equal(gate.claim('homelab', CORRELATION), true)
  assert.equal(gate.claim('homelab', '22345678-1234-4678-9234-567812345678'), false)
  assert.throws(() => gate.assertCanDial('homelab'), /paused/)
  assert.doesNotThrow(() => gate.assertCanDial('homelab', CORRELATION))
  gate.release('homelab', '22345678-1234-4678-9234-567812345678')
  assert.equal(gate.owner('homelab'), CORRELATION)
  gate.release('homelab', CORRELATION)
  assert.doesNotThrow(() => gate.assertCanDial('homelab'))
})

test('durable recovery claim survives memory release and fences ordinary dials, edits, and removal', () => {
  let durable: string | null = CORRELATION
  const gate = new ManagedConnectionUpdateGate(id => (id === 'homelab' ? durable : null))

  assert.equal(gate.claim('homelab', CORRELATION), true)
  gate.release('homelab', CORRELATION)
  assert.equal(gate.owner('homelab'), CORRELATION)
  assert.throws(() => gate.assertCanDial('homelab'), /paused/)
  assert.throws(() => gate.assertCanMutate('homelab'), /edited or removed/)
  assert.doesNotThrow(() => gate.assertCanDial('homelab', CORRELATION))
  assert.equal(gate.claim('homelab', '22345678-1234-4678-9234-567812345678'), false)

  durable = null
  assert.doesNotThrow(() => gate.assertCanDial('homelab'))
  assert.doesNotThrow(() => gate.assertCanMutate('homelab'))
})

test('inactive SSH crash recovery keeps ordinary dials fenced until positive clearance', async () => {
  let durableOwner: string | null = CORRELATION
  const relaunchedGate = new ManagedConnectionUpdateGate(id => (id === 'homelab' ? durableOwner : null))
  let releaseClearance!: () => void

  const clearance = new Promise<void>(resolve => {
    releaseClearance = resolve
  })

  let restoreCalls = 0
  let journalCleared = false

  const recovery = recoverManagedSshScopes({
    scopes: [] as Array<{ profile: string }>,
    awaitClearance: () => clearance,
    restoreScope: async () => {
      restoreCalls += 1
    },
    completeRecovery: async () => {
      journalCleared = true
      durableOwner = null
    }
  })

  await Promise.resolve()
  assert.throws(() => relaunchedGate.assertCanDial('homelab'), /paused/)
  assert.equal(journalCleared, false)
  releaseClearance()
  const { disposition, results } = await recovery

  assert.equal(disposition, 'complete')
  assert.deepEqual(results, [])
  assert.equal(restoreCalls, 0)
  assert.equal(journalCleared, true)
  assert.doesNotThrow(() => relaunchedGate.assertCanDial('homelab'))
})

test('durable recovery disposition is bounded once the remote install is clear', () => {
  const max = MAX_MANAGED_SSH_RECOVERY_ATTEMPTS

  assert.equal(managedSshRecoveryDisposition({ attempts: 0, restoreFailures: 0, updateSucceeded: false }), 'complete')
  assert.equal(managedSshRecoveryDisposition({ attempts: 1, restoreFailures: 1, updateSucceeded: false }), 'retry')
  assert.equal(
    managedSshRecoveryDisposition({ attempts: max - 1, restoreFailures: 1, updateSucceeded: false }),
    'retry'
  )
  assert.equal(managedSshRecoveryDisposition({ attempts: max, restoreFailures: 1, updateSucceeded: false }), 'abandon')
  assert.equal(managedSshRecoveryDisposition({ attempts: 1, restoreFailures: 2, updateSucceeded: true }), 'abandon')
  assert.equal(
    managedSshRecoveryDisposition({ attempts: 2, maxAttempts: 5, restoreFailures: 1, updateSucceeded: false }),
    'retry'
  )
})

async function relaunchRecoveryWithStuckScope(options: {
  attempts: number
  clearance?: { exitCode: null | number; receipt: null | { correlationId: string; outcome: string } }
}) {
  let durableOwner: string | null = CORRELATION
  let recordedAttempts: null | number = null
  const gate = new ManagedConnectionUpdateGate(id => (id === 'homelab' ? durableOwner : null))

  const outcome = await recoverManagedSshScopes({
    attempts: options.attempts,
    scopes: [{ profile: 'default' }, { profile: 'stuck' }],
    awaitClearance: async () => options.clearance ?? { exitCode: null, receipt: null },
    restoreScope: async scope => {
      if (scope.profile === 'stuck') {
        throw new Error('serve never became healthy')
      }
    },
    recordFailedAttempt: async next => {
      recordedAttempts = next
    },
    completeRecovery: async () => {
      durableOwner = null
    }
  })

  return { gate, outcome, recordedAttempts }
}

test('a scope that never restores stops fencing the connection after bounded relaunch attempts (#107827)', async () => {
  const first = await relaunchRecoveryWithStuckScope({ attempts: 0 })

  assert.equal(first.outcome.disposition, 'retry')
  assert.equal(first.recordedAttempts, 1)
  assert.throws(() => first.gate.assertCanDial('homelab'), /paused/)
  assert.throws(() => first.gate.assertCanMutate('homelab'), /edited or removed/)

  const last = await relaunchRecoveryWithStuckScope({ attempts: MAX_MANAGED_SSH_RECOVERY_ATTEMPTS - 1 })

  assert.equal(last.outcome.disposition, 'abandon')
  assert.equal(last.outcome.attempts, MAX_MANAGED_SSH_RECOVERY_ATTEMPTS)
  assert.equal(last.recordedAttempts, null)
  assert.deepEqual(
    last.outcome.results.map(result => result.status),
    ['fulfilled', 'rejected']
  )
  assert.doesNotThrow(() => last.gate.assertCanDial('homelab'))
  assert.doesNotThrow(() => last.gate.assertCanMutate('homelab'))
})

test('a correlated remote exit 0 ends durable recovery on the first failed restore', async () => {
  const success = await relaunchRecoveryWithStuckScope({
    attempts: 0,
    clearance: { exitCode: 0, receipt: { correlationId: CORRELATION, outcome: 'success' } }
  })

  assert.equal(success.outcome.disposition, 'abandon')
  assert.doesNotThrow(() => success.gate.assertCanDial('homelab'))

  // Reported shape: `.update_exit_code.<correlation>` is 0 but no receipt was written.
  const exitOnly = await relaunchRecoveryWithStuckScope({ attempts: 0, clearance: { exitCode: 0, receipt: null } })

  assert.equal(exitOnly.outcome.disposition, 'abandon')
  assert.doesNotThrow(() => exitOnly.gate.assertCanDial('homelab'))

  const failed = await relaunchRecoveryWithStuckScope({
    attempts: 0,
    clearance: { exitCode: 1, receipt: { correlationId: CORRELATION, outcome: 'failed' } }
  })

  assert.equal(failed.outcome.disposition, 'retry')
  assert.throws(() => failed.gate.assertCanDial('homelab'), /paused/)
})

test('managed update joins a pre-claim bootstrap until its final gate check rolls back the serve', async () => {
  const gate = new ManagedConnectionUpdateGate()
  const coordinator = createBootstrapCoordinator()
  let releaseLifecycle!: () => void

  const lifecycle = new Promise<void>(resolve => {
    releaseLifecycle = resolve
  })

  let serveLive = false
  let rollbackComplete = false

  const startPromise = coordinator.start(
    '',
    'fingerprint',
    async () => {
      await lifecycle
      serveLive = true
      await fenceManagedSshBootstrapPublication({
        assertCanPublish: () => gate.assertCanDial('homelab'),
        publish: () => {},
        rollback: async () => {
          serveLive = false
          rollbackComplete = true
        }
      })
    },
    { managedScope: 'primary', registryConnectionId: 'homelab' }
  )

  // The fence rethrows managed-update-in-progress after rolling back — that
  // rejection propagating out of start() is the CONTRACT under test, not an
  // accident. Swallow it here so vitest doesn't flag the floating promise as
  // an unhandled rejection while the assertions below verify the rollback.
  startPromise.catch(() => {})
  const barrier = waitForManagedSshBootstrapFence(coordinator.active, 'homelab')
  let barrierComplete = false
  void barrier.then(() => {
    barrierComplete = true
  })

  assert.equal(gate.claim('homelab', CORRELATION), true)
  await Promise.resolve()
  assert.equal(barrierComplete, false)
  releaseLifecycle()
  await barrier

  assert.equal(rollbackComplete, true)
  assert.equal(serveLive, false)
  assert.equal(barrierComplete, true)
})

test('bootstrap publication stays in the same turn as its final gate assertion', async () => {
  const gate = new ManagedConnectionUpdateGate()
  let published = false
  let rolledBack = false

  queueMicrotask(() => {
    gate.claim('homelab', CORRELATION)
  })
  await fenceManagedSshBootstrapPublication({
    assertCanPublish: () => gate.assertCanDial('homelab'),
    publish: () => {
      published = true
    },
    rollback: async () => {
      rolledBack = true
    }
  })

  assert.equal(published, true)
  assert.equal(rolledBack, false)
  assert.equal(gate.owner('homelab'), CORRELATION)
})

test('registry-qualified primary token adoption stays reusable across consecutive launches', () => {
  const legacy: Record<string, string> = {}
  const registry: Record<string, string> = {}

  for (const servedToken of ['served-on-first-launch', 'served-on-second-launch']) {
    const plan = managedSshTokenPersistencePlan('profile', 'homelab')

    assert.equal(plan.legacySource, 'profile')
    assert.equal(plan.registryConnectionId, 'homelab')
    legacy[plan.legacySource!] = servedToken
    registry[plan.registryConnectionId] = servedToken
    assert.equal(legacy.profile, registry.homelab)
  }

  assert.equal(legacy.profile, 'served-on-second-launch')
  assert.equal(registry.homelab, 'served-on-second-launch')
  assert.deepEqual(managedSshTokenPersistencePlan('registry:homelab'), {
    legacySource: null,
    registryConnectionId: 'homelab'
  })
})

test('actual primary and matching bare/composite pools keep distinct managed scope roles', () => {
  const base = { connectionId: 'homelab', prefix: 'conn:homelab::' }

  assert.equal(
    managedSshScopeRole({
      ...base,
      key: '',
      state: { primaryRegistryScope: true, registryConnectionId: 'homelab' }
    }),
    'primary'
  )
  assert.equal(
    managedSshScopeRole({
      ...base,
      key: 'research',
      routeConnectionId: 'homelab',
      state: { primaryRegistryScope: false, registryConnectionId: 'homelab' }
    }),
    'pool'
  )
  assert.equal(managedSshScopeRole({ ...base, key: 'conn:homelab::research' }), 'pool')
})

test('durable recovery preserves every same-profile primary, registry, and legacy scope', () => {
  const scopes = managedSshRecoveryScopes(
    [
      { key: '', profile: 'default', primary: true },
      { key: 'conn:homelab::default', profile: 'default' },
      { key: 'default', profile: 'default' }
    ],
    'conn:homelab::'
  )

  assert.deepEqual(scopes, [
    { key: '', kind: 'primary', profile: 'default' },
    { key: 'conn:homelab::default', kind: 'registry', profile: 'default' },
    { key: 'default', kind: 'legacy', profile: 'default' }
  ])
})

test('update-all deduplicates the same recovery scope and keeps primary precedence', () => {
  assert.deepEqual(
    managedSshRecoveryScopes(
      [
        { key: 'conn:homelab::default', profile: 'default' },
        { key: 'conn:homelab::default', profile: 'default', primary: true },
        { key: 'conn:homelab::default', profile: 'default' }
      ],
      'conn:homelab::'
    ),
    [{ key: 'conn:homelab::default', kind: 'primary', profile: 'default' }]
  )
})

test.runIf(process.platform !== 'win32').each([0, 23])(
  'POSIX managed launcher executes the updater command and atomically publishes status %i',
  async (exitCode: number): Promise<void> => {
    const home: string = await mkdtemp(path.join(os.tmpdir(), 'hermes managed launch '))
    const shell: string = (await exec('command -v bash', { shell: 'bash' })).stdout.trim()
    const launcher: string = path.join(home, 'hermes launcher')

    try {
      await writeFile(
        launcher,
        `#!${shell}\nprintf '%s\\n' "$@" "$HERMES_HOME" "$HERMES_UPDATE_CORRELATION_ID" "$HERMES_UPDATE_ORIGIN_PROFILE" "$HERMES_UPDATE_ORIGIN_HOME" "$HERMES_UPDATE_OUTPUT_PATH"\nexit ${exitCode}\n`,
        { encoding: 'utf8', mode: 0o700 }
      )

      const command: string = buildPosixManagedUpdateLaunch(
        {
          ssh: { exec: async (): Promise<string> => '' },
          platform: 'Linux',
          hermesPath: launcher,
          hermesHome: home
        },
        CORRELATION
      )

      const { stdout, stderr } = await exec(command, { shell, env: { ...process.env, HOME: home, HERMES_HOME: home } })
      const statusPath: string = path.join(home, `.update_exit_code.${CORRELATION}`)
      const logPath: string = path.join(home, 'logs', `desktop-update-${CORRELATION}.log`)
      let status: string | undefined

      for (let attempt: number = 0; attempt < 100; attempt += 1) {
        try {
          status = await readFile(statusPath, 'utf8')

          break
        } catch (error: unknown) {
          if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) {
            throw error
          }

          await new Promise<void>(resolve => setTimeout(resolve, 25))
        }
      }

      const log: string = await readFile(logPath, 'utf8')
      assert.match(stdout, /MANAGED_UPDATE_STARTED/)
      assert.match(log, /managed-update start pid=/)
      assert.match(log, new RegExp(`managed-update exit rc=${exitCode}`))
      assert.equal(status, String(exitCode), `updater output: ${log}`)
      const logLines: string[] = log.trimEnd().split('\n')
      assert.match(logLines.shift() ?? '', /managed-update start pid=/)
      assert.match(logLines.pop() ?? '', new RegExp(`managed-update exit rc=${exitCode}`))
      assert.deepEqual(logLines, ['update', '--yes', home, CORRELATION, 'default', home, logPath])
      assert.equal((await stat(statusPath)).mode & 0o777, 0o600)
      assert.equal(
        (await readdir(home)).some((name: string): boolean => name.endsWith('.tmp')),
        false
      )
    } finally {
      await rm(home, { force: true, recursive: true })
    }
  }
)
test('Windows managed launcher starts a hidden child and leaves exit 75 to the external coordinator', () => {
  const command = buildWindowsManagedUpdateLaunch(
    {
      ssh: { exec: async () => '' },
      platform: 'Windows',
      hermesPath: 'C:\\Hermes\\hermes.exe',
      hermesHome: 'C:\\Users\\alice\\.hermes',
      pythonPath: 'C:\\Hermes\\python.exe'
    },
    CORRELATION
  )

  const outer = Buffer.from(command.split(' ').at(-1) || '', 'base64').toString('utf16le')
  const wrapperBase64 = outer.match(/"-EncodedCommand",'([^']+)'/)?.[1]

  assert.match(outer, /Start-Process/)
  assert.match(outer, /WindowStyle Hidden/)
  assert.ok(wrapperBase64, 'outer launcher carries a separately encoded detached wrapper')
  const wrapper = Buffer.from(wrapperBase64!, 'base64').toString('utf16le')

  assert.match(wrapper, /update --yes/)
  assert.doesNotMatch(wrapper, /update --yes --gateway/)
  assert.match(wrapper, /HERMES_UPDATE_WINDOWS_DETACHED/)
  assert.match(wrapper, /HERMES_UPDATE_TAURI_READY_PATH/)
  assert.match(wrapper, /HERMES_UPDATE_TAURI_OUTCOME_PATH/)
  assert.match(wrapper, /\$rc -ne 75/)
  assert.match(wrapper, /\$handoffAccepted=/)
  assert.match(wrapper, new RegExp(`update_launch_intent\\.${CORRELATION}`))
  assert.ok(wrapper.indexOf('update_launch_intent') < wrapper.indexOf('update --yes'))
  assert.match(outer, /launcher intent timed out/)
})

test('remote observation rejects a receipt for another correlation', () => {
  assert.throws(
    () =>
      parseRemoteUpdateObservation(
        observation({
          exitCode: 0,
          receipt: { correlationId: '22345678-1234-4678-9234-567812345678', outcome: 'success' }
        }),
        CORRELATION
      ),
    /did not match/
  )
})

test.runIf(process.platform !== 'win32')(
  'POSIX observer reads the exact correlation receipt and terminal marker from disk',
  async () => {
    const home = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-update-'))

    try {
      const receipts = path.join(home, 'logs', 'update_receipts')
      await mkdir(receipts, { recursive: true })
      await writeFile(path.join(home, `.update_exit_code.${CORRELATION}`), '0')
      await writeFile(
        path.join(receipts, `update_${CORRELATION}.json`),
        JSON.stringify({
          correlation_id: CORRELATION,
          outcome: 'success',
          started_at: '2026-08-23T00:00:00Z',
          finished_at: '2026-08-23T00:01:00Z',
          pre_update: { sha: 'old' },
          post_update: { sha: 'new' }
        })
      )

      const command = buildRemoteUpdateObservationCommand(
        {
          ssh: { exec: async () => '' },
          platform: 'Linux',
          hermesPath: '/opt/hermes/hermes',
          hermesHome: home
        },
        CORRELATION
      )

      const { stdout } = await exec(command, { shell: 'sh' })
      const parsed = parseRemoteUpdateObservation(stdout, CORRELATION)

      assert.equal(parsed.marker, 'absent')
      assert.equal(parsed.exitCode, 0)
      assert.equal(parsed.receipt?.correlationId, CORRELATION)
      assert.equal(parsed.receipt?.preSha, 'old')
      assert.equal(parsed.receipt?.postSha, 'new')
    } finally {
      await rm(home, { force: true, recursive: true })
    }
  }
)

test.runIf(process.platform !== 'win32')(
  'a committed update that still owes a step stays a success but names the owed step',
  async () => {
    const home = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-owed-'))

    const run = async (
      followups: unknown[],
      { exitCode = 0, outcome = 'success', restoreFails = false, userAction = null as unknown } = {}
    ) => {
      const receipts = path.join(home, 'logs', 'update_receipts')
      await mkdir(receipts, { recursive: true })
      await writeFile(path.join(home, `.update_exit_code.${CORRELATION}`), String(exitCode))
      await writeFile(
        path.join(receipts, 'update_x.json'),
        JSON.stringify({
          correlation_id: CORRELATION,
          outcome,
          started_at: '2026-08-23T00:00:00Z',
          finished_at: '2026-08-23T00:01:00Z',
          followups,
          user_action: userAction
        })
      )

      const command = buildRemoteUpdateObservationCommand(
        { ssh: { exec: async () => '' }, platform: 'Linux', hermesPath: '/opt/hermes/hermes', hermesHome: home },
        CORRELATION
      )

      const observed = parseRemoteUpdateObservation((await exec(command, { shell: 'sh' })).stdout, CORRELATION)

      return runManagedSshUpdate({
        connectionId: 'home',
        correlationId: CORRELATION,
        scopes: [{ key: 'conn:home::default', profile: 'default' }],
        preflightRemote: async () => {},
        drainScope: async () => {},
        updateRemote: async () => ({ exitCode: observed.exitCode!, receipt: observed.receipt! }),
        awaitRestoreClearance: async () => {},
        closeTransports: async () => {},
        restoreScope: async () => {
          if (restoreFails) {
            throw new Error('restore refused')
          }
        },
        releaseGate: () => {}
      })
    }

    try {
      const owed = await run([{ step: 'dependencies', reason: 'sync failed', at: '2026-08-23T00:01:00Z' }])

      assert.equal(owed.ok, true)
      assert.equal(owed.updateOk, true)
      assert.equal(owed.restoreOk, true)
      assert.deepEqual(owed.owed, [{ step: 'dependencies', reason: 'sync failed' }])
      assert.match(owed.message || '', /still owed: dependencies \(sync failed\)/)
      assert.match(owed.message || '', /hermes update/)
      assert.match(managedSshUpdateAllRow({ id: 'home' }, owed).detail || '', /dependencies/)

      const clean = await run([])

      assert.equal(clean.ok, true)
      assert.equal(clean.owed, undefined)
      assert.equal(clean.message, 'Remote Hermes updated and every managed SSH profile is ready.')

      // The debt survives a failed restoration: update and restore verdicts stay independent.
      const unrestored = await run([{ step: 'dependencies', reason: 'sync failed' }], { restoreFails: true })

      assert.equal(unrestored.updateOk, true)
      assert.equal(unrestored.restoreOk, false)
      assert.deepEqual(unrestored.owed, [{ step: 'dependencies', reason: 'sync failed' }])
      assert.match(
        managedSshUpdateAllRow({ id: 'home' }, unrestored).detail || '',
        /still owed: dependencies \(sync failed\)/
      )

      // record_user_action turns a success into partial/exit 1: its exact manual instruction is shown,
      // and the rerun advice is never offered for a parked stash.
      const instruction = 'Run `git stash apply stash@{0}` in /opt/hermes to restore your local changes.'

      const partial = await run([], {
        exitCode: 1,
        outcome: 'partial',
        userAction: { step: 'local_changes', reason: instruction }
      })

      assert.equal(partial.updateOk, false)
      assert.equal(partial.restoreOk, true)
      assert.deepEqual(partial.owed, [{ step: 'local_changes', reason: instruction }])
      const detail = managedSshUpdateAllRow({ id: 'home' }, partial).detail || ''
      assert.ok(detail.includes(instruction), detail)
      assert.match(detail, /local_changes/)
      assert.doesNotMatch(detail, /Re-run `hermes update`/)
    } finally {
      await rm(home, { force: true, recursive: true })
    }
  }
)

test.runIf(process.platform !== 'win32')(
  'managed observer unwraps a named profile home for the install-wide marker',
  async () => {
    const root = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-profile-marker-'))
    const profileHome = path.join(root, 'profiles', 'research')

    // A v2 claim whose owner is gone and whose delegate is alive at its creation time.
    const delegate = spawn('python3', ['-c', 'import time;print(time.time(),flush=True);time.sleep(30)'], {
      stdio: ['ignore', 'pipe', 'inherit']
    })

    try {
      const ct = await new Promise<string>(resolve =>
        delegate.stdout.once('data', chunk => resolve(String(chunk).trim()))
      )

      const now = Math.floor(Date.now() / 1000)
      await mkdir(profileHome, { recursive: true })
      await writeFile(
        path.join(root, '.hermes-update-in-progress'),
        `0\n${now}\nct:${ct}\ndelegate:${delegate.pid} ct:${ct}\nrun:desk-1\n`
      )

      const command = buildRemoteUpdateObservationCommand(
        {
          ssh: { exec: async () => '' },
          platform: 'Linux',
          hermesPath: '/opt/hermes/hermes',
          hermesHome: profileHome
        },
        CORRELATION
      )

      const { stdout } = await exec(command, { shell: 'sh' })
      const parsed = parseRemoteUpdateObservation(stdout, CORRELATION)

      assert.equal(parsed.marker, 'live')
      assert.equal(parsed.markerPid, delegate.pid)
    } finally {
      delegate.kill('SIGKILL')
      await rm(root, { force: true, recursive: true })
    }
  }
)

test.runIf(process.platform !== 'win32')(
  'managed observer reads a named profile update receipt from the root home the CLI writes',
  async () => {
    const root = await mkdtemp(path.join(os.tmpdir(), 'hermes-managed-profile-receipt-'))
    const profileHome = path.join(root, 'profiles', 'research')

    const receipt = (outcome: string) =>
      JSON.stringify({
        correlation_id: CORRELATION,
        outcome,
        started_at: '2026-08-23T00:00:00Z',
        finished_at: '2026-08-23T00:01:00Z'
      })

    try {
      // The CLI's receipt store is install-wide; a stale copy under the profile is not its source.
      await mkdir(path.join(root, 'logs', 'update_receipts'), { recursive: true })
      await mkdir(path.join(profileHome, 'logs', 'update_receipts'), { recursive: true })
      await writeFile(path.join(profileHome, `.update_exit_code.${CORRELATION}`), '0')
      await writeFile(path.join(root, 'logs', 'update_receipts', `update_${CORRELATION}.json`), receipt('success'))
      await writeFile(
        path.join(profileHome, 'logs', 'update_receipts', `update_${CORRELATION}.json`),
        receipt('failed')
      )

      const command = buildRemoteUpdateObservationCommand(
        {
          ssh: { exec: async () => '' },
          platform: 'Linux',
          hermesPath: '/opt/hermes/hermes',
          hermesHome: profileHome
        },
        CORRELATION
      )

      const { stdout } = await exec(command, { shell: 'sh' })
      const parsed = parseRemoteUpdateObservation(stdout, CORRELATION)

      assert.equal(parsed.exitCode, 0)
      assert.equal(parsed.receipt?.correlationId, CORRELATION)
      assert.equal(parsed.receipt?.outcome, 'success')
    } finally {
      await rm(root, { force: true, recursive: true })
    }
  }
)

test('Windows coordinator handoff is pending until its marker clears and correlated receipt is durable', async () => {
  const replies = [
    observation({ marker: 'live', markerPid: 44 }),
    observation({
      marker: 'live',
      markerPid: 88,
      coordinatorReady: { correlationId: CORRELATION, pid: 88 }
    }),
    observation({
      marker: 'absent',
      exitCode: null,
      receipt: {
        correlationId: CORRELATION,
        outcome: 'success',
        finishedAt: '2026-08-23T00:00:00Z'
      },
      coordinatorReady: { correlationId: CORRELATION, pid: 88 }
    })
  ]

  let calls = 0

  const target = {
    platform: 'Windows' as const,
    hermesPath: 'C:\\Hermes\\hermes.exe',
    hermesHome: 'C:\\Users\\alice\\.hermes',
    pythonPath: 'C:\\Hermes\\python.exe',
    ssh: {
      exec: async () => {
        const reply = replies[Math.min(calls, replies.length - 1)]
        calls += 1

        return reply
      }
    }
  }

  const proof = await waitForManagedRemoteUpdate(target, CORRELATION, {
    pollMs: 0,
    sleep: async () => {}
  })

  assert.equal(calls, 3)
  assert.equal(proof.exitCode, 0)
  assert.equal(proof.receipt.correlationId, CORRELATION)
})

test('terminal status without its durable receipt fails instead of claiming success', async () => {
  let now = 0

  const target = {
    platform: 'Linux' as const,
    hermesPath: '~/.local/bin/hermes',
    hermesHome: '~/.hermes',
    ssh: { exec: async () => observation({ marker: 'absent', exitCode: 0 }) }
  }

  await assert.rejects(
    waitForManagedRemoteUpdate(target, CORRELATION, {
      now: () => now,
      pollMs: 1,
      sleep: async () => {
        now += RECEIPT_GRACE_MS
      }
    }),
    /without a correlated durable receipt/
  )
})

test('live or malformed remote markers fail actionably at bounded update and recovery deadlines', async () => {
  for (const marker of ['live', 'malformed'] as const) {
    let now = 0

    const target = {
      platform: 'Linux' as const,
      hermesPath: '~/.local/bin/hermes',
      hermesHome: '~/.hermes',
      ssh: { exec: async () => observation({ marker, ...(marker === 'live' ? { markerPid: 44 } : {}) }) }
    }

    const clock = {
      timeoutMs: 5,
      pollMs: 1,
      now: () => now,
      sleep: async () => {
        now += 5
      }
    }

    await assert.rejects(waitForManagedRemoteUpdate(target, CORRELATION, clock), /services remain stopped/)
    now = 0
    await assert.rejects(waitForManagedRemoteClearance(target, CORRELATION, clock), /durable recovery record/)
  }
})

test('a journaled launch requires correlated terminal proof or an observed live-owner transition before restore', async () => {
  let now = 0

  const target = {
    platform: 'Linux' as const,
    hermesPath: '~/.local/bin/hermes',
    hermesHome: '~/.hermes',
    ssh: { exec: async () => observation({ marker: 'absent' }) }
  }

  await assert.rejects(
    waitForManagedRemoteClearance(target, CORRELATION, {
      requireTerminal: true,
      timeoutMs: 5,
      pollMs: 1,
      now: () => now,
      sleep: async () => {
        now += 5
      }
    }),
    /durable recovery record/
  )

  target.ssh.exec = async () =>
    observation({
      marker: 'absent',
      exitCode: 0,
      receipt: { correlationId: CORRELATION, outcome: 'success', finishedAt: '2026-08-23T00:00:00Z' }
    })
  await assert.doesNotReject(
    waitForManagedRemoteClearance(target, CORRELATION, { requireTerminal: true, timeoutMs: 0 })
  )
})

test('remote launch intent fences crash recovery even before the local journal records launch proof', async () => {
  let now = 0

  const target = {
    platform: 'Linux' as const,
    hermesPath: '~/.local/bin/hermes',
    hermesHome: '~/.hermes',
    ssh: { exec: async () => observation({ marker: 'absent', launchIntent: 'present' }) }
  }

  await assert.rejects(
    waitForManagedRemoteClearance(target, CORRELATION, {
      timeoutMs: 5,
      pollMs: 1,
      now: () => now,
      sleep: async () => {
        now += 5
      }
    }),
    /durable recovery record/
  )

  target.ssh.exec = async () => observation({ marker: 'absent', launchIntent: 'absent' })
  await assert.doesNotReject(waitForManagedRemoteClearance(target, CORRELATION, { timeoutMs: 0 }))
  target.ssh.exec = async () => observation({ marker: 'absent', launchIntent: 'dead' })
  await assert.doesNotReject(
    waitForManagedRemoteClearance(target, CORRELATION, { requireTerminal: true, timeoutMs: 0 })
  )
})

test('before-quit join observes operations added while an earlier update settles', async () => {
  let releaseFirst!: () => void
  let releaseSecond!: () => void
  const operations = new Set<Promise<void>>()

  const first = new Promise<void>(resolve => {
    releaseFirst = resolve
  })

  operations.add(first)
  const joined = waitForManagedUpdateOperations(() => operations)

  const second = new Promise<void>(resolve => {
    releaseSecond = resolve
  })

  operations.add(second)
  first.finally(() => operations.delete(first))
  second.finally(() => operations.delete(second))
  releaseFirst()
  await Promise.resolve()
  let settled = false
  joined.then(() => {
    settled = true
  })
  await Promise.resolve()
  assert.equal(settled, false)
  releaseSecond()
  await joined
  assert.equal(settled, true)
})

test('managed lifecycle restores every captured profile after update failure before releasing the gate', async () => {
  const events: string[] = []

  const scopes = [
    { key: 'conn:home::default', profile: 'default' },
    { key: 'conn:home::research', profile: 'research' }
  ]

  const result = await runManagedSshUpdate({
    connectionId: 'home',
    correlationId: CORRELATION,
    scopes,
    preflightRemote: async () => {
      events.push('preflight')
    },
    drainScope: async scope => {
      events.push(`drain:${scope.profile}`)
    },
    updateRemote: async () => {
      events.push('update')
      throw new Error('fetch failed')
    },
    awaitRestoreClearance: async () => {
      events.push('clear')
    },
    closeTransports: async () => {
      events.push('close')
    },
    restoreScope: async scope => {
      events.push(`restore:${scope.profile}`)
    },
    releaseGate: () => {
      events.push('release')
    }
  })

  assert.deepEqual(events, [
    'preflight',
    'drain:default',
    'drain:research',
    'update',
    'clear',
    'close',
    'restore:default',
    'restore:research',
    'release'
  ])
  assert.equal(result.ok, false)
  assert.equal(result.updateOk, false)
  assert.equal(result.restoreOk, true)
  assert.equal(result.outcome, 'update-failed')
  assert.deepEqual(
    result.scopes.map(scope => [scope.profile, scope.restored]),
    [
      ['default', true],
      ['research', true]
    ]
  )
})

test('inactive managed update journals before launch and clears only after remote clearance', async () => {
  const events: string[] = []

  const result = await runManagedSshUpdate({
    connectionId: 'homelab',
    correlationId: CORRELATION,
    scopes: [],
    preflightRemote: async () => {
      events.push('preflight')
    },
    prepareRecovery: async () => {
      events.push('journal')
    },
    drainScope: async () => {
      events.push('unexpected-drain')
    },
    updateRemote: async () => {
      events.push('update')

      return {
        exitCode: 0,
        receipt: { correlationId: CORRELATION, outcome: 'success' }
      }
    },
    awaitRestoreClearance: async () => {
      events.push('clearance')
    },
    closeTransports: async () => {
      events.push('close')
    },
    restoreScope: async () => {
      events.push('unexpected-restore')
    },
    completeRecovery: async () => {
      events.push('clear-journal')
    },
    releaseGate: () => {
      events.push('release')
    }
  })

  assert.deepEqual(events, ['preflight', 'journal', 'update', 'clearance', 'close', 'clear-journal', 'release'])
  assert.equal(result.ok, true)
  assert.deepEqual(result.scopes, [])
})

test('managed lifecycle attempts every drain and every restore when one ownership proof fails', async () => {
  const drained: string[] = []
  const restored: string[] = []
  let updateCalled = false

  const result = await runManagedSshUpdate({
    connectionId: 'home',
    correlationId: CORRELATION,
    scopes: [
      { key: 'a', profile: 'default' },
      { key: 'b', profile: 'research' }
    ],
    preflightRemote: async () => {},
    drainScope: async scope => {
      drained.push(scope.profile)

      if (scope.profile === 'default') {
        throw new Error('foreign owner')
      }
    },
    updateRemote: async () => {
      updateCalled = true

      return {
        exitCode: 0,
        receipt: { correlationId: CORRELATION, outcome: 'success' }
      }
    },
    awaitRestoreClearance: async () => {},
    closeTransports: async () => {},
    restoreScope: async scope => {
      restored.push(scope.profile)
    },
    releaseGate: () => {}
  })

  assert.deepEqual(drained, ['default', 'research'])
  assert.deepEqual(restored, ['default', 'research'])
  assert.equal(updateCalled, false)
  assert.match(result.error || '', /foreign owner/)
})

async function updateWithStuckRestore(exitCode: number, outcome: string) {
  let journalCleared = false

  const result = await runManagedSshUpdate({
    connectionId: 'home',
    correlationId: CORRELATION,
    scopes: [
      { key: 'a', profile: 'default' },
      { key: 'b', profile: 'stuck' }
    ],
    preflightRemote: async () => {},
    prepareRecovery: async () => {},
    drainScope: async () => {},
    updateRemote: async () => ({ exitCode, receipt: { correlationId: CORRELATION, outcome } }),
    awaitRestoreClearance: async () => {},
    closeTransports: async () => {},
    restoreScope: async scope => {
      if (scope.profile === 'stuck') {
        throw new Error('serve never became healthy')
      }
    },
    completeRecovery: async () => {
      journalCleared = true
    },
    releaseGate: () => {}
  })

  return { journalCleared, result }
}

test('a successful update clears the journal even when one scope fails to restore (#107827)', async () => {
  const { journalCleared, result } = await updateWithStuckRestore(0, 'success')

  assert.equal(journalCleared, true)
  assert.equal(result.updateOk, true)
  assert.equal(result.restoreOk, false)
  assert.equal(result.outcome, 'restore-failed')
  assert.deepEqual(
    result.scopes.map(scope => [scope.profile, scope.restored]),
    [
      ['default', true],
      ['stuck', false]
    ]
  )
})

test('a failed update keeps the journal for relaunch recovery when a scope fails to restore', async () => {
  const { journalCleared, result } = await updateWithStuckRestore(1, 'failed')

  assert.equal(journalCleared, false)
  assert.equal(result.outcome, 'update-and-restore-failed')
})

test('managed lifecycle journals before drain and leaves scopes stopped when clearance cannot be proved', async () => {
  const events: string[] = []

  const result = await runManagedSshUpdate({
    connectionId: 'home',
    correlationId: CORRELATION,
    scopes: [{ key: 'a', profile: 'default' }],
    preflightRemote: async () => {
      events.push('preflight')
    },
    prepareRecovery: async () => {
      events.push('journal')
    },
    drainScope: async () => {
      events.push('drain')
    },
    updateRemote: async () => {
      events.push('update')
      throw new Error('observer timed out')
    },
    awaitRestoreClearance: async () => {
      events.push('clear')
      throw new Error('marker unavailable; durable recovery retained')
    },
    closeTransports: async () => {
      events.push('close')
    },
    restoreScope: async () => {
      events.push('restore')
    },
    completeRecovery: async () => {
      events.push('clear-journal')
    },
    releaseGate: () => {
      events.push('release')
    }
  })

  assert.deepEqual(events, ['preflight', 'journal', 'drain', 'update', 'clear', 'close', 'release'])
  assert.equal(result.restoreOk, false)
  assert.equal(result.scopes[0].restored, false)
  assert.match(result.scopes[0].error || '', /durable recovery retained/)
})

test('journal cleanup failure prevents a false successful update result', async () => {
  const result = await runManagedSshUpdate({
    connectionId: 'home',
    correlationId: CORRELATION,
    scopes: [{ key: 'a', profile: 'default' }],
    preflightRemote: async () => {},
    prepareRecovery: async () => {},
    drainScope: async () => {},
    updateRemote: async () => ({
      exitCode: 0,
      receipt: { correlationId: CORRELATION, outcome: 'success' }
    }),
    awaitRestoreClearance: async () => {},
    closeTransports: async () => {},
    restoreScope: async () => {},
    completeRecovery: async () => {
      throw new Error('disk remained busy')
    },
    releaseGate: () => {}
  })

  assert.equal(result.updateOk, true)
  assert.equal(result.restoreOk, false)
  assert.equal(result.ok, false)
  assert.equal(result.outcome, 'restore-failed')
  assert.match(result.error || '', /recovery record/)
})

test('preflight refusal leaves a healthy primary scope untouched and releases without waiting', async () => {
  const events: string[] = []

  const result = await runManagedSshUpdate({
    connectionId: 'home',
    correlationId: CORRELATION,
    scopes: [{ key: '', primary: true, profile: 'default' }],
    preflightRemote: async () => {
      events.push('preflight')
      throw new Error('live foreign update')
    },
    drainScope: async () => {
      events.push('drain')
    },
    updateRemote: async () => {
      events.push('update')

      return { exitCode: 0, receipt: { correlationId: CORRELATION, outcome: 'success' } }
    },
    awaitRestoreClearance: async () => {
      events.push('clear')
    },
    closeTransports: async () => {
      events.push('close')
    },
    restoreScope: async () => {
      events.push('restore')
    },
    releaseGate: () => {
      events.push('release')
    }
  })

  assert.deepEqual(events, ['preflight', 'close', 'release'])
  assert.equal(result.updateOk, false)
  assert.equal(result.restoreOk, true)
})

test('refused result is structured and has no managed scopes to restore', () => {
  const result = refusedManagedSshUpdate('cloud', CORRELATION, 'not managed')

  assert.equal(result.outcome, 'refused')
  assert.equal(result.restoreOk, true)
  assert.deepEqual(result.scopes, [])
})

test('a live Desktop-owned serve on a macOS remote is a per-row skip, not a batch failure (#124617)', () => {
  // Darwin cannot bind a signal to the verified PID, so the drain refuses to
  // stop the serve. Detect that before any scope is touched and report the
  // connection as skipped instead of an aborted update.
  const blocker = managedSshDrainBlocker([
    { profile: 'default', state: { remotePlatform: 'Darwin' } },
    { profile: 'work', state: null }
  ])

  assert.equal(blocker?.reason, 'darwin-drain-unsupported')
  assert.match(blocker?.message || '', /macOS remote \(default\)/)

  const refused = refusedManagedSshUpdate('mac', CORRELATION, blocker!.message, blocker!.reason)
  const row: any = managedSshUpdateAllRow({ connectionId: 'mac', label: 'Mac', kind: 'ssh' }, refused)

  assert.equal(row.skipped, true)
  assert.equal(row.reason, 'darwin-drain-unsupported')
  assert.equal(row.detail, blocker!.message)
  assert.equal(row.error, undefined)
})

test('macOS remotes with no live serve and Linux remotes still update', () => {
  assert.equal(managedSshDrainBlocker([{ profile: 'default', state: null }]), null)
  assert.equal(managedSshDrainBlocker([{ profile: 'default', state: { remotePlatform: 'Linux' } }]), null)

  const failed = { ...refusedManagedSshUpdate('box', CORRELATION, 'boom'), outcome: 'update-failed' as const }
  const row: any = managedSshUpdateAllRow({ connectionId: 'box' }, failed)

  assert.equal(row.skipped, undefined)
  assert.equal(row.ok, false)
  assert.equal(row.error, 'boom')
})

// Review H3: the join had no bound of its own, so one unresolved (or a
// late-registered) managed operation parked a local apply forever. A bounded
// join answers false at its deadline and abandons nothing; unbounded (quit)
// and an empty or settled set still answer as before.
test('a bounded managed-operation join answers at its deadline without dropping the operation', async () => {
  const operations = new Set<Promise<unknown>>()
  let releaseLate!: () => void
  const late = new Promise<void>(resolve => (releaseLate = resolve))

  const first = Promise.resolve().then(() => {
    operations.add(late)
  })

  operations.add(first)
  first.finally(() => operations.delete(first))

  const outcome = await Promise.race([
    waitForManagedUpdateOperations(() => operations, { timeoutMs: 50 }),
    new Promise(resolve => setTimeout(resolve, 2_000, 'still parked'))
  ])

  assert.equal(outcome, false)
  assert.equal(operations.has(late), true, 'the pending operation keeps its registration')
  releaseLate()
  late.finally(() => operations.delete(late))
  await late
  await Promise.resolve()
  assert.equal(await waitForManagedUpdateOperations(() => operations, { timeoutMs: 50 }), true)
})

test('a local apply refuses, without spawning anything, while a managed operation outlives the bounded join', async () => {
  const pending = new Promise<void>(() => {})
  const logs: string[] = []

  const refusal = await joinManagedUpdatesForApply(
    () => [pending],
    line => logs.push(line),
    20
  )

  assert.equal(refusal?.ok, false)
  assert.equal(refusal?.error, 'managed-update-running')
  assert.match(logs.join('\n'), /refused until it finishes/)
  const settling = new Set<Promise<unknown>>()
  const done = Promise.resolve().then(() => settling.delete(done))

  settling.add(done)
  assert.equal(
    await joinManagedUpdatesForApply(
      () => settling,
      () => {},
      1_000
    ),
    null
  )
})