test(desktop): verify probe responsiveness and stale-start cancellation

(cherry picked from commit c12f7e0f677a234e7e5057a13485e39346da0d03)

apps/desktop/e2e/runtime-probe-race-sitecustomize.py

@@ -0,0 +1,40 @@
+"""Sandbox-only gate: let the real interpreter and Hermes CLI run unchanged."""
+import atexit
+import json
+import os
+from pathlib import Path
+import sys
+import time
+
+
+def record(directory, name, **extra):
+    # Separate, atomically published records avoid partial reads/concurrent
+    # appends when Python's Windows venv redirector starts several children.
+    target = directory / name
+    target.mkdir(exist_ok=True)
+    pending = target / f"{os.getpid()}.tmp"
+    pending.write_text(
+        json.dumps({"pid": os.getpid(), "argv": sys.orig_argv, **extra}),
+        encoding="utf-8",
+    )
+    pending.replace(target / f"{os.getpid()}.json")
+
+
+directory = os.environ.get("HERMES_E2E_RUNTIME_RACE_DIR")
+if directory:
+    directory = Path(directory)
+    # Production supplies this secret only to actual backend spawns. Do not
+    # change or short-circuit their startup: the record observes a real child.
+    if os.environ.get("HERMES_DASHBOARD_SESSION_TOKEN"):
+        record(directory, "spawned")
+    record(directory, "all")
+    if sys.orig_argv[1:] == ["-m", "hermes_cli.main", "serve", "--help"]:
+        record(directory, "entered", parent_pid=os.getppid())
+        deadline = time.monotonic() + 25
+        while not (directory / "release").exists():
+            if time.monotonic() >= deadline:
+                record(directory, "timed-out")
+                break
+            time.sleep(0.02)
+        record(directory, "released")
+        atexit.register(record, directory, "exited")

apps/desktop/e2e/runtime-probe-races.spec.ts

@@ -0,0 +1,178 @@
+import { execFileSync } from 'node:child_process'
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+import { expect, test } from '@playwright/test'
+
+import { buildAppEnv, createSandbox, launchDesktop, writeEnvFile, writeMockProviderConfig } from './fixtures'
+
+const repo = path.resolve(import.meta.dirname, '../../..')
+interface ChildRecord {
+  pid: number
+  argv: string[]
+  parent_pid?: number
+}
+
+function records(directory: string, name: string): ChildRecord[] {
+  const folder = path.join(directory, name)
+
+  return fs.existsSync(folder)
+    ? fs
+        .readdirSync(folder)
+        .filter(file => file.endsWith('.json'))
+        .map(file => JSON.parse(fs.readFileSync(path.join(folder, file), 'utf8')))
+    : []
+}
+
+// Both cases use the real renderer/preload/main IPC and provisioned Python.
+// The empty namespace directory forces source-inspection fallback WITHOUT
+// modifying the real checkout or venv. Python still imports the real package.
+for (const owner of ['primary', 'pool'] as const) {
+  test(`${owner} invalidation during serve help prevents stale backend spawn`, async () => {
+    test.setTimeout(75_000)
+    const python = process.env.HERMES_DESKTOP_PYTHON
+    expect(python, 'Set HERMES_DESKTOP_PYTHON to a provisioned Hermes venv').toBeTruthy()
+    const sandbox = createSandbox(`runtime-race-${owner}`)
+    let running: Awaited<ReturnType<typeof launchDesktop>> | undefined
+    const hook = path.join(sandbox.root, 'hook')
+    fs.mkdirSync(hook)
+
+    try {
+      const prefix = execFileSync(python!, ['-c', 'import sys; print(sys.prefix)'], {
+        encoding: 'utf8',
+        env: buildAppEnv(sandbox),
+        timeout: 15_000,
+        windowsHide: true
+      }).trim()
+
+      const active = path.join(sandbox.hermesHome, 'hermes-agent')
+      fs.mkdirSync(path.join(active, 'hermes_cli'), { recursive: true })
+      fs.copyFileSync(path.join(repo, 'hermes_cli', 'main.py'), path.join(active, 'hermes_cli', 'main.py'))
+      fs.symlinkSync(prefix, path.join(active, 'venv'), process.platform === 'win32' ? 'junction' : 'dir')
+      fs.copyFileSync(
+        path.join(import.meta.dirname, 'runtime-probe-race-sitecustomize.py'),
+        path.join(hook, 'sitecustomize.py')
+      )
+      fs.mkdirSync(path.join(sandbox.hermesHome, 'profiles', 'race-pool'), { recursive: true })
+      writeMockProviderConfig(sandbox.hermesHome, 'http://127.0.0.1:1')
+      writeEnvFile(sandbox.hermesHome)
+
+      const env = buildAppEnv(sandbox, {
+        HOME: sandbox.root,
+        USERPROFILE: sandbox.root,
+        HERMES_DESKTOP_IS_PACKAGED: '1',
+        HERMES_DESKTOP_HERMES_ROOT: '',
+        HERMES_DESKTOP_HERMES: '',
+        HERMES_DESKTOP_PYTHON: '',
+        HERMES_PROBE_TIMEOUT_MS: '30000',
+        HERMES_E2E_RUNTIME_RACE_DIR: hook,
+        PYTHONPATH: [hook, repo].join(path.delimiter),
+        PYTHONDONTWRITEBYTECODE: '1',
+        NODE_OPTIONS: ''
+      })
+
+      delete env.HERMES_DESKTOP_DEV_SERVER
+      expect(env.HERMES_DASHBOARD_SESSION_TOKEN).toBeUndefined()
+
+      const imported = execFileSync(python!, ['-c', 'import hermes_cli; print(hermes_cli.__file__)'], {
+        cwd: active,
+        env,
+        encoding: 'utf8',
+        timeout: 15_000,
+        windowsHide: true
+      }).trim()
+
+      expect(path.resolve(imported)).toBe(path.join(repo, 'hermes_cli', '__init__.py'))
+      running = await launchDesktop(env)
+      // Keep preload but remove automatic UI reconnects: only the explicit
+      // bridge requests below own the test's connection attempts.
+      await running.page.goto('about:blank')
+      await running.page.waitForFunction(() => Boolean((window as any).hermesDesktop))
+
+      const pending = running.page
+        .evaluate(async scope => {
+          try {
+            await (window as any).hermesDesktop.getConnection(scope === 'pool' ? 'race-pool' : undefined)
+
+            return { status: 'resolved', error: '' }
+          } catch (error) {
+            return { status: 'rejected', error: String(error) }
+          }
+        }, owner)
+        .catch(error => ({ status: 'harness-error', error: String(error) }))
+
+      await expect.poll(() => records(hook, 'entered').length, { timeout: 20_000 }).toBeGreaterThan(0)
+      expect(records(hook, 'released')).toEqual([])
+
+      // Windows venv python.exe can be a redirector that remains the worker's
+      // immediate parent. Verify the real Electron-owned chain, not POSIX PPID.
+      const probeParents = await running.app.evaluate(() => [
+        process.pid,
+        ...(process as any)
+          ._getActiveHandles()
+          .filter(
+            (handle: any) =>
+              JSON.stringify(handle.spawnargs?.slice(1)) ===
+              JSON.stringify(['-m', 'hermes_cli.main', 'serve', '--help'])
+          )
+          .map((child: any) => child.pid)
+      ])
+
+      expect(probeParents).toContain(records(hook, 'entered')[0].parent_pid)
+
+      if (owner === 'pool') {
+        // Primary and pool share the serve-support promise. Wait for the pool's
+        // own import probe to close and its promise continuations to drain, so
+        // invalidation occurs at serve-help, not earlier runtime discovery.
+        await expect
+          .poll(
+            () =>
+              records(hook, 'all').filter(
+                record => record.argv.at(-1) === 'import yaml; import dotenv; import hermes_cli.config'
+              ).length
+          )
+          .toBe(2)
+        await running.app.evaluate(async () => {
+          const children = (process as any)
+            ._getActiveHandles()
+            .filter(
+              (handle: any) => handle.spawnargs?.at(-1) === 'import yaml; import dotenv; import hermes_cli.config'
+            )
+
+          await Promise.all(children.map((child: any) => new Promise<void>(resolve => child.once('close', resolve))))
+          await new Promise<void>(resolve => setImmediate(resolve))
+        })
+      }
+
+      const applied = await running.page.evaluate(
+        scope =>
+          (window as any).hermesDesktop.applyConnectionConfig({
+            mode: 'local',
+            ...(scope === 'pool' ? { profile: 'race-pool' } : {})
+          }),
+        owner
+      )
+
+      expect(applied.mode).toBe('local')
+      expect(records(hook, 'released')).toEqual([])
+      fs.writeFileSync(path.join(hook, 'release'), '')
+      // Rejection is the completion barrier for the actual awaiting spawn path,
+      // not an arbitrary sleep followed by an absence assertion.
+      const result = await pending
+      expect(result.status, result.error).toBe('rejected')
+      await expect.poll(() => records(hook, 'exited').length, { timeout: 15_000 }).toBeGreaterThan(0)
+      expect(records(hook, 'timed-out')).toEqual([])
+      const stale = records(hook, 'spawned').filter(record => owner === 'primary' || record.argv.includes('race-pool'))
+      expect(stale).toEqual([])
+      console.log(`${owner}: real serve-help exited; stale backend Python spawns=${stale.length}`)
+    } finally {
+      fs.writeFileSync(path.join(hook, 'release'), '')
+
+      if (running) {
+        await running.app.close()
+      }
+
+      sandbox.cleanup()
+    }
+  })
+}

apps/desktop/e2e/runtime-probe-sitecustomize.py

@@ -0,0 +1,49 @@
+"""Sandbox-only hook: require the real import probe's Electron parent to reply.
+
+Loaded by Python itself via PYTHONPATH; never replaces an interpreter or imports.
+A fatal exit is intentional: Python otherwise ignores sitecustomize exceptions.
+"""
+import json
+import os
+from pathlib import Path
+import socket
+import sys
+import time
+
+
+def _gate_runtime_probe():
+    gate = os.environ.get("HERMES_E2E_RUNTIME_PROBE_DIR")
+    argv = sys.orig_argv
+    if not gate or "-c" not in argv:
+        return
+    code = argv[argv.index("-c") + 1]
+    if "import yaml" not in code or "import hermes_cli.config" not in code:
+        return
+    root = Path(gate)
+    record = {"pid": os.getpid(), "argv": argv, "home": os.environ.get("HERMES_HOME")}
+    (root / "entered.json").write_text(json.dumps(record), encoding="utf-8")
+    deadline = time.monotonic() + 25
+    port_file = root / "port"
+    while not port_file.exists():
+        if time.monotonic() >= deadline:
+            raise TimeoutError("Electron parent never started its loopback service")
+        time.sleep(0.01)
+    with socket.create_connection(("127.0.0.1", int(port_file.read_text())), timeout=20) as connection:
+        connection.sendall(str(os.getpid()).encode("ascii"))
+        with connection.makefile("rb") as response:
+            reply = response.read(128).decode("ascii")
+        if not reply.startswith("parent:"):
+            raise RuntimeError(f"Unexpected parent response: {reply!r}")
+        record["parent_pid"] = int(reply.removeprefix("parent:"))
+    pending = root / "replied.json.tmp"
+    pending.write_text(json.dumps(record), encoding="utf-8")
+    pending.replace(root / "replied.json")
+
+
+try:
+    _gate_runtime_probe()
+except Exception as error:
+    gate = os.environ.get("HERMES_E2E_RUNTIME_PROBE_DIR")
+    if gate:
+        (Path(gate) / "failed.txt").write_text(str(error), encoding="utf-8")
+    os._exit(91)

apps/desktop/e2e/runtime-probes.spec.ts

@@ -0,0 +1,147 @@
+import { execFileSync } from 'node:child_process'
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+import { expect, test } from '@playwright/test'
+
+import { startMockServer } from '../../../tests-js/scripts/mock-server'
+
+import {
+  buildAppEnv,
+  createSandbox,
+  launchDesktop,
+  waitForAppReady,
+  writeEnvFile,
+  writeMockProviderConfig
+} from './fixtures'
+
+const repo = path.resolve(import.meta.dirname, '../../..')
+
+/**
+ * Real local active-install discovery, not a remote-connection reproduction.
+ * Saved remotes bypass this resolver; this does not establish AppHangB1's cause.
+ * Run after npm run build, with HERMES_DESKTOP_PYTHON set to a provisioned venv.
+ * No setup/update commands or writes through the source/venv links are needed.
+ */
+test('runtime import probe can complete I/O with its actual Electron parent', async () => {
+  test.setTimeout(90_000)
+  const python = process.env.HERMES_DESKTOP_PYTHON
+  expect(python, 'Set HERMES_DESKTOP_PYTHON to an existing Hermes test venv interpreter').toBeTruthy()
+  const sandbox = createSandbox('runtime-probe')
+  const mock = await startMockServer()
+  let running: Awaited<ReturnType<typeof launchDesktop>> | undefined
+
+  try {
+    // Link only the required source package and the actual venv (Windows needs
+    // its Scripts/python.exe redirector and pyvenv.cfg, not a copied executable).
+    const prefix = execFileSync(python!, ['-c', 'import sys; print(sys.prefix)'], {
+      encoding: 'utf8',
+      env: buildAppEnv(sandbox),
+      timeout: 15_000,
+      windowsHide: true
+    }).trim()
+
+    const active = path.join(sandbox.hermesHome, 'hermes-agent')
+    fs.mkdirSync(active)
+    const linkType = process.platform === 'win32' ? 'junction' : 'dir'
+    fs.symlinkSync(path.join(repo, 'hermes_cli'), path.join(active, 'hermes_cli'), linkType)
+    fs.symlinkSync(prefix, path.join(active, 'venv'), linkType)
+    const hook = path.join(sandbox.root, 'python-hook')
+    fs.mkdirSync(hook)
+    fs.copyFileSync(
+      path.join(import.meta.dirname, 'runtime-probe-sitecustomize.py'),
+      path.join(hook, 'sitecustomize.py')
+    )
+    writeMockProviderConfig(sandbox.hermesHome, mock.url)
+    writeEnvFile(sandbox.hermesHome)
+
+    const env = buildAppEnv(sandbox, {
+      HOME: sandbox.root,
+      USERPROFILE: sandbox.root,
+      HERMES_DESKTOP_IS_PACKAGED: '1',
+      HERMES_DESKTOP_HERMES_ROOT: '',
+      HERMES_DESKTOP_HERMES: '',
+      HERMES_DESKTOP_PYTHON: '',
+      HERMES_PROBE_TIMEOUT_MS: '15000',
+      HERMES_E2E_RUNTIME_PROBE_DIR: hook,
+      PYTHONPATH: [hook, repo].join(path.delimiter),
+      PYTHONDONTWRITEBYTECODE: '1',
+      NODE_OPTIONS: ''
+    })
+
+    delete env.HERMES_DESKTOP_DEV_SERVER
+    running = await launchDesktop(env)
+    await expect.poll(() => fs.existsSync(path.join(hook, 'entered.json')), { timeout: 20_000 }).toBe(true)
+
+    // The Python child is still waiting for a port. Exercise the existing
+    // renderer → preload → main IPC bridge before allowing that child to exit.
+    const limits = await running.page.evaluate(() =>
+      (
+        window as unknown as {
+          hermesDesktop: { getPoolLimits: () => Promise<{ maxBackends: number }> }
+        }
+      ).hermesDesktop.getPoolLimits()
+    )
+
+    expect(limits.maxBackends).toBeGreaterThan(0)
+    expect(fs.existsSync(path.join(hook, 'replied.json'))).toBe(false)
+    // This server MUST live in Electron main, not the Playwright runner. A
+    // synchronous child probe prevents main from accepting/replying and fails.
+    await running.app.evaluate(async (_electron, directory) => {
+      const net = process.getBuiltinModule('net')
+      const fs = process.getBuiltinModule('fs')
+      const path = process.getBuiltinModule('path')
+      await new Promise<void>((resolve, reject) => {
+        const server = net.createServer(socket => {
+          socket.on('error', () => undefined)
+          socket.once('data', data => {
+            fs.appendFileSync(
+              path.join(directory, 'accepted.jsonl'),
+              JSON.stringify({
+                parentPid: process.pid,
+                childPid: Number(data.toString())
+              }) + '\n'
+            )
+            socket.end(`parent:${process.pid}`)
+          })
+        })
+
+        server.once('error', reject)
+        server.listen(0, '127.0.0.1', () => {
+          const address = server.address()
+
+          if (!address || typeof address === 'string') {
+            return reject(new Error('No TCP address'))
+          }
+
+          fs.writeFileSync(path.join(directory, 'port.tmp'), String(address.port))
+          fs.renameSync(path.join(directory, 'port.tmp'), path.join(directory, 'port'))
+          server.unref()
+          resolve()
+        })
+      })
+    }, hook)
+    await expect.poll(() => fs.existsSync(path.join(hook, 'replied.json')), { timeout: 20_000 }).toBe(true)
+    const reply = JSON.parse(fs.readFileSync(path.join(hook, 'replied.json'), 'utf8'))
+
+    const accepted = fs
+      .readFileSync(path.join(hook, 'accepted.jsonl'), 'utf8')
+      .trim()
+      .split('\n')
+      .map(line => JSON.parse(line))
+
+    expect(reply.home).toBe(sandbox.hermesHome)
+    expect(reply.pid).not.toBe(running.app.process().pid)
+    expect(reply.parent_pid).toBe(running.app.process().pid)
+    expect(accepted).toContainEqual({ parentPid: reply.parent_pid, childPid: reply.pid })
+    expect(fs.existsSync(path.join(hook, 'failed.txt'))).toBe(false)
+    await waitForAppReady({ ...running, mock, mockUrl: mock.url, sandbox, cleanup: async () => {} })
+  } finally {
+    if (running) {
+      await running.app.close()
+    }
+
+    await mock.close()
+    sandbox.cleanup()
+  }
+})

apps/desktop/electron/backend-claim-exec.test.ts

@@ -0,0 +1,34 @@
+import assert from 'node:assert/strict'
+
+import { test } from 'vitest'
+
+import { execText } from './backend-claim'
+
+test('execText closes noninteractive stdin and trims output after EOF', async () => {
+  const output = await execText(
+    process.execPath,
+    ['-e', "process.stdin.resume(); process.stdin.on('end', () => process.stdout.write('  eof  \\n'))"],
+    { timeout: 2000 }
+  )
+
+  assert.equal(output, 'eof')
+}, 10_000)
+
+// Windows forcibly terminates children; a graceful SIGTERM handler is POSIX-only.
+test.skipIf(process.platform === 'win32')(
+  'execText rejects timed-out output even when SIGTERM exits zero',
+  async () => {
+    await assert.rejects(
+      execText(
+        process.execPath,
+        [
+          '-e',
+          "process.on('SIGTERM', () => process.exit(0)); process.stdout.write('candidate'); setInterval(() => {}, 1000)"
+        ],
+        { timeout: 2000 }
+      ),
+      /timed out/i
+    )
+  },
+  10_000
+)

apps/desktop/electron/backend-claim.ts

@@ -26,13 +26,19 @@ import { hiddenWindowsChildOptions } from './windows-child-options'
 
 export function execText(command: string, args: string[], { timeout = 3000 } = {}): Promise<string> {
   return new Promise<string>((resolve, reject) => {
-    execFile(command, args, hiddenWindowsChildOptions({ encoding: 'utf8', timeout }), (error, stdout) => {
+    const child = execFile(command, args, hiddenWindowsChildOptions({ encoding: 'utf8', timeout }), (error, stdout) => {
       if (error) {
         reject(error)
+      } else if (timeout > 0 && child.killed) {
+        // A SIGTERM handler can exit zero after execFile's timeout fired.
+        reject(new Error(`${command} timed out after ${timeout}ms`))
       } else {
         resolve(String(stdout || '').trim())
       }
     })
+
+    // These probes are noninteractive; do not leave readers waiting for input.
+    child.stdin?.end()
   })
 }
 

apps/desktop/electron/backend-probes.windows.test.ts

@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+import { test } from 'vitest'
+
+import { execText } from './backend-claim'
+import { verifyHermesCli } from './backend-probes'
+
+// These exercise native Windows commands, not a mocked process.platform.
+const windowsTest = test.skipIf(process.platform !== 'win32')
+
+windowsTest(
+  'runtime discovery reads native registry and Python launcher output',
+  async () => {
+    const registry = await execText(
+      'reg.exe',
+      ['query', 'HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion', '/v', 'ProductName'],
+      { timeout: 15_000 }
+    )
+
+    assert.match(registry, /ProductName\s+REG_SZ\s+\S/)
+
+    const python = await execText('py.exe', ['-3', '-c', 'import sys; print(sys.executable)'], { timeout: 15_000 })
+    assert.ok(path.isAbsolute(python), 'the launcher must return a usable interpreter, not a command shim')
+    assert.ok(fs.statSync(python).isFile())
+  },
+  40_000
+)
+
+windowsTest(
+  'CLI probe preserves native cmd and bat success and failure results',
+  async () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-native-cli-'))
+
+    try {
+      for (const extension of ['cmd', 'bat']) {
+        const command = path.join(root, `hermes.${extension}`)
+        fs.writeFileSync(command, '@echo off\r\nif "%~1"=="--version" (exit /b 0) else (exit /b 1)\r\n')
+        assert.equal(await verifyHermesCli(command, { shell: true }), true, extension)
+        fs.writeFileSync(command, '@echo off\r\nexit /b 1\r\n')
+        assert.equal(await verifyHermesCli(command, { shell: true }), false, extension)
+      }
+    } finally {
+      fs.rmSync(root, { recursive: true, force: true })
+    }
+  },
+  40_000
+)