From 78399c88da3a987e9eb3ea757f59ce0daeb05f41 Mon Sep 17 00:00:00 2001 From: jango <91889514+jangomango76@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:48:17 +0200 Subject: [PATCH] test(desktop): verify probe responsiveness and stale-start cancellation (cherry picked from commit c12f7e0f677a234e7e5057a13485e39346da0d03) --- .../e2e/runtime-probe-race-sitecustomize.py | 40 ++++ apps/desktop/e2e/runtime-probe-races.spec.ts | 178 ++++++++++++++++++ .../e2e/runtime-probe-sitecustomize.py | 49 +++++ apps/desktop/e2e/runtime-probes.spec.ts | 147 +++++++++++++++ .../electron/backend-claim-exec.test.ts | 34 ++++ apps/desktop/electron/backend-claim.ts | 8 +- .../electron/backend-probes.windows.test.ts | 50 +++++ 7 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/e2e/runtime-probe-race-sitecustomize.py create mode 100644 apps/desktop/e2e/runtime-probe-races.spec.ts create mode 100644 apps/desktop/e2e/runtime-probe-sitecustomize.py create mode 100644 apps/desktop/e2e/runtime-probes.spec.ts create mode 100644 apps/desktop/electron/backend-claim-exec.test.ts create mode 100644 apps/desktop/electron/backend-probes.windows.test.ts diff --git a/apps/desktop/e2e/runtime-probe-race-sitecustomize.py b/apps/desktop/e2e/runtime-probe-race-sitecustomize.py new file mode 100644 index 0000000000..8346f5963b --- /dev/null +++ b/apps/desktop/e2e/runtime-probe-race-sitecustomize.py @@ -0,0 +1,40 @@ +"""Sandbox-only gate: let the real interpreter and Hermes CLI run unchanged.""" +import atexit +import json +import os +from pathlib import Path +import sys +import time + + +def record(directory, name, **extra): + # Separate, atomically published records avoid partial reads/concurrent + # appends when Python's Windows venv redirector starts several children. + target = directory / name + target.mkdir(exist_ok=True) + pending = target / f"{os.getpid()}.tmp" + pending.write_text( + json.dumps({"pid": os.getpid(), "argv": sys.orig_argv, **extra}), + encoding="utf-8", + ) + pending.replace(target / f"{os.getpid()}.json") + + +directory = os.environ.get("HERMES_E2E_RUNTIME_RACE_DIR") +if directory: + directory = Path(directory) + # Production supplies this secret only to actual backend spawns. Do not + # change or short-circuit their startup: the record observes a real child. + if os.environ.get("HERMES_DASHBOARD_SESSION_TOKEN"): + record(directory, "spawned") + record(directory, "all") + if sys.orig_argv[1:] == ["-m", "hermes_cli.main", "serve", "--help"]: + record(directory, "entered", parent_pid=os.getppid()) + deadline = time.monotonic() + 25 + while not (directory / "release").exists(): + if time.monotonic() >= deadline: + record(directory, "timed-out") + break + time.sleep(0.02) + record(directory, "released") + atexit.register(record, directory, "exited") diff --git a/apps/desktop/e2e/runtime-probe-races.spec.ts b/apps/desktop/e2e/runtime-probe-races.spec.ts new file mode 100644 index 0000000000..311e8db570 --- /dev/null +++ b/apps/desktop/e2e/runtime-probe-races.spec.ts @@ -0,0 +1,178 @@ +import { execFileSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as path from 'node:path' + +import { expect, test } from '@playwright/test' + +import { buildAppEnv, createSandbox, launchDesktop, writeEnvFile, writeMockProviderConfig } from './fixtures' + +const repo = path.resolve(import.meta.dirname, '../../..') +interface ChildRecord { + pid: number + argv: string[] + parent_pid?: number +} + +function records(directory: string, name: string): ChildRecord[] { + const folder = path.join(directory, name) + + return fs.existsSync(folder) + ? fs + .readdirSync(folder) + .filter(file => file.endsWith('.json')) + .map(file => JSON.parse(fs.readFileSync(path.join(folder, file), 'utf8'))) + : [] +} + +// Both cases use the real renderer/preload/main IPC and provisioned Python. +// The empty namespace directory forces source-inspection fallback WITHOUT +// modifying the real checkout or venv. Python still imports the real package. +for (const owner of ['primary', 'pool'] as const) { + test(`${owner} invalidation during serve help prevents stale backend spawn`, async () => { + test.setTimeout(75_000) + const python = process.env.HERMES_DESKTOP_PYTHON + expect(python, 'Set HERMES_DESKTOP_PYTHON to a provisioned Hermes venv').toBeTruthy() + const sandbox = createSandbox(`runtime-race-${owner}`) + let running: Awaited> | undefined + const hook = path.join(sandbox.root, 'hook') + fs.mkdirSync(hook) + + try { + const prefix = execFileSync(python!, ['-c', 'import sys; print(sys.prefix)'], { + encoding: 'utf8', + env: buildAppEnv(sandbox), + timeout: 15_000, + windowsHide: true + }).trim() + + const active = path.join(sandbox.hermesHome, 'hermes-agent') + fs.mkdirSync(path.join(active, 'hermes_cli'), { recursive: true }) + fs.copyFileSync(path.join(repo, 'hermes_cli', 'main.py'), path.join(active, 'hermes_cli', 'main.py')) + fs.symlinkSync(prefix, path.join(active, 'venv'), process.platform === 'win32' ? 'junction' : 'dir') + fs.copyFileSync( + path.join(import.meta.dirname, 'runtime-probe-race-sitecustomize.py'), + path.join(hook, 'sitecustomize.py') + ) + fs.mkdirSync(path.join(sandbox.hermesHome, 'profiles', 'race-pool'), { recursive: true }) + writeMockProviderConfig(sandbox.hermesHome, 'http://127.0.0.1:1') + writeEnvFile(sandbox.hermesHome) + + const env = buildAppEnv(sandbox, { + HOME: sandbox.root, + USERPROFILE: sandbox.root, + HERMES_DESKTOP_IS_PACKAGED: '1', + HERMES_DESKTOP_HERMES_ROOT: '', + HERMES_DESKTOP_HERMES: '', + HERMES_DESKTOP_PYTHON: '', + HERMES_PROBE_TIMEOUT_MS: '30000', + HERMES_E2E_RUNTIME_RACE_DIR: hook, + PYTHONPATH: [hook, repo].join(path.delimiter), + PYTHONDONTWRITEBYTECODE: '1', + NODE_OPTIONS: '' + }) + + delete env.HERMES_DESKTOP_DEV_SERVER + expect(env.HERMES_DASHBOARD_SESSION_TOKEN).toBeUndefined() + + const imported = execFileSync(python!, ['-c', 'import hermes_cli; print(hermes_cli.__file__)'], { + cwd: active, + env, + encoding: 'utf8', + timeout: 15_000, + windowsHide: true + }).trim() + + expect(path.resolve(imported)).toBe(path.join(repo, 'hermes_cli', '__init__.py')) + running = await launchDesktop(env) + // Keep preload but remove automatic UI reconnects: only the explicit + // bridge requests below own the test's connection attempts. + await running.page.goto('about:blank') + await running.page.waitForFunction(() => Boolean((window as any).hermesDesktop)) + + const pending = running.page + .evaluate(async scope => { + try { + await (window as any).hermesDesktop.getConnection(scope === 'pool' ? 'race-pool' : undefined) + + return { status: 'resolved', error: '' } + } catch (error) { + return { status: 'rejected', error: String(error) } + } + }, owner) + .catch(error => ({ status: 'harness-error', error: String(error) })) + + await expect.poll(() => records(hook, 'entered').length, { timeout: 20_000 }).toBeGreaterThan(0) + expect(records(hook, 'released')).toEqual([]) + + // Windows venv python.exe can be a redirector that remains the worker's + // immediate parent. Verify the real Electron-owned chain, not POSIX PPID. + const probeParents = await running.app.evaluate(() => [ + process.pid, + ...(process as any) + ._getActiveHandles() + .filter( + (handle: any) => + JSON.stringify(handle.spawnargs?.slice(1)) === + JSON.stringify(['-m', 'hermes_cli.main', 'serve', '--help']) + ) + .map((child: any) => child.pid) + ]) + + expect(probeParents).toContain(records(hook, 'entered')[0].parent_pid) + + if (owner === 'pool') { + // Primary and pool share the serve-support promise. Wait for the pool's + // own import probe to close and its promise continuations to drain, so + // invalidation occurs at serve-help, not earlier runtime discovery. + await expect + .poll( + () => + records(hook, 'all').filter( + record => record.argv.at(-1) === 'import yaml; import dotenv; import hermes_cli.config' + ).length + ) + .toBe(2) + await running.app.evaluate(async () => { + const children = (process as any) + ._getActiveHandles() + .filter( + (handle: any) => handle.spawnargs?.at(-1) === 'import yaml; import dotenv; import hermes_cli.config' + ) + + await Promise.all(children.map((child: any) => new Promise(resolve => child.once('close', resolve)))) + await new Promise(resolve => setImmediate(resolve)) + }) + } + + const applied = await running.page.evaluate( + scope => + (window as any).hermesDesktop.applyConnectionConfig({ + mode: 'local', + ...(scope === 'pool' ? { profile: 'race-pool' } : {}) + }), + owner + ) + + expect(applied.mode).toBe('local') + expect(records(hook, 'released')).toEqual([]) + fs.writeFileSync(path.join(hook, 'release'), '') + // Rejection is the completion barrier for the actual awaiting spawn path, + // not an arbitrary sleep followed by an absence assertion. + const result = await pending + expect(result.status, result.error).toBe('rejected') + await expect.poll(() => records(hook, 'exited').length, { timeout: 15_000 }).toBeGreaterThan(0) + expect(records(hook, 'timed-out')).toEqual([]) + const stale = records(hook, 'spawned').filter(record => owner === 'primary' || record.argv.includes('race-pool')) + expect(stale).toEqual([]) + console.log(`${owner}: real serve-help exited; stale backend Python spawns=${stale.length}`) + } finally { + fs.writeFileSync(path.join(hook, 'release'), '') + + if (running) { + await running.app.close() + } + + sandbox.cleanup() + } + }) +} diff --git a/apps/desktop/e2e/runtime-probe-sitecustomize.py b/apps/desktop/e2e/runtime-probe-sitecustomize.py new file mode 100644 index 0000000000..239f27d8e6 --- /dev/null +++ b/apps/desktop/e2e/runtime-probe-sitecustomize.py @@ -0,0 +1,49 @@ +"""Sandbox-only hook: require the real import probe's Electron parent to reply. + +Loaded by Python itself via PYTHONPATH; never replaces an interpreter or imports. +A fatal exit is intentional: Python otherwise ignores sitecustomize exceptions. +""" +import json +import os +from pathlib import Path +import socket +import sys +import time + + +def _gate_runtime_probe(): + gate = os.environ.get("HERMES_E2E_RUNTIME_PROBE_DIR") + argv = sys.orig_argv + if not gate or "-c" not in argv: + return + code = argv[argv.index("-c") + 1] + if "import yaml" not in code or "import hermes_cli.config" not in code: + return + root = Path(gate) + record = {"pid": os.getpid(), "argv": argv, "home": os.environ.get("HERMES_HOME")} + (root / "entered.json").write_text(json.dumps(record), encoding="utf-8") + deadline = time.monotonic() + 25 + port_file = root / "port" + while not port_file.exists(): + if time.monotonic() >= deadline: + raise TimeoutError("Electron parent never started its loopback service") + time.sleep(0.01) + with socket.create_connection(("127.0.0.1", int(port_file.read_text())), timeout=20) as connection: + connection.sendall(str(os.getpid()).encode("ascii")) + with connection.makefile("rb") as response: + reply = response.read(128).decode("ascii") + if not reply.startswith("parent:"): + raise RuntimeError(f"Unexpected parent response: {reply!r}") + record["parent_pid"] = int(reply.removeprefix("parent:")) + pending = root / "replied.json.tmp" + pending.write_text(json.dumps(record), encoding="utf-8") + pending.replace(root / "replied.json") + + +try: + _gate_runtime_probe() +except Exception as error: + gate = os.environ.get("HERMES_E2E_RUNTIME_PROBE_DIR") + if gate: + (Path(gate) / "failed.txt").write_text(str(error), encoding="utf-8") + os._exit(91) diff --git a/apps/desktop/e2e/runtime-probes.spec.ts b/apps/desktop/e2e/runtime-probes.spec.ts new file mode 100644 index 0000000000..336252fbcf --- /dev/null +++ b/apps/desktop/e2e/runtime-probes.spec.ts @@ -0,0 +1,147 @@ +import { execFileSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as path from 'node:path' + +import { expect, test } from '@playwright/test' + +import { startMockServer } from '../../../tests-js/scripts/mock-server' + +import { + buildAppEnv, + createSandbox, + launchDesktop, + waitForAppReady, + writeEnvFile, + writeMockProviderConfig +} from './fixtures' + +const repo = path.resolve(import.meta.dirname, '../../..') + +/** + * Real local active-install discovery, not a remote-connection reproduction. + * Saved remotes bypass this resolver; this does not establish AppHangB1's cause. + * Run after npm run build, with HERMES_DESKTOP_PYTHON set to a provisioned venv. + * No setup/update commands or writes through the source/venv links are needed. + */ +test('runtime import probe can complete I/O with its actual Electron parent', async () => { + test.setTimeout(90_000) + const python = process.env.HERMES_DESKTOP_PYTHON + expect(python, 'Set HERMES_DESKTOP_PYTHON to an existing Hermes test venv interpreter').toBeTruthy() + const sandbox = createSandbox('runtime-probe') + const mock = await startMockServer() + let running: Awaited> | undefined + + try { + // Link only the required source package and the actual venv (Windows needs + // its Scripts/python.exe redirector and pyvenv.cfg, not a copied executable). + const prefix = execFileSync(python!, ['-c', 'import sys; print(sys.prefix)'], { + encoding: 'utf8', + env: buildAppEnv(sandbox), + timeout: 15_000, + windowsHide: true + }).trim() + + const active = path.join(sandbox.hermesHome, 'hermes-agent') + fs.mkdirSync(active) + const linkType = process.platform === 'win32' ? 'junction' : 'dir' + fs.symlinkSync(path.join(repo, 'hermes_cli'), path.join(active, 'hermes_cli'), linkType) + fs.symlinkSync(prefix, path.join(active, 'venv'), linkType) + const hook = path.join(sandbox.root, 'python-hook') + fs.mkdirSync(hook) + fs.copyFileSync( + path.join(import.meta.dirname, 'runtime-probe-sitecustomize.py'), + path.join(hook, 'sitecustomize.py') + ) + writeMockProviderConfig(sandbox.hermesHome, mock.url) + writeEnvFile(sandbox.hermesHome) + + const env = buildAppEnv(sandbox, { + HOME: sandbox.root, + USERPROFILE: sandbox.root, + HERMES_DESKTOP_IS_PACKAGED: '1', + HERMES_DESKTOP_HERMES_ROOT: '', + HERMES_DESKTOP_HERMES: '', + HERMES_DESKTOP_PYTHON: '', + HERMES_PROBE_TIMEOUT_MS: '15000', + HERMES_E2E_RUNTIME_PROBE_DIR: hook, + PYTHONPATH: [hook, repo].join(path.delimiter), + PYTHONDONTWRITEBYTECODE: '1', + NODE_OPTIONS: '' + }) + + delete env.HERMES_DESKTOP_DEV_SERVER + running = await launchDesktop(env) + await expect.poll(() => fs.existsSync(path.join(hook, 'entered.json')), { timeout: 20_000 }).toBe(true) + + // The Python child is still waiting for a port. Exercise the existing + // renderer → preload → main IPC bridge before allowing that child to exit. + const limits = await running.page.evaluate(() => + ( + window as unknown as { + hermesDesktop: { getPoolLimits: () => Promise<{ maxBackends: number }> } + } + ).hermesDesktop.getPoolLimits() + ) + + expect(limits.maxBackends).toBeGreaterThan(0) + expect(fs.existsSync(path.join(hook, 'replied.json'))).toBe(false) + // This server MUST live in Electron main, not the Playwright runner. A + // synchronous child probe prevents main from accepting/replying and fails. + await running.app.evaluate(async (_electron, directory) => { + const net = process.getBuiltinModule('net') + const fs = process.getBuiltinModule('fs') + const path = process.getBuiltinModule('path') + await new Promise((resolve, reject) => { + const server = net.createServer(socket => { + socket.on('error', () => undefined) + socket.once('data', data => { + fs.appendFileSync( + path.join(directory, 'accepted.jsonl'), + JSON.stringify({ + parentPid: process.pid, + childPid: Number(data.toString()) + }) + '\n' + ) + socket.end(`parent:${process.pid}`) + }) + }) + + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + + if (!address || typeof address === 'string') { + return reject(new Error('No TCP address')) + } + + fs.writeFileSync(path.join(directory, 'port.tmp'), String(address.port)) + fs.renameSync(path.join(directory, 'port.tmp'), path.join(directory, 'port')) + server.unref() + resolve() + }) + }) + }, hook) + await expect.poll(() => fs.existsSync(path.join(hook, 'replied.json')), { timeout: 20_000 }).toBe(true) + const reply = JSON.parse(fs.readFileSync(path.join(hook, 'replied.json'), 'utf8')) + + const accepted = fs + .readFileSync(path.join(hook, 'accepted.jsonl'), 'utf8') + .trim() + .split('\n') + .map(line => JSON.parse(line)) + + expect(reply.home).toBe(sandbox.hermesHome) + expect(reply.pid).not.toBe(running.app.process().pid) + expect(reply.parent_pid).toBe(running.app.process().pid) + expect(accepted).toContainEqual({ parentPid: reply.parent_pid, childPid: reply.pid }) + expect(fs.existsSync(path.join(hook, 'failed.txt'))).toBe(false) + await waitForAppReady({ ...running, mock, mockUrl: mock.url, sandbox, cleanup: async () => {} }) + } finally { + if (running) { + await running.app.close() + } + + await mock.close() + sandbox.cleanup() + } +}) diff --git a/apps/desktop/electron/backend-claim-exec.test.ts b/apps/desktop/electron/backend-claim-exec.test.ts new file mode 100644 index 0000000000..7652f95c03 --- /dev/null +++ b/apps/desktop/electron/backend-claim-exec.test.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { execText } from './backend-claim' + +test('execText closes noninteractive stdin and trims output after EOF', async () => { + const output = await execText( + process.execPath, + ['-e', "process.stdin.resume(); process.stdin.on('end', () => process.stdout.write(' eof \\n'))"], + { timeout: 2000 } + ) + + assert.equal(output, 'eof') +}, 10_000) + +// Windows forcibly terminates children; a graceful SIGTERM handler is POSIX-only. +test.skipIf(process.platform === 'win32')( + 'execText rejects timed-out output even when SIGTERM exits zero', + async () => { + await assert.rejects( + execText( + process.execPath, + [ + '-e', + "process.on('SIGTERM', () => process.exit(0)); process.stdout.write('candidate'); setInterval(() => {}, 1000)" + ], + { timeout: 2000 } + ), + /timed out/i + ) + }, + 10_000 +) diff --git a/apps/desktop/electron/backend-claim.ts b/apps/desktop/electron/backend-claim.ts index 16793d33ae..a3542c7220 100644 --- a/apps/desktop/electron/backend-claim.ts +++ b/apps/desktop/electron/backend-claim.ts @@ -26,13 +26,19 @@ import { hiddenWindowsChildOptions } from './windows-child-options' export function execText(command: string, args: string[], { timeout = 3000 } = {}): Promise { return new Promise((resolve, reject) => { - execFile(command, args, hiddenWindowsChildOptions({ encoding: 'utf8', timeout }), (error, stdout) => { + const child = execFile(command, args, hiddenWindowsChildOptions({ encoding: 'utf8', timeout }), (error, stdout) => { if (error) { reject(error) + } else if (timeout > 0 && child.killed) { + // A SIGTERM handler can exit zero after execFile's timeout fired. + reject(new Error(`${command} timed out after ${timeout}ms`)) } else { resolve(String(stdout || '').trim()) } }) + + // These probes are noninteractive; do not leave readers waiting for input. + child.stdin?.end() }) } diff --git a/apps/desktop/electron/backend-probes.windows.test.ts b/apps/desktop/electron/backend-probes.windows.test.ts new file mode 100644 index 0000000000..94cdb109ca --- /dev/null +++ b/apps/desktop/electron/backend-probes.windows.test.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import { test } from 'vitest' + +import { execText } from './backend-claim' +import { verifyHermesCli } from './backend-probes' + +// These exercise native Windows commands, not a mocked process.platform. +const windowsTest = test.skipIf(process.platform !== 'win32') + +windowsTest( + 'runtime discovery reads native registry and Python launcher output', + async () => { + const registry = await execText( + 'reg.exe', + ['query', 'HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion', '/v', 'ProductName'], + { timeout: 15_000 } + ) + + assert.match(registry, /ProductName\s+REG_SZ\s+\S/) + + const python = await execText('py.exe', ['-3', '-c', 'import sys; print(sys.executable)'], { timeout: 15_000 }) + assert.ok(path.isAbsolute(python), 'the launcher must return a usable interpreter, not a command shim') + assert.ok(fs.statSync(python).isFile()) + }, + 40_000 +) + +windowsTest( + 'CLI probe preserves native cmd and bat success and failure results', + async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-native-cli-')) + + try { + for (const extension of ['cmd', 'bat']) { + const command = path.join(root, `hermes.${extension}`) + fs.writeFileSync(command, '@echo off\r\nif "%~1"=="--version" (exit /b 0) else (exit /b 1)\r\n') + assert.equal(await verifyHermesCli(command, { shell: true }), true, extension) + fs.writeFileSync(command, '@echo off\r\nexit /b 1\r\n') + assert.equal(await verifyHermesCli(command, { shell: true }), false, extension) + } + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } + }, + 40_000 +)