From 1a09c424141939d109c00fbfc9e82c25abc62acd Mon Sep 17 00:00:00 2001 From: ethernet Date: Sun, 6 Sep 2026 22:21:06 -0400 Subject: [PATCH] refactor(bundle): share Python payload assembly across desktop and Termux --- .github/workflows/desktop-bundled-release.yml | 18 +- apps/desktop/BUILDING.md | 47 +- apps/desktop/electron-builder.config.cjs | 2 +- apps/desktop/electron/payload-backend.ts | 2 +- apps/desktop/package.json | 2 +- apps/desktop/scripts/after-pack.mjs | 30 +- apps/desktop/scripts/before-build.mjs | 8 +- apps/desktop/scripts/cli-launchers.test.mjs | 147 +----- .../scripts/materialize-payload-links.mjs | 109 ----- .../materialize-payload-links.test.mjs | 144 ------ apps/desktop/scripts/stage-payload.mjs | 47 -- docs/shared-bundle-builds.md | 51 ++ pm/cli.py | 171 +------ scripts/build-bundled-desktop.mjs | 437 ------------------ scripts/bundles/desktop.py | 123 +++++ .../launcher_wrapper.py} | 2 +- .../mint_launchers.py} | 4 +- scripts/bundles/native.py | 187 ++++++++ scripts/bundles/payload.py | 228 +++++++++ scripts/bundles/stage.py | 30 ++ scripts/desktop-cli/cli-entrypoints.mjs | 130 ------ scripts/termux/launchers.py | 42 +- scripts/termux/payload_facts.py | 27 +- scripts/termux/termux_build.sh | 8 +- tests/pm/test_pm_core.py | 16 +- tests/scripts/test_bundle_native.py | 54 +++ tests/scripts/test_bundle_payload.py | 92 ++++ tests/scripts/test_desktop_cli_wrapper.py | 4 +- tests/scripts/test_mint_launchers.py | 16 +- tests/test_termux_launchers.py | 2 +- 30 files changed, 868 insertions(+), 1312 deletions(-) delete mode 100644 apps/desktop/scripts/materialize-payload-links.mjs delete mode 100644 apps/desktop/scripts/materialize-payload-links.test.mjs delete mode 100644 apps/desktop/scripts/stage-payload.mjs create mode 100644 docs/shared-bundle-builds.md delete mode 100644 scripts/build-bundled-desktop.mjs create mode 100644 scripts/bundles/desktop.py rename scripts/{desktop-cli/launcher-wrapper.py => bundles/launcher_wrapper.py} (98%) rename scripts/{desktop-cli/mint-launchers.py => bundles/mint_launchers.py} (97%) create mode 100644 scripts/bundles/native.py create mode 100644 scripts/bundles/payload.py create mode 100644 scripts/bundles/stage.py delete mode 100644 scripts/desktop-cli/cli-entrypoints.mjs mode change 100644 => 100755 scripts/termux/termux_build.sh create mode 100644 tests/scripts/test_bundle_native.py create mode 100644 tests/scripts/test_bundle_payload.py diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 0449f7ba59..bdefc1e73a 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -44,7 +44,7 @@ name: Desktop Bundled Release # Apple credentials, and FAIL rather than publish unsigned — forks without # the credentials can only build (upload_release=false), never publish. # -# scripts/build-bundled-desktop.mjs is the one driver. Local and CI run +# scripts/bundles/desktop.py is the one driver. Local and CI run # the same command. This workflow adds caching and upload only. # # Triggers: workflow_dispatch with an explicit tag. A tag push does not @@ -237,7 +237,7 @@ jobs: shell: bash # The host toolchain that BUILDS the artifact comes from the same # pin table as the embedded runtimes (pm/lock.json), so gate == pin - # by construction in build-bundled-desktop.mjs's toolchain gates. + # by construction in bundles/desktop.py's toolchain gates. run: | python -c ' import json @@ -420,12 +420,12 @@ jobs: AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_TOKEN_CREDENTIALS: prod run: | - node scripts/build-bundled-desktop.mjs --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + uv run --no-project --python 3.11 python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled # The Store-submission MSIX is the same bundled payload re-packed # with the Partner Center packaging identity (publish-win32-store # bundles these into the universal Store .msixbundle and submits it; # they also land in the tag archive, never a feed dir). - node scripts/build-bundled-desktop.mjs --tag="$HERMES_PAYLOAD_TAG" --variant=store + uv run --no-project --python 3.11 python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=store - name: Audit bundle architecture shell: bash @@ -476,7 +476,7 @@ jobs: done # ── macOS builders (REAL) ────────────────────────────────────────────────── - # Native per-arch darwin builds via scripts/build-bundled-desktop.mjs (the + # Native per-arch darwin builds via scripts/bundles/desktop.py (the # one driver: same `--mac dmg zip` pass a local mac build runs). Each leg # signs (CSC_LINK) and notarizes (afterSign notarize.mjs) when the # release-signing environment carries the Apple credentials; a publishing @@ -534,7 +534,7 @@ jobs: shell: bash # The host toolchain that BUILDS the artifact comes from the same # pin table as the embedded runtimes (pm/lock.json), so gate == pin - # by construction in build-bundled-desktop.mjs's toolchain gates. + # by construction in bundles/desktop.py's toolchain gates. run: | python3 -c ' import json @@ -715,7 +715,7 @@ jobs: # every Mach-O) — raise the fd limit and let DEBUG show progress. ulimit -n 16384 2>/dev/null || true echo "file descriptor limit: soft=$(ulimit -Sn) hard=$(ulimit -Hn)" - node scripts/build-bundled-desktop.mjs --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + uv run --no-project --python 3.11 python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled - name: Audit bundle architecture shell: bash @@ -1374,9 +1374,7 @@ jobs: run: | npm ci --workspace ui-tui --include=dev --no-fund --no-audit --silent npm run build --workspace ui-tui - test -f ui-tui/dist/entry.js - mkdir -p termux-build/payload/app/hermes_cli/tui_dist - cp ui-tui/dist/entry.js termux-build/payload/app/hermes_cli/tui_dist/entry.js + python3 scripts/bundles/payload.py surfaces termux-build/payload --repo-dir app --tui-only - name: Assemble the .deb shell: bash diff --git a/apps/desktop/BUILDING.md b/apps/desktop/BUILDING.md index 07b3925b15..c6ae22e4e9 100644 --- a/apps/desktop/BUILDING.md +++ b/apps/desktop/BUILDING.md @@ -30,7 +30,7 @@ cua-driver) is digest-pinned and staged at build time. | Platform | Artifact | Notes | |---|---|---| | Windows | MSIX `.msix` / `.msixbundle` | The shipping artifact. Signed with Azure Trusted Signing. Out-of-store installs update via the OS App Installer (.appinstaller source; the app checks + prompts, the OS applies). Store-submission builds (HERMES_DESKTOP_VARIANT=store) use the Partner Center identity and update via the Store. | -| macOS | `.dmg` | Signed and notarized when the `APPLE_*` / `CSC_*` secrets are set. Updated via electron-updater against the `latest-mac.yml` feed. | +| macOS | `.dmg` | Signed and notarized when the `APPLE_*` / `CSC_*` secrets are set. Updated via electron-updater against the stable/canary macOS feed. | | Linux | unpacked / AppImage | Unsigned. | NSIS is intentionally dead (D1 decision): Windows ships MSIX only. @@ -40,29 +40,24 @@ NSIS is intentionally dead (D1 decision): Windows ships MSIX only. One script drives the whole build: ``` -node scripts/build-bundled-desktop.mjs --tag=vX.Y.Z +uv run --no-project --python 3.11 python scripts/bundles/desktop.py --tag=vX.Y.Z ``` -The script always runs every step: +The shared Python builder performs these steps: -1. **Gate the toolchain.** The host `node` and `npm` must satisfy - `package.json` engines, and the toolchain pins resolve from - `pm/lock.json` (the "Resolve toolchain pins" CI step). The payload embeds - these exact pinned versions, so gate == embed. -2. **Build the JS surfaces.** ui-tui (with hermes-ink) and the dashboard SPA. -3. **Build the desktop app.** `npm run build` in `apps/desktop`: vite, - electron-main bundle, native deps, then payload staging. -4. **Stage the agent payload** (`pm bundle`). This snapshots the repo at the - tag with `git archive`, copies the prebuilt JS surfaces in, installs the - pinned CPython and `site-packages`, stages the pinned managed tools, and - writes `manifest.json` plus the install stamp. Each staged binary must - prove the target architecture in its own version banner. A wrong- - architecture binary fails the build. -5. **Package with electron-builder.** MSIX on Windows, DMG on macOS. +1. Validate the release tag and checkout identity. Check native Node architecture. +2. Install the locked JS workspace and validate its declared engine constraints. +3. Build the TUI and dashboard outputs. +4. Stage the native payload through PM, place JS assets, relocate links, and + generate launchers from the archived project's script declarations. +5. Build the Electron app and package MSIX, DMG/ZIP, or AppImage. -Payload staging stays dormant unless `HERMES_DESKTOP_VARIANT=bundled` is -set. The build script sets it. A normal `npm run dev` or `npm run pack` -without the script does not stage payloads. +Light omits the embedded runtime by definition. Its app is built and packaged +without staging the unused Python payload. The normal development loop remains +separate from release bundle assembly. + +See [shared bundle builds](../../docs/shared-bundle-builds.md) for the modules +shared with Termux and the target-specific boundaries. ## Code signing (Windows) @@ -102,12 +97,12 @@ in sync with app-builder-lib when electron-builder bumps. ## Code signing (macOS) -The macOS build signs and notarizes with electron-builder's builtin -notarization when the `APPLE_ID` / `APPLE_APP_SPECIFIC_PASSWORD` / -`APPLE_TEAM_ID` secrets are present. The `sign-nested-chromium` path signs -the payload's nested Chromium Mach-O binaries (see `sign-nested-chromium.mjs`, -wired from `after-pack.mjs`); the outer app bundle is signed by the -electron-builder signing pass. +The existing after-sign hook owns notarization using `APPLE_API_KEY`, +`APPLE_API_KEY_ID`, and `APPLE_API_ISSUER`, or a keychain profile. The workflow +writes the key from its `APPLE_API_KEY_P8` secret. The outer app and nested +Mach-O executables must be signed before publication. See +[macOS bundle updates](../../docs/macos-bundle-updates.md) for the feed contract +and publishing gates. ## Local build diff --git a/apps/desktop/electron-builder.config.cjs b/apps/desktop/electron-builder.config.cjs index d433ea2c9f..0bb9461326 100644 --- a/apps/desktop/electron-builder.config.cjs +++ b/apps/desktop/electron-builder.config.cjs @@ -203,7 +203,7 @@ module.exports = { // scripts/msix-shared.mjs). setBuildNumber makes getVersionInWeirdWindowsForm // use the BUILD_NUMBER env (4th component) instead of hardcoding ".0" — a // stable build sets no BUILD_NUMBER and stays X.Y.Z.0, a canary build sets - // it via build-bundled-desktop.mjs so App Installer updates over equal + // it via scripts/bundles/desktop.py so App Installer updates over equal // canary-over-canary versions instead of refusing them. setBuildNumber: true, // Floor Windows 11 22H2. Below build 18307 the manifest schema caps diff --git a/apps/desktop/electron/payload-backend.ts b/apps/desktop/electron/payload-backend.ts index 1b46512c9c..b12d3a5f63 100644 --- a/apps/desktop/electron/payload-backend.ts +++ b/apps/desktop/electron/payload-backend.ts @@ -70,7 +70,7 @@ export function resolvePayload( const toolsDir = path.join(root, manifest.store) const venvDir = path.join(root, manifest.venv) // The CLI trampoline staged into bin/ (hermes/hermes-agent/hermes-acp — - // build-bundled-desktop.mjs 5b: distlib-minted launchers on win32, + // scripts/bundles/desktop.py 5b: distlib-minted launchers on win32, // $0-relative bash trampolines on POSIX). It execs the store python with // the payload's own PYTHONPATH, so it is the single bundled entry point. const shim = path.join(root, 'bin', deps.isWindows ? 'hermes.exe' : 'hermes') diff --git a/apps/desktop/package.json b/apps/desktop/package.json index e5fa014805..6cbb5ab430 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -34,7 +34,7 @@ "builder": "cross-env NODE_OPTIONS=--max-old-space-size=16384 node scripts/run-electron-builder.mjs", "pack": "npm run build && npm run builder -- --dir --publish never", "dist": "npm run build && npm run builder", - "payload": "node scripts/stage-payload.mjs", + "payload": "uv run --no-project --python 3.11 python ../../scripts/bundles/stage.py --out build/agent-payload", "dist:bundled": "npm run payload && npm run dist", "dist:mac": "npm run build && npm run builder -- --mac", "dist:mac:dmg": "npm run build && npm run builder -- --mac dmg", diff --git a/apps/desktop/scripts/after-pack.mjs b/apps/desktop/scripts/after-pack.mjs index f003e17c7b..8eb56cb02f 100644 --- a/apps/desktop/scripts/after-pack.mjs +++ b/apps/desktop/scripts/after-pack.mjs @@ -20,8 +20,9 @@ */ import path from 'node:path' +import fs from 'node:fs' +import { execFileSync } from 'node:child_process' -import { findPackedPayload, relativizePayloadLinks } from './materialize-payload-links.mjs' import { batchSignAppTree } from './batch-sign-binaries.mjs' import { resolveSigningIdentity, signNestedChromium } from './sign-nested-chromium.mjs' import { sanitizeTree } from './sanitize-pe-signatures.mjs' @@ -29,34 +30,27 @@ import { stampExeIdentity } from './set-exe-identity.mjs' export default async function afterPack(context) { const platform = context.electronPlatformName + const resources = platform === 'darwin' + ? path.join(context.appOutDir, `${context.packager.appInfo.productFilename}.app`, 'Contents', 'Resources') + : path.join(context.appOutDir, 'resources') + const payload = path.join(resources, 'agent-payload') + if (platform !== 'win32' && fs.existsSync(path.join(payload, 'manifest.json'))) { + execFileSync('uv', ['run', '--no-project', '--python', '3.11', 'python', + path.resolve(import.meta.dirname, '../../../scripts/bundles/payload.py'), 'relocate', payload], { stdio: 'inherit' }) + } if (platform === 'darwin') { - const payload = findPackedPayload(context.appOutDir, platform) - if (payload) { - const n = relativizePayloadLinks(payload) + if (fs.existsSync(payload)) { const entitlements = path.join(import.meta.dirname, '..', 'electron', 'entitlements.mac.inherit.plist') const { identity, keychain } = await resolveSigningIdentity(context.packager) const nested = signNestedChromium(payload, { entitlements, identity, keychain }) console.log( - `[after-pack] relativized ${n} payload links; repaired ${nested.repaired} framework links; signed ${nested.signed} nested chromium targets` + + `[after-pack] repaired ${nested.repaired} framework links; signed ${nested.signed} nested chromium targets` + (identity ? ` as ${identity}` : ' (no Developer ID in the builder keychain)') ) } return } if (platform === 'linux') { - // Linux has no codesign, but the relocatable venv's bin/python* are - // ABSOLUTE symlinks onto the build runner's store interpreter, so they - // dangle once the unpacked tree moves (first-boot smoke, or a user - // installing to a different path). Rewrite them to RELATIVE symlinks - // (the target lives inside the payload) — a relative link survives - // relocation AND keeps the interpreter's real prefix resolution (a - // copied binary would fall back to the baked build prefix and lose its - // stdlib). Out-of-payload targets fail the build. - const payload = findPackedPayload(context.appOutDir, platform) - if (payload) { - const n = relativizePayloadLinks(payload) - console.log(`[after-pack] relativized ${n} payload links so the relocatable venv survives relocation`) - } return } if (platform !== 'win32') { diff --git a/apps/desktop/scripts/before-build.mjs b/apps/desktop/scripts/before-build.mjs index c984105a44..93a3b09c2e 100644 --- a/apps/desktop/scripts/before-build.mjs +++ b/apps/desktop/scripts/before-build.mjs @@ -21,7 +21,6 @@ import fs from 'node:fs' import path from 'node:path' import { createRequire } from 'node:module' -import { CLI_LAUNCHER_SPECS } from '../../../scripts/desktop-cli/cli-entrypoints.mjs' const require = createRequire(import.meta.url) const { @@ -73,7 +72,8 @@ function writeMsixExtensions() { const manifest = path.join(desktop, 'build', 'agent-payload', 'manifest.json') const payload = fs.existsSync(manifest) ? JSON.parse(fs.readFileSync(manifest, 'utf8')) : null const launchers = light || !payload || payload.external - ? [] : CLI_LAUNCHER_SPECS.map(spec => spec.name) + ? [] : payload.launchers + if (!Array.isArray(launchers)) throw new Error('Bundled payload has no declared launchers') const aliases = appExecutionAliasExtensions(launchers) // The uap3:AppExtension fragment that registers the app as a Windows // Copilot hardware key provider. The press activates hermes://copilot-key/start. @@ -111,9 +111,9 @@ ${aliases}` * One uap5:Extension block per payload CLI launcher, each naming its own * Executable (the distlib-minted launcher exes under bin/) and the alias * that exe serves. Exported pure for tests. - * @param {{ name: string }[]} [launchers] exe stems under bin/ + * @param {string[]} launchers exe stems under bin/ */ -export function appExecutionAliasExtensions(launchers = CLI_LAUNCHER_SPECS.map((s) => s.name)) { +export function appExecutionAliasExtensions(launchers) { if (launchers.length === 0) return '' const bs = String.fromCharCode(92) const executable = (name) => ['app', 'resources', 'agent-payload', 'bin', `${name}.exe`].join(bs) diff --git a/apps/desktop/scripts/cli-launchers.test.mjs b/apps/desktop/scripts/cli-launchers.test.mjs index c75f2f2a00..7405b32bf4 100644 --- a/apps/desktop/scripts/cli-launchers.test.mjs +++ b/apps/desktop/scripts/cli-launchers.test.mjs @@ -1,143 +1,14 @@ import assert from 'node:assert/strict' -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - import { test } from 'vitest' - -import { - CLI_LAUNCHER_SPECS, - posixTrampolineScripts, - renderWinWrapper -} from '../../../scripts/desktop-cli/cli-entrypoints.mjs' import { appExecutionAliasExtensions } from './before-build.mjs' -const scriptDir = path.dirname(fileURLToPath(import.meta.url)) - -// The payload layout facts every generator receives (bin-relative, forward -// slashes — the same composition build-bundled-desktop.mjs step 5b feeds in). -const RELS = { - relPython: '../tools/cpython-3.11.16+20260814-win32-x64/bin/python3', - relSite: '../venv/lib/python3.11/site-packages', - relRepo: '../hermes-agent' -} - -test('three launcher specs mirror [project.scripts] in pyproject.toml', () => { - assert.deepEqual( - CLI_LAUNCHER_SPECS.map(s => s.name), - ['hermes', 'hermes-agent', 'hermes-acp'] - ) - assert.deepEqual( - CLI_LAUNCHER_SPECS.map(s => `${s.module}:${s.func}`), - ['hermes_cli.main:main', 'run_agent:main', 'acp_adapter.entry:main'] - ) -}) - -const scripts = posixTrampolineScripts(RELS) - -test('one POSIX trampoline per entry, named plainly (no suffix)', () => { - assert.deepEqual( - scripts.map(s => s.name), - ['hermes', 'hermes-agent', 'hermes-acp'] - ) -}) - -test('trampoline is $0-relative: shebang, symlink-chain resolution, own-dir cd', () => { - const text = scripts[0].text - assert.ok(text.startsWith('#!/usr/bin/env bash\n')) - // The whole relocatability contract: nothing in the script may be an - // absolute path or a Windows path — everything resolves off $0. - for (const line of text.split('\n')) { - const code = line.replace(/^#/, '').trim() - assert.ok(!/[A-Za-z]:\\/.test(code), `windows path in generated script: ${line}`) - assert.ok(!/=\s*\/(?!usr\/bin\/env)/.test(code), `absolute path in generated script: ${line}`) - } - assert.match(text, /self="\$0"/) - assert.match(text, /while \[ -L "\$self" \]/) - assert.match(text, /BIN_DIR="\$\(cd -- "\$\(dirname -- "\$self"\)" && pwd\)"/) - assert.match(text, /PYTHON="\$BIN_DIR"\/\.\.\/tools\//) -}) - -test('trampoline composes the payload import roots (repo first) and replaces inherited PYTHONPATH', () => { - const text = scripts[0].text - assert.match(text, /unset PYTHONPATH PYTHONHOME/) - assert.match(text, /export PYTHONPATH="\$REPO:\$SITE"/) - assert.match(text, /REPO="\$BIN_DIR"\/\.\.\/hermes-agent/) - assert.match(text, /SITE="\$BIN_DIR"\/\.\.\/venv\/lib\/python3\.11\/site-packages/) -}) - -test('trampoline keeps pycache out of the payload and execs the entry module', () => { - const text = scripts[0].text - assert.match(text, /PYTHONPYCACHEPREFIX="\$HOME\/\.cache\/hermes-pycache"/) - assert.match(text, /if \[ -z "\$\{PYTHONPYCACHEPREFIX:-\}" \]/, 'user-set prefix must win') - assert.match(text, /exec "\$PYTHON" -m hermes_cli\.main "\$@"/) -}) - -test('trampoline fails loudly (exit 2) when the bundled interpreter is missing', () => { - const text = scripts[0].text - assert.match(text, /bundled interpreter missing at \$PYTHON/) - assert.match(text, /exit 2/) -}) - -test('each trampoline execs its own entry module', () => { - const modules = scripts.map(s => /exec "\$PYTHON" -m (\S+) "\$@"/.exec(s.text)?.[1]) - assert.deepEqual(modules, ['hermes_cli.main', 'run_agent', 'acp_adapter.entry']) -}) - -test('win wrapper substitution bakes the entry module and payload layout facts', () => { - const text = renderWinWrapper(CLI_LAUNCHER_SPECS[0], RELS.relRepo, RELS.relSite) - assert.ok(!text.includes('__HERMES_'), 'placeholders must be fully substituted') - assert.match(text, /HERMES_ENTRY_MODULE = "hermes_cli\.main"/) - assert.match(text, /HERMES_ENTRY_FUNC = "main"/) - assert.match(text, /HERMES_REPO_REL = "\.\.\/hermes-agent"/) - assert.match(text, /HERMES_SITE_REL = "\.\.\/venv\/lib\/python3\.11\/site-packages"/) -}) - -test('win wrapper rejects values that still contain placeholders', () => { - assert.throws(() => renderWinWrapper({ module: '__HERMES_REPO_REL__', func: 'main' }, RELS.relRepo, RELS.relSite)) -}) - -test('MSIX: ONE uap5 alias Extension naming every launcher alias', () => { - const xml = appExecutionAliasExtensions() - const blocks = xml.match(/`), - `no ExecutionAlias for ${spec.name}.exe` - ) - } - assert.equal((xml.match(/ { - // The light gating lives in writeMsixExtensions (light ? '' : ...); the - // pure builder must stay gating-free, so just pin its shape here. - assert.ok(appExecutionAliasExtensions(['hermes']).includes('windows.appExecutionAlias')) - assert.ok(scriptDir.length > 0) -}) - -// ── HermesGateway Windows Service fragment (removed) ────────────────────── -// The MSIX SCM service feature was removed (settled 2026-09-03: the existing -// user-logon Scheduled Task supervises the gateway; a desktop6:Service -// cannot run as the installing user — the desktop6 schema requires -// StartAccount in localSystem|localService|networkService). This pins the -// removal so the invalid fragment cannot silently come back. - -test('the manifest extension writer registers no windows.service', () => { - // writeMsixExtensions is the one writer; its source must carry no - // desktop6:Service emission (this is a removal pin, not a shape snapshot: - // any NEW extension fragments may be added freely). - const source = fs.readFileSync(new URL('./before-build.mjs', import.meta.url), 'utf8') - assert.ok(!source.includes('windows.service'), 'no windows.service Category emitted') - assert.ok(!source.includes('desktop6:Service'), 'no desktop6:Service fragment emitted') +test('one MSIX extension consumes the launchers declared by the payload', () => { + const names = ['custom-cli', 'another-cli'] + const xml = appExecutionAliasExtensions(names) + assert.equal((xml.match(//bin/python3), -// so venv/bin/python -> ../tools//bin/python3 survives moving the -// whole tree, and the interpreter resolves its real prefix through the -// link (stdlib found, no /install fallback). -// -// Only venv/bin is rewritten. Do not walk the rest of the payload: a -// file-symlink at Foo.framework/Foo is the framework binary, and -// flattening it makes codesign report "bundle format is ambiguous". -// -// A symlink whose target points OUTSIDE the bundle is a build bug (the -// payload's own venv must never reference the builder's machine) — fail -// loudly rather than ship a dangling link. - -import fs from 'node:fs' -import path from 'node:path' - -export function findPackedPayload(appOutDir, platform) { - if (!appOutDir) return null - const candidates = - platform === 'darwin' - ? [ - path.join(appOutDir, 'Contents', 'Resources', 'agent-payload'), - path.join(appOutDir, 'Hermes.app', 'Contents', 'Resources', 'agent-payload'), - path.join(appOutDir, 'HermesBundled.app', 'Contents', 'Resources', 'agent-payload') - ] - : [path.join(appOutDir, 'resources', 'agent-payload')] - return candidates.find(p => fs.existsSync(p)) || null -} - -/** - * Rewrite payload venv/bin symlinks to RELATIVE targets that resolve - * inside the bundle. Returns the count rewritten. - * - * The single rule: a bundled venv's links must point at the payload's own - * store. Links whose target already resolves inside the payload are - * relativized (or left if already relative). Links whose target is the - * BUILD staging path (build/agent-payload/tools//...) — which - * doesn't exist in the unpacked app — are rewritten to the matching - * store entry under /tools/. Anything else fails the build. - */ -export function relativizePayloadLinks(root) { - if (!root || !fs.existsSync(root)) return 0 - const bin = path.join(root, 'venv', 'bin') - if (!fs.existsSync(bin)) return 0 - const payloadRoot = path.resolve(root) - const toolsDir = path.join(payloadRoot, 'tools') - let count = 0 - let entries - try { - entries = fs.readdirSync(bin, { withFileTypes: true }) - } catch { - return 0 - } - for (const ent of entries) { - const p = path.join(bin, ent.name) - let st - try { - st = fs.lstatSync(p) - } catch { - continue - } - if (!st.isSymbolicLink()) continue - let target - try { - target = fs.readlinkSync(p) - } catch { - continue - } - const absTarget = path.resolve(bin, target) - const insidePayload = absTarget !== payloadRoot && absTarget.startsWith(payloadRoot + path.sep) - let resolvedTarget = absTarget - if (!insidePayload) { - // The BUILD staging form: build/agent-payload/tools//... . - // The tail names a store entry that must exist inside THIS payload. - const m = target.match(/(?:^|\/)tools\/(.+)$/) - if (!m) { - throw new Error( - `venv/bin/${ent.name} -> ${target} points outside the payload and does not name ` + - 'a store entry (tools//...); a bundled venv must reference the payload store', - ) - } - const inPayload = path.join(toolsDir, m[1]) - if (!fs.existsSync(inPayload)) { - throw new Error( - `venv/bin/${ent.name} -> ${target}: store entry ${m[1]} is not present in the payload ` + - `(${inPayload}); a bundled venv must reference the payload store`, - ) - } - resolvedTarget = inPayload - } - const rel = path.relative(bin, resolvedTarget) - if (target === rel) continue - fs.unlinkSync(p) - fs.symlinkSync(rel, p) - count += 1 - } - return count -} diff --git a/apps/desktop/scripts/materialize-payload-links.test.mjs b/apps/desktop/scripts/materialize-payload-links.test.mjs deleted file mode 100644 index 1284e65c14..0000000000 --- a/apps/desktop/scripts/materialize-payload-links.test.mjs +++ /dev/null @@ -1,144 +0,0 @@ -import assert from 'node:assert/strict' -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' -import { test } from 'vitest' - -import { findPackedPayload, relativizePayloadLinks } from './materialize-payload-links.mjs' - -function tempRoot() { - return fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-relativize-')) -} - -test('findPackedPayload locates the mac nested payload', () => { - const root = tempRoot() - try { - const app = path.join(root, 'Hermes.app') - const payload = path.join(app, 'Contents', 'Resources', 'agent-payload') - fs.mkdirSync(payload, { recursive: true }) - assert.equal(findPackedPayload(app, 'darwin'), payload) - assert.equal(findPackedPayload(root, 'darwin'), payload) - assert.equal(findPackedPayload(root, 'linux'), null) - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) - -test('relativizePayloadLinks rewrites an absolute build-staging symlink to a payload-relative one', () => { - if (process.platform === 'win32') return - const root = tempRoot() - try { - const store = path.join(root, 'tools', 'python', 'bin') - const venv = path.join(root, 'venv', 'bin') - fs.mkdirSync(store, { recursive: true }) - fs.mkdirSync(venv, { recursive: true }) - const real = path.join(store, 'python3.11') - fs.writeFileSync(real, 'interpreter-bytes') - const link = path.join(venv, 'python3') - // The absolute build-staging form uv --relocatable writes: points at - // build/agent-payload/tools/... which does NOT exist here — but the - // store entry it names (tools/python/bin/python3.11) DOES exist in the - // payload root, which is what the rewrite keys on. - fs.symlinkSync('/somewhere/build/agent-payload/tools/python/bin/python3.11', link) - assert.equal(fs.lstatSync(link).isSymbolicLink(), true) - - const n = relativizePayloadLinks(root) - assert.equal(n, 1) - assert.equal(fs.lstatSync(link).isSymbolicLink(), true) - const target = fs.readlinkSync(link) - assert.equal(target.startsWith(path.sep), false) // now relative - assert.equal(path.resolve(venv, target), real) - assert.equal(fs.readFileSync(link, 'utf8'), 'interpreter-bytes') // resolves - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) - -test('relativizePayloadLinks throws when the named store entry is missing from the payload', () => { - if (process.platform === 'win32') return - const root = tempRoot() - try { - const venv = path.join(root, 'venv', 'bin') - fs.mkdirSync(venv, { recursive: true }) - // Names tools/python/... but no such entry exists under the payload. - fs.symlinkSync('/somewhere/build/agent-payload/tools/python/bin/python3.11', path.join(venv, 'python3')) - assert.throws(() => relativizePayloadLinks(root), /store entry .* is not present in the payload/) - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) - -test('relativizePayloadLinks leaves a sibling link (python3 -> python) alone', () => { - if (process.platform === 'win32') return - const root = tempRoot() - try { - const store = path.join(root, 'tools', 'python', 'bin') - const venv = path.join(root, 'venv', 'bin') - fs.mkdirSync(store, { recursive: true }) - fs.mkdirSync(venv, { recursive: true }) - const real = path.join(store, 'python3.11') - fs.writeFileSync(real, 'interpreter-bytes') - // python -> store link; python3 -> python sibling. - fs.symlinkSync('/somewhere/build/agent-payload/tools/python/bin/python3.11', path.join(venv, 'python')) - fs.symlinkSync('python', path.join(venv, 'python3')) - - const n = relativizePayloadLinks(root) - assert.equal(n, 1) // only the store link rewritten - assert.equal(fs.readlinkSync(path.join(venv, 'python3')), 'python') // sibling untouched - assert.equal(fs.readFileSync(path.join(venv, 'python3'), 'utf8'), 'interpreter-bytes') // resolves via chain - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) - -test('relativizePayloadLinks throws when the target does not name a store entry', () => { - if (process.platform === 'win32') return - const root = tempRoot() - try { - const venv = path.join(root, 'venv', 'bin') - fs.mkdirSync(venv, { recursive: true }) - fs.symlinkSync('/usr/bin/python3.11', path.join(venv, 'python3')) - assert.throws(() => relativizePayloadLinks(root), /does not name a store entry/) - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) - -test('relativizePayloadLinks leaves already-relative links alone', () => { - if (process.platform === 'win32') return - const root = tempRoot() - try { - const store = path.join(root, 'tools', 'python', 'bin') - const venv = path.join(root, 'venv', 'bin') - fs.mkdirSync(store, { recursive: true }) - fs.mkdirSync(venv, { recursive: true }) - const real = path.join(store, 'python3.11') - fs.writeFileSync(real, 'interpreter-bytes') - const link = path.join(venv, 'python3') - fs.symlinkSync(path.relative(venv, real), link) - - assert.equal(relativizePayloadLinks(root), 0) - assert.equal(fs.readlinkSync(link), path.relative(venv, real)) - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) - -test('relativizePayloadLinks does not touch a framework file-symlink (not under venv/bin)', () => { - if (process.platform === 'win32') return - const root = tempRoot() - try { - const versioned = path.join(root, 'tools', 'chromium-1208', 'F.framework', 'Versions', 'A') - fs.mkdirSync(versioned, { recursive: true }) - const real = path.join(versioned, 'F') - fs.writeFileSync(real, 'machO') - const link = path.join(root, 'tools', 'chromium-1208', 'F.framework', 'F') - fs.symlinkSync(path.join('Versions', 'A', 'F'), link) - assert.equal(fs.lstatSync(link).isSymbolicLink(), true) - - assert.equal(relativizePayloadLinks(root), 0) - assert.equal(fs.lstatSync(link).isSymbolicLink(), true) - } finally { - fs.rmSync(root, { recursive: true, force: true }) - } -}) diff --git a/apps/desktop/scripts/stage-payload.mjs b/apps/desktop/scripts/stage-payload.mjs deleted file mode 100644 index bc5122a16f..0000000000 --- a/apps/desktop/scripts/stage-payload.mjs +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Stage the pm payload into build/agent-payload for a BUNDLED desktop - * build. Runs `hermes pm bundle` with the repo's own venv interpreter — - * the payload carries the repo snapshot (committed state), the tool - * store + facts, and a relocatable venv on the pinned interpreter. - * - * Plain `npm run dist` (no payload) still works: before-build.mjs writes - * an external:true stub and the app resolves a runtime at first launch. - */ -import { execFileSync } from 'node:child_process' -import fs from 'node:fs' -import path from 'node:path' - -const desktopRoot = path.join(import.meta.dirname, '..') -const repoRoot = path.join(desktopRoot, '..', '..') -const payloadDir = path.join(desktopRoot, 'build', 'agent-payload') - -function venvPython() { - for (const venv of ['.venv', 'venv']) { - for (const rel of [ - ['Scripts', 'python.exe'], - ['bin', 'python'] - ]) { - const candidate = path.join(repoRoot, venv, ...rel) - - if (fs.existsSync(candidate)) { - return candidate - } - } - } - - return null -} - -const python = venvPython() - -if (!python) { - console.error('stage-payload: no repo venv found — run `uv sync` in the repo root first') - process.exit(1) -} - -console.log(`stage-payload: ${python} -m pm.cli bundle --out ${payloadDir}`) -execFileSync(python, ['-m', 'pm.cli', 'bundle', '--out', payloadDir], { - cwd: repoRoot, - stdio: 'inherit', - env: { ...process.env, PYTHONUTF8: '1' } -}) diff --git a/docs/shared-bundle-builds.md b/docs/shared-bundle-builds.md new file mode 100644 index 0000000000..f33e02feb9 --- /dev/null +++ b/docs/shared-bundle-builds.md @@ -0,0 +1,51 @@ +# Shared bundle builds + +The build-only Python modules live in `scripts/bundles/`. They use PM for +package installation and dependency resolution. They do not implement a +second package manager. + +| Module | Responsibility | Consumers | +|---|---|---| +| `payload.py` | Git snapshots, manifest/facts, JS output placement, relocation and launcher generation | Native desktop and Termux | +| `native.py` | Native tool-store staging and dependency venv assembly | `hermes pm bundle`, desktop builder | +| `desktop.py` | Build the JS surfaces, assemble the payload, invoke Electron packaging | Native release workflow, local builds | +| `stage.py` | Stage a payload and its launchers without creating an Electron package | `npm run payload` | +| `mint_launchers.py` | Mint Windows launchers with the payload interpreter's distlib | Shared launcher assembly | +| `launcher_wrapper.py` | Runtime template for the minted executable | Shared launcher renderer | + +Build a desktop bundle from a checkout at its release tag: + +```sh +uv run --no-project --python 3.11 python scripts/bundles/desktop.py --tag=vX.Y.Z +``` + +The builder derives console entrypoints from the archived `pyproject.toml`. +It records launcher names in the payload manifest. The MSIX hook reads those +names rather than carrying a second entrypoint list. POSIX launchers follow +links to the installed payload and call each declared function directly. +Windows launchers are minted by the payload's own Python, preserving native +architecture and relocation behavior. + +Termux uses the same snapshot, manifest/fact writer, JS asset placement and +POSIX launcher generator. Its package-manager hooks stay in `scripts/termux/`. +Its bionic wheel compilation and offline installation remain target-specific: +a glibc host cannot execute the shipped interpreter, and Termux has a fixed +installation prefix. Native desktop staging instead resolves on the target OS. +Neither path compiles dependencies on the user's machine. + +Electron-specific work stays with Electron: renderer/main-process bundling, +Node native bindings, MSIX metadata, signing, notarization and app packaging. +The after-pack hook invokes the shared Python relocation command instead of +maintaining its own link rewrite algorithm. + +Ordinary bundle staging does not scan user plugin trees. Runtime plugin +admission is a separate PM transaction. Build output cleanup is confined to +its payload store; it must not prune the machine-wide downloader partials. + +## Verification boundary + +Tests execute shared snapshot/manifest helpers on real git fixtures, stage +real subprocesses and venvs at controlled boundaries, and mint/run Windows +launchers after relocation. POSIX launcher and symlink tests run on POSIX. +A local helper test is not proof of a signed installer, bionic wheel build, +or stable-release upgrade. The release workflows own those native receipts. diff --git a/pm/cli.py b/pm/cli.py index fdb56d5f66..c69bedfec2 100644 --- a/pm/cli.py +++ b/pm/cli.py @@ -102,16 +102,6 @@ def _install_names(names: list[str], target: str | None = None) -> int: -def _bundle_package_names() -> list[str]: - names = [ - n - for n in _lockfile().names() - if not get_package(n).internal or n == "uv" - ] - if "python" not in names: - names.append("python") - return names - def cmd_install(args) -> int: cross_target = getattr(args, "target", None) @@ -549,166 +539,9 @@ def cmd_status(args) -> int: def cmd_bundle(args) -> int: - """Stage a complete payload for THIS machine's target into --out: - repo snapshot + store + facts (via the normal install path, redirected) - + a relocatable venv built on the staged interpreter and synced from - uv.lock. Built natively per (os, arch); there is no cross-target - staging.""" - import os + from scripts.bundles.native import stage_native + return stage_native(args) - from pm import paths - - out = Path(args.out).resolve() - store_dir = out / "tools" - store_dir.mkdir(parents=True, exist_ok=True) - # A manifest from a previous run would make this payload look sealed - # and refuse its own staging; it is rewritten at the end. - (out / "manifest.json").unlink(missing_ok=True) - - repo_dir = out / "hermes-agent" - ref = args.ref or "HEAD" - if repo_dir.exists(): - shutil.rmtree(repo_dir) - repo_dir.mkdir(parents=True) - print(f"staging repo snapshot ({ref})…", flush=True) - archive = subprocess.run( - ["git", "archive", "--format=tar", ref], - cwd=paths.repo_root(), capture_output=True, timeout=600, - ) - if archive.returncode != 0: - print(f"✗ repo: git archive {ref} failed: {archive.stderr.decode()[-500:]}") - return 1 - import io - import tarfile - - with tarfile.open(fileobj=io.BytesIO(archive.stdout)) as tar: - tar.extractall(repo_dir, filter="data") - print(f"✓ repo ({ref})") - - os.environ["HERMES_RUNTIME_DIR"] = str(store_dir) - - names = _bundle_package_names() - failed = _install_names( - [n for n in names if get_package(n).missing_reason(current_target()) is None] - ) - - # Prune the staged store BEFORE the venv sync and packaging: drop the - # fetch- download-cache archives (needed only at install time — dead - # weight in the shipped payload AND in the CI cache that restores this - # dir) and any orphaned package versions left over from an older lock - # the cache carried in. A lean staged store = a lean CI cache. - removed, kept = _gc_store(_store(), _facts()) - print(f"✓ gc: pruned {removed} fetch/stale entries, kept {kept}") - - uv_bin, env = pm_uv() - if uv_bin is None: - print("✗ venv: uv did not stage") - return 1 - - python_fact = _facts().get("python") - if python_fact is None: - print("✗ venv: no staged interpreter to build on") - return 1 - python_bin = get_package("python").binary( - _store().entry(python_fact["entry"]), current_target() - ) - - # Build + sync INSIDE the staged repo: the editable project install - # must point at the payload's own tree, not this checkout. - venv_dir = out / "venv" - if venv_dir.exists(): - shutil.rmtree(venv_dir) - env["VIRTUAL_ENV"] = str(venv_dir) - env.pop("UV_NO_CONFIG", None) - if current_target().startswith("darwin"): - # python-build-standalone bakes phantom toolchain paths (its build - # dir's llvm-ar) into sysconfig; sdist builds then fail with - # "No such file or directory: .../tools/llvm/bin/llvm-ar". Point - # sdist builds at the machine's real toolchain. - env.setdefault("AR", "/usr/bin/ar") - env.setdefault("CC", "clang") - for cmd in ( - [uv_bin, "venv", "--relocatable", "--python", str(python_bin), str(venv_dir)], - [uv_bin, "sync", "--frozen", "--all-extras", "--active"], - ): - print(f" venv: $ {' '.join(cmd)}", flush=True) - code, tail = _run_live(cmd, cwd=repo_dir, env=env) - if code != 0: - print(f"✗ venv: {' '.join(cmd[1:3])} failed:\n{tail}") - return 1 - print("✓ venv (relocatable, all extras, on the staged interpreter)") - - # The frozen feature set: the EXACT extras that installed on this - # target (markers gate some off per-platform). This file is the - # lazy-off contract — pm sync never deviates from it. - from pm.features import installed_extras, write_features - - features = installed_extras(repo_dir, venv_dir) - write_features(features, out) - print(f"✓ enabled-features.json ({len(features)} extras recorded)") - - # Ship the uv cache: the staged venv sync just warmed the hermes-owned - # cache with every wheel this payload needs. Copying it in makes a - # mutable-venv rebuild from the bundle near-free (`uv sync --offline` - # from a warm cache probed at 0.4s vs 1.2s cold) — the blow-away-on- - # update contract depends on it. - from pm.packages import uv_cache_dir as bundle_uv_cache_dir - - payload_cache = out / "uv-cache" - if payload_cache.exists(): - shutil.rmtree(payload_cache, ignore_errors=True) - src_cache = bundle_uv_cache_dir() - if src_cache.is_dir(): - print(f" uv-cache: copying {src_cache} → payload...", flush=True) - shutil.copytree(src_cache, payload_cache) - print("✓ uv-cache (staged — warm rebuilds for the mutable venv)") - else: - print(" uv-cache: none warm (first bundle on this machine?)") - - bad = _arch_guard(store_dir) - for line in bad: - print(f"✗ arch: {line}") - failed += 1 - - manifest = { - "schema": 1, - "target": current_target(), - "ref": ref, - "repo": "hermes-agent", - "venv": "venv", - "store": "tools", - } - (out / "manifest.json").write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - print(f"✓ manifest ({out / 'manifest.json'})") - return 1 if failed else 0 - - -def _arch_guard(store_dir: Path) -> list[str]: - """Every staged binary must be built for this machine's target — a - payload staged with a mismatched interpreter or PATH tool ships an - artifact that cannot run. Reads facts, probes each entry binary.""" - from pm.lock import Facts - from pm.package import machine_matches_binary - - facts = Facts(store_dir / "facts.json") - problems = [] - target = current_target() - for name in _lockfile().names(): - package = get_package(name) - fact = facts.get(name) - if fact is None or "entry" not in fact: - continue - binary = package.binary(store_dir / fact["entry"], target) - if binary is None or not binary.is_file(): - continue - verdict = machine_matches_binary(binary, target) - # A package that declares this target as emulated (x64 binary run - # under Windows ARM64 built-in emulation) is fine with the x64 PE. - if verdict is False and target not in package.emulated_arch_targets: - problems.append(f"{name}: {binary.name} is not a {target} binary") - return problems def main(argv=None) -> int: diff --git a/scripts/build-bundled-desktop.mjs b/scripts/build-bundled-desktop.mjs deleted file mode 100644 index a6275b8341..0000000000 --- a/scripts/build-bundled-desktop.mjs +++ /dev/null @@ -1,437 +0,0 @@ -#!/usr/bin/env node -// build-bundled-desktop.mjs — one local=CI driver for a bundled desktop -// installer. Every step always runs. A skipped step is a different artifact. -// -// 1. preflight: git + npm; a release tag is resolvable -// 2. npm ci at the repo root (stamp-gated) -// 3. build ui-tui and the dashboard SPA -// 4. pm bundle (repo snapshot + tool store + relocatable venv) -// 5. plant the just-built JS surfaces into the staged payload -// (git archive only carries committed files; those dists are gitignored) -// 6. npm run build + builder in apps/desktop -// -// Usage: -// node scripts/build-bundled-desktop.mjs --tag=v0.20.5 -// node scripts/build-bundled-desktop.mjs --tag=v0.20.5 --variant=bundled -// -// Signing is CI's job. Local builds are unsigned. - -import { createHash } from 'node:crypto' -import { execSync, spawnSync } from 'node:child_process' -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -import { CLI_LAUNCHER_SPECS, posixTrampolineScripts, renderWinWrapper } from './desktop-cli/cli-entrypoints.mjs' -import { windowsFileVersion } from '../apps/desktop/scripts/windows-file-version.mjs' -import { relativizePayloadLinks } from '../apps/desktop/scripts/materialize-payload-links.mjs' -import { canaryBuildMinutes } from './msix-shared.mjs' - -const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') -const PAYLOAD_DIR = path.join(REPO_ROOT, 'apps', 'desktop', 'build', 'agent-payload') - -const args = process.argv.slice(2) -for (const flag of ['--no-install', '--no-package']) { - if (args.includes(flag)) { - fail(`${flag} is retired: every release build runs every step`) - } -} -const tagArg = args.find(a => a.startsWith('--tag='))?.slice('--tag='.length) -const variant = args.find(a => a.startsWith('--variant='))?.slice('--variant='.length) || 'bundled' -const dashDash = process.argv.indexOf('--') -const extraBuilderArgs = dashDash === -1 ? [] : process.argv.slice(dashDash + 1) - -if (!['bundled', 'light', 'store'].includes(variant)) { - fail(`--variant must be 'bundled', 'light', or 'store', got '${variant}'`) -} - -function fail(message) { - console.error(`[build-bundled] ${message}`) - process.exit(1) -} - -function run(cmd, argv, opts = {}) { - console.log(`[build-bundled] $ ${cmd} ${argv.join(' ')}`) - const shell = process.platform === 'win32' - if (shell) { - const bad = argv.find(a => /\s/.test(a)) - if (bad) { - fail( - `argument with whitespace cannot cross the Windows shell: ${JSON.stringify(bad)} — pass it via environment instead` - ) - } - } - const result = spawnSync(cmd, argv, { stdio: 'inherit', cwd: REPO_ROOT, shell, ...opts }) - if (result.status !== 0) { - fail(`${cmd} exited ${result.status}`) - } -} - -function capture(cmd) { - return execSync(cmd, { cwd: REPO_ROOT, encoding: 'utf8' }).trim() -} - -function pythonMinorFromLock() { - const lock = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'pm', 'lock.json'), 'utf8')) - const version = lock?.packages?.python?.version - if (typeof version !== 'string') { - fail('pm/lock.json has no python version') - } - return version.split('+')[0].split('.').slice(0, 2).join('.') -} - -function requireOnPath(tool) { - const probe = spawnSync(tool, ['--version'], { stdio: 'ignore', shell: process.platform === 'win32' }) - if (probe.status !== 0) { - fail(`required tool missing: ${tool}`) - } -} - -// ── 1. preflight ──────────────────────────────────────────────────────────── - -for (const tool of ['git', 'npm', 'uv']) { - requireOnPath(tool) -} - -// Toolchain gates. The build's output depends on these tools, so a wrong -// version makes a silently different artifact (the first Windows build -// shipped a wrong-arch uv exactly this way). The rules come from ONE -// source — package.json "engines". The EMBEDDED runtimes are a separate -// concern: they come from pm/lock.json via `pm bundle` (the payload -// python/node/uv are the pinned artifacts in the pm store), never from -// the host toolchain — the gates below only approve the tools that BUILD -// the artifact (the JS surfaces are built and npm-installed by the host -// node; the payload interpreter is installed by the host uv). CI installs -// the pinned versions from pm/lock.json as the host toolchain, so -// gate == pin there by construction. -export function parseVersion(text) { - const match = String(text).match(/(\d+)\.(\d+)\.(\d+)/) - return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null -} - -export function compareVersions(a, b) { - for (let i = 0; i < 3; i += 1) { - if (a[i] !== b[i]) return a[i] - b[i] - } - return 0 -} - -// The subset of semver ranges that package.json engines actually uses: -// space-separated comparators AND together, `||` separates alternatives. -// An unparseable comparator fails closed. -export function satisfiesRange(version, range) { - return String(range).split('||').some(alternative => { - const comparators = alternative.trim().split(/\s+/).filter(Boolean) - if (comparators.length === 0) return false - return comparators.every(comparator => { - const m = comparator.match(/^(>=|<=|>|<|=)?v?(\d+)\.(\d+)\.(\d+)$/) - if (!m) return false - const cmp = compareVersions(version, [Number(m[2]), Number(m[3]), Number(m[4])]) - switch (m[1]) { - case '>=': return cmp >= 0 - case '<=': return cmp <= 0 - case '>': return cmp > 0 - case '<': return cmp < 0 - default: return cmp === 0 - } - }) - }) -} - -export function uvBannerProblem(banner) { - // A build triple is three dash-joined words that end in letters - // (aarch64-pc-windows-msvc). Its position varies: nix builds print it - // first in the parens, official builds put a commit hash and a date - // before it. Match it anywhere — the date (2026-07-31) cannot match - // because its last segment is digits. - return /[a-z0-9_]+-[a-z0-9]+-[a-z][a-z0-9-]*/.test(String(banner)) - ? null - : 'its --version prints no build triple; the payload arch guard needs one (official uv 0.12+, or any nix/source build)' -} - -const engines = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')).engines || {} - -for (const tool of ['node', 'npm']) { - const text = tool === 'node' ? process.version : capture('npm --version') - const version = parseVersion(text) - if (!version) { - fail(`${tool}: cannot parse a version from ${JSON.stringify(text)}`) - } - const range = engines[tool] - if (range && !satisfiesRange(version, range)) { - fail(`${tool} ${version.join('.')} does not satisfy package.json engines ${JSON.stringify(range)} — the build would make a different artifact`) - } - console.log(`[build-bundled] ${tool} ${version.join('.')} (engines: ${range || 'unconstrained'})`) -} - -{ - const uvBanner = capture('uv --version') - const problem = uvBannerProblem(uvBanner) - if (problem) { - fail(`uv (${uvBanner}) would make a broken artifact: ${problem}`) - } - console.log(`[build-bundled] ${uvBanner} (build-host uv; the payload uv comes from pm/lock.json)`) -} - -let tag = tagArg -if (!tag) { - try { - tag = capture('git describe --tags --exact-match') - } catch { - fail('no --tag=vX.Y.Z given and HEAD is not at an exact release tag') - } -} -if (!/^v(?:0|[1-9]\d{0,2})\.\d+\.\d+(?:-canary\.20\d{6}(?:\d{6})?)?$/.test(tag)) { - fail(`'${tag}' is not a release tag (vX.Y.Z or vX.Y.0-canary.YYYYMMDDHHMMSS)`) -} - -const pyprojectVersion = fs - .readFileSync(path.join(REPO_ROOT, 'pyproject.toml'), 'utf8') - .match(/^version\s*=\s*"([^"]+)"/m)?.[1] -if (!pyprojectVersion) { - fail('could not read version from pyproject.toml') -} -const isCanary = tag.includes('-canary.') -if (!isCanary && tag !== `v${pyprojectVersion}`) { - fail(`tag ${tag} does not match pyproject.toml version ${pyprojectVersion}`) -} -const artifactVersion = isCanary ? tag.slice(1) : pyprojectVersion -const fileVersion = windowsFileVersion(tag) - -const passes = { - linux: [{ targets: '--linux AppImage' }], - darwin: [{ targets: '--mac dmg zip' }], - win32: [{ targets: '--win msix' }] -}[process.platform] -if (!passes) { - fail(`unsupported platform: ${process.platform}`) -} - -console.log(`[build-bundled] tag=${tag} variant=${variant} platform=${process.platform}-${process.arch}`) - -// ── 2. npm ci ─────────────────────────────────────────────────────────────── - -const installStamp = [ - `lock=${createHash('sha256').update(fs.readFileSync(path.join(REPO_ROOT, 'package-lock.json'))).digest('hex')}`, - `node=${process.version}`, - `npm=${execSync('npm --version', { encoding: 'utf8', shell: process.platform === 'win32' }).trim()}`, - `target=${process.platform}-${process.arch}` -].join(' ') -const installStampPath = path.join(REPO_ROOT, 'node_modules', '.install-stamp') -if (fs.existsSync(installStampPath) && fs.readFileSync(installStampPath, 'utf8') === installStamp) { - console.log('[build-bundled] node_modules matches its install stamp — npm ci output already present') -} else { - fs.rmSync(installStampPath, { force: true }) - run('npm', ['ci', '--no-audit', '--no-fund', '--fetch-retries=5', '--prefer-offline'], { - env: { ...process.env, CI: 'true' } - }) - fs.writeFileSync(installStampPath, installStamp) -} - -// ── 3. JS surfaces ────────────────────────────────────────────────────────── - -run('npm', ['run', 'build', '--workspace', 'ui-tui']) -run('npm', ['run', 'build', '--workspace', 'web']) - -const tuiEntry = path.join(REPO_ROOT, 'ui-tui', 'dist', 'entry.js') -const webDist = path.join(REPO_ROOT, 'hermes_cli', 'web_dist') -if (!fs.existsSync(tuiEntry)) { - fail(`ui-tui build did not write ${tuiEntry}`) -} -if (!fs.existsSync(path.join(webDist, 'index.html'))) { - fail(`web build did not write ${webDist}/index.html`) -} - -// ── 4. pm bundle ──────────────────────────────────────────────────────────── - -const pyMinor = pythonMinorFromLock() -run( - 'uv', - ['run', '--no-project', '--python', pyMinor, 'python', '-m', 'pm.cli', 'bundle', '--out', PAYLOAD_DIR, '--ref', tag], - { env: { ...process.env, PYTHONUTF8: '1' } } -) - -// ── 5. plant JS into the staged snapshot ──────────────────────────────────── -// git archive only carries committed files. The TUI and dashboard dists -// are gitignored, so they must be copied into the payload after staging. - -const payloadRepo = path.join(PAYLOAD_DIR, 'hermes-agent') -if (!fs.existsSync(path.join(payloadRepo, 'pyproject.toml'))) { - fail(`pm bundle did not write a repo snapshot at ${payloadRepo}`) -} - -const plantedTui = path.join(payloadRepo, 'hermes_cli', 'tui_dist', 'entry.js') -fs.mkdirSync(path.dirname(plantedTui), { recursive: true }) -fs.copyFileSync(tuiEntry, plantedTui) - -const plantedWeb = path.join(payloadRepo, 'hermes_cli', 'web_dist') -fs.rmSync(plantedWeb, { recursive: true, force: true }) -fs.cpSync(webDist, plantedWeb, { recursive: true, dereference: true }) -if (!fs.existsSync(path.join(plantedWeb, 'index.html'))) { - fail('planted web_dist is missing index.html') -} -console.log('[build-bundled] planted hermes_cli/tui_dist/entry.js and hermes_cli/web_dist into the payload') - -if (process.platform === 'darwin') { - const n = relativizePayloadLinks(PAYLOAD_DIR) - console.log(`[build-bundled] relativized ${n} payload links so codesign can sign each path once`) -} - -// ── 5b. stage the payload CLI entrypoints (hermes / hermes-agent / hermes-acp) -// The bundled payload's CLI entrypoints are fully self-relative and need NO -// toolchain at package time — the rust shim (apps/desktop/shim) is gone: -// -// win32 — distlib-MINTED launchers (scripts/desktop-cli/mint-launchers.py -// run by the payload's own store python, so the minting architecture is -// the target architecture by construction). Each minted exe is a plain -// PE + shebang `#!\..\tools\\python.exe` + a zip -// overlay of scripts/desktop-cli/launcher-wrapper.py, which resolves the -// launcher's own dir from sys.argv[0], puts the payload repo + venv -// site-packages on sys.path, and defaults sys.pycache_prefix to the -// user-level cache — everything the rust shim did, no cargo. distlib -// comes from the store python's pip (pip._vendor.distlib); no extra dep. -// -// POSIX — $0-relative bash trampolines generated by -// scripts/desktop-cli/cli-entrypoints.mjs (exported pure for tests). -// -// A sealed payload has no working editable install (the venv's editable -// pointer names the BUILD machine), so BOTH kinds set the payload's own -// import roots (repo, then venv site-packages) themselves and keep -// __pycache__ writes out of the payload. -function stageCliLaunchers(outDir, rels) { - const binDir = path.join(outDir, 'bin') - fs.rmSync(binDir, { recursive: true, force: true }) - fs.mkdirSync(binDir, { recursive: true }) - if (process.platform === 'win32') { - stageWinLaunchers(binDir, rels) - console.log(`[build-bundled] minted CLI launchers (${CLI_LAUNCHER_SPECS.map(s => s.name + '.exe').join(', ')}) → ${binDir}`) - } else { - for (const { name, text } of posixTrampolineScripts(rels)) { - const file = path.join(binDir, name) - fs.writeFileSync(file, text) - fs.chmodSync(file, 0o755) - } - console.log(`[build-bundled] staged POSIX CLI trampolines (${CLI_LAUNCHER_SPECS.map(s => s.name).join(', ')}) → ${binDir}`) - } -} - -function stageWinLaunchers(binDir, rels) { - // Mint with the payload's own store python: same distribution the - // launchers will execute, and its arch IS the target arch (distlib - // picks the launcher stub — incl. the -arm variant — by the MINTING - // interpreter's platform). distlib rides in that python's pip. - const interpreter = path.join(PAYLOAD_DIR, 'tools', pythonEntry, 'python.exe') - if (!fs.existsSync(interpreter)) { - fail(`mint interpreter missing at ${interpreter}`) - } - if (/\s/.test(interpreter) || /\s/.test(PAYLOAD_DIR)) { - // spawnSync(shell: true) cannot carry a whitespace path on win32; the - // repo (and thus the payload) must live in a space-free directory. - fail(`mint paths must be whitespace-free: ${interpreter}`) - } - const mintScript = path.join(REPO_ROOT, 'scripts', 'desktop-cli', 'mint-launchers.py') - const minted = [] - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-mint-')) - try { - for (const spec of CLI_LAUNCHER_SPECS) { - const wrapper = path.join(tmp, `${spec.name}-wrapper.py`) - fs.writeFileSync(wrapper, renderWinWrapper(spec, rels.relRepo, rels.relSite)) - run(interpreter, [mintScript], { - env: { - ...process.env, - HERMES_MINT_BIN_DIR: binDir, - HERMES_MINT_SPECS: JSON.stringify([spec]), - HERMES_MINT_WRAPPER: wrapper, - // The shebang: backslashes, bin-relative, literal - // first — the launcher resolves it against its own dir at runtime. - HERMES_MINT_PYTHON: `\\${rels.relPython.replace(/\//g, '\\')}` - } - }) - minted.push(path.join(binDir, `${spec.name}.exe`)) - } - } finally { - fs.rmSync(tmp, { recursive: true, force: true }) - } - for (const exe of minted) { - if (!fs.existsSync(exe)) { - fail(`mint produced no launcher at ${exe}`) - } - } -} - -const pythonEntry = (() => { - try { - const facts = JSON.parse(fs.readFileSync(path.join(PAYLOAD_DIR, 'tools', 'facts.json'), 'utf8')) - return facts.packages?.python?.entry - } catch { - return undefined - } -})() -if (!pythonEntry) { - fail('payload facts.json has no python entry — cannot stage the CLI launchers') -} -// The launchers' python is the store interpreter (platform layout: -// bin/python3 on posix, python.exe on win32), bin-relative. -const relStorePython = path.join('tools', pythonEntry, process.platform === 'win32' ? 'python.exe' : 'bin', process.platform === 'win32' ? '' : 'python3') - .replace(/\\/g, '/') - .replace(/\/+$/, '') -// The venv site-packages hold the dependency tree (uv sync). POSIX nests -// under lib/python3.X/, so the version comes from the staged interpreter's -// entry name. The entry prefix varies by target (cpython-3.11.15-... on -// win32/linux, python-3.11.16+... on darwin) — grab the first dotted -// numeric pair, which is the python version in every shape. -const pyMinorFromEntry = pythonEntry.match(/\d+\.\d+/)?.[0] -if (!pyMinorFromEntry) { - fail(`cannot derive python minor version from entry ${pythonEntry} — cannot stage the CLI launchers`) -} -const venvSitePackages = process.platform === 'win32' - ? '../venv/Lib/site-packages' - : `../venv/lib/python${pyMinorFromEntry}/site-packages` -// The repo snapshot dir name comes from the payload manifest (pm bundle -// writes repo: "hermes-agent"). -const payloadManifest = JSON.parse(fs.readFileSync(path.join(PAYLOAD_DIR, 'manifest.json'), 'utf8')) -if (typeof payloadManifest.repo !== 'string') { - fail('payload manifest.json has no repo field — cannot stage the CLI launchers') -} -stageCliLaunchers(PAYLOAD_DIR, { - relPython: `../${relStorePython}`, - relSite: venvSitePackages, - relRepo: `../${payloadManifest.repo}` -}) - -// ── 6. desktop build + package ────────────────────────────────────────────── - -const env = { - ...process.env, - HERMES_DESKTOP_VARIANT: variant, - HERMES_PAYLOAD_TAG: tag, - // The MSIX 4th version component is minutes-since-stable (see - // msix-shared.mjs). electron-builder reads BUILD_NUMBER for it when - // msix.setBuildNumber is on; a stable build leaves it unset and ships - // X.Y.Z.0. Computed here so the manifest, the .msixbundle /bv and the - // .appinstaller feed all agree (same derivation, same value). - ...(isCanary ? { BUILD_NUMBER: String(canaryBuildMinutes(tag, REPO_ROOT)) } : {}) -} -const desktop = path.join(REPO_ROOT, 'apps', 'desktop') - -for (const pass of passes) { - console.log(`[build-bundled] pass: ${pass.targets}`) - run('npm', ['run', 'build'], { cwd: desktop, env }) - run( - 'npm', - [ - 'run', - 'builder', - '--', - ...pass.targets.split(' '), - `-c.extraMetadata.version=${artifactVersion}`, - ...(fileVersion - ? [`-c.extraMetadata.shortVersion=${fileVersion}`, `-c.extraMetadata.shortVersionWindows=${fileVersion}`] - : []), - ...extraBuilderArgs - ], - { cwd: desktop, env } - ) -} -console.log(`[build-bundled] artifacts: ${path.join(desktop, 'release')}`) diff --git a/scripts/bundles/desktop.py b/scripts/bundles/desktop.py new file mode 100644 index 0000000000..4f0e5e3aa9 --- /dev/null +++ b/scripts/bundles/desktop.py @@ -0,0 +1,123 @@ +"""Build a complete desktop bundle with the shared Python payload tools.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.bundles.payload import plant_surfaces, relativize_links, stage_launchers + + +def run(argv: list[str], *, cwd: Path, env: dict[str, str]) -> None: + print("bundle: " + subprocess.list2cmdline(argv), flush=True) + subprocess.run(argv, cwd=cwd, env=env, check=True) + + +def capture(argv: list[str], repo: Path) -> str: + return subprocess.check_output(argv, cwd=repo, text=True, encoding="utf-8").strip() + + +def release_version(repo: Path, tag: str) -> str: + from scripts.termux.deb_version import channel_for_tag + + channel_for_tag(tag) # shared release tag grammar, not a second version parser + version = tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8-sig"))["project"]["version"] + if "-canary." not in tag and tag != "v" + version: + raise ValueError(f"tag {tag} does not match project version {version}") + return tag[1:] + + +def npm_command(node: str) -> list[str]: + # npm.cmd needs cmd.exe; Node's CLI accepts argv directly, including spaces. + npm = shutil.which("npm") + if not npm: + raise FileNotFoundError("npm is required") + prefix = Path(npm).resolve().parent + candidates = [prefix / "node_modules/npm/bin/npm-cli.js", prefix.parent / "lib/node_modules/npm/bin/npm-cli.js"] + for candidate in candidates: + if candidate.is_file(): + return [node, str(candidate)] + # POSIX npm is normally a symlink to its CLI file. + if os.name != "nt": + return [node, str(Path(npm).resolve())] + raise FileNotFoundError(f"npm CLI missing beside {npm}") + + +def build(repo: Path, tag: str, variant: str, builder_args: list[str]) -> None: + from pm.store import current_target + + repo = repo.resolve() + version = release_version(repo, tag) + commit = capture(["git", "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}"], repo) + if capture(["git", "rev-parse", "HEAD"], repo) != commit: + raise ValueError("the build checkout must be at the release tag") + node = shutil.which("node") + if not node or not shutil.which("uv"): + raise FileNotFoundError("Node and uv are required") + npm = npm_command(node) + env = {**os.environ, "CI": "true", "PYTHONUTF8": "1", "GITHUB_SHA": commit, + "HERMES_DESKTOP_VARIANT": variant, "HERMES_PAYLOAD_TAG": tag} + target = current_target() + node_arch = capture([node, "-p", "process.arch"], repo) + if node_arch != target.split("-")[1]: + raise ValueError(f"Node {node_arch} does not match build target {target}") + stamp = json.dumps({"lock": hashlib.sha256((repo / "package-lock.json").read_bytes()).hexdigest(), + "node": capture([node, "--version"], repo), + "npm": capture([*npm, "--version"], repo), "target": target}, sort_keys=True) + stamp_path = repo / "node_modules/.install-stamp" + if not stamp_path.is_file() or stamp_path.read_text(encoding="utf-8") != stamp: + stamp_path.unlink(missing_ok=True) + run([*npm, "ci", "--no-audit", "--no-fund", "--fetch-retries=5", "--prefer-offline"], cwd=repo, env=env) + stamp_path.write_text(stamp, encoding="utf-8") + # Use the installed semver implementation for package.json's actual grammar. + run([node, "-e", "const s=require('semver'),p=require('./package.json'); for(const [n,v] of [['node',process.versions.node],['npm',process.argv[1]]]) if(!s.satisfies(v,p.engines[n])) throw Error(n+' violates '+p.engines[n])", capture([*npm, "--version"], repo)], cwd=repo, env=env) + payload = repo / "apps/desktop/build/agent-payload" + if variant == "light": + shutil.rmtree(payload, ignore_errors=True) + payload.mkdir(parents=True) + (payload / "manifest.json").write_text('{"schema":1,"external":true}\n', encoding="utf-8") + else: + run([*npm, "run", "build", "--workspace", "ui-tui"], cwd=repo, env=env) + run([*npm, "run", "build", "--workspace", "web"], cwd=repo, env=env) + run([sys.executable, "-m", "pm.cli", "bundle", "--out", str(payload), "--ref", tag], cwd=repo, env=env) + manifest = json.loads((payload / "manifest.json").read_text(encoding="utf-8-sig")) + plant_surfaces(payload / manifest["repo"], repo) + relativize_links(payload) + stage_launchers(payload, manifest) + desktop = repo / "apps/desktop" + # Windows file-version and MSIX build-number policy remains with its packager. + version_args = [] + if sys.platform == "win32": + script = "const w=require('./apps/desktop/scripts/windows-file-version.mjs');const m=require('./scripts/msix-shared.mjs');console.log(JSON.stringify({file:w.windowsFileVersion(process.argv[1]),build:process.argv[1].includes('-canary.')?m.canaryBuildMinutes(process.argv[1],process.cwd()):null}))" + metadata = json.loads(capture([node, "-e", script, tag], repo)) + if metadata["build"] is not None: + env["BUILD_NUMBER"] = str(metadata["build"]) + if metadata["file"]: + version_args = [f'-c.extraMetadata.shortVersion={metadata["file"]}', f'-c.extraMetadata.shortVersionWindows={metadata["file"]}'] + targets = {"win32": ["--win", "msix"], "darwin": ["--mac", "dmg", "zip"], "linux": ["--linux", "AppImage"]}[sys.platform] + run([*npm, "run", "build"], cwd=desktop, env=env) + run([*npm, "run", "builder", "--", *targets, f"-c.extraMetadata.version={version}", *version_args, *builder_args], cwd=desktop, env=env) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tag", required=True) + parser.add_argument("--variant", choices=["bundled", "store", "light"], default="bundled") + parser.add_argument("--repo", type=Path, default=ROOT) + parser.add_argument("builder_args", nargs=argparse.REMAINDER) + args = parser.parse_args() + build(args.repo, args.tag, args.variant, [v for v in args.builder_args if v != "--"]) + + +if __name__ == "__main__": + main() diff --git a/scripts/desktop-cli/launcher-wrapper.py b/scripts/bundles/launcher_wrapper.py similarity index 98% rename from scripts/desktop-cli/launcher-wrapper.py rename to scripts/bundles/launcher_wrapper.py index b5a441c73b..d167f90837 100644 --- a/scripts/desktop-cli/launcher-wrapper.py +++ b/scripts/bundles/launcher_wrapper.py @@ -1,6 +1,6 @@ """Bundled-payload CLI entry wrapper — the distlib launcher's zip overlay. -scripts/build-bundled-desktop.mjs reads this file, substitutes the four +scripts/bundles/payload.py reads this file, substitutes the four HERMES_* placeholders (see the constants below), and hands the result to a distlib ScriptMaker as the script text. On win32 the minted artifact is a real PE: the distlib diff --git a/scripts/desktop-cli/mint-launchers.py b/scripts/bundles/mint_launchers.py similarity index 97% rename from scripts/desktop-cli/mint-launchers.py rename to scripts/bundles/mint_launchers.py index 4fb5cd4089..fce0768a1a 100644 --- a/scripts/desktop-cli/mint-launchers.py +++ b/scripts/bundles/mint_launchers.py @@ -1,6 +1,6 @@ """Mint the bundled payload's win32 CLI launchers with distlib. -Run by scripts/build-bundled-desktop.mjs (step 5b) with the payload's OWN +Run by scripts/bundles/desktop.py (step 5b) with the payload's OWN store python as the minting interpreter — the store python is the same distribution the launchers will execute, and its architecture is by construction the target architecture, which is exactly what distlib's @@ -22,7 +22,7 @@ win32 argv, so argv is not an option): module, "func": entry function} — mirrors [project.scripts] in pyproject.toml HERMES_MINT_WRAPPER path of the RENDERED launcher-wrapper.py for THIS - entry (substitution is cli-entrypoints.mjs's job, + entry (substitution is scripts/bundles/payload.py's job, one implementation, one test) HERMES_MINT_PYTHON bin-relative path of the store python, BACKslashes, e.g. \..\tools\\python.exe — diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py new file mode 100644 index 0000000000..afd3840929 --- /dev/null +++ b/scripts/bundles/native.py @@ -0,0 +1,187 @@ +"""Native payload staging through PM's existing package authority.""" +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +from pm.cli import _install_names, _run_live +from pm.ensure import _store, _facts, _lockfile, uv as pm_uv +from pm.registry import get_package +from pm.store import current_target + +def _bundle_package_names() -> list[str]: + names = [ + n + for n in _lockfile().names() + if not get_package(n).internal or n == "uv" + ] + if "python" not in names: + names.append("python") + return names + + +def _arch_guard(store_dir: Path) -> list[str]: + """Every staged binary must be built for this machine's target — a + payload staged with a mismatched interpreter or PATH tool ships an + artifact that cannot run. Reads facts, probes each entry binary.""" + from pm.lock import Facts + from pm.package import machine_matches_binary + + facts = Facts(store_dir / "facts.json") + problems = [] + target = current_target() + for name in _lockfile().names(): + package = get_package(name) + fact = facts.get(name) + if fact is None or "entry" not in fact: + continue + binary = package.binary(store_dir / fact["entry"], target) + if binary is None or not binary.is_file(): + continue + verdict = machine_matches_binary(binary, target) + # A package that declares this target as emulated (x64 binary run + # under Windows ARM64 built-in emulation) is fine with the x64 PE. + if verdict is False and target not in package.emulated_arch_targets: + problems.append(f"{name}: {binary.name} is not a {target} binary") + return problems + + + +def stage_native(args) -> int: + previous = os.environ.get("HERMES_RUNTIME_DIR") + try: + return _stage_native(args) + finally: + if previous is None: + os.environ.pop("HERMES_RUNTIME_DIR", None) + else: + os.environ["HERMES_RUNTIME_DIR"] = previous + + +def _stage_native(args) -> int: + """Stage a complete payload for THIS machine's target into --out: + repo snapshot + store + facts (via the normal install path, redirected) + + a relocatable venv built on the staged interpreter and synced from + uv.lock. Built natively per (os, arch); there is no cross-target + staging.""" + import os + + from pm import paths + + out = Path(args.out).resolve() + store_dir = out / "tools" + store_dir.mkdir(parents=True, exist_ok=True) + # A manifest from a previous run would make this payload look sealed + # and refuse its own staging; it is rewritten at the end. + (out / "manifest.json").unlink(missing_ok=True) + + repo_dir = out / "hermes-agent" + ref = args.ref or "HEAD" + from scripts.bundles.payload import snapshot, write_manifest, relativize_links + print(f"staging repo snapshot ({ref})…", flush=True) + snapshot(paths.repo_root(), ref, repo_dir) + + os.environ["HERMES_RUNTIME_DIR"] = str(store_dir) + + names = _bundle_package_names() + failed = _install_names( + [n for n in names if get_package(n).missing_reason(current_target()) is None] + ) + + # Prune the staged store BEFORE the venv sync and packaging: drop the + # fetch- download-cache archives (needed only at install time — dead + # weight in the shipped payload AND in the CI cache that restores this + # dir) and any orphaned package versions left over from an older lock + # the cache carried in. A lean staged store = a lean CI cache. + if failed: + return 1 + # Only this build's store is ours to prune; machine-wide partials are not. + store = _store() + keep = _facts().entries_in_use() + for entry in store.root.iterdir(): + if entry.is_dir() and not entry.name.startswith(".") and entry.name not in keep: + shutil.rmtree(entry) + + + uv_bin, env = pm_uv() + if uv_bin is None: + print("✗ venv: uv did not stage") + return 1 + + python_fact = _facts().get("python") + if python_fact is None: + print("✗ venv: no staged interpreter to build on") + return 1 + python_bin = get_package("python").binary( + _store().entry(python_fact["entry"]), current_target() + ) + + # Build + sync INSIDE the staged repo: the editable project install + # must point at the payload's own tree, not this checkout. + venv_dir = out / "venv" + if venv_dir.exists(): + shutil.rmtree(venv_dir) + env["VIRTUAL_ENV"] = str(venv_dir) + env.pop("UV_NO_CONFIG", None) + if current_target().startswith("darwin"): + # python-build-standalone bakes phantom toolchain paths (its build + # dir's llvm-ar) into sysconfig; sdist builds then fail with + # "No such file or directory: .../tools/llvm/bin/llvm-ar". Point + # sdist builds at the machine's real toolchain. + env.setdefault("AR", "/usr/bin/ar") + env.setdefault("CC", "clang") + for cmd in ( + [uv_bin, "venv", "--relocatable", "--python", str(python_bin), str(venv_dir)], + [uv_bin, "sync", "--frozen", "--all-extras", "--active"], + ): + print(f" venv: $ {' '.join(cmd)}", flush=True) + code, tail = _run_live(cmd, cwd=repo_dir, env=env) + if code != 0: + print(f"✗ venv: {' '.join(cmd[1:3])} failed:\n{tail}") + return 1 + print("✓ venv (relocatable, all extras, on the staged interpreter)") + + # The frozen feature set: the EXACT extras that installed on this + # target (markers gate some off per-platform). This file is the + # lazy-off contract — pm sync never deviates from it. + from pm.features import installed_extras, write_features + + features = installed_extras(repo_dir, venv_dir) + write_features(features, out) + print(f"✓ enabled-features.json ({len(features)} extras recorded)") + + # Ship the uv cache: the staged venv sync just warmed the hermes-owned + # cache with every wheel this payload needs. Copying it in makes a + # mutable-venv rebuild from the bundle near-free (`uv sync --offline` + # from a warm cache probed at 0.4s vs 1.2s cold) — the blow-away-on- + # update contract depends on it. + from pm.packages import uv_cache_dir as bundle_uv_cache_dir + + payload_cache = out / "uv-cache" + if payload_cache.exists(): + shutil.rmtree(payload_cache, ignore_errors=True) + src_cache = bundle_uv_cache_dir() + if src_cache.is_dir(): + print(f" uv-cache: copying {src_cache} → payload...", flush=True) + shutil.copytree(src_cache, payload_cache) + print("✓ uv-cache (staged — warm rebuilds for the mutable venv)") + else: + print(" uv-cache: none warm (first bundle on this machine?)") + + bad = _arch_guard(store_dir) + for line in bad: + print(f"✗ arch: {line}") + failed += 1 + + if failed: + return 1 + relativize_links(out) + from scripts.bundles.payload import record_tools + recorded = {name: fact["entry"] for name in names if (fact := _facts().get(name)) and "entry" in fact} + record_tools(out, paths.lockfile_path(), current_target(), recorded) + write_manifest(out, target=current_target(), repo="hermes-agent", ref=ref) + print(f"✓ manifest ({out / 'manifest.json'})") + return 1 if failed else 0 + + diff --git a/scripts/bundles/payload.py b/scripts/bundles/payload.py new file mode 100644 index 0000000000..90bc65cfb1 --- /dev/null +++ b/scripts/bundles/payload.py @@ -0,0 +1,228 @@ +"""Shared payload layout, source snapshots and generated launchers.""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def write_manifest(root: Path, *, target: str, repo: str, ref: str | None = None) -> dict: + manifest = {"schema": 1, "target": target, "repo": repo, "venv": "venv", "store": "tools"} + if ref is not None: + manifest["ref"] = ref + (root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest + + +def snapshot(repo: Path, ref: str, destination: Path) -> None: + """Archive a resolved git revision without carrying checkout metadata.""" + repo, destination = repo.resolve(), destination.resolve() + if repo == destination or repo.is_relative_to(destination): + raise ValueError("the snapshot destination must not contain the source checkout") + with tempfile.TemporaryDirectory(prefix="hermes-archive-") as temp: + archive = Path(temp) / "source.tar" + subprocess.run(["git", "archive", "--format=tar", "--output", str(archive), ref], cwd=repo, check=True) + if destination.exists(): + shutil.rmtree(destination) + destination.mkdir(parents=True) + with tarfile.open(archive) as source: + source.extractall(destination, filter="data") + + +def project_entries(repo: Path) -> dict[str, str]: + return tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8-sig"))["project"]["scripts"] + + +def record_tools(root: Path, lock_path: Path, target: str, entries: dict[str, str]) -> None: + from pm.lock import Facts, Lockfile + from pm.registry import get_package + from pm.store import tree_digest + + store = root / "tools" + facts, lock = Facts(store / "facts.json"), Lockfile(lock_path) + for name, entry_name in entries.items(): + entry = store / entry_name + version, artifacts = lock.version(name), lock.artifacts(name, target) + if not entry.is_dir() or not version or not artifacts: + raise ValueError(f"incomplete payload tool: {name}") + facts.record(name, version, entry_name, get_package(name).env(entry, target), store, + target=target, artifacts=[a["sha256"] for a in artifacts], digest=tree_digest(entry)) + + +def plant_surfaces(repo: Path, source: Path, *, dashboard: bool = True) -> None: + tui = source / "ui-tui/dist/entry.js" + if not tui.is_file(): + raise FileNotFoundError(tui) + destination = repo / "hermes_cli/tui_dist" + destination.mkdir(parents=True, exist_ok=True) + shutil.copy2(tui, destination / "entry.js") + if dashboard: + web = source / "hermes_cli/web_dist" + if not (web / "index.html").is_file(): + raise FileNotFoundError(web / "index.html") + shutil.rmtree(repo / "hermes_cli/web_dist", ignore_errors=True) + shutil.copytree(web, repo / "hermes_cli/web_dist") + + +def relativize_links(root: Path) -> int: + """Only dependency-venv links move; framework links belong to codesign.""" + root = root.resolve() + directory = root / "venv/bin" + count = 0 + if not directory.is_dir(): + return count + for link in directory.iterdir(): + if not link.is_symlink(): + continue + target = os.readlink(link) + if not os.path.isabs(target): + # Keep sibling chains intact; their absolute store link is rewritten separately. + if not Path(os.path.abspath(directory / target)).is_relative_to(root): + raise ValueError(f"link escapes payload: {link} -> {target}") + continue + resolved = (directory / target).resolve() + if not resolved.is_relative_to(root): + parts = Path(target).parts + if "tools" not in parts: + raise ValueError(f"link escapes payload: {link} -> {target}") + resolved = root.joinpath(*parts[parts.index("tools"):]).resolve() + if not resolved.is_relative_to(root) or not resolved.exists(): + raise ValueError(f"missing payload link target: {link} -> {target}") + relative = os.path.relpath(resolved, directory) + if relative != target: + link.unlink() + link.symlink_to(relative) + count += 1 + for link in directory.iterdir(): + if link.is_symlink() and (not link.resolve().is_relative_to(root) or not link.exists()): + raise ValueError(f"invalid relative payload link: {link}") + return count + + +def render_wrapper(entry: str, repo: str, site: str) -> str: + module, func = entry.split(":", 1) + text = (Path(__file__).with_name("launcher_wrapper.py")).read_text(encoding="utf-8-sig") + for key, value in {"ENTRY_MODULE": module, "ENTRY_FUNC": func, "REPO_REL": repo, "SITE_REL": site}.items(): + if '"' in value or "\n" in value or "__" in value: + raise ValueError(f"invalid launcher value: {key}") + text = text.replace(f"__HERMES_{key}__", value) + return text + + +def posix_launcher(name: str, entry: str, *, python: str, repo: str, site: str, target: str) -> str: + import shlex + + module, func = entry.split(":", 1) + bionic = target.endswith("-bionic") + header = "#!/data/data/com.termux/files/usr/bin/sh" if bionic else "#!/usr/bin/env bash" + extra = "" + if bionic: + extra = '''PREFIX="${PREFIX:-/data/data/com.termux/files/usr}" +export PREFIX +export LD_LIBRARY_PATH="$root/tools/python$PREFIX/lib:$root/tools/node$PREFIX/lib:$root/tools/ffmpeg$PREFIX/lib:$root/runtime-libs/lib:$PREFIX/lib" +export HERMES_PYTHON_SRC_ROOT="$REPO" +export HERMES_PYTHON="$PYTHON" +export HERMES_NODE="$root/tools/node$PREFIX/bin/node" +export HERMES_RUNTIME_DIR="$root/tools" +export PATH="$root/tools/npm/bin:$root/tools/node$PREFIX/bin:$root/tools/ffmpeg$PREFIX/bin:$root/tools/ripgrep:$PATH" +''' + code = f"import sys; sys.argv[0]={name!r}; from {module} import {func}; sys.exit({func}())" + return f'''{header} +set -eu +self="$0" +while [ -L "$self" ]; do + target="$(readlink "$self")" + case "$target" in + /*) self="$target" ;; + *) self="$(dirname "$self")/$target" ;; + esac +done +root="$(cd "$(dirname "$self")/.." && pwd)" +PYTHON="$root/{python}" +REPO="$root/{repo}" +SITE="$root/{site}" +[ -x "$PYTHON" ] || {{ printf '%s\\n' 'Bundled interpreter missing; reinstall Hermes.' >&2; exit 2; }} +unset PYTHONPATH PYTHONHOME +export PYTHONPATH="$REPO:$SITE" +export PYTHONPYCACHEPREFIX="${{PYTHONPYCACHEPREFIX:-${{XDG_CACHE_HOME:-$HOME/.cache}}/hermes-pycache}}" +{extra}exec "$PYTHON" -P -c {shlex.quote(code)} "$@" +''' + + +def stage_launchers(root: Path, manifest: dict, *, run=subprocess.run) -> list[str]: + from pm.lock import Facts + from pm.registry import get_package + + target = manifest["target"] + repo = root / manifest["repo"] + entries = project_entries(repo) + facts = Facts(root / manifest["store"] / "facts.json") + python_fact = facts.get("python") + if not python_fact: + raise ValueError("payload has no Python fact") + python = get_package("python").binary(root / manifest["store"] / python_fact["entry"], target) + if python is None or not python.is_file(): + raise FileNotFoundError("payload interpreter missing") + windows = target.startswith("win32") + minor = python_fact["version"].split("+")[0].rsplit(".", 1)[0] + site = f'{manifest["venv"]}/' + ("Lib/site-packages" if windows else f"lib/python{minor}/site-packages") + bindir = root / "bin" + bindir.mkdir(parents=True, exist_ok=True) + relative_python = python.relative_to(root).as_posix() + for name, entry in entries.items(): + if windows: + with tempfile.TemporaryDirectory(prefix="hermes-mint-") as temp: + wrapper = Path(temp) / "wrapper.py" + wrapper.write_text(render_wrapper(entry, f'../{manifest["repo"]}', f"../{site}"), encoding="utf-8") + module, func = entry.split(":", 1) + env = {**os.environ, "HERMES_MINT_BIN_DIR": str(bindir), + "HERMES_MINT_SPECS": json.dumps([{"name": name, "module": module, "func": func}]), + "HERMES_MINT_WRAPPER": str(wrapper), + "HERMES_MINT_PYTHON": "\\..\\" + relative_python.replace("/", "\\")} + run([str(python), str(Path(__file__).with_name("mint_launchers.py"))], env=env, check=True) + else: + # Termux's prefix is contractual; its venv interpreter is built at that prefix. + launch_python = f'{manifest["venv"]}/bin/python' if target.endswith("-bionic") else relative_python + script = posix_launcher(name, entry, python=launch_python, repo=manifest["repo"], site=site, target=target) + output = bindir / name + output.write_text(script, encoding="utf-8") + output.chmod(0o755) + manifest["launchers"] = list(entries) + (root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return list(entries) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["launchers", "relocate", "surfaces"]) + parser.add_argument("payload", type=Path) + parser.add_argument("--source", type=Path, default=ROOT) + parser.add_argument("--tui-only", action="store_true") + parser.add_argument("--repo-dir", help="staged repository directory when no manifest exists yet") + args = parser.parse_args() + if args.action == "relocate": + relativize_links(args.payload) + return + if args.action == "surfaces" and args.repo_dir: + plant_surfaces(args.payload / args.repo_dir, args.source, dashboard=not args.tui_only) + return + manifest = json.loads((args.payload / "manifest.json").read_text(encoding="utf-8-sig")) + if args.action == "launchers": + stage_launchers(args.payload.resolve(), manifest) + else: + plant_surfaces(args.payload / manifest["repo"], args.source, dashboard=not args.tui_only) + + +if __name__ == "__main__": + main() diff --git a/scripts/bundles/stage.py b/scripts/bundles/stage.py new file mode 100644 index 0000000000..e530e15383 --- /dev/null +++ b/scripts/bundles/stage.py @@ -0,0 +1,30 @@ +"""Stage the shared native payload and launchers without an Electron package.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +from scripts.bundles.native import stage_native +from scripts.bundles.payload import stage_launchers + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True) + parser.add_argument("--ref", default="HEAD") + args = parser.parse_args() + code = stage_native(args) + if code: + return code + root = Path(args.out).resolve() + manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8-sig")) + stage_launchers(root, manifest) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/desktop-cli/cli-entrypoints.mjs b/scripts/desktop-cli/cli-entrypoints.mjs deleted file mode 100644 index 599df9ec47..0000000000 --- a/scripts/desktop-cli/cli-entrypoints.mjs +++ /dev/null @@ -1,130 +0,0 @@ -/** - * cli-entrypoints.mjs — the bundled payload's CLI entrypoint generators. - * - * The bundled payload ships three CLI entrypoints (hermes / hermes-agent / - * hermes-acp — mirrors [project.scripts] in pyproject.toml) staged into - * agent-payload/bin/. They are fully self-relative, so a bundled artifact - * works wherever it lands (and read-only, MSIX included): - * - * win32 — distlib-MINTED launchers (scripts/desktop-cli/mint-launchers.py - * runs on the payload's own store python, whose architecture is by - * construction the target's). The minted exe is a plain PE + shebang + - * zip overlay of scripts/desktop-cli/launcher-wrapper.py. The shebang is - * `#!\..\tools\\python.exe` — the - * literal is resolved by the launcher at run time relative to its own - * directory, which is the relocatability mechanism (live-proved). - * - * POSIX — $0-relative bash trampolines generated below. No PE is needed: - * a plain script exec'ing the store python is the entrypoint. They follow - * the $0 symlink chain so a link out on PATH (e.g. ~/.local/bin) resolves - * back into the install. - * - * Pure module: no side effects, importable from tests. - */ - -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -const HERE = path.dirname(fileURLToPath(import.meta.url)) - -/** The three CLI entrypoints, mirroring [project.scripts] in pyproject.toml. */ -export const CLI_LAUNCHER_SPECS = [ - { name: 'hermes', module: 'hermes_cli.main', func: 'main' }, - { name: 'hermes-agent', module: 'run_agent', func: 'main' }, - { name: 'hermes-acp', module: 'acp_adapter.entry', func: 'main' } -] - -/** - * Render scripts/desktop-cli/launcher-wrapper.py for one entry. The - * relative paths are the payload's bin-relative layout facts, forward - * slashes (same convention as the payload manifest — the wrapper splits - * them itself, so one text works cross-platform). - * - * @param {{ module: string, func: string }} spec - * @param {string} relRepo bin-relative repo dir, e.g. ../hermes-agent - * @param {string} relSite bin-relative venv site-packages - * @returns {string} the rendered wrapper source - */ -export function renderWinWrapper(spec, relRepo, relSite) { - const template = fs.readFileSync(path.join(HERE, 'launcher-wrapper.py'), 'utf8') - const substitutions = { - __HERMES_ENTRY_MODULE__: spec.module, - __HERMES_ENTRY_FUNC__: spec.func, - __HERMES_REPO_REL__: relRepo, - __HERMES_SITE_REL__: relSite - } - let text = template - for (const [placeholder, value] of Object.entries(substitutions)) { - if (value.includes('__')) { - throw new Error(`bad substitution for ${placeholder}: ${JSON.stringify(value)}`) - } - text = text.replaceAll(placeholder, value) - } - for (const placeholder of Object.keys(substitutions)) { - if (text.includes(placeholder)) { - throw new Error(`launcher-wrapper.py has an unsubstituted ${placeholder}`) - } - } - return text -} - -/** - * One POSIX trampoline. Rel paths are bin-relative with FORWARD slashes - * (the same strings the win32 side bakes); the bash resolves them against - * the trampoline's own directory. - * - * @param {{ name: string, module: string }} spec - * @param {{ relPython: string, relSite: string, relRepo: string }} rels - * @returns {string} executable bash text - */ -export function posixTrampolineScript(spec, rels) { - const join = (rel) => ['"$BIN_DIR"', ...rel.split('/')].join('/') - return `#!/usr/bin/env bash -# Generated by scripts/build-bundled-desktop.mjs — the bundled payload's -# POSIX CLI entrypoint (${spec.name}). Fully $0-relative: follows the -# symlink chain so a link out on PATH (~/.local/bin) resolves back into -# the install, then execs the store python with the payload's own import -# roots. Do not edit the generated copy — change the generator. -set -euo pipefail -self="$0" -while [ -L "$self" ]; do - target="$(readlink "$self")" - case "$target" in - /*) self="$target" ;; - *) self="$(dirname "$self")/$target" ;; - esac -done -BIN_DIR="$(cd -- "$(dirname -- "$self")" && pwd)" - -PYTHON=${join(rels.relPython)} -REPO=${join(rels.relRepo)} -SITE=${join(rels.relSite)} - -[ -x "$PYTHON" ] || { - echo "${spec.name}: bundled interpreter missing at $PYTHON — the Hermes install is damaged; reinstall from the website" >&2 - exit 2 -} - -# A sealed payload has no working editable install (its pointer names the -# BUILD machine), so its own import roots REPLACE any inherited PYTHONPATH. -# Repo first — its hermes_cli wins over anything stale in site-packages. -unset PYTHONPATH PYTHONHOME -export PYTHONPATH="$REPO:$SITE" -# Keep __pycache__ writes out of the (possibly read-only) payload. -if [ -z "\${PYTHONPYCACHEPREFIX:-}" ] && [ -n "\${HOME:-}" ]; then - export PYTHONPYCACHEPREFIX="$HOME/.cache/hermes-pycache" -fi - -exec "$PYTHON" -m ${spec.module} "$@" -` -} - -/** - * All three POSIX trampolines. - * @param {{ relPython: string, relSite: string, relRepo: string }} rels - * @returns {{ name: string, text: string }[]} - */ -export function posixTrampolineScripts(rels) { - return CLI_LAUNCHER_SPECS.map((spec) => ({ name: spec.name, text: posixTrampolineScript(spec, rels) })) -} diff --git a/scripts/termux/launchers.py b/scripts/termux/launchers.py index f5a3fb9e21..d6d7702499 100644 --- a/scripts/termux/launchers.py +++ b/scripts/termux/launchers.py @@ -5,42 +5,28 @@ import argparse from pathlib import Path import shlex import tomllib +import sys - -_LAUNCHER = '''#!/data/data/com.termux/files/usr/bin/sh -set -eu -self="$0" -while [ -L "$self" ]; do - target="$(readlink "$self")" - case "$target" in - /*) self="$target" ;; - *) self="$(dirname "$self")/$target" ;; - esac -done -root="$(cd "$(dirname "$self")/.." && pwd)" -PREFIX="${PREFIX:-/data/data/com.termux/files/usr}" -export PREFIX -unset PYTHONHOME -export LD_LIBRARY_PATH="$root/tools/python/data/data/com.termux/files/usr/lib:$root/tools/node/data/data/com.termux/files/usr/lib:$root/tools/ffmpeg/data/data/com.termux/files/usr/lib:$root/runtime-libs/lib:$PREFIX/lib" -export PYTHONPATH="$root/app" -export HERMES_PYTHON_SRC_ROOT="$root/app" -export HERMES_PYTHON="$root/venv/bin/python" -export HERMES_NODE="$root/tools/node/data/data/com.termux/files/usr/bin/node" -export HERMES_RUNTIME_DIR="$root/tools" -export PATH="$root/tools/npm/bin:$root/tools/node/data/data/com.termux/files/usr/bin:$root/tools/ffmpeg/data/data/com.termux/files/usr/bin:$root/tools/ripgrep:$PATH" -export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-${XDG_CACHE_HOME:-$HOME/.cache}/hermes-pycache}" -exec "$HERMES_PYTHON" -P -c __ENTRY__ "$@" -''' +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) def write_launchers(payload: Path, entries: dict[str, str]) -> None: + from scripts.bundles.payload import posix_launcher + + lock = payload / "app/pm/lock.json" + if lock.is_file(): + import json + version = json.loads(lock.read_text(encoding="utf-8-sig"))["packages"]["python"]["version"] + minor = version.split("+")[0].rsplit(".", 1)[0] + else: + minor = f"{sys.version_info.major}.{sys.version_info.minor}" bindir = payload / "bin" bindir.mkdir(parents=True, exist_ok=True) for name, entry in entries.items(): - module, func = entry.split(":", 1) - script = f"import sys; sys.argv[0] = {name!r}; from {module} import {func}; sys.exit({func}())" + text = posix_launcher(name, entry, python="venv/bin/python", repo="app", + site=f"venv/lib/python{minor}/site-packages", target="linux-arm64-bionic") path = bindir / name - path.write_text(_LAUNCHER.replace("__ENTRY__", shlex.quote(script)), encoding="utf-8") + path.write_text(text, encoding="utf-8") path.chmod(0o755) diff --git a/scripts/termux/payload_facts.py b/scripts/termux/payload_facts.py index e34e3678df..17612db6c0 100644 --- a/scripts/termux/payload_facts.py +++ b/scripts/termux/payload_facts.py @@ -8,34 +8,17 @@ import sys sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from pm.lock import Facts, Lockfile -from pm.store import tree_digest -from pm.registry import get_package -import pm.packages # Registers the runtime definitions. + def write_facts(payload: Path, lock_path: Path, build_set: Path) -> None: target = "linux-arm64-bionic" - lock = Lockfile(lock_path) - store = payload / "tools" - facts = Facts(store / "facts.json") - for name in ("python", "node", "uv", "npm", "ffmpeg", "ripgrep"): - entry = store / name - if not entry.is_dir(): - raise RuntimeError(f"missing payload tool: {name}") - version = lock.version(name) - artifacts = lock.artifacts(name, target) - if not version or not artifacts: - raise RuntimeError(f"missing pin: {name} on {target}") - facts.record( - name, version, name, get_package(name).env(entry, target), store, - target=target, artifacts=[a["sha256"] for a in artifacts], - digest=tree_digest(entry), - ) + from scripts.bundles.payload import record_tools + record_tools(payload, lock_path, target, {name: name for name in ("python", "node", "uv", "npm", "ffmpeg", "ripgrep")}) natives = [line.strip() for line in build_set.read_text(encoding="utf-8").splitlines() if line.strip()] (payload / "native-wheels.json").write_text(json.dumps(natives) + "\n", encoding="utf-8") - manifest = {"schema": 1, "target": target, "repo": "app", "venv": "venv", "store": "tools"} - (payload / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + from scripts.bundles.payload import write_manifest + write_manifest(payload, target=target, repo="app") if __name__ == "__main__": diff --git a/scripts/termux/termux_build.sh b/scripts/termux/termux_build.sh old mode 100644 new mode 100755 index 1f4ae2ef28..8a14ef2567 --- a/scripts/termux/termux_build.sh +++ b/scripts/termux/termux_build.sh @@ -189,7 +189,13 @@ mkdir -p "$WORK/tree" "$WHEELHOUSE" # [c] Stage the tag as a gitless tree. log "Archiving $TAG into $WORK/tree" -git -C "$REPO_ABS" archive --format=tar "$TAG" | tar -xf - -C "$WORK/tree" +python3 - "$REPO_ABS" "$TAG" "$WORK/tree" <<'PY' +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from scripts.bundles.payload import snapshot +snapshot(Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3])) +PY [ -f "$WORK/tree/pyproject.toml" ] || fail "archived tag tree has no pyproject.toml -- bad tag?" REPO_ROOT="$(cd "$HERE/../.." && pwd)" DIGEST="$(cd "$REPO_ROOT" && python3 -c 'from pm.lock import termux_docker_digest; print(termux_docker_digest())')" diff --git a/tests/pm/test_pm_core.py b/tests/pm/test_pm_core.py index 82e44e58c4..41b40d781c 100644 --- a/tests/pm/test_pm_core.py +++ b/tests/pm/test_pm_core.py @@ -657,14 +657,14 @@ def test_check_reports_venv_drift_and_missing_tools(venv_env): def test_bundle_package_names_include_browsers(monkeypatch, tmp_path): - from pm.cli import _bundle_package_names + from scripts.bundles.native import _bundle_package_names from pm.lock import Lockfile lock = Lockfile(tmp_path / "lock.json") for name in ("uv", "python", "ripgrep", "chromium", "chromium-headless-shell", "node", "npm"): lock.set_pin(name, "1", {"any": {"url": "x", "sha256": "0" * 64}}) lock.save() - monkeypatch.setattr("pm.cli._lockfile", lambda: lock) + monkeypatch.setattr("scripts.bundles.native._lockfile", lambda: lock) names = _bundle_package_names() # Browsers now ship in every payload (win32-arm64 runs the x64 build # under emulation); nothing is excluded from the bundle. @@ -681,7 +681,7 @@ def test_bundle_package_names_include_browsers(monkeypatch, tmp_path): def test_bundle_closure_uv_stays_internal_node_npm_ship(monkeypatch, tmp_path): """Locks the semantics split: uv is pm's install machinery and never ships by closure, node/npm are runtime tools and always do.""" - from pm.cli import _bundle_package_names + from scripts.bundles.native import _bundle_package_names from pm.lock import Lockfile from pm.registry import get_package @@ -689,7 +689,7 @@ def test_bundle_closure_uv_stays_internal_node_npm_ship(monkeypatch, tmp_path): for name in ("uv", "node", "npm"): lock.set_pin(name, "1", {"any": {"url": "x", "sha256": "0" * 64}}) lock.save() - monkeypatch.setattr("pm.cli._lockfile", lambda: lock) + monkeypatch.setattr("scripts.bundles.native._lockfile", lambda: lock) names = _bundle_package_names() # uv stays internal (off PATH, off the default install) but ships in # the bundle via the explicit whitelist — install machinery rides along. @@ -703,7 +703,7 @@ def test_bundle_closure_uv_stays_internal_node_npm_ship(monkeypatch, tmp_path): def test_arch_guard_allows_emulated_x64_on_win32_arm64(monkeypatch, tmp_path): """agent-browser on win32-arm64 ships the x64 PE (emulated). The guard must not reject it when the package declares the target emulated.""" - import pm.cli as cli + from scripts.bundles import native as cli from pm.lock import Facts, Lockfile from pm.registry import get_package @@ -721,9 +721,9 @@ def test_arch_guard_allows_emulated_x64_on_win32_arm64(monkeypatch, tmp_path): lock = Lockfile(tmp_path / "lock.json") lock.set_pin("agent-browser", "0.35.1", {"any": {"url": "x", "sha256": "0" * 64}}) lock.save() - monkeypatch.setattr("pm.cli._lockfile", lambda: lock) - monkeypatch.setattr("pm.cli.current_target", lambda: "win32-arm64") - monkeypatch.setattr("pm.cli.get_package", lambda name: get_package(name)) + monkeypatch.setattr("scripts.bundles.native._lockfile", lambda: lock) + monkeypatch.setattr("scripts.bundles.native.current_target", lambda: "win32-arm64") + monkeypatch.setattr("scripts.bundles.native.get_package", lambda name: get_package(name)) facts = Facts(store / "facts.json") facts.record("agent-browser", "0.35.1", entry.name, {}, store) diff --git a/tests/scripts/test_bundle_native.py b/tests/scripts/test_bundle_native.py new file mode 100644 index 0000000000..9dc035967a --- /dev/null +++ b/tests/scripts/test_bundle_native.py @@ -0,0 +1,54 @@ +"""The native bundle pipeline publishes only after a real staged sync succeeds.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +from scripts.bundles import native + + +def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_path, monkeypatch): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\n') + subprocess.run(["git", "init", str(repo)], check=True, capture_output=True) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-m", "fixture"], cwd=repo, check=True, capture_output=True) + output = tmp_path / "payload" + monkeypatch.setattr("pm.paths.repo_root", lambda: repo) + monkeypatch.setattr(native, "_bundle_package_names", lambda: []) + monkeypatch.setattr(native, "_install_names", lambda names: 0) + monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: Path(sys.executable).parent)) + monkeypatch.setattr(native, "_facts", lambda: SimpleNamespace(get=lambda _: {"entry": "python"}, entries_in_use=lambda: [])) + monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: Path(sys.executable))) + monkeypatch.setattr(native, "pm_uv", lambda: (sys.executable, dict(os.environ))) + monkeypatch.setattr(native, "_arch_guard", lambda store: []) + monkeypatch.setattr("scripts.bundles.payload.relativize_links", lambda root: 0) + monkeypatch.setattr("pm.features.installed_extras", lambda *args: []) + monkeypatch.setattr("pm.packages.uv_cache_dir", lambda: tmp_path / "empty-cache") + real_run = native._run_live + calls = [] + + def child(argv, *, cwd, env): + calls.append(argv[1]) + assert not (output / "manifest.json").exists() + # Execute a real child and a real venv, without network or tool-store writes. + command = [sys.executable, "-m", "venv", "--without-pip", str(output / "venv")] if argv[1] == "venv" else [sys.executable, "-c", "print('sync fixture complete')"] + return real_run(command, cwd=cwd, env=env) + + monkeypatch.setattr(native, "_run_live", child) + monkeypatch.setenv("HERMES_RUNTIME_DIR", str(tmp_path / "original")) + assert native.stage_native(SimpleNamespace(out=str(output), ref="HEAD")) == 0 + assert calls == ["venv", "sync"] + assert (output / "hermes-agent/pyproject.toml").is_file() + assert json.loads((output / "manifest.json").read_text())["repo"] == "hermes-agent" + assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original") + + monkeypatch.setattr(native, "_run_live", lambda *a, **kw: (1, "injected failure")) + assert native.stage_native(SimpleNamespace(out=str(output), ref="HEAD")) == 1 + assert not (output / "manifest.json").exists() + assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original") diff --git a/tests/scripts/test_bundle_payload.py b/tests/scripts/test_bundle_payload.py new file mode 100644 index 0000000000..52084b9c98 --- /dev/null +++ b/tests/scripts/test_bundle_payload.py @@ -0,0 +1,92 @@ +"""Shared bundle operations use real filesystem and entrypoint contracts.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys + + +import pytest + +from scripts.bundles.payload import plant_surfaces, posix_launcher, project_entries, relativize_links, snapshot, write_manifest +from scripts.bundles.desktop import release_version + + +def test_snapshot_and_manifest_are_shared_by_both_layouts(tmp_path): + source = tmp_path / "source" + source.mkdir() + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + project = '[project]\nname="fixture"\nversion="1.2.3"\n[project.scripts]\ncustom="entry:run"\n' + (source / "pyproject.toml").write_text(project, encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=source, check=True) + subprocess.run(["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-m", "fixture"], cwd=source, check=True, capture_output=True) + (source / "untracked").write_text("must not ship") + for repo_name, target in [("app", "linux-arm64-bionic"), ("hermes-agent", "win32-arm64")]: + root = tmp_path / repo_name + root.mkdir() + snapshot(source, "HEAD", root / repo_name) + manifest = write_manifest(root, target=target, repo=repo_name) + assert project_entries(root / manifest["repo"]) == {"custom": "entry:run"} + assert not (root / repo_name / "untracked").exists() + assert not (root / repo_name / ".git").exists() + assert json.loads((root / "manifest.json").read_text()) == manifest + assert release_version(source, "v1.2.3") == "1.2.3" + with pytest.raises(ValueError): + release_version(source, "v1.2.4") + + +def test_surfaces_require_complete_outputs_and_replace_stale_files(tmp_path): + source, repo = tmp_path / "build", tmp_path / "payload" + tui = source / "ui-tui/dist" + web = source / "hermes_cli/web_dist" + tui.mkdir(parents=True) + web.mkdir(parents=True) + (tui / "entry.js").write_text("built tui") + (web / "index.html").write_text("built web") + plant_surfaces(repo, source) + (repo / "hermes_cli/web_dist/stale").write_text("old") + plant_surfaces(repo, source) + assert not (repo / "hermes_cli/web_dist/stale").exists() + assert (repo / "hermes_cli/tui_dist/entry.js").read_text() == "built tui" + (web / "index.html").unlink() + with pytest.raises(FileNotFoundError): + plant_surfaces(repo, source) + + +@pytest.mark.platforms("posix") +def test_relocation_preserves_sibling_and_framework_links(tmp_path): + root = tmp_path / "payload" + store = root / "tools/python/bin" + venv = root / "venv/bin" + store.mkdir(parents=True) + venv.mkdir(parents=True) + (store / "python3").write_text("interpreter") + (venv / "python").symlink_to("/builder/tools/python/bin/python3") + (venv / "python3").symlink_to("python") + framework = root / "tools/framework" + framework.symlink_to("python/bin/python3") + assert relativize_links(root) == 1 + assert (venv / "python3").read_text() == "interpreter" + assert os.readlink(venv / "python3") == "python" + assert os.readlink(framework) == "python/bin/python3" + assert relativize_links(root) == 0 + (venv / "bad").symlink_to("/usr/bin/python") + with pytest.raises(ValueError, match="escapes payload"): + relativize_links(root) + + +@pytest.mark.platforms("posix") +@pytest.mark.parametrize("target", ["linux-x64", "linux-arm64-bionic"]) +def test_both_launchers_keep_active_home_and_call_declared_function(tmp_path, target): + root = tmp_path / "payload with spaces" + (root / "bin").mkdir(parents=True) + (root / "app").mkdir() + (root / "python").symlink_to(sys.executable) + (root / "app/entry.py").write_text("import json,os,sys\ndef run():\n print(json.dumps([os.environ['HERMES_HOME'],sys.argv[1:]])); return 7\n") + launcher = root / "bin/custom" + launcher.write_text(posix_launcher("custom", "entry:run", python="python", repo="app", site="deps", target=target)) + result = subprocess.run(["sh", str(launcher), "two words", "$(nope)", ""], cwd=tmp_path, + env={**os.environ, "HERMES_HOME": str(tmp_path / "custom/profiles/memory")}, capture_output=True, text=True) + assert result.returncode == 7, result.stderr + assert json.loads(result.stdout) == [str(tmp_path / "custom/profiles/memory"), ["two words", "$(nope)", ""]] diff --git a/tests/scripts/test_desktop_cli_wrapper.py b/tests/scripts/test_desktop_cli_wrapper.py index 5f2b2bc66b..e3c789200b 100644 --- a/tests/scripts/test_desktop_cli_wrapper.py +++ b/tests/scripts/test_desktop_cli_wrapper.py @@ -1,6 +1,6 @@ """Unit tests for the bundled payload's win32 launcher wrapper. -scripts/desktop-cli/launcher-wrapper.py is the zip overlay a distlib +scripts/bundles/launcher_wrapper.py is the zip overlay a distlib ScriptMaker packs into every minted CLI launcher exe (the rust shim's replacement). The wrapper is import-safe on purpose, so these tests drive its real logic — path resolution, sys.path order, the pycache_prefix @@ -21,7 +21,7 @@ from pathlib import Path import pytest _REPO = Path(__file__).resolve().parents[2] -_WRAPPER = _REPO / "scripts" / "desktop-cli" / "launcher-wrapper.py" +_WRAPPER = _REPO / "scripts" / "bundles" / "launcher_wrapper.py" def _load(env=None): diff --git a/tests/scripts/test_mint_launchers.py b/tests/scripts/test_mint_launchers.py index 54f34cbab7..1527cbe629 100644 --- a/tests/scripts/test_mint_launchers.py +++ b/tests/scripts/test_mint_launchers.py @@ -1,4 +1,4 @@ -"""Live tests for the win32 launcher mint (scripts/desktop-cli/mint-launchers.py). +"""Live tests for the win32 launcher mint (scripts/bundles/mint_launchers.py). These RUN the real mint on the current interpreter and then execute the minted launcher — the strongest proof the mechanism is intact (the research @@ -27,7 +27,7 @@ from pathlib import Path import pytest _REPO = Path(__file__).resolve().parents[2] -_MINT = _REPO / "scripts" / "desktop-cli" / "mint-launchers.py" +_MINT = _REPO / "scripts" / "bundles" / "mint_launchers.py" pytestmark = [ pytest.mark.platforms("windows"), @@ -113,16 +113,8 @@ def _mint(bin_dir: Path, wrapper: Path, specs) -> list[str]: def _render_wrapper(tmp_path: Path) -> Path: - """cli-entrypoints.mjs's renderWinWrapper, replicated (the .mjs cannot - be imported from python) — same placeholders, same substitution.""" - text = (_REPO / "scripts" / "desktop-cli" / "launcher-wrapper.py").read_text(encoding="utf-8") - for placeholder, value in { - "__HERMES_ENTRY_MODULE__": "hermes_cli.main", - "__HERMES_ENTRY_FUNC__": "main", - "__HERMES_REPO_REL__": "../repo", - "__HERMES_SITE_REL__": "../venv/Lib/site-packages", - }.items(): - text = text.replace(placeholder, value) + from scripts.bundles.payload import render_wrapper + text = render_wrapper("hermes_cli.main:main", "../repo", "../venv/Lib/site-packages") out = tmp_path / "wrapper.py" out.write_text(text, encoding="utf-8") return out diff --git a/tests/test_termux_launchers.py b/tests/test_termux_launchers.py index d2b4bd3bb5..8259d88cdb 100644 --- a/tests/test_termux_launchers.py +++ b/tests/test_termux_launchers.py @@ -45,7 +45,7 @@ def test_launchers_forward_arguments_and_export_payload_environment(tmp_path): assert Path(env["HERMES_PYTHON"]).resolve() == Path(sys.executable).resolve() assert Path(env["HERMES_NODE"]) == payload / "tools/node/data/data/com.termux/files/usr/bin/node" assert Path(env["HERMES_RUNTIME_DIR"]) == payload / "tools" - assert Path(env["PYTHONPATH"]) == payload / "app" + assert env["PYTHONPATH"].split(os.pathsep)[0] == str(payload / "app") assert env["PYTHONHOME"] is None assert not Path(env["PYTHONPYCACHEPREFIX"]).is_relative_to(payload)