diff --git a/.github/actions/save-pm-cache/README.md b/.github/actions/save-pm-cache/README.md index ef5e498b9e..ead9136344 100644 --- a/.github/actions/save-pm-cache/README.md +++ b/.github/actions/save-pm-cache/README.md @@ -1,8 +1,8 @@ # Bundle dependency caches `setup-pm` restores uv's real cache, not the installed virtual environment. -Its fallback keys retain the native target, OS version, Python version, and -pruning mode. A metadata or dependency change can reuse compatible wheels; +Its fallback keys retain the native target, OS version, and Python version. +A metadata or dependency change can reuse compatible wheels; uv still resolves and installs from the frozen project lock. Bundle jobs set `save-python-cache: false` and call `save-pm-cache` after their @@ -22,10 +22,14 @@ automatic cache path and its exact-hit smoke tests remain available. Smoke namespaces precede the native/dependency identity, outside production's restore prefix. PM Toolchain cleanup removes its run-scoped snapshots. -Before saving, `python -m pm.build_env --prune-cache --cache PATH` removes -dangling entries. It does not use `--ci`: that option discards downloaded wheels, while bundles copy the full -cache for offline dependency installation. Pruning happens after payload staging -and packaging, and it does not change the staged payload. +Before saving, `python -m pm.build_env --exact-lock --cache PATH --lock-source REPO` +deletes every entry the project's `uv.lock` cannot resolve. It keeps downloaded +wheels the lock resolves (bundles copy the full cache for offline dependency +installation) — `uv cache prune --ci` would discard them. Pruning happens after +payload staging and packaging, and it does not change the staged payload. +The same lock-exactness contract governs the bundle ship gate +(`stage_uv_cache`) and CI's rolling snapshots, so no snapshot accumulates +sediment for superseded pins. This is not a lockfile-aware or size-bounded cache. Old, still-referenced package versions can remain inside a snapshot and be copied forward. GitHub evicts whole diff --git a/.github/actions/save-pm-cache/action.yml b/.github/actions/save-pm-cache/action.yml index 4f42f17e2c..3adc096538 100644 --- a/.github/actions/save-pm-cache/action.yml +++ b/.github/actions/save-pm-cache/action.yml @@ -1,5 +1,5 @@ name: Save the PM Python dependency cache -description: Prune dangling entries and save a rolling cache after the consumer, even when it failed. +description: Prune to the lock and save a rolling cache after the consumer, even when it failed. inputs: python: description: Prepared Python used to invoke PM. @@ -15,16 +15,18 @@ runs: steps: # Status checks are needed inside the composite too: the caller can # enter after a failed build. Never prune while cancellation kills uv. - - name: Prune dangling uv cache entries + - name: Prune cache entries outside the project lock id: prune if: ${{ !cancelled() }} shell: bash env: PM_PYTHON: ${{ inputs.python }} PM_CACHE: ${{ inputs.path }} - # Keep downloaded wheels as well as source-built wheels. Bundles copy - # this cache for offline installs; --ci would discard required inputs. - run: '"$PM_PYTHON" -m pm.build_env --prune-cache --cache "$PM_CACHE"' + PM_LOCK_SOURCE: ${{ github.workspace }} + # Exact-to-lock: downloaded wheels the lock resolves survive (bundles + # copy this cache for offline installs), while superseded pins do not + # accumulate. --ci was never usable here — it discards downloaded wheels. + run: '"$PM_PYTHON" -m pm.build_env --exact-lock --cache "$PM_CACHE" --lock-source "$PM_LOCK_SOURCE"' - name: Save reusable Python dependencies if: ${{ !cancelled() && steps.prune.outcome == 'success' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/actions/setup-pm/README.md b/.github/actions/setup-pm/README.md index b9abb8597c..a29ac669ab 100644 --- a/.github/actions/setup-pm/README.md +++ b/.github/actions/setup-pm/README.md @@ -50,7 +50,11 @@ The official, SHA-pinned `actions/cache` transports three independent caches: All restores use the exact primary key, without fallback prefixes, matching setup-uv and setup-node's npm behavior. Successful jobs save at teardown; -exact hits are not saved again. PM re-verifies restored tools before use. +exact hits are not saved again. Only dependency-carrying callers +(`extras` set) auto-save the uv cache: a tool-only job never runs a +dependency operation, so letting it save would freeze an empty cache under +the production key, where an immutable exact hit blocks real saves forever. +PM re-verifies restored tools before use. Dependency caches never contain `node_modules` or virtual environments. Keep an installed-tree cache in the caller if that job needs one. @@ -62,12 +66,16 @@ that toolchain. `python-cache-dependency-glob` defaults to `pyproject.toml` and An npm cache without a matching lockfile fails, rather than caching an unversioned dependency set. -`prune-python-cache: true` registers PM's CI cache-pruning operation at teardown, +`prune-python-cache: true` registers PM's lock-exact cache-pruning operation at teardown, after the caller's installs and before the cache save. It is skipped on an exact hit. The default is `false`, as in setup-uv v9; migrated v8 callers opt in to retain their former policy. The small nested JavaScript action exists only because GitHub composite actions cannot declare their own post step. It uses -Node's standard library and has no bundled dependencies. +Node's standard library and has no bundled dependencies. Pruning deletes cache +entries the project's `uv.lock` cannot resolve (the same exactness contract the +bundle ship gate enforces) and keeps downloaded wheels the lock still needs — +`uv cache prune --ci` would discard them, leaving a snapshot that warms almost +nothing. Outputs include `python-version`, `uv-version`, `node-version`, `npm-version`, `python-path`, `venv`, `target`, and the three `*-cache-hit` flags. diff --git a/.github/actions/setup-pm/action.yml b/.github/actions/setup-pm/action.yml index 4d27818cf1..84dc60bff6 100644 --- a/.github/actions/setup-pm/action.yml +++ b/.github/actions/setup-pm/action.yml @@ -149,15 +149,18 @@ runs: - name: Cache uv dependency downloads and builds id: python-cache - if: inputs.toolchain != 'node' && inputs.cache-python == 'true' && inputs.save-python-cache == 'true' + if: inputs.toolchain != 'node' && inputs.cache-python == 'true' && inputs.save-python-cache == 'true' && inputs.extras != '' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.install.outputs.uv-cache-path }} - key: setup-pm-uv-v2-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ inputs.prune-python-cache }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }} + # pruned/unpruned share one namespace: the pre-save prune is now exact + # to the lock (keeps lock-required wheels, drops superseded pins), so + # both variants carry the same content contract. + key: setup-pm-uv-v3-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }} # Isolated writes also need a distinct prefix: production's broad # fallback must not restore a smoke run's partial dependency set. - restore-keys: ${{ inputs.cache-suffix == '' && format('setup-pm-uv-v2-production-{0}-{1}-{2}-{3}-', steps.prepare.outputs.target, steps.prepare.outputs.os-version, steps.prepare.outputs.python-version, inputs.prune-python-cache) || '' }} + restore-keys: ${{ inputs.cache-suffix == '' && format('setup-pm-uv-v3-production-{0}-{1}-{2}-', steps.prepare.outputs.target, steps.prepare.outputs.os-version, steps.prepare.outputs.python-version) || '' }} - name: Restore uv dependencies for an explicit save id: python-cache-restore @@ -167,10 +170,10 @@ runs: path: ${{ steps.install.outputs.uv-cache-path }} # Caches are immutable. Each producer needs its own snapshot, even # when a caller run contains both PM Bundle and Desktop Release. - key: setup-pm-uv-v2-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ inputs.prune-python-cache }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + key: setup-pm-uv-v3-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} restore-keys: | - setup-pm-uv-v2-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ inputs.prune-python-cache }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }}- - ${{ inputs.cache-suffix == '' && format('setup-pm-uv-v2-production-{0}-{1}-{2}-{3}-', steps.prepare.outputs.target, steps.prepare.outputs.os-version, steps.prepare.outputs.python-version, inputs.prune-python-cache) || '' }} + setup-pm-uv-v3-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }}- + ${{ inputs.cache-suffix == '' && format('setup-pm-uv-v3-production-{0}-{1}-{2}-', steps.prepare.outputs.target, steps.prepare.outputs.os-version, steps.prepare.outputs.python-version) || '' }} # Post steps run in reverse registration order: prune before cache save. - name: Register uv cache pruning @@ -179,6 +182,7 @@ runs: with: python: ${{ steps.install.outputs.python-path }} cache: ${{ steps.install.outputs.uv-cache-path }} + lock-source: ${{ github.workspace }} - name: Require an npm dependency lock for caching if: inputs.toolchain != 'python' && inputs.cache-node == 'true' && hashFiles(inputs.node-cache-dependency-path) == '' @@ -194,6 +198,10 @@ runs: with: path: ${{ steps.install.outputs.npm-cache-path }} key: node-cache-${{ runner.os }}-${{ steps.prepare.outputs.arch }}-npm-${{ hashFiles(inputs.node-cache-dependency-path) }}${{ inputs.cache-suffix != '' && format('-{0}', inputs.cache-suffix) || '' }} + # A lockfile bump misses the exact key, but the previous download set + # is still mostly valid — npm verifies every tarball it reinstalls. + restore-keys: | + node-cache-${{ runner.os }}-${{ steps.prepare.outputs.arch }}-npm- - name: Restore npm dependencies for an explicit save id: node-cache-restore diff --git a/.github/actions/setup-pm/prune/action.yml b/.github/actions/setup-pm/prune/action.yml index eb3729d223..e47bc095ba 100644 --- a/.github/actions/setup-pm/prune/action.yml +++ b/.github/actions/setup-pm/prune/action.yml @@ -1,5 +1,5 @@ name: Prune the PM uv cache at job teardown -description: Register uv pruning before the enclosing cache action saves. +description: Register lock-exact pruning before the enclosing cache action saves. inputs: python: description: Prepared Python used to invoke the PM cache operation. @@ -7,6 +7,9 @@ inputs: cache: description: Cache directory restored by the enclosing setup action. required: true + lock-source: + description: Repository checkout whose uv.lock selects the kept entries. + required: true runs: using: node24 main: index.mjs diff --git a/.github/actions/setup-pm/prune/index.mjs b/.github/actions/setup-pm/prune/index.mjs index 80b3984861..5fb6968bea 100644 --- a/.github/actions/setup-pm/prune/index.mjs +++ b/.github/actions/setup-pm/prune/index.mjs @@ -6,13 +6,18 @@ import { pathToFileURL } from 'node:url' // this action's post runs first, pruning the cache just before it is saved. export function run(env, execute = spawnSync) { if (!env.STATE_python) { - for (const [key, value] of Object.entries({ python: env.INPUT_PYTHON, cache: env.INPUT_CACHE })) { + for (const [key, value] of Object.entries({ python: env.INPUT_PYTHON, cache: env.INPUT_CACHE, lockSource: env.INPUT_LOCK_SOURCE })) { if (!value || /[\r\n\0]/.test(value)) throw new Error(`invalid ${key}`) appendFileSync(env.GITHUB_STATE, `${key}=${value}\n`, 'utf8') } return } - const result = execute(env.STATE_python, ['-m', 'pm.build_env', '--prune-cache', '--cache', env.STATE_cache, '--ci'], { + // Exact-to-lock pruning keeps every wheel the project's uv.lock resolves — + // including downloaded ones. `--ci` pruning discarded downloaded wheels so + // the saved snapshot warmed almost nothing; bundling later ships this same + // cache only after its own exact-lock gate, so exactness is the shared + // contract, and sediment for superseded pins never accumulates. + const result = execute(env.STATE_python, ['-m', 'pm.build_env', '--exact-lock', '--cache', env.STATE_cache, '--lock-source', env.STATE_lockSource], { env, stdio: 'inherit', }) diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 26f14459d1..9a1e347388 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -424,6 +424,27 @@ jobs: CHANNEL_REQUEST_SHA256: ${{ needs.validate.outputs.channel-request-sha256 }} run: python -m scripts.releases.channel_publish request --out "$RUNNER_TEMP/channel-request.json" + - name: Resolve the builder toolchain versions + id: electron-tools + shell: bash + run: node -e ' + const l = require("./package-lock.json") + const eb = l.packages["node_modules/electron-builder"].version + if (!eb) process.exit(1) + console.log("eb=" + eb) + ' + + - name: Restore the electron-builder toolchain cache + id: electron-tools-restore + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # winCodeSign kit, ATS dlib, dotnet runtime and rcedit land in + # ELECTRON_BUILDER_CACHE; the builder version selects the content. + path: ${{ github.workspace }}/.cache/electron-builder + key: eb3-${{ runner.os }}-${{ runner.arch }}-builder-${{ steps.electron-tools.outputs.eb }} + restore-keys: | + eb3-${{ runner.os }}-${{ runner.arch }}- + - name: Restore desktop dependency inputs id: build-cache uses: ./.github/actions/desktop-build-cache @@ -458,6 +479,15 @@ jobs: producer: desktop key: ${{ steps.build-cache.outputs.cache-key }} + # Commit builds run these steps with a read-only cache token; the + # exact-hit-skip semantics of actions/cache keep a read token harmless. + - name: Save the electron-builder toolchain + if: ${{ !cancelled() && steps.prepare.outcome == 'success' && inputs.build_commit == '' && inputs.channel_build == '' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.cache/electron-builder + key: eb3-${{ runner.os }}-${{ runner.arch }}-builder-${{ steps.electron-tools.outputs.eb }} + - name: Restore verified payload signatures id: payload-signatures uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -692,6 +722,27 @@ jobs: CHANNEL_REQUEST_SHA256: ${{ needs.validate.outputs.channel-request-sha256 }} run: python -m scripts.releases.channel_publish request --out "$RUNNER_TEMP/channel-request.json" + - name: Resolve the builder toolchain versions + id: electron-tools + shell: bash + run: node -e ' + const l = require("./package-lock.json") + const eb = l.packages["node_modules/electron-builder"].version + if (!eb) process.exit(1) + console.log("eb=" + eb) + ' + + - name: Restore the electron-builder toolchain cache + id: electron-tools-restore + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # winCodeSign kit, ATS dlib, dotnet runtime and rcedit land in + # ELECTRON_BUILDER_CACHE; the builder version selects the content. + path: ${{ github.workspace }}/.cache/electron-builder + key: eb3-${{ runner.os }}-${{ runner.arch }}-builder-${{ steps.electron-tools.outputs.eb }} + restore-keys: | + eb3-${{ runner.os }}-${{ runner.arch }}- + - name: Restore desktop dependency inputs id: build-cache uses: ./.github/actions/desktop-build-cache @@ -726,6 +777,15 @@ jobs: producer: desktop key: ${{ steps.build-cache.outputs.cache-key }} + # Commit builds run these steps with a read-only cache token; the + # exact-hit-skip semantics of actions/cache keep a read token harmless. + - name: Save the electron-builder toolchain + if: ${{ !cancelled() && steps.prepare.outcome == 'success' && inputs.build_commit == '' && inputs.channel_build == '' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.cache/electron-builder + key: eb3-${{ runner.os }}-${{ runner.arch }}-builder-${{ steps.electron-tools.outputs.eb }} + # Commit artifacts are downloadable too. Require signing for them, # candidates, and published tags before building any payload. - name: Require signing credentials when publishing @@ -1098,25 +1158,16 @@ jobs: # Cache reuse is optional. Bundle scripts provision the pinned SDK # and signing dependencies through electron-builder on a cache miss. # This smaller consumer has its own cache, separate from desktop inputs. - - name: Resolve electron's default download cache path - shell: bash - run: | - case "$RUNNER_OS" in - Windows) echo "ELECTRON_DEFAULT_CACHE=$LOCALAPPDATA/electron/Cache" >> "$GITHUB_ENV" ;; - macOS) echo "ELECTRON_DEFAULT_CACHE=$HOME/Library/Caches/electron" >> "$GITHUB_ENV" ;; - *) echo "ELECTRON_DEFAULT_CACHE=$HOME/.cache/electron" >> "$GITHUB_ENV" ;; - esac - - name: Restore electron + electron-builder toolchain uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ${{ github.workspace }}/.cache/electron-builder - ${{ github.workspace }}/.cache/electron - ${{ env.ELECTRON_DEFAULT_CACHE }} - key: eb2-${{ runner.os }}-${{ runner.arch }}-electron-${{ steps.toolchain.outputs.electron }}-builder-${{ steps.toolchain.outputs.builder }} + # ensureWindowsBundleTools resolves the winCodeSign kit, ATS dlib + # and dotnet runtime through electron-builder's toolset cache. + path: ${{ github.workspace }}/.cache/electron-builder + # Saved by the build-win32-release legs during preparation. + key: eb3-${{ runner.os }}-${{ runner.arch }}-builder-${{ steps.toolchain.outputs.builder }} restore-keys: | - eb2-${{ runner.os }}-${{ runner.arch }}- + eb3-${{ runner.os }}-${{ runner.arch }}- - name: Retrieve Windows packages from R2 shell: bash diff --git a/pm/build_env.py b/pm/build_env.py index 2bf34738c2..7a46dcf953 100644 --- a/pm/build_env.py +++ b/pm/build_env.py @@ -26,7 +26,11 @@ def main(argv: Sequence[str] | None = None) -> int: operation.add_argument("--check-lock", action="store_true") operation.add_argument("--export-requirements", type=Path) operation.add_argument("--prune-cache", action="store_true") + operation.add_argument("--exact-lock", action="store_true", + help="prune to the project lock: entries the uv.lock cannot resolve are deleted") operation.add_argument("--manager-runtime", action="store_true") + parser.add_argument("--lock-source", type=Path, default=None, + help="repo whose uv.lock selects the kept entries for --exact-lock (default: cwd)") parser.add_argument("--upgrade", action="store_true") parser.add_argument("--ci", action="store_true") parser.add_argument("--sealed", action="store_true") @@ -40,17 +44,31 @@ def main(argv: Sequence[str] | None = None) -> int: if args.requirements is not None: requirements.extend(line.strip() for line in args.requirements.read_text(encoding="utf-8").splitlines() if line.strip() and not line.lstrip().startswith("#")) - if args.prune_cache: + if args.exact_lock: + if args.ci or args.prune_cache: + parser.error("--exact-lock replaces --ci/--prune-cache: downloaded wheels the lock keeps must survive") + if args.lock_source is None: + parser.error("--exact-lock requires --lock-source") + if args.cache is None: + parser.error("--exact-lock requires --cache") + elif args.prune_cache: if args.cache is None: parser.error("--prune-cache requires --cache") elif not requirements and args.source is None: parser.error("--source is required for project operations") - build = not (args.lock_only or args.check_lock or args.export_requirements or args.prune_cache) + build = not (args.lock_only or args.check_lock or args.export_requirements + or args.prune_cache or args.exact_lock) if build and args.out is None: parser.error("--out is required when building an environment") if args.manager_runtime and args.python is None: parser.error("--manager-runtime requires the target --python") try: + if args.exact_lock: + from pm.cache_lock import prune_uv_cache_to_lock + + pruned = prune_uv_cache_to_lock(args.cache, args.lock_source) + print(f"pruned {pruned} cache entries outside the lock") + return 0 if args.prune_cache: pm.prune_cache(args.cache, ci=args.ci) return 0 diff --git a/pm/cache_lock.py b/pm/cache_lock.py new file mode 100644 index 0000000000..7472e6436b --- /dev/null +++ b/pm/cache_lock.py @@ -0,0 +1,59 @@ +"""Exact-to-lock pruning for uv caches. + +The lock is the contract for anything downstream consumes wholesale: a +shipped bundle payload and a rolling CI cache snapshot both must not carry +wheels the project's uv.lock cannot resolve. Rolling caches accumulate +entries for superseded pins (prefix restores across dependency changes); +pruning to the lock keeps every consumer exact without discarding wheels +the current lock still needs (unlike ``uv cache prune --ci``). +""" +from __future__ import annotations + +import re +import shutil +from pathlib import Path + + +def lock_package_names(source_repo: Path) -> set[str]: + """Dist names the lock can resolve; the exactness contract for a cache.""" + import tomllib + + data = tomllib.loads((source_repo / "uv.lock").read_text(encoding="utf-8")) + return {entry["name"].lower().replace("_", "-") for entry in data["package"]} + + +def prune_uv_cache_to_lock(cache: Path, source_repo: Path) -> int: + """Delete cache entries for dists the lock cannot resolve. + + Unidentifiable buckets (no dist-info) survive — pruning fails open for + unknown layouts, never for identifiable stale pins. Returns the pruned + entry count for the build log. + """ + keep = lock_package_names(source_repo) + dist_info = re.compile(r"([A-Za-z0-9_.]+?)-\d[^-]*\.dist-info") + + def dist_name(bucket: Path) -> str | None: + for marker_file in bucket.glob("*.dist-info"): + match = dist_info.match(marker_file.name) + if match: + return match.group(1).lower().replace("_", "-") + return None + + pruned = 0 + archive = cache / "archive-v0" + if archive.is_dir(): + for bucket in archive.iterdir(): + if not bucket.is_dir(): + continue + name = dist_name(bucket) + if name is not None and name not in keep: + shutil.rmtree(bucket, ignore_errors=True) + pruned += 1 + for family_dir in (*cache.glob("wheels-v*/pypi"), *cache.glob("sdists-v*/pypi")): + if not family_dir.is_dir(): + continue + for entry in family_dir.iterdir(): + if entry.is_dir() and entry.name.lower().replace("_", "-") not in keep: + shutil.rmtree(entry, ignore_errors=True) + pruned += 1 + return pruned diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index cd504a6ca6..33b708a0b2 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -52,52 +52,9 @@ def _arch_guard(store_dir: Path) -> list[str]: -def _lock_package_names(source_repo: Path) -> set[str]: - """Dist names the shipped lock can resolve; the ship contract for the cache.""" - import tomllib +from pm.cache_lock import lock_package_names, prune_uv_cache_to_lock - data = tomllib.loads((source_repo / "uv.lock").read_text(encoding="utf-8")) - return {entry["name"].lower().replace("_", "-") for entry in data["package"]} - - -def prune_uv_cache_to_lock(cache: Path, source_repo: Path) -> int: - """Delete cache entries for dists the lock cannot resolve. - - The CI cache is a rolling snapshot restored by prefix after dependency - changes, so it accumulates wheels for superseded pins. Shipping that - sediment would bloat every bundle; the lock is the ship contract. - Returns the pruned entry count for the build log. - """ - import re - - keep = _lock_package_names(source_repo) - dist_info = re.compile(r"([A-Za-z0-9_.]+?)-\d[^-]*\.dist-info") - - def dist_name(bucket: Path) -> str | None: - for marker_file in bucket.glob("*.dist-info"): - match = dist_info.match(marker_file.name) - if match: - return match.group(1).lower().replace("_", "-") - return None - - pruned = 0 - archive = cache / "archive-v0" - if archive.is_dir(): - for bucket in archive.iterdir(): - if not bucket.is_dir(): - continue - name = dist_name(bucket) - if name is not None and name not in keep: - shutil.rmtree(bucket, ignore_errors=True) - pruned += 1 - for family_dir in (*cache.glob("wheels-v*/pypi"), *cache.glob("sdists-v*/pypi")): - if not family_dir.is_dir(): - continue - for entry in family_dir.iterdir(): - if entry.is_dir() and entry.name.lower().replace("_", "-") not in keep: - shutil.rmtree(entry, ignore_errors=True) - pruned += 1 - return pruned +__all__ = ["prune_uv_cache_to_lock", "lock_package_names", "stage_uv_cache"] def stage_uv_cache(source: Path, destination: Path) -> None: diff --git a/scripts/ci/cleanup_pm_toolchain_caches.py b/scripts/ci/cleanup_pm_toolchain_caches.py index 3ae3eb7e5f..729e0fb258 100644 --- a/scripts/ci/cleanup_pm_toolchain_caches.py +++ b/scripts/ci/cleanup_pm_toolchain_caches.py @@ -16,7 +16,7 @@ def cleanup_run_caches(run_id: str, request, *, page_size: int = 100) -> list[in raise ValueError("cleanup requires a completed PM Toolchain run") pattern = re.compile( rf"^(?:(?:setup-pm-tools-|node-cache-).*-smoke(?:-prune|-consumers)?-{run_id}-[1-9][0-9]*" - rf"|setup-pm-uv-v2-smoke(?:-prune|-consumers)?-{run_id}-[1-9][0-9]*-.+)$" + rf"|setup-pm-uv-v[23]-smoke(?:-prune|-consumers)?-{run_id}-[1-9][0-9]*-.+)$" ) def collect(): diff --git a/tests-js/setup-pm-cache.test.mjs b/tests-js/setup-pm-cache.test.mjs index 9ece1b407b..be2673f3e7 100644 --- a/tests-js/setup-pm-cache.test.mjs +++ b/tests-js/setup-pm-cache.test.mjs @@ -19,10 +19,10 @@ it('restores compatible wheels without freezing a partial build under its depend expect(prefixes[1]).toContain(`steps.prepare.outputs.${boundary}`) } expect(prefixes[1]).toContain("inputs.cache-suffix == ''") - expect(prefixes[1]).toContain('inputs.prune-python-cache') expect(prefixes[1]).not.toContain('hashFiles') - expect(restored.uses.split('@')[0]).toBe('actions/cache/restore') - expect(cached.if).toContain("inputs.save-python-cache == 'true'") + // Only dependency-carrying callers save; tool-only jobs must not freeze an + // empty cache under the production key (a stub exact-hit blocks real saves). + expect(cached.if).toContain("inputs.extras != ''") expect(restored.if).toContain("inputs.save-python-cache == 'false'") expect(setup.outputs['python-cache-key'].value).toContain('steps.python-cache-restore.outputs.cache-primary-key') @@ -32,19 +32,20 @@ it('restores compatible wheels without freezing a partial build under its depend for (const template of [cached.with.key, restored.with.key, rollingPrefix]) { const production = template.replace(namespace, 'production') const smoke = template.replace(namespace, 'smoke-42-1') - expect(production.startsWith('setup-pm-uv-v2-production-')).toBe(true) - expect(smoke.startsWith('setup-pm-uv-v2-smoke-42-1-')).toBe(true) - expect(smoke.startsWith('setup-pm-uv-v2-production-')).toBe(false) - expect(production.startsWith('setup-pm-uv-v2-smoke-42-1-')).toBe(false) + expect(production.startsWith('setup-pm-uv-v3-production-')).toBe(true) + expect(smoke.startsWith('setup-pm-uv-v3-smoke-42-1-')).toBe(true) + expect(smoke.startsWith('setup-pm-uv-v3-production-')).toBe(false) + expect(production.startsWith('setup-pm-uv-v3-smoke-42-1-')).toBe(false) } }) -it('explicit PM saves preserve downloaded offline wheels and do not prune during cancellation', () => { +it('explicit PM saves prune to the lock and do not prune during cancellation', () => { const [prune, upload] = save.runs.steps expect(prune.if).toBe('${{ !cancelled() }}') - expect(prune.run).toBe('"$PM_PYTHON" -m pm.build_env --prune-cache --cache "$PM_CACHE"') + expect(prune.run).toBe('"$PM_PYTHON" -m pm.build_env --exact-lock --cache "$PM_CACHE" --lock-source "$PM_LOCK_SOURCE"') expect(prune.env.PM_PYTHON).toBe('${{ inputs.python }}') expect(prune.env.PM_CACHE).toBe('${{ inputs.path }}') + expect(prune.env.PM_LOCK_SOURCE).toBe('${{ github.workspace }}') expect(upload.if).toBe(`\${{ !cancelled() && steps.${prune.id}.outcome == 'success' }}`) expect(upload.uses.split('@')[0]).toBe('actions/cache/save') expect(upload.with).toEqual({ path: '${{ inputs.path }}', key: '${{ inputs.key }}' }) diff --git a/tests-js/setup-pm-post.test.mjs b/tests-js/setup-pm-post.test.mjs index 9aaaca386e..96e9e2c40f 100644 --- a/tests-js/setup-pm-post.test.mjs +++ b/tests-js/setup-pm-post.test.mjs @@ -10,21 +10,22 @@ afterEach(() => { for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }) }) -it('prunes the saved uv cache at teardown, never during registration', () => { +it('prunes the saved uv cache to the lock at teardown, never during registration', () => { const directory = mkdtempSync(join(tmpdir(), 'pm-post-')) directories.push(directory) const state = join(directory, 'state') const execute = vi.fn(() => ({ status: 0 })) const cache = join(directory, 'cache with spaces') const python = join(directory, 'prepared Python') - run({ GITHUB_STATE: state, INPUT_PYTHON: python, INPUT_CACHE: cache }, execute) + const lockSource = join(directory, 'checkout') + run({ GITHUB_STATE: state, INPUT_PYTHON: python, INPUT_CACHE: cache, INPUT_LOCK_SOURCE: lockSource }, execute) expect(execute).not.toHaveBeenCalled() const saved = Object.fromEntries(readFileSync(state, 'utf8').trim().split('\n').map(line => { const index = line.indexOf('=') return [`STATE_${line.slice(0, index)}`, line.slice(index + 1)] })) run({ ...saved, UV_CACHE_DIR: 'a later unrelated cache' }, execute) - expect(execute).toHaveBeenCalledWith(python, ['-m', 'pm.build_env', '--prune-cache', '--cache', cache, '--ci'], expect.objectContaining({ + expect(execute).toHaveBeenCalledWith(python, ['-m', 'pm.build_env', '--exact-lock', '--cache', cache, '--lock-source', lockSource], expect.objectContaining({ env: expect.objectContaining(saved), stdio: 'inherit', }))