diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 432135dd57..24a7d84488 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -4,6 +4,7 @@ OpenAI-style internals. Auth: API keys (``sk-ant-api*``) -> x-api-key; OAuth set payload conversion and credentials live in ``agent/anthropic_{endpoints,message_convert, credentials}.py``; import them from there.""" +from pm import install_hint import logging import math import os @@ -60,7 +61,7 @@ def _require_sdk(purpose: str, verb: str = "Install it with"): sdk = _get_anthropic_sdk() if sdk is None: raise ImportError(f"The 'anthropic' package is required for {purpose}. {verb}: " - "python -c \"from pm import sync_venv; sync_venv(['anthropic'], explicit=True)\"") + f"{install_hint('anthropic')}") return sdk diff --git a/agent/azure_identity_adapter.py b/agent/azure_identity_adapter.py index 5bf2ef2b45..dd89c60cea 100644 --- a/agent/azure_identity_adapter.py +++ b/agent/azure_identity_adapter.py @@ -11,6 +11,7 @@ Reference: https://learn.microsoft.com/azure/ai-foundry/foundry-models/how-to/co from __future__ import annotations +from pm import install_hint import contextvars import functools import logging @@ -32,7 +33,7 @@ SCOPE_AI_AZURE_DEFAULT = "https://ai.azure.com/.default" _AZURE_IDENTITY_FEATURE = "azure-identity" _INSTALL_MSG = "The 'azure-identity' package is required for Azure AI Foundry Entra ID authentication. " _LAZY_INSTALL_HINT = ( - "Run: python -c \"from pm import sync_venv; sync_venv(['azure-identity'], explicit=True)\"" + f"Run: {install_hint('azure-identity')}" ) _AUTH_HEADERS = ("Authorization", "authorization", "Api-Key", "api-key", "X-Api-Key", "x-api-key") diff --git a/agent/bedrock_adapter.py b/agent/bedrock_adapter.py index 69cb1d967c..dbc8046d7a 100644 --- a/agent/bedrock_adapter.py +++ b/agent/bedrock_adapter.py @@ -5,6 +5,7 @@ control-plane model discovery. OpenAI-format messages/tools are converted to Con and responses normalized back to OpenAI-shaped objects. """ +from pm import install_hint import base64 import importlib import json @@ -97,7 +98,7 @@ def _require_boto3(): except ImportError: raise ImportError( "The 'boto3' package is required for the AWS Bedrock provider. " - "Run: python -c \"from pm import sync_venv; sync_venv(['bedrock'], explicit=True)\"" + f"Run: {install_hint('bedrock')}" ) try: version = tuple(int(x) for x in boto3.__version__.split(".")[:3]) diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py index 7ac7a6a82c..bbd6489682 100644 --- a/agent/monitoring/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -9,6 +9,7 @@ subscriber runs fail-isolated on the emitter thread; ``event_filter`` keeps othe from __future__ import annotations +from pm import install_hint import importlib import logging import os @@ -66,7 +67,7 @@ def _require_sdk(names: Iterable[str] = _SPAN_SDK, *, auto_install: bool = True) except Exception as e: # ImportError or partial install raise OTLPUnavailable( "OTLP export requires the optional dependency. Install with:\n" - " python -c \"from pm import sync_venv; sync_venv(['otlp'], explicit=True)\"\n" + f" {install_hint('otlp')}\n" f"(import error: {e})" ) diff --git a/agent/trace_upload.py b/agent/trace_upload.py index 0e8e987a79..7da260ad49 100644 --- a/agent/trace_upload.py +++ b/agent/trace_upload.py @@ -7,6 +7,7 @@ programmatic callers use :func:`build_trace_jsonl` + :func:`_do_upload`.""" from __future__ import annotations +from pm import install_hint import json import logging import os @@ -195,7 +196,7 @@ def _do_upload( from huggingface_hub import HfApi except ImportError: return ("Hugging Face upload needs the `huggingface_hub` package. Run: " - "python -c \"from pm import sync_venv; sync_venv(['trace-upload'], explicit=True)\"") + f"{install_hint('trace-upload')}") api = HfApi(token=token) try: who = api.whoami() diff --git a/gateway/platforms/qqbot/adapter.py b/gateway/platforms/qqbot/adapter.py index 7d8eaf9a15..1db05011f4 100644 --- a/gateway/platforms/qqbot/adapter.py +++ b/gateway/platforms/qqbot/adapter.py @@ -10,6 +10,7 @@ tries QQ's free ``asr_refer_text`` first, then the configured STT provider. from __future__ import annotations +from pm import install_hint import asyncio import contextlib import json @@ -189,7 +190,7 @@ class QQAdapter(OwnAccessPolicyMixin, BasePlatformAdapter): is accepted for interface conformance only (QQBot has no server-side update queue).""" for ok, code, what, hint in ( (AIOHTTP_AVAILABLE, "qq_missing_dependency", "aiohttp not installed", - ". Run: python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\""), + f". Run: {install_hint('messaging')}"), (HTTPX_AVAILABLE, "qq_missing_dependency", "httpx not installed", ". Run: hermes pm repair"), (self._app_id and self._client_secret, "qq_missing_credentials", "QQ_APP_ID and QQ_CLIENT_SECRET are required", "")): @@ -1136,7 +1137,7 @@ class QQAdapter(OwnAccessPolicyMixin, BasePlatformAdapter): import pilk except ImportError: logger.warning("[%s] pilk not installed — cannot decode SILK audio. Run: " - "python -c \"from pm import sync_venv; sync_venv(['silk'], explicit=True)\"", self._log_tag) + f"{install_hint('silk')}", self._log_tag) return None silk_path = src_path.rsplit(".", 1)[0] + ".silk" diff --git a/gateway/platforms/qqbot/onboard.py b/gateway/platforms/qqbot/onboard.py index 0a3b848acc..73c31f6a46 100644 --- a/gateway/platforms/qqbot/onboard.py +++ b/gateway/platforms/qqbot/onboard.py @@ -3,6 +3,7 @@ from __future__ import annotations +from pm import install_hint import logging import time from enum import IntEnum @@ -98,7 +99,7 @@ def qr_register(timeout_seconds: int = 600) -> Optional[dict]: else: print(f" Open this URL in QQ on your phone:\n {url}") print(" For a scannable QR code, run: " - "python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\"") + f"{install_hint('messaging')}") print() while time.monotonic() < deadline: try: diff --git a/gateway/run_turn.py b/gateway/run_turn.py index bc7350eb0e..0e3a7cfab8 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -5,6 +5,7 @@ are imported lazily inside method bodies (import cycle) so ``patch("gateway.run. from __future__ import annotations +from pm import install_hint import logging from typing import TYPE_CHECKING import asyncio @@ -2721,7 +2722,7 @@ class GatewayTurnMixin: from aiohttp import ClientSession as _AioClientSession, ClientTimeout except ImportError: return self._proxy_error_result("⚠️ Proxy mode requires aiohttp. Run: " - "python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\"") + f"{install_hint('messaging')}") proxy_url = self._get_proxy_url() if not proxy_url: diff --git a/gateway/run_voice.py b/gateway/run_voice.py index 7441d49ebe..06a8df1137 100644 --- a/gateway/run_voice.py +++ b/gateway/run_voice.py @@ -4,6 +4,7 @@ the MRO). ``gateway.run`` internals are imported lazily inside method bodies (im from __future__ import annotations +from pm import install_hint import asyncio import functools import json @@ -164,7 +165,7 @@ class GatewayVoiceMixin: if not any(tok in str(e).lower() for tok in ("pynacl", "nacl", "davey")): return f"Failed to join voice channel: {e}" return ("Voice dependencies are missing (PyNaCl / davey). " - "Run: `python -c \"from pm import sync_venv; sync_venv(['discord'], explicit=True)\"`") + f"Run: `{install_hint('discord')}`") if not success: adapter._voice_input_callback = None return "Failed to join voice channel. Check bot permissions (Connect + Speak)." diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 82299e8e41..dca01172ee 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -11,6 +11,7 @@ from __future__ import annotations +from pm import install_hint import json import logging import os @@ -2090,7 +2091,7 @@ def _get_azure_foundry_auth_status() -> Dict[str, Any]: "is skipped here. Run `hermes doctor` to verify token acquisition." ) if installed else ( "azure-identity not installed. From the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['azure-identity'], explicit=True)\". " + f"{install_hint('azure-identity')}. " "Then restart Hermes.")) except Exception as exc: info["logged_in"] = False diff --git a/hermes_cli/auth_commands.py b/hermes_cli/auth_commands.py index 02b380afb8..75d52b7aa4 100644 --- a/hermes_cli/auth_commands.py +++ b/hermes_cli/auth_commands.py @@ -1,6 +1,7 @@ """Credential-pool auth subcommands.""" from __future__ import annotations +from pm import install_hint from hermes_cli.cli_output import line_input import math @@ -753,7 +754,7 @@ def _print_azure_entra_status() -> None: if not has_azure_identity_installed(): print(" Status: ⚠ azure-identity not installed") print(" From the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['azure-identity'], explicit=True)\"") + f"{install_hint('azure-identity')}") print(" Then restart Hermes.") else: info = describe_active_credential(config=EntraIdentityConfig(scope=scope), timeout_seconds=10.0) diff --git a/hermes_cli/doctor_connectivity.py b/hermes_cli/doctor_connectivity.py index f54af5026b..f3a939f5fd 100644 --- a/hermes_cli/doctor_connectivity.py +++ b/hermes_cli/doctor_connectivity.py @@ -6,6 +6,7 @@ print and issue strings to append. No printing inside workers — the caller pri from __future__ import annotations +from pm import install_hint import concurrent.futures import errno import functools @@ -279,7 +280,7 @@ def _probe_bedrock() -> ProbeResult: return _row(name, "ok", f"({auth_var}, {region}, {n} models)", label=label) except ImportError: hint = ("From the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['bedrock'], explicit=True)\". " + f"{install_hint('bedrock')}. " "Then restart Hermes.") return _row(name, "warn", "(boto3 not installed)", [hint], label=label) except Exception as e: @@ -313,7 +314,7 @@ def _probe_azure_entra() -> ProbeResult: return _row(name, "warn", f"(adapter import failed: {exc})", [f"Azure Foundry adapter import failed: {exc}"], label=label) if not has_azure_identity_installed(): return _row(name, "warn", "(azure-identity not installed)", ["From the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['azure-identity'], explicit=True)\". " + f"{install_hint('azure-identity')}. " "Then restart Hermes."], label=label) entra_cfg = model_cfg.get("entra") or {} scope = (str(entra_cfg.get("scope") or "").strip() if isinstance(entra_cfg, dict) else "") or SCOPE_AI_AZURE_DEFAULT diff --git a/hermes_cli/main_agent_cmds.py b/hermes_cli/main_agent_cmds.py index 0086a329fa..17a2e49b65 100644 --- a/hermes_cli/main_agent_cmds.py +++ b/hermes_cli/main_agent_cmds.py @@ -4,6 +4,7 @@ Split out of ``hermes_cli/main.py``. Names that still live in main (``PROJECT_RO are imported lazily inside the functions that use them (avoids an import cycle). """ +from pm import install_hint import sys @@ -84,7 +85,7 @@ def cmd_acp(args): except ImportError as e: print("The ACP server can't start: its protocol packages are missing from this install.", file=sys.stderr) print("From the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['acp'], explicit=True)\"", file=sys.stderr) + f"{install_hint('acp')}", file=sys.stderr) print("Then restart Hermes.", file=sys.stderr) print(f"Details: {e}", file=sys.stderr) sys.exit(1) diff --git a/hermes_cli/model_setup_flows_azure.py b/hermes_cli/model_setup_flows_azure.py index 9af67ce3ef..8d32c8daa0 100644 --- a/hermes_cli/model_setup_flows_azure.py +++ b/hermes_cli/model_setup_flows_azure.py @@ -6,6 +6,7 @@ Prompt strings and config write order are behavior. from __future__ import annotations +from pm import install_hint from dataclasses import dataclass, field from hermes_cli.config import clear_model_endpoint_credentials @@ -56,7 +57,7 @@ def _azure_entra_preflight(current_entra: dict): _say("◐ The 'azure-identity' package is not installed yet.", " The preflight requests it through PM if lazy installs are enabled.", " To install explicitly, run from the Hermes environment:", - " python -c \"from pm import sync_venv; sync_venv(['azure-identity'], explicit=True)\"", + f" {install_hint('azure-identity')}", " Then restart Hermes.") # Only the optional scope override is persisted; identity selection (tenant, diff --git a/hermes_cli/runtime_state.py b/hermes_cli/runtime_state.py index e8dc35aa4c..813c7bd233 100644 --- a/hermes_cli/runtime_state.py +++ b/hermes_cli/runtime_state.py @@ -4,7 +4,8 @@ from __future__ import annotations import atexit import errno import base64 -from contextlib import contextmanager +from collections.abc import Callable +from contextlib import contextmanager, suppress import hashlib import json import logging @@ -182,25 +183,40 @@ def finish_publication(project: Path) -> None: recover_publication(project) -def lease_generation(environment: Path) -> None: - """Hold a kernel lock until process exit. +def lease_generation(environment: Path) -> Callable[[], None]: + """Hold a kernel lock until process exit; the returned callable releases it early. - Call under ``runtime_lock`` at boot; when that lock times out it is still safe to lease - without it, because the collector only removes generations that are NOT selected and are - older than a day — the generation being leased here is the selected one. + Call under ``runtime_lock`` at boot. Without the lock (``runtime_lock`` timed out) the + caller must re-read the selection after leasing: an installer may have moved it in between, + and an unselected, unleased generation is exactly what the collector removes. """ - generation = environment.parent + return lease_directory(environment.parent) + + +def lease_directory(generation: Path) -> Callable[[], None]: + """Pin a lease-managed generation directory for this process's lifetime.""" if not (generation / ".lease-managed").is_file(): - return # Generations produced before leases stay conservatively retained. + return lambda: None # Generations produced before leases stay conservatively retained. leases = generation / ".leases" leases.mkdir(exist_ok=True) - fd = os.open(leases / uuid.uuid4().hex, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + lease = leases / uuid.uuid4().hex + fd = os.open(lease, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) try: _lock(fd, wait=True) except BaseException: os.close(fd) raise - atexit.register(os.close, fd) + + def release() -> None: + atexit.unregister(release) + os.close(fd) + # Best effort: the collector ignores unlocked lease files, but one per + # invocation would otherwise accumulate for the life of the generation. + with suppress(OSError): + lease.unlink() + + atexit.register(release) + return release def collect_generations(project: Path, *, min_age_seconds: float = 86400) -> list[Path]: @@ -224,16 +240,19 @@ def collect_generations(project: Path, *, min_age_seconds: float = 86400) -> lis marker = generation / ".lease-managed" if not marker.is_file() or time.time() - marker.stat().st_mtime < min_age_seconds: continue - active = False - for lease in (generation / ".leases").glob("*"): - fd = os.open(lease, os.O_RDWR) - try: - if not _lock(fd, wait=False): - active = True - break - finally: - os.close(fd) - if not active: + if not leases_held(generation): shutil.rmtree(generation) removed.append(generation) return removed + + +def leases_held(generation: Path) -> bool: + """True while any process still holds a lease taken by ``lease_directory``.""" + for lease in (generation / ".leases").glob("*"): + fd = os.open(lease, os.O_RDWR) + try: + if not _lock(fd, wait=False): + return True + finally: + os.close(fd) + return False diff --git a/hermes_cli/telegram_managed_bot.py b/hermes_cli/telegram_managed_bot.py index 9aa9bca65f..3225e862d5 100644 --- a/hermes_cli/telegram_managed_bot.py +++ b/hermes_cli/telegram_managed_bot.py @@ -3,6 +3,7 @@ service (no BotFather copy-paste); the raw Telegram token is saved locally after from __future__ import annotations +from pm import install_hint import os import re import sys @@ -78,7 +79,7 @@ def print_qr_code(url: str, *, include_link: bool = True) -> None: """Print a QR code to stdout, with URL fallback if qrcode is missing.""" print(render_qr_terminal(url) or ( " (QR code unavailable. From the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\". " + f"{install_hint('messaging')}. " "Then restart Hermes.)")) if include_link: print(f" Link: {url}") diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py index f135178e38..ab2414bde8 100644 --- a/plugins/platforms/feishu/adapter.py +++ b/plugins/platforms/feishu/adapter.py @@ -18,6 +18,7 @@ Session keys prefer union_id (user_id_alt) over open_id (user_id) for stability. from __future__ import annotations +from pm import install_hint import asyncio import collections import concurrent.futures @@ -4236,7 +4237,7 @@ def _qr_register_inner(*, initial_domain: str, timeout_seconds: int) -> Optional else: print(f" Open this URL in Feishu / Lark on your phone:\n\n {qr_url}\n") print(" Tip: from the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\" " + f"{install_hint('messaging')} " "to display a scannable QR code here next time") print() result = _poll_registration( diff --git a/plugins/platforms/wecom/adapter.py b/plugins/platforms/wecom/adapter.py index a5fdc6b28f..c7b0ba13fa 100644 --- a/plugins/platforms/wecom/adapter.py +++ b/plugins/platforms/wecom/adapter.py @@ -6,6 +6,7 @@ Config (``platforms.wecom.extra``): ``bot_id``/``secret`` (or WECOM_BOT_ID / WEC from __future__ import annotations +from pm import install_hint import asyncio import json import logging @@ -675,7 +676,7 @@ def qr_scan_for_bot_info(*, timeout_seconds: int = _QR_POLL_TIMEOUT) -> Optional except Exception: print(f" Open this URL in WeCom on your phone:\n\n {page_url}\n") print(" Tip: from the Hermes environment, run: " - "python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\" " + f"{install_hint('messaging')} " "to display a scannable QR code here next time") print("\n Fetching configuration results...", end="", flush=True) deadline = time.monotonic() + timeout_seconds diff --git a/pm/__init__.py b/pm/__init__.py index 03e20e7f72..310eb8e5a9 100644 --- a/pm/__init__.py +++ b/pm/__init__.py @@ -31,7 +31,7 @@ _EXPORTS = { "prepare_tools", ), "pm.operations": ("environment_python", "python_tool"), - "pm.extras": ("available", "ensure_import"), + "pm.extras": ("available", "ensure_import", "install_hint"), "pm.lock": ("Facts", "Lockfile"), "pm.package": ("InstallError", "Package", "Runner", "compose_env"), "pm.registry": ("all_packages", "get_package", "register", "walk"), diff --git a/pm/build_env.py b/pm/build_env.py index 7a46dcf953..65c41da0e4 100644 --- a/pm/build_env.py +++ b/pm/build_env.py @@ -64,7 +64,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.error("--manager-runtime requires the target --python") try: if args.exact_lock: - from pm.cache_lock import prune_uv_cache_to_lock + from pm.uv_cache_prune import prune_uv_cache_to_lock pruned = prune_uv_cache_to_lock(args.cache, args.lock_source) print(f"pruned {pruned} cache entries outside the lock") diff --git a/pm/cli.py b/pm/cli.py index dbd446997c..4011ff8c2d 100644 --- a/pm/cli.py +++ b/pm/cli.py @@ -164,9 +164,13 @@ def cmd_install(args) -> int: return 1 # Source-install launchers require the store interpreter, even though # Python remains optional when provisioning individual tools. - names = args.names or source_install_packages(_lockfile().names()) + extras = list(dict.fromkeys(getattr(args, "extra", None) or ())) + if extras and cross_target: + print("✗ --extra syncs this install's venv and cannot combine with --target") + return 1 + names = args.names if args.names or extras else source_install_packages(_lockfile().names()) failed = _install_names(names, target=cross_target) - if not args.names: + if extras or not args.names: from pm.install import sync_venv try: @@ -175,8 +179,8 @@ def cmd_install(args) -> int: # the installers' old `--extra all` did. sync_venv unions, so # any lazy extras already recorded survive this; it only makes # a fresh bootstrap match what the first update would do. - sync_venv(["all"], explicit=True) - print("✓ venv") + sync_venv(extras or ["all"], explicit=True) + print(f"✓ venv{' +' + ' +'.join(extras) if extras else ''}") except InstallError as e: print(f"✗ {e}") failed += 1 @@ -264,7 +268,13 @@ def _gc_store(store, facts) -> tuple[int, int]: keep = facts.entries_in_use() collect_partials(partials_dir) for item in sorted(store.root.iterdir()): - if not item.is_dir() or item.name.startswith("."): + if not item.is_dir(): + continue + # Scratch dirs are created and removed under this same lock, so any + # that remain belong to a killed installer. Other dot-dirs stay: + # .previous-* is the restore point the next install of that entry + # consumes, and it is only safe to drop after that verification. + if item.name.startswith(".") and not item.name.startswith(".staging-"): continue if item.name in keep: continue @@ -282,9 +292,13 @@ def cmd_gc(args) -> int: facts = _facts() if store.root == _store().root else Facts(store.root / "facts.json") removed, kept = _gc_store(store, facts) from hermes_cli.runtime_state import collect_generations + from pm.environments import install_state_dir from pm.paths import repo_root + from pm.runtime import collect_runtime_generations generations = collect_generations(repo_root()) - print(f"gc: removed {removed}, kept {kept}; removed {len(generations)} dependency generations") + runtimes = collect_runtime_generations(install_state_dir(repo_root()) / "pm-runtime") + print(f"gc: removed {removed}, kept {kept}; removed {len(generations)} dependency generations, " + f"{len(runtimes)} PM runtime generations") return 0 @@ -533,6 +547,8 @@ def main(argv=None) -> int: p = sub.add_parser("install", help="install packages (default: all required)") p.add_argument("names", nargs="*") + p.add_argument("--extra", action="append", default=[], metavar="NAME", + help="enable a declared dependency extra in the venv (repeatable)") p.add_argument( "--target", help="stage for a cross target (e.g. linux-arm64-bionic on a glibc " diff --git a/pm/client.py b/pm/client.py index eff4735da5..114cf079fb 100644 --- a/pm/client.py +++ b/pm/client.py @@ -65,14 +65,22 @@ def _request(operation, arguments, *, callbacks=None, pause_event=None, project_ # PM is itself a missing prerequisite, not permission to bootstrap tools. try: command = runtime_command(worker, bootstrap=False, cache=cache) - except InstallError as exc: + except InstallError as cold: + exc = cold + if arguments.get("extras"): + # The user asked for an extra, not for PM's own runtime: name + # the command that provisions both. + from pm.extras import install_hint + + exc = InstallError(cold.package, f"{cold.cause} while enabling {list(arguments['extras'])}", + "run `" + "`, `".join(install_hint(extra) for extra in arguments["extras"]) + "`") token = receipt.begin("sync") try: receipt.record_refusal("lazy-install", str(exc)) receipt.record_step("dependency-sync", False, f"{type(exc).__name__}: {exc}") finally: receipt.finalize("failed", 1, token=token) - raise + raise exc from None environment["HERMES_DISABLE_LAZY_INSTALLS"] = "1" elif spec.bootstrap == "never": command = runtime_command(worker, bootstrap=False, cache=cache) diff --git a/pm/downloader.py b/pm/downloader.py index 5ee83b7181..382bbfbce6 100644 --- a/pm/downloader.py +++ b/pm/downloader.py @@ -54,7 +54,11 @@ class _HttpsRedirectHandler(urllib.request.HTTPRedirectHandler): carry the payload in the clear.""" def redirect_request(self, req, fp, code, msg, headers, newurl): - if not (newurl.startswith("https://") or newurl.startswith(_LOOPBACK)): + # Plain-http loopback exists for test servers, and only a download that + # started on loopback may stay there: an https origin bouncing to a local + # listener would hand an unpinned model file to whatever is bound there. + loopback = req.full_url.startswith(_LOOPBACK) and newurl.startswith(_LOOPBACK) + if not (newurl.startswith("https://") or loopback): raise DownloadError(f"refusing redirect to non-https url: {newurl}") # urllib preserves our User-Agent and range headers. Do not re-add # arbitrary headers after its redirect policy has processed them. diff --git a/pm/environment.py b/pm/environment.py index ac89ab3876..735b3e73cc 100644 --- a/pm/environment.py +++ b/pm/environment.py @@ -173,15 +173,20 @@ def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], def _base_environment(env: Mapping[str, str] | None = None) -> dict[str, str]: - """Ambient UV settings are never policy; explicit build index settings are.""" - index_settings = { - "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL", "UV_NO_INDEX", "UV_FIND_LINKS", - "UV_INSECURE_HOST", "UV_KEYRING_PROVIDER", "UV_NATIVE_TLS", - } - return {key: value for key, value in (os.environ if env is None else env).items() + """Ambient UV settings never select the project, interpreter or cache. + + Index and transport settings are the exception (pm.index_config): without + them mirrored and air-gapped networks cannot resolve anything. + """ + from pm.index_config import bridged_index_settings, is_forwarded + + source = os.environ if env is None else env + base = {key: value for key, value in source.items() if not key.startswith("PYTHON") and key != "VIRTUAL_ENV" - and (not key.startswith("UV_") or - (env is not None and (key.startswith("UV_INDEX") or key in index_settings)))} + and (not key.startswith("UV_") or is_forwarded(key))} + if env is None: + base.update(bridged_index_settings(os.environ)) + return base def managed_environment(destination: Path, *, python: Path | None = None, @@ -255,12 +260,23 @@ class PythonEnvironment: command.append("--no-config") if self.offline: command.append("--offline") - if self.output is not None: - # uv hides build-backend output until failure without verbose mode. - command.append("--verbose") - return _run_streaming(command, cwd=cwd, env=env, timeout=timeout, output=self.output) - return subprocess.run(command, cwd=str(cwd), env=env, capture_output=True, - text=True, encoding="utf-8", errors="replace", timeout=timeout) + try: + if self.output is not None: + # uv hides build-backend output until failure without verbose mode, + # but --verbose alone is uv's DEBUG level: ~200 lines of interpreter + # and cache internals on every streamed run. RUST_LOG scopes it to the + # build frontend, so only the backend's own lines reach the user. + command.append("--verbose") + env.setdefault("RUST_LOG", "uv_build_frontend=debug") + return _run_streaming(command, cwd=cwd, env=env, timeout=timeout, output=self.output) + return subprocess.run(command, cwd=str(cwd), env=env, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=timeout) + except subprocess.TimeoutExpired as exc: + from pm.index_config import TIMEOUT_HINT + + # A silent stall against an unreachable index is the #95608 shape; + # name the mirror knobs instead of surfacing a raw TimeoutExpired. + raise InstallError("venv", f"uv {args[0]} timed out after {timeout}s", TIMEOUT_HINT) from exc def create(self) -> None: """Create at the final destination; callers must not move a live venv.""" diff --git a/pm/environments.py b/pm/environments.py index 8cc5510020..6945dc4bf7 100644 --- a/pm/environments.py +++ b/pm/environments.py @@ -208,7 +208,12 @@ def activate_dependencies(project_root: Path) -> None: if held: recover_publication(project_root) environment = selected_venv(project_root) - lease_generation(environment) + release = lease_generation(environment) + # Without the lock, an installer may commit a new generation between the + # read and the lease, leaving the leased one unselected and collectable. + while not held and (current := selected_venv(project_root)) != environment: + release() + environment, release = current, lease_generation(current) selected = site_packages(environment) if not selected.is_dir() and not runtime_facts_path(project_root).is_file(): return diff --git a/pm/extras.py b/pm/extras.py index 9f1709a45c..b7de900db1 100644 --- a/pm/extras.py +++ b/pm/extras.py @@ -158,6 +158,11 @@ def extra_supported(extra: str, *, environment: dict[str, str] | None = None, return True +def install_hint(extra: str) -> str: + """The one command users are told to run for a missing extra.""" + return f"hermes pm install --extra {extra}" + + def ensure_import(extra: str) -> None: """Make an extra available: no-op when the anchor imports, otherwise sync the venv with the extra enabled. Raises InstallError on failure diff --git a/pm/index_config.py b/pm/index_config.py new file mode 100644 index 0000000000..dc86d36a59 --- /dev/null +++ b/pm/index_config.py @@ -0,0 +1,107 @@ +"""Forward the user's package-index and transport configuration into uv. + +PM strips ambient ``UV_*`` so a caller's uv settings cannot steer which +project, interpreter or cache an operation uses (pm.environment). Index and +transport knobs are different: on mirrored or air-gapped networks they are the +only way any dependency resolves at all (#88453, #94613, #95608). Only those +cross the boundary; the lockfile stays authoritative for what gets installed. + +uv never reads pip's configuration, so a pip-only mirror (``PIP_INDEX_URL`` or +``index-url`` in pip.conf) is bridged to ``UV_INDEX_URL`` unless uv already has +an index of its own. Stdlib only: the bootstrap runner imports this before any +dependency exists. +""" +from __future__ import annotations + +from collections.abc import Mapping +import configparser +import os +from pathlib import Path +import sys + +# Explicit uv index / transport settings that survive into uv. UV_INDEX__ +# {USERNAME,PASSWORD} credentials match by prefix in is_forwarded(). +FORWARDED_UV_SETTINGS = frozenset({ + "UV_INDEX_URL", "UV_EXTRA_INDEX_URL", "UV_DEFAULT_INDEX", "UV_INDEX", "UV_NO_INDEX", + "UV_FIND_LINKS", "UV_INDEX_STRATEGY", "UV_KEYRING_PROVIDER", + "UV_NATIVE_TLS", "UV_INSECURE_HOST", "UV_HTTP_TIMEOUT", +}) + +_UV_INDEX_KNOBS = ("UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_INDEX") + +# pip knob → uv knob, applied only when uv has no value of its own. +_PIP_TO_UV = ( + ("PIP_EXTRA_INDEX_URL", "UV_EXTRA_INDEX_URL"), + ("PIP_TRUSTED_HOST", "UV_INSECURE_HOST"), +) + +TIMEOUT_HINT = ("uv timed out. If your network needs a package mirror, set index-url in " + "pip.conf (bridged to uv automatically) or UV_INDEX_URL; raise UV_HTTP_TIMEOUT " + "for slow links.") + + +def is_forwarded(key: str) -> bool: + return key in FORWARDED_UV_SETTINGS or key.startswith("UV_INDEX_") + + +def pip_config_candidates(env: Mapping[str, str]) -> list[Path]: + """pip's config files, lowest precedence first, as ``pip._internal.configuration`` ranks them. + + Global, then user (skipped entirely when ``PIP_CONFIG_FILE`` names an existing file), then + the interpreter's ``sys.prefix`` site file, then ``PIP_CONFIG_FILE`` itself on top. + ``RawConfigParser.read`` applies them in order, so the last file wins. + ``PIP_CONFIG_FILE=os.devnull`` disables all of them. + """ + explicit = env.get("PIP_CONFIG_FILE", "") + if explicit == os.devnull: + return [] + home = Path.home() + if sys.platform == "win32": + name = "pip.ini" + global_files = [Path(env.get("ProgramData") or r"C:\ProgramData") / "pip" / name] + user_files = [home / "pip" / name, + Path(env.get("APPDATA") or home / "AppData" / "Roaming") / "pip" / name] + elif sys.platform == "darwin": + name = "pip.conf" + global_files = [Path("/Library/Application Support/pip") / name] + app_support = home / "Library" / "Application Support" / "pip" + user_files = [home / ".pip" / name, + (app_support if app_support.is_dir() else home / ".config" / "pip") / name] + else: + name = "pip.conf" + xdg_dirs = (env.get("XDG_CONFIG_DIRS") or "/etc/xdg").split(os.pathsep) + global_files = [Path(d) / "pip" / name for d in xdg_dirs if d] + [Path("/etc") / name] + user_files = [home / ".pip" / name, Path(env.get("XDG_CONFIG_HOME") or home / ".config") / "pip" / name] + explicit_files = [Path(explicit)] if explicit else [] + if explicit_files and explicit_files[0].is_file(): + user_files = [] + return global_files + user_files + [Path(sys.prefix) / name] + explicit_files + + +def pip_conf_index_url(env: Mapping[str, str]) -> str | None: + # Raw: pip does not interpolate, and mirror URLs carry percent-encoded credentials. + parser = configparser.RawConfigParser() + try: + parser.read(str(path) for path in pip_config_candidates(env)) + if not parser.has_section("global"): + return None + return parser.get("global", "index-url", fallback="").strip() or None + except configparser.Error: + return None + + +def bridged_index_settings(ambient: Mapping[str, str]) -> dict[str, str]: + """The uv index/transport settings *ambient* asks for, pip knobs translated. + + ``PIP_INDEX_URL`` beats pip.conf, as in pip; any explicit uv index knob beats both. + """ + settings = {key: value for key, value in ambient.items() if is_forwarded(key)} + if not any(settings.get(key) for key in _UV_INDEX_KNOBS): + index_url = (ambient.get("PIP_INDEX_URL") or "").strip() or pip_conf_index_url(ambient) + if index_url: + settings["UV_INDEX_URL"] = index_url + for pip_key, uv_key in _PIP_TO_UV: + value = (ambient.get(pip_key) or "").strip() + if value and not settings.get(uv_key): + settings[uv_key] = value + return settings diff --git a/pm/launch.py b/pm/launch.py index 96aa6bd7c2..ce4dcee3ac 100644 --- a/pm/launch.py +++ b/pm/launch.py @@ -9,6 +9,8 @@ truststore.inject_into_ssl() sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from pm.cli import main +from pm.runtime import lease_current_runtime if __name__ == "__main__": + lease_current_runtime() raise SystemExit(main()) diff --git a/pm/plugins_state.py b/pm/plugins_state.py index 87a8796404..af30c531e8 100644 --- a/pm/plugins_state.py +++ b/pm/plugins_state.py @@ -88,8 +88,14 @@ def _is_directory(path: Path) -> bool: def dependency_homes() -> list[Path]: - """Every home whose selection feeds the shared venv: the default home plus each profile. - Enumerates the complete union or refuses; a partial scan cannot remove members.""" + """Every home whose selection feeds the shared venv: the default home plus each LIVE profile. + Enumerates the complete union or refuses; a partial scan cannot remove members. + + Live means what ``hermes profile`` lists (hermes_constants): a valid id carrying an identity + marker and no tombstone. Staging dirs (``.work.staging-*``), deleted profiles and stray + marker-less dirs must not put plugins into the shared environment. + """ + from hermes_constants import PROFILE_ID_RE, named_profile_is_live from pm.environments import dependency_home_root homes = [dependency_home_root()] @@ -100,7 +106,9 @@ def dependency_homes() -> list[Path]: return homes except OSError as exc: raise ValueError(f"could not enumerate profiles: {root}") from exc - homes.extend(profile for profile in profiles if _is_directory(profile)) + homes.extend(profile for profile in profiles + if _is_directory(profile) and profile.name != "default" + and PROFILE_ID_RE.match(profile.name) and named_profile_is_live(profile)) return homes diff --git a/pm/runtime.py b/pm/runtime.py index f6d7af7c36..0b92d80c6c 100644 --- a/pm/runtime.py +++ b/pm/runtime.py @@ -137,6 +137,7 @@ def prepare_runtime(uv: Path, python: Path, root: Path, *, offline: bool = False try: print("Preparing the isolated PM runtime…", file=sys.stderr, flush=True) executable = stage_runtime(uv, python, environment, project=project, offline=offline, cache=cache) + (environment / ".lease-managed").touch() _write(environment / "pm-runtime.json", {"inputs": identity}) _write(selected, {"inputs": identity, "generation": generation.as_posix()}) return executable @@ -145,6 +146,46 @@ def prepare_runtime(uv: Path, python: Path, root: Path, *, offline: bool = False raise +def lease_current_runtime() -> None: + """Pin the PM runtime this process runs from so the collector leaves it alone.""" + if (Path(sys.prefix) / "pm-runtime.json").is_file(): + from hermes_cli.runtime_state import lease_directory + + lease_directory(Path(sys.prefix)) + + +def collect_runtime_generations(root: Path) -> list[Path]: + """Remove PM runtime generations nothing can run from any more. + + Staging happens under ``.prepare.lock``, so with it held an unpublished generation + (no ``pm-runtime.json``) is an aborted stage. A superseded published generation goes + once every worker launched from it has exited; generations published before leases + existed stay, as the application collector keeps its own. + """ + from hermes_cli.runtime_state import _lock, leases_held + + generations = root / "generations" + removed: list[Path] = [] + if not generations.is_dir(): + return removed + with (root / ".prepare.lock").open("a+b") as lock: + if not _lock(lock.fileno(), wait=False): + return removed # a stage is in flight; maintenance skips rather than queues + try: + selected = json.loads((root / "selected.json").read_text(encoding="utf-8-sig")).get("generation", "") + except FileNotFoundError: + selected = "" + for generation in sorted(generations.iterdir()): + if not generation.is_dir() or generation.is_symlink() or generation == root / selected: + continue + published = (generation / "pm-runtime.json").is_file() + if published and (not (generation / ".lease-managed").is_file() or leases_held(generation)): + continue + shutil.rmtree(generation) + removed.append(generation) + return removed + + def runtime_python(*, bootstrap: bool = True, cache: Path | None = None) -> Path: """Resolve PM without selecting, repairing, or importing the app environment.""" diff --git a/pm/security_packages.py b/pm/security_packages.py index 45057efcfc..6b6110961d 100644 --- a/pm/security_packages.py +++ b/pm/security_packages.py @@ -94,9 +94,9 @@ class Tirith(_SignedBinary): return [archive, *[f"{base}/{name}" for name in ("checksums.txt", "checksums.txt.sig", "checksums.txt.pem")]] def verify_provenance(self, directory: Path) -> None: - from tools.tirith_security import _verify_release_provenance + from tools.tirith_security import verify_release_provenance - _, reason = _verify_release_provenance(directory, logging.getLogger(__name__).warning) + _, reason = verify_release_provenance(directory, logging.getLogger(__name__).warning) if reason: raise InstallError(self.name, reason) diff --git a/pm/store.py b/pm/store.py index 7b4a69211b..b775160c5b 100644 --- a/pm/store.py +++ b/pm/store.py @@ -365,9 +365,14 @@ class Store: """Serialize writers using the same advisory lock as runtime publication.""" from hermes_cli.runtime_state import _lock self.root.mkdir(parents=True, exist_ok=True) - fd = os.open(self.root / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600) + lock = self.root / ".install.lock" + fd = os.open(lock, os.O_CREAT | os.O_RDWR, 0o600) try: - _lock(fd, wait=True) + # A second `hermes pm install` behind an sdist build otherwise sits + # silent for minutes; say what it is waiting on. + if not _lock(fd, wait=True, timeout=2): + print(f"waiting for {lock} (another PM operation holds it)", file=sys.stderr, flush=True) + _lock(fd, wait=True) yield finally: os.close(fd) diff --git a/pm/cache_lock.py b/pm/uv_cache_prune.py similarity index 100% rename from pm/cache_lock.py rename to pm/uv_cache_prune.py diff --git a/pm/worker.py b/pm/worker.py index 83d42775d5..6d56f05ddd 100644 --- a/pm/worker.py +++ b/pm/worker.py @@ -73,7 +73,9 @@ def main(): from pm import paths, receipt from pm.package import InstallError from pm.registry import load_package_definitions + from pm.runtime import lease_current_runtime from pm.worker_operations import OPERATIONS + lease_current_runtime() context = request["context"] paths.repo_root = lambda: Path(context["repo"]) paths.lockfile_path = lambda: Path(context["lockfile"]) diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index 664d55dad9..aa773d5f1d 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -52,7 +52,7 @@ def _arch_guard(store_dir: Path) -> list[str]: -from pm.cache_lock import lock_package_names, prune_uv_cache_to_lock +from pm.uv_cache_prune import lock_package_names, prune_uv_cache_to_lock __all__ = ["prune_uv_cache_to_lock", "lock_package_names", "stage_uv_cache"] diff --git a/tests/agent/test_azure_identity_adapter.py b/tests/agent/test_azure_identity_adapter.py index 14fb5301d1..c732155f94 100644 --- a/tests/agent/test_azure_identity_adapter.py +++ b/tests/agent/test_azure_identity_adapter.py @@ -545,7 +545,7 @@ class TestDescribeActiveCredential: ) assert info["ok"] is False assert "lazy installs disabled" in info["error"] - assert "sync_venv(['azure-identity'], explicit=True)" in info["hint"] + assert "hermes pm install --extra azure-identity" in info["hint"] def test_reports_env_sources_for_managed_identity(self, fake_azure_identity, monkeypatch): from agent.azure_identity_adapter import describe_active_credential diff --git a/tests/gateway/test_feishu_onboard.py b/tests/gateway/test_feishu_onboard.py index 1624b41a2a..2d85ffe2ef 100644 --- a/tests/gateway/test_feishu_onboard.py +++ b/tests/gateway/test_feishu_onboard.py @@ -244,7 +244,7 @@ class TestQrRegister: output = capsys.readouterr().out assert "https://example.com/qr" in output - assert "sync_venv(['messaging'], explicit=True)" in output + assert "hermes pm install --extra messaging" in output assert "pip install" not in output # -- Contract: expected errors → None, unexpected errors → propagate -- diff --git a/tests/hermes_cli/test_telegram_managed_bot.py b/tests/hermes_cli/test_telegram_managed_bot.py index b7b10b49f0..c104ff70ca 100644 --- a/tests/hermes_cli/test_telegram_managed_bot.py +++ b/tests/hermes_cli/test_telegram_managed_bot.py @@ -33,7 +33,7 @@ class TestQRCode: print_qr_code("https://example.com") output = capsys.readouterr().out assert "https://example.com" in output - assert "sync_venv" in output + assert "hermes pm install --extra messaging" in output assert "pip install" not in output def test_print_qr_code_with_url(self, capsys): @@ -49,7 +49,7 @@ class TestQRCode: with patch.dict("sys.modules", {"qrcode": None}): print_qr_code("https://t.me/newbot/Bot/test_bot") captured = capsys.readouterr() - assert "sync_venv(['messaging'], explicit=True)" in captured.out + assert "hermes pm install --extra messaging" in captured.out assert "pip install" not in captured.out diff --git a/tests/pm/test_downloader.py b/tests/pm/test_downloader.py index f89b45cd90..a6899bc932 100644 --- a/tests/pm/test_downloader.py +++ b/tests/pm/test_downloader.py @@ -186,6 +186,12 @@ def test_redirect_to_non_https_refused(): # An https redirect resolves through the default handler. assert handler.redirect_request(req, None, 302, "Found", {}, "https://example.com/b") is not None + # Loopback is a test-server affordance: an https origin may not land there. + with pytest.raises(DownloadError): + handler.redirect_request(req, None, 302, "Found", {}, "http://127.0.0.1:8000/b") + local = urllib.request.Request("http://127.0.0.1:8000/a") + assert handler.redirect_request(local, None, 302, "Found", {}, + "http://localhost:8000/b") is not None # ── pause ───────────────────────────────────────────────────── diff --git a/tests/pm/test_environment_build.py b/tests/pm/test_environment_build.py index 57580f2031..6b9ab6c999 100644 --- a/tests/pm/test_environment_build.py +++ b/tests/pm/test_environment_build.py @@ -465,7 +465,9 @@ def streaming_runner(request, tmp_path): env=dict(os.environ), timeout=timeout) return environment._run(["-c", script], cwd=tmp_path, timeout=timeout) - return run, output, RuntimeError if request.param == "cli" else subprocess.TimeoutExpired + from pm.package import InstallError + + return run, output, RuntimeError if request.param == "cli" else InstallError @pytest.mark.parametrize("parent_exits", [True, False]) diff --git a/tests/pm/test_index_bridging.py b/tests/pm/test_index_bridging.py new file mode 100644 index 0000000000..5983ede607 --- /dev/null +++ b/tests/pm/test_index_bridging.py @@ -0,0 +1,84 @@ +"""Mirrored and air-gapped networks configure indexes through pip or uv; PM forwards +exactly that into uv while still refusing every other ambient uv setting.""" +from __future__ import annotations + +import os +import subprocess + +import pytest + +from pm.environment import PythonEnvironment, _base_environment +from pm.package import InstallError + + +@pytest.fixture +def clean_index_env(monkeypatch, tmp_path): + for key in list(os.environ): + if key.startswith(("UV_", "PIP_")): + monkeypatch.delenv(key) + monkeypatch.setenv("PIP_CONFIG_FILE", os.devnull) + return tmp_path + + +def test_pip_index_reaches_uv_but_ambient_uv_selection_does_not(clean_index_env, monkeypatch): + monkeypatch.setenv("PIP_INDEX_URL", "https://mirror.example/simple") + monkeypatch.setenv("PIP_TRUSTED_HOST", "mirror.example") + monkeypatch.setenv("UV_HTTP_TIMEOUT", "300") + monkeypatch.setenv("UV_INDEX_CORP_PASSWORD", "s3cret") + monkeypatch.setenv("UV_PYTHON", "/poison/python") + monkeypatch.setenv("UV_CACHE_DIR", "/poison/cache") + monkeypatch.setenv("UV_PROJECT_ENVIRONMENT", "/poison/venv") + + env = _base_environment() + + assert env["UV_INDEX_URL"] == "https://mirror.example/simple" + assert env["UV_INSECURE_HOST"] == "mirror.example" + assert env["UV_HTTP_TIMEOUT"] == "300" + assert env["UV_INDEX_CORP_PASSWORD"] == "s3cret" + assert not {"UV_PYTHON", "UV_CACHE_DIR", "UV_PROJECT_ENVIRONMENT"} & env.keys() + + +def test_pip_conf_is_bridged_only_when_uv_has_no_index(clean_index_env, monkeypatch): + pip_conf = clean_index_env / "pip.conf" + # Percent-encoded credentials: pip reads its config raw, so must the bridge. + pip_conf.write_text("[global]\nindex-url = https://user:p%40ss@mirror.example/simple\n", encoding="utf-8") + monkeypatch.setenv("PIP_CONFIG_FILE", str(pip_conf)) + + assert _base_environment()["UV_INDEX_URL"] == "https://user:p%40ss@mirror.example/simple" + + monkeypatch.setenv("UV_DEFAULT_INDEX", "https://explicit.example/simple") + env = _base_environment() + assert env["UV_DEFAULT_INDEX"] == "https://explicit.example/simple" + assert "UV_INDEX_URL" not in env + + +def test_streamed_runs_do_not_request_uv_debug_output(tmp_path, monkeypatch): + import io + from pm import environment + + seen: list[list[str]] = [] + kwargs_seen: list[dict] = [] + + def record(command, **kwargs): + seen.append(command) + kwargs_seen.append(kwargs) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(environment, "_run_streaming", record) + PythonEnvironment(uv=tmp_path / "uv", python=tmp_path / "python", destination=tmp_path / "venv", + cache=tmp_path / "cache", env={}, output=io.StringIO())._run(["sync"], cwd=tmp_path, timeout=5) + (command,), (kwargs,) = seen, kwargs_seen + # Verbose only where the build backend speaks; uv's own DEBUG stays silent. + assert "--verbose" in command and kwargs["env"]["RUST_LOG"] == "uv_build_frontend=debug" + + +def test_uv_timeout_names_the_mirror_knobs(tmp_path, monkeypatch): + def stall(*args, **kwargs): + raise subprocess.TimeoutExpired(args[0], kwargs["timeout"]) + + monkeypatch.setattr(subprocess, "run", stall) + environment = PythonEnvironment(uv=tmp_path / "uv", python=tmp_path / "python", + destination=tmp_path / "venv", cache=tmp_path / "cache", env={}) + with pytest.raises(InstallError, match="UV_INDEX_URL") as info: + environment._run(["sync"], cwd=tmp_path, timeout=7) + assert "timed out after 7s" in str(info.value) diff --git a/tests/pm/test_install_extra.py b/tests/pm/test_install_extra.py new file mode 100644 index 0000000000..980ff50232 --- /dev/null +++ b/tests/pm/test_install_extra.py @@ -0,0 +1,52 @@ +"""`hermes pm install --extra NAME` is the one command every missing-extra hint names.""" +from types import SimpleNamespace + +import pytest + +from pm import install_hint + + +def test_hint_names_a_command_the_cli_accepts(monkeypatch): + from pm import cli, runtime + from pm import install as install_mod + + synced = [] + monkeypatch.setattr(runtime, "is_runtime", lambda: True) + monkeypatch.setattr(install_mod, "sync_venv", lambda extras, **kwargs: synced.append((list(extras), kwargs))) + monkeypatch.setattr(cli, "_install_names", + lambda names, target=None: 0 if not names else pytest.fail(f"tools installed: {names}")) + + argv = install_hint("anthropic").split()[2:] + assert cli.main(argv) == 0 + assert synced == [(["anthropic"], {"explicit": True})] + + +def test_extra_syncs_only_the_named_extras(monkeypatch): + from pm import cli + from pm import install as install_mod + + synced = [] + monkeypatch.setattr(install_mod, "sync_venv", lambda extras, **kwargs: synced.append(list(extras))) + monkeypatch.setattr(cli, "_install_names", lambda names, target=None: 0) + assert cli.cmd_install(SimpleNamespace(names=[], extra=["otlp", "mcp", "otlp"], target=None)) == 0 + assert synced == [["otlp", "mcp"]] + + +def test_cold_runtime_refusal_names_the_extra(monkeypatch, tmp_path): + import pm.client as client + from pm import receipt + from pm.package import InstallError + + monkeypatch.setattr("pm.install.lazy_installs_allowed", lambda: False) + monkeypatch.setattr(client, "runtime_environment", lambda: {}) + monkeypatch.setattr("pm.registry.package_definitions", lambda names: []) + monkeypatch.setattr(client, "runtime_command", lambda *args, **kwargs: (_ for _ in ()).throw( + InstallError("pm-runtime", "not installed or outdated and lazy installs are disabled"))) + for name in ("begin", "record_refusal", "record_step", "finalize"): + monkeypatch.setattr(receipt, name, lambda *args, **kwargs: None) + + with pytest.raises(InstallError) as info: + client._request("sync_venv", {"extras": ["bedrock"], "explicit": False, "repair": False}, + project_root=tmp_path) + assert install_hint("bedrock") in info.value.remedy + assert "bedrock" in info.value.cause diff --git a/tests/pm/test_lease_reselection.py b/tests/pm/test_lease_reselection.py new file mode 100644 index 0000000000..5a46ffd089 --- /dev/null +++ b/tests/pm/test_lease_reselection.py @@ -0,0 +1,75 @@ +"""A reader that lost the install lock still ends up leasing the generation it imports from.""" +import contextlib +import json +import os +import sys + +import pytest + + +@pytest.fixture +def generations(tmp_path, monkeypatch): + from pm.environments import install_state_dir, runtime_facts_path, site_packages + + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) + state = install_state_dir(repo) + for name in ("first", "second"): + venv = state / "environments" / name / "venv" + venv.mkdir(parents=True) + (venv / "pyvenv.cfg").write_text("version = 3.11", encoding="utf-8") + site_packages(venv).mkdir(parents=True) + (venv.parent / ".lease-managed").touch() + + def select(name): + environment = state / "environments" / name / "venv" + runtime_facts_path(repo).write_text(json.dumps({"packages": {"venv": {"environment": str(environment)}}}), encoding="utf-8") + return environment.resolve() + + return repo, state, select + + +def test_unlocked_reader_releases_a_lease_the_installer_moved_away_from(generations, monkeypatch): + from hermes_cli import runtime_state + from pm.environments import activate_dependencies + + repo, state, select = generations + first = select("first") + real_lease = runtime_state.lease_generation + leased = [] + + def racing_lease(environment): + # The installer commits between the reader's selection read and its lease. + if not leased: + select("second") + leased.append(environment) + return real_lease(environment) + + @contextlib.contextmanager + def lost_lock(project, **kwargs): + yield False + + monkeypatch.setattr(runtime_state, "runtime_lock", lost_lock) + monkeypatch.setattr(runtime_state, "lease_generation", racing_lease) + # Activation rewrites this process's import path and environment; keep it scoped. + monkeypatch.setattr(sys, "path", list(sys.path)) + for key in ("PYTHONPATH", "PATH", "VIRTUAL_ENV"): + monkeypatch.setenv(key, os.environ.get(key, "")) + + activate_dependencies(repo) + + assert leased[0] == first and leased[-1] == state / "environments" / "second" / "venv" + assert not any((first.parent / ".leases").iterdir()), "the stale lease must be released" + assert len(list((state / "environments" / "second" / ".leases").iterdir())) == 1 + + +def test_release_removes_the_lease_file(generations): + from hermes_cli.runtime_state import lease_generation + + _, state, select = generations + environment = select("first") + release = lease_generation(environment) + assert list((environment.parent / ".leases").iterdir()) + release() + assert not list((environment.parent / ".leases").iterdir()) diff --git a/tests/pm/test_plugins_state.py b/tests/pm/test_plugins_state.py index 023aabd0c3..ea9c6dc159 100644 --- a/tests/pm/test_plugins_state.py +++ b/tests/pm/test_plugins_state.py @@ -45,6 +45,25 @@ def test_enabled_plugins_ordered_reads_all_homes(homes): assert by_root.get(profile_home / "plugins") == ["c-plug"] +def test_only_live_profiles_join_the_dependency_union(homes): + from hermes_constants import mark_named_profile_deleted + + default_home, profile_home = homes + profiles = profile_home.parent + _write_config(default_home, ["a-plug"]) + _write_config(profile_home, ["live-plug"]) + for name in (".work.staging-123", "Bad Name", "retired", "ghost"): + (profiles / name).mkdir() + for name in (".work.staging-123", "Bad Name", "retired"): + _write_config(profiles / name, [f"{name}-plug"]) + mark_named_profile_deleted(profiles / "retired") + (profiles / "ghost" / "plugins").mkdir() # runtime side-effect dir, no identity marker + + assert pstate.dependency_homes() == [default_home, profile_home] + by_root = pstate.enabled_plugins_ordered() + assert set(by_root) == {default_home / "plugins", profile_home / "plugins"} + + @pytest.mark.parametrize("boundary", ["profile-listing", "profile-stat", "config-read", "plugin-stat", "manifest-read", "manifest-stat", "provider-stat"]) def test_unreadable_profile_state_is_not_an_empty_selection(homes, monkeypatch, boundary): from pm.workspace import enabled_member_dirs diff --git a/tests/pm/test_pm_core.py b/tests/pm/test_pm_core.py index 479dce4467..1335ea14d7 100644 --- a/tests/pm/test_pm_core.py +++ b/tests/pm/test_pm_core.py @@ -14,7 +14,7 @@ import pm.paths as paths import pm.registry as registry from pm.lock import Facts, Lockfile from pm.package import InstallError, compose_env -from pm.packages import BinaryPackage +from pm.packages import BinaryPackage, Venv from pm.store import Store, current_target from tests.pm._fixtures import make_tar, served as served @@ -71,9 +71,12 @@ def pm_env(tmp_path, served, monkeypatch): ensure_mod = importlib.import_module("pm.install") monkeypatch.setattr(ensure_mod, "lazy_installs_allowed", lambda: True) + # Restore the real registry wholesale on teardown. Tests must not + # monkeypatch.setitem into the cleared dict: that undo runs after this + # restore and deletes the built-in entry from the live registry. saved = dict(registry._packages) registry._packages.clear() - for cls in (FakeTool, DepTool, TopTool, MultiTool): + for cls in (FakeTool, DepTool, TopTool, MultiTool, Venv): registry._packages[cls.name] = cls() FakeTool.base_url = base_url MultiTool.base_url = base_url @@ -243,7 +246,6 @@ def test_install_forgets_verification_when_state_operation_releases_lock(pm_env, import os from hermes_cli.runtime_state import _lock from pm.cli import _install_names - from pm.packages import Venv ensure = importlib.import_module("pm.install") lockfile_path, runtime, docroot, _ = pm_env @@ -266,7 +268,6 @@ def test_install_forgets_verification_when_state_operation_releases_lock(pm_env, os.close(fd) monkeypatch.setattr(ensure, "sync_venv", sync) - monkeypatch.setitem(registry._packages, "venv", Venv()) assert _install_names(["deptool", "venv", "toptool"]) == 0 assert binary.read_text(encoding="utf-8") == "deptool" @@ -400,8 +401,13 @@ def test_gc_keeps_used_removes_orphans(pm_env): ensure("faketool", base_env={}) orphan = runtime / "orphan-9.9-nowhere" orphan.mkdir() + # A killed installer's scratch dir; its restore point must survive gc. + (runtime / ".staging-abandoned" / "tree").mkdir(parents=True) + (runtime / ".previous-faketool-1.0").mkdir() cmd_gc(None) assert not orphan.exists() + assert not (runtime / ".staging-abandoned").exists() + assert (runtime / ".previous-faketool-1.0").is_dir() assert any(p.name.startswith("faketool-1.0") for p in runtime.iterdir()) diff --git a/tests/pm/test_runtime_gc.py b/tests/pm/test_runtime_gc.py new file mode 100644 index 0000000000..4851a1f719 --- /dev/null +++ b/tests/pm/test_runtime_gc.py @@ -0,0 +1,49 @@ +"""Superseded and aborted PM runtime generations are collected; running workers are not.""" +import json +from pathlib import Path + +from pm.runtime import collect_runtime_generations + + +def _generation(root: Path, name: str, *, published: bool = True, leased: bool = True) -> Path: + generation = root / "generations" / name + generation.mkdir(parents=True) + if leased: + (generation / ".lease-managed").touch() + if published: + (generation / "pm-runtime.json").write_text(json.dumps({"inputs": name}), encoding="utf-8") + return generation + + +def test_collector_keeps_selected_leased_and_pre_lease_generations(tmp_path): + from hermes_cli.runtime_state import lease_directory + + root = tmp_path / "pm-runtime" + selected = _generation(root, "selected") + busy = _generation(root, "busy") + idle = _generation(root, "idle") + legacy = _generation(root, "legacy", leased=False) + aborted = _generation(root, "aborted", published=False) + (root / "selected.json").write_text(json.dumps({"generation": "generations/selected"}), encoding="utf-8") + release = lease_directory(busy) + lease_directory(idle)() + + removed = collect_runtime_generations(root) + + assert set(removed) == {idle, aborted} + assert selected.is_dir() and busy.is_dir() and legacy.is_dir() + release() + assert collect_runtime_generations(root) == [busy] + + +def test_collector_yields_to_an_in_flight_stage(tmp_path): + from hermes_cli.runtime_state import _lock + + root = tmp_path / "pm-runtime" + aborted = _generation(root, "aborted", published=False) + root.mkdir(exist_ok=True) + with (root / ".prepare.lock").open("a+b") as lock: + assert _lock(lock.fileno(), wait=False) + assert collect_runtime_generations(root) == [] + assert aborted.is_dir() + assert collect_runtime_generations(root) == [aborted] diff --git a/tests/pm/test_store_lock_wait.py b/tests/pm/test_store_lock_wait.py new file mode 100644 index 0000000000..a47d21c449 --- /dev/null +++ b/tests/pm/test_store_lock_wait.py @@ -0,0 +1,32 @@ +"""A writer queued behind the store lock says so instead of blocking silently.""" +import os +import threading +import time + +from hermes_cli.runtime_state import _lock +from pm.store import Store + + +def test_install_lock_reports_what_it_waits_on(tmp_path, capsys): + store = Store(tmp_path / "store") + store.root.mkdir() + holder = os.open(store.root / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600) + assert _lock(holder, wait=False) + entered = threading.Event() + + def contend(): + with store.install_lock(): + entered.set() + + waiter = threading.Thread(target=contend, daemon=True) + waiter.start() + deadline = time.monotonic() + 15 + err = "" + while "waiting for" not in err and time.monotonic() < deadline: + time.sleep(0.1) + err += capsys.readouterr().err + assert str(store.root / ".install.lock") in err + assert not entered.is_set() + os.close(holder) + assert entered.wait(timeout=10), "the message must not replace the wait" + waiter.join(timeout=5) diff --git a/tests/pm/test_venv_sync_ambient_config.py b/tests/pm/test_venv_sync_ambient_config.py index 7aa000fe23..24f846b1eb 100644 --- a/tests/pm/test_venv_sync_ambient_config.py +++ b/tests/pm/test_venv_sync_ambient_config.py @@ -31,7 +31,7 @@ def test_ambient_uv_config_does_not_affect_pm_venv_sync(tmp_path, monkeypatch): (config / "uv" / "uv.toml").write_text('required-version="<0.0.1"\n') for key, value in { "UV_NO_CONFIG": "1", "UV_CONFIG_FILE": "/poison/uv.toml", - "UV_DEFAULT_INDEX": "https://poison.invalid/simple", "UV_PYTHON": "/poison/python", + "UV_PYTHON": "/poison/python", "UV_PROJECT_ENVIRONMENT": str(tmp_path / "unrelated-environment"), "UV_CACHE_DIR": str(tmp_path / "hostile-cache"), "UV_PROJECT": "/poison/project", "VIRTUAL_ENV": "/poison/venv", diff --git a/tests/tools/test_wake_word.py b/tests/tools/test_wake_word.py index 46d2b354ad..88e08fce37 100644 --- a/tests/tools/test_wake_word.py +++ b/tests/tools/test_wake_word.py @@ -373,7 +373,7 @@ def test_requirements_lazy_disabled_returns_remedy_not_nameerror(monkeypatch): r = ww.check_wake_word_requirements({"provider": "openwakeword"}) assert r["available"] is False assert r["deps_available"] is False - assert "sync_venv(['wake-openwakeword'], explicit=True)" in r["hint"] + assert "hermes pm install --extra wake-openwakeword" in r["hint"] def test_requirements_deps_present_but_no_audio_hint(monkeypatch): diff --git a/tools/mcp_oauth.py b/tools/mcp_oauth.py index fd3827853d..c0e5d89d82 100644 --- a/tools/mcp_oauth.py +++ b/tools/mcp_oauth.py @@ -8,6 +8,7 @@ Document URL (CIMD) when the server supports it, else RFC 7591 DCR. ``mcp_server (all optional): client_id, client_secret, scope, redirect_port, redirect_uri (proxy callback), redirect_host, client_name, client_metadata_url, cimd, user_agent, timeout.""" +from pm import install_hint import asyncio import contextlib import contextvars @@ -1227,7 +1228,7 @@ def build_oauth_auth(server_name: str, server_url: str, oauth_config: dict | Non global HermesOAuthClientProvider if not _OAUTH_AVAILABLE or _sdk_class("OAuthClientProvider") is None: logger.warning("MCP OAuth requested for '%s' but SDK auth types are not available. Run: " - "python -c \"from pm import sync_venv; sync_venv(['mcp'], explicit=True)\"", server_name) + f"{install_hint('mcp')}", server_name) return None from tools.mcp_oauth_provider import build_provider_kwargs, prepare_oauth_config diff --git a/tools/send_message_senders.py b/tools/send_message_senders.py index cfa21e4efc..f53822dfac 100644 --- a/tools/send_message_senders.py +++ b/tools/send_message_senders.py @@ -1,5 +1,6 @@ """Standalone per-platform senders and error helpers for send_message.""" +from pm import install_hint import asyncio import contextlib import logging @@ -304,7 +305,7 @@ async def _send_telegram(token, chat_id, message, media_files=None, thread_id=No return _success("telegram", chat_id, warnings, message_id=str(last_msg.message_id)) except ImportError: return {"error": "python-telegram-bot not installed. Run: " - "python -c \"from pm import sync_venv; sync_venv(['telegram'], explicit=True)\""} + f"{install_hint('telegram')}"} except Exception as e: return _error(f"Telegram send failed: {e}") @@ -367,7 +368,7 @@ async def _resolve_slack_user_target(token, chat_id): import aiohttp except ImportError: return None, {"error": "aiohttp not installed. Run: " - "python -c \"from pm import sync_venv; sync_venv(['messaging'], explicit=True)\""} + f"{install_hint('messaging')}"} try: from gateway.platforms.base import resolve_proxy_url, proxy_kwargs_for_aiohttp _sess_kw, _req_kw = proxy_kwargs_for_aiohttp(resolve_proxy_url()) @@ -540,7 +541,7 @@ async def _send_matrix_via_adapter(pconfig, chat_id, message, media_files=None, from plugins.platforms.matrix.adapter import MatrixAdapter except ImportError: return {"error": "Matrix dependencies not installed. Run: " - "python -c \"from pm import sync_venv; sync_venv(['matrix'], explicit=True)\""} + f"{install_hint('matrix')}"} adapter = MatrixAdapter(pconfig) try: if not await adapter.connect(): diff --git a/tools/tirith_security.py b/tools/tirith_security.py index 07613e5b09..8d3e8cc6fc 100644 --- a/tools/tirith_security.py +++ b/tools/tirith_security.py @@ -107,7 +107,7 @@ def _verify_cosign(checksums_path: str, sig_path: str, cert_path: str) -> bool | return True -def _verify_release_provenance(directory: Path, log) -> tuple[bool, str]: +def verify_release_provenance(directory: Path, log) -> tuple[bool, str]: """Verify PM-acquired checksum provenance; this function never downloads. Missing/broken cosign is optional; an explicit rejection is fatal. diff --git a/tools/tts_tool.py b/tools/tts_tool.py index a1d9c9da5b..ab36cf1a1d 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -8,6 +8,7 @@ Sibling ``tts_tool_*`` modules hold backends/delivery/lifecycle; they read the s here (config, provider resolution, lazy SDK importers) through ``_origin()`` at call time. """ +from pm import install_hint import asyncio import contextlib import datetime @@ -169,7 +170,7 @@ _FFMPEG_OPUS_PROVIDERS = frozenset({"edge", "neutts", "minimax", "xai", "kittent _BUILTIN_DISPATCH: Dict[str, tuple] = { "elevenlabs": (lambda: _importable(_import_elevenlabs), "ElevenLabs", "_generate_elevenlabs", "ElevenLabs provider selected but 'elevenlabs' package not installed. Run: " - "python -c \"from pm import sync_venv; sync_venv(['tts-premium'], explicit=True)\""), + f"{install_hint('tts-premium')}"), "openai": (lambda: _importable(_import_openai_client), "OpenAI TTS", "_generate_openai_tts", "OpenAI provider selected but 'openai' package not installed."), "deepinfra": (lambda: _importable(_import_openai_client), "DeepInfra TTS", "_generate_deepinfra_tts", @@ -220,7 +221,7 @@ def _select_builtin_engine(provider: str) -> tuple: return "neutts", None return provider, _error_json( "No TTS provider available. Enable Edge TTS with: " - "python -c \"from pm import sync_venv; sync_venv(['edge-tts'], explicit=True)\" " + f"{install_hint('edge-tts')} " "or run 'hermes setup tts' and choose NeuTTS for local synthesis.") diff --git a/tools/wake_word.py b/tools/wake_word.py index d38e5d12ee..69f68c2150 100644 --- a/tools/wake_word.py +++ b/tools/wake_word.py @@ -10,6 +10,7 @@ idle (two input streams on one device is unreliable cross-platform). from __future__ import annotations +from pm import install_hint import logging import os import queue @@ -391,7 +392,7 @@ def check_wake_word_requirements(cfg: Optional[Dict[str, Any]] = None, *, key_ok = False hint = "Set PORCUPINE_ACCESS_KEY (free key at https://console.picovoice.ai)." elif not deps_ok and not lazy_ok: - hint = f"python -c \"from pm import sync_venv; sync_venv(['{feature}'], explicit=True)\"" + hint = install_hint(feature) elif deps_ok and not audio_ok and resolve_capture_mode(cfg) == "local": hint = "Microphone capture needs sounddevice + numpy and a working audio device." elif not stt_ok or not tts_ok: