From babbec1c4c990f02d6c6ab60f08b726b7c8591ec Mon Sep 17 00:00:00 2001 From: ethernet Date: Wed, 23 Sep 2026 18:13:38 -0400 Subject: [PATCH] feat(pm): isolate developer test environment from runtime extras --- .github/actions/setup-pm/README.md | 14 +-- .github/actions/setup-pm/action.yml | 16 +++- .github/workflows/docker.yml | 3 +- .github/workflows/e2e-desktop.yml | 3 +- .github/workflows/js-tests.yml | 3 +- .github/workflows/termux-verify.yml | 3 +- .github/workflows/tests-os.yml | 3 +- .github/workflows/tests.yml | 6 +- .github/workflows/windows-venv-e2e.yml | 3 +- AGENTS.md | 2 +- CONTRIBUTING.es.md | 2 +- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- activate | 28 +++++- activate.ps1 | 6 +- evals/codebase_navigability/README.md | 2 +- pm/__init__.py | 2 +- pm/cli.py | 23 ++++- pm/client.py | 11 +++ pm/environments.py | 15 ++- pm/operations.py | 90 +++++++++++++----- pm/testenv.py | 45 +++++++++ pm/worker_operations.py | 1 + pyproject.toml | 27 +++--- scripts/_activation.sh | 50 ++++++++++ scripts/_hermes-python | 38 ++------ scripts/ci/setup_toolchain.py | 25 ++--- scripts/run_tests.sh | 91 ++++++++----------- setup-hermes.ps1 | 6 +- setup-hermes.sh | 23 +++-- .../references/contributor-guide.md | 2 +- .../hermes-agent/references/windows-quirks.md | 2 +- .../python-debugpy/SKILL.md | 2 +- tests/gateway/conftest.py | 4 +- .../test_update_launch_completion.py | 2 +- tests/pm/test_activation_runtime.py | 10 +- tests/pm/test_activation_setup.py | 12 ++- tests/pm/test_install_default_closure.py | 2 +- tests/pm/test_runtime_entrypoints.py | 13 +-- tests/scripts/test_hermes_python_prologue.py | 26 +++++- tests/scripts/test_setup_toolchain.py | 36 ++++++-- tests/test_packaging_metadata.py | 18 +++- uv.lock | 44 ++++----- website/docs/developer-guide/contributing.md | 2 +- website/docs/getting-started/nix-setup.md | 2 +- website/docs/reference/package-management.md | 39 ++++---- .../software-development-python-debugpy.md | 2 +- .../current/developer-guide/contributing.md | 2 +- .../current/getting-started/nix-setup.md | 2 +- .../software-development-python-debugpy.md | 2 +- 50 files changed, 516 insertions(+), 253 deletions(-) create mode 100644 pm/testenv.py create mode 100644 scripts/_activation.sh diff --git a/.github/actions/setup-pm/README.md b/.github/actions/setup-pm/README.md index f4bd3159a9..dbc60b122d 100644 --- a/.github/actions/setup-pm/README.md +++ b/.github/actions/setup-pm/README.md @@ -10,17 +10,19 @@ not resolve a version range, install another setup action, or modify the lock. - uses: ./.github/actions/setup-pm with: toolchain: all - extras: '["dev"]' + extras: '[]' + test-environment: 'true' - run: python --version && node --version && npm --version ``` `toolchain` defaults to `python` (Python and PM's private installer). `node` installs Node and npm; `all` installs both pairs. There are no version overrides. `extras` is a JSON array because GitHub action inputs are strings. Omit it for tools only; `[]` -installs the core Python dependencies, and `["dev"]` adds the dev extra. PM -checks `uv.lock`, installs the requested dependencies, and validates the environment. -The dev extra uses an independent test environment including the test dependency -group; only non-test installs publish an application selection. It does not enable plugins. +installs the core Python dependencies. Set `test-environment: 'true'` to build an +independent interpreter with the `dev` and `test` dependency groups; any `extras` +then select runtime features in that interpreter. PM checks `uv.lock`, installs +the requested dependencies, and validates the environment. Only non-test installs +publish an application selection. It does not enable plugins. Subsequent steps get `python`, `python3`, `node`, `npm` and `npx` for the selected toolchain on PATH. The pinned npm precedes Node's bundled npm. @@ -51,7 +53,7 @@ The official, SHA-pinned `actions/cache` transports three independent caches: All restores use the exact primary key, without fallback prefixes, matching setup-uv and setup-node's npm behavior. Successful jobs save at teardown; exact hits are not saved again. Only dependency-carrying callers -(`extras` set) auto-save the uv cache: a tool-only job never runs a +(`extras` set or `test-environment: 'true'`) auto-save the uv cache: a tool-only job never runs a dependency operation, so letting it save would freeze an empty cache under the production key, where an immutable exact hit blocks real saves forever. PM re-verifies restored tools before use. diff --git a/.github/actions/setup-pm/action.yml b/.github/actions/setup-pm/action.yml index 614bc7f5a7..af86ab8643 100644 --- a/.github/actions/setup-pm/action.yml +++ b/.github/actions/setup-pm/action.yml @@ -11,8 +11,11 @@ inputs: description: Use protected R2 credentials to preserve and retrieve the selected toolchain before installation. default: 'false' extras: - description: 'JSON extras: omit for tools; [] installs core; dev selects an isolated test environment plus the test dependency group.' + description: 'JSON runtime extras: omit for tools; [] installs core. Used as test coverage extras with test-environment.' default: '' + test-environment: + description: 'Build an isolated interpreter with the dev and test groups; never publish these to the runtime environment.' + default: 'false' cache: description: Cache the verified PM tool store. default: 'true' @@ -152,7 +155,7 @@ runs: - name: Cache uv dependency downloads and builds id: python-cache - if: inputs.toolchain != 'node' && inputs.cache-python == 'true' && inputs.save-python-cache == 'true' && inputs.extras != '' + if: inputs.toolchain != 'node' && inputs.cache-python == 'true' && inputs.save-python-cache == 'true' && (inputs.extras != '' || inputs.test-environment == 'true') uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.install.outputs.uv-cache-path }} @@ -216,14 +219,17 @@ runs: key: setup-pm-npm-v1-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.npm-version }}-${{ hashFiles(inputs.node-cache-dependency-path) }}-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: setup-pm-npm-v1-${{ inputs.cache-suffix || 'production' }}-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.npm-version }}-${{ hashFiles(inputs.node-cache-dependency-path) }}-${{ github.job }}- - - name: Install the requested Python extras through PM + - name: Install the requested Python dependencies through PM id: dependencies - if: inputs.extras != '' + if: inputs.extras != '' || inputs.test-environment == 'true' shell: bash env: _PM_ACTION: ${{ github.action_path }} _PM_TOOLCHAIN: ${{ inputs.toolchain }} _PM_EXTRAS: ${{ inputs.extras }} + _PM_TEST_ENVIRONMENT: ${{ inputs.test-environment }} run: | + args=() + if [ "$_PM_TEST_ENVIRONMENT" = true ]; then args+=(--test-environment); fi "$HERMES_PYTHON" -S "$_PM_ACTION/../../../scripts/ci/setup_toolchain.py" dependencies \ - --toolchain "$_PM_TOOLCHAIN" --extras "$_PM_EXTRAS" --home "$HERMES_HOME" + --toolchain "$_PM_TOOLCHAIN" --extras "$_PM_EXTRAS" --home "$HERMES_HOME" "${args[@]}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1d363717a5..2ed6b2ccc0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -238,7 +238,8 @@ jobs: - name: Set up locked Python and test dependencies uses: ./.github/actions/setup-pm with: - extras: '["dev"]' + extras: '[]' + test-environment: 'true' prune-python-cache: true - name: Restore the image install stamp for the docker tests diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml index 473060b7a6..02c2d9c13a 100644 --- a/.github/workflows/e2e-desktop.yml +++ b/.github/workflows/e2e-desktop.yml @@ -43,7 +43,8 @@ jobs: uses: ./.github/actions/setup-pm with: toolchain: all - extras: '["all", "dev"]' + extras: '["all"]' + test-environment: 'true' prune-python-cache: true # Full npm ci (not --ignore-scripts): electron's postinstall diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 0b9a7a2c20..7e613a2afe 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -26,7 +26,8 @@ jobs: # the electron contracts drive real hermes_cli/pm code through # HERMES_PYTHON, which needs the application dependencies. toolchain: all - extras: '["dev"]' + extras: '[]' + test-environment: 'true' - name: Install zsh and a virtual display for Electron if: runner.os == 'Linux' diff --git a/.github/workflows/termux-verify.yml b/.github/workflows/termux-verify.yml index 9f2ee8c6da..936a540ab3 100644 --- a/.github/workflows/termux-verify.yml +++ b/.github/workflows/termux-verify.yml @@ -51,7 +51,8 @@ jobs: sudo apt-get install -y patchelf - uses: ./.github/actions/setup-pm with: - extras: '["dev"]' + extras: '[]' + test-environment: 'true' cache-python: false - name: Run the native linker and wheel contracts run: | diff --git a/.github/workflows/tests-os.yml b/.github/workflows/tests-os.yml index 998f39ee10..1cb17c95f6 100644 --- a/.github/workflows/tests-os.yml +++ b/.github/workflows/tests-os.yml @@ -82,7 +82,8 @@ jobs: uses: ./.github/actions/setup-pm with: packages: ripgrep - extras: '["all", "dev", "telegram", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + extras: '["all", "telegram", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + test-environment: 'true' prune-python-cache: true - name: Run ${{ matrix.marker }} tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a5e7138ac9..99a25e7a52 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,8 @@ jobs: - name: Set up locked Python and test dependencies uses: ./.github/actions/setup-pm with: - extras: '["all", "dev", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + extras: '["all", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + test-environment: 'true' prune-python-cache: true - name: Restore per-file duration cache @@ -137,7 +138,8 @@ jobs: - name: Set up locked Python and test dependencies uses: ./.github/actions/setup-pm with: - extras: '["all", "dev", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + extras: '["all", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + test-environment: 'true' prune-python-cache: true - name: Run e2e tests diff --git a/.github/workflows/windows-venv-e2e.yml b/.github/workflows/windows-venv-e2e.yml index 4643eb0535..04728274f7 100644 --- a/.github/workflows/windows-venv-e2e.yml +++ b/.github/workflows/windows-venv-e2e.yml @@ -51,7 +51,8 @@ jobs: - name: Set up locked Python and test dependencies uses: ./.github/actions/setup-pm with: - extras: '["dev", "messaging"]' + extras: '["messaging"]' + test-environment: 'true' prune-python-cache: true - name: Run venv-holder live E2E diff --git a/AGENTS.md b/AGENTS.md index 4ef4c0ba05..6573cbc319 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -367,7 +367,7 @@ with API keys set has caused repeated "works locally, fails in CI" incidents (an Prepare a test interpreter with the checkout's bootstrapped Python: ```bash -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test ``` This is a fresh build, not an in-place sync. If the disposable output exists, diff --git a/CONTRIBUTING.es.md b/CONTRIBUTING.es.md index 7768e855db..fc1e0d7ea0 100644 --- a/CONTRIBUTING.es.md +++ b/CONTRIBUTING.es.md @@ -94,7 +94,7 @@ hermes --version Usa el Python preparado por PM para crear un entorno nuevo: ```bash -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test scripts/run_tests.sh tests/agent/ -v ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 281f463b5d..fc30e0daa1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,7 +153,7 @@ architecture before building source dependencies. Build an independent interpreter for tests and editor tools: ```bash -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test ``` PM builds from the committed lock and checks dependency consistency before diff --git a/Dockerfile b/Dockerfile index b8e24a1d87..e6c5aa5619 100644 --- a/Dockerfile +++ b/Dockerfile @@ -279,7 +279,7 @@ RUN cd plugins/platforms/photon/sidecar && \ # # `pm.build_env --no-install-project --extra all --extra messaging --extra otlp` # installs the deps reachable through the composite `[all]` extra -# (handpicked set intended for the production image — excludes `[dev]`), +# (handpicked set intended for the production image; dependency groups are not selected), # plus gateway messaging adapters that should work in the published image # without a first-boot lazy install. We do NOT use `--all-extras`: # that would pull in `[rl]` (atroposlib + tinker + torch + wandb from diff --git a/activate b/activate index 1a1d27979a..bb00431057 100755 --- a/activate +++ b/activate @@ -18,15 +18,41 @@ # install` and `hermes update` keep that check. # ============================================================================ _HERMES_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Options: `--test-extras a,b` selects the test environment's runtime extras +# (default: [all]); `--` ends options. +# A bare `source ./activate` inherits the caller's positional parameters. +# Ignore unrelated words so scripts with their own arguments can source it. +_hermes_test_env="--test-environment" +_hermes_expect_extras="" +for _hermes_arg in "$@"; do + if [ -n "$_hermes_expect_extras" ]; then + _hermes_test_env="--test-environment=$_hermes_arg" + _hermes_expect_extras="" + continue + fi + case "$_hermes_arg" in + --) break ;; + --test-extras) _hermes_expect_extras=1 ;; + --test-extras=*) _hermes_test_env="--test-environment=${_hermes_arg#--test-extras=}" ;; + *) : ;; + esac +done +if [ -n "$_hermes_expect_extras" ]; then + printf '%s\n' 'activate: --test-extras needs a comma-separated list' >&2 + unset _hermes_arg _hermes_test_env _hermes_expect_extras + return 2 2>/dev/null || exit 2 +fi # Setup runs in a child: failures must not exit or partially activate the # caller's shell, and activation must not republish launchers or shell config. if ! ( unset PYTHONHOME PYTHONPATH VIRTUAL_ENV - bash "$_HERMES_REPO/setup-hermes.sh" --runtime-only + bash "$_HERMES_REPO/setup-hermes.sh" --runtime-only "$_hermes_test_env" ) >&2; then printf '%s\n' 'activate: setup failed; shell environment unchanged' >&2 + unset _hermes_arg _hermes_test_env _hermes_expect_extras return 1 2>/dev/null || exit 1 fi +unset _hermes_arg _hermes_test_env _hermes_expect_extras # Guard against double-sourcing: venv activate precedent (re-activating is a # no-op re-save; we keep it idempotent by deactivating first). Key on the diff --git a/activate.ps1 b/activate.ps1 index b219950a59..e58ed04c91 100755 --- a/activate.ps1 +++ b/activate.ps1 @@ -1,5 +1,7 @@ # Source this file to sync and apply the PM environment; deactivate restores it. # Trusts the recorded tool digest. `hermes pm install` re-checks the bytes. +# -TestExtras a,b selects runtime extras in the test environment (default: [all]). +param([string]$TestExtras = '') $ErrorActionPreference = 'Stop' $OutputEncoding = [System.Console]::OutputEncoding = [System.Console]::InputEncoding = [System.Text.Encoding]::UTF8 @@ -14,7 +16,9 @@ foreach ($key in @('PYTHONPATH', 'PYTHONHOME', 'VIRTUAL_ENV')) { try { # Run separately so setup's exit/failure cannot terminate the sourced shell. $shell = (Get-Process -Id $PID).Path - & $shell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "$repo\setup-hermes.ps1" -RuntimeOnly | Out-Host + $testArgs = @() + if ($TestExtras) { $testArgs = @('-TestExtras', $TestExtras) } + & $shell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "$repo\setup-hermes.ps1" -RuntimeOnly @testArgs | Out-Host if ($LASTEXITCODE -ne 0) { throw 'activate: setup failed; shell environment unchanged' } } finally { foreach ($key in $bootstrapSaved.Keys) { diff --git a/evals/codebase_navigability/README.md b/evals/codebase_navigability/README.md index 04cbdfab5c..0d0cebeabc 100644 --- a/evals/codebase_navigability/README.md +++ b/evals/codebase_navigability/README.md @@ -51,7 +51,7 @@ NAV_OUT=out/ python evals/codebase_navigability/runtime_bench.py . head Use a fresh benchmark path; do not replace an existing environment. Runtime and pytest-collection measurements also require the target tree's application/test dependencies. Prepare those in a separate caller-owned output with -`python -m pm.build_env --source --out --extra dev --group test` +`python -m pm.build_env --source --out --group dev --group test` from a PM-prepared checkout, rather than injecting benchmark packages into Hermes. `bench.py` and `static_metrics.py` take ~2 min each on a 1M-line tree; `lookup_sim.py` ~10 min for diff --git a/pm/__init__.py b/pm/__init__.py index 310eb8e5a9..acb3ece679 100644 --- a/pm/__init__.py +++ b/pm/__init__.py @@ -26,7 +26,7 @@ _EXPORTS = { ), "pm.client": ( "ensure", "sync_venv", "build_environment", "lock_project", "stage_manager_runtime", - "ensure_environment", "ensure_python_tool", "venv_is_current", "check_project_lock", + "ensure_environment", "ensure_project_environment", "ensure_python_tool", "venv_is_current", "check_project_lock", "export_requirements", "build_requirements_environment", "prune_cache", "stage_tools", "prepare_tools", ), diff --git a/pm/cli.py b/pm/cli.py index 51ea0c4677..452f85e0ae 100644 --- a/pm/cli.py +++ b/pm/cli.py @@ -167,6 +167,10 @@ def cmd_install(args) -> int: extras = list(dict.fromkeys(getattr(args, "extra", None) or ())) tools_only = bool(getattr(args, "tools_only", False)) trust_recorded = bool(getattr(args, "trust_recorded", False)) + test_environment = getattr(args, "test_environment", None) + if test_environment is not None and (extras or cross_target or args.names or tools_only): + print("✗ --test-environment builds beside the default closure; it does not take names, --extra, --target, or --tools-only") + return 1 if tools_only and (extras or cross_target or args.names): print("✗ --tools-only installs the tool closure and then stops; it does not take names, --extra, or --target") return 1 @@ -214,10 +218,24 @@ def cmd_install(args) -> int: except InstallError as e: print(f"✗ {e}") failed += 1 + if test_environment is not None and not failed: + from pm import check_project_lock + from pm.testenv import ensure_testenv, parse_extras + + # Before the input stamps: they then cover this environment too, so + # the shebang/run_tests.sh staleness check rebuilds it when it drifts. + try: + check_project_lock(repo_root(), explicit=True) + ensure_testenv(repo_root(), parse_extras(test_environment)) + print("✓ test environment") + except InstallError as e: + print(f"✗ {e}") + failed += 1 if full_closure and not failed: from pm.environments import activation_inputs_dir, record_activation_inputs - record_activation_inputs(activation_inputs_dir(repo_root()), input_mtimes) + record_activation_inputs(activation_inputs_dir(repo_root()), input_mtimes, repo_root(), + test_environment=test_environment is not None) return 1 if failed else 0 @@ -588,6 +606,9 @@ def main(argv=None) -> int: p.add_argument("--trust-recorded", action="store_true", help="trust the recorded tool digest instead of re-hashing every entry. " "shell activation only. a deliberate install re-checks the bytes") + p.add_argument("--test-environment", nargs="?", const="", default=None, metavar="EXTRAS", + help="also make this checkout's isolated test environment current (activation). " + "EXTRAS is comma-separated; omitted selects [all]") p.add_argument( "--target", help="stage for a cross target (e.g. linux-arm64-bionic on a glibc " diff --git a/pm/client.py b/pm/client.py index 114cf079fb..8f8ef1f307 100644 --- a/pm/client.py +++ b/pm/client.py @@ -290,6 +290,17 @@ def ensure_environment( })) +def ensure_project_environment( + name: str, project: Path, *, extras: Sequence[str] = (), groups: Sequence[str] = (), + root: Path | None = None, explicit: bool = False, timeout: int = 1800, +) -> Path: + """Select an isolated environment of the project's locked dependencies.""" + return Path(_python_operation("ensure_project_environment", { + "name": name, "project": Path(project), "extras": list(extras), "groups": list(groups), + "root": root, "explicit": explicit, "timeout": timeout, + })) + + def ensure_python_tool( name: str, requirements: Sequence[str], executable: str, *, root: Path | None = None, explicit: bool = False, timeout: int = 1800, diff --git a/pm/environments.py b/pm/environments.py index 214bfede39..512fa8192b 100644 --- a/pm/environments.py +++ b/pm/environments.py @@ -56,7 +56,7 @@ def activation_input_mtimes(project_root: Path) -> dict[str, int]: return {name: (root / name).stat().st_mtime_ns for name in ACTIVATION_INPUTS if (root / name).is_file()} -def record_activation_inputs(stamps: Path, mtimes: dict[str, int]) -> None: +def record_activation_inputs(stamps: Path, mtimes: dict[str, int], project_root: Path, *, test_environment: bool) -> None: """Give each stamp the exact mtime of the input the install was verified against. Recorded on every successful install, including no-op syncs: a checkout that @@ -67,6 +67,12 @@ def record_activation_inputs(stamps: Path, mtimes: dict[str, int]) -> None: import shutil shutil.rmtree(stamps, ignore_errors=True) + # The sentinel is inherited by child shells; equal input mtimes in another + # checkout must never make their test interpreter appear current here. + stamps.mkdir(parents=True, exist_ok=True) + (stamps / ".project-root").write_text(str(Path(project_root).resolve()), encoding="utf-8") + if test_environment: + (stamps / ".test-environment").touch() for name, mtime in mtimes.items(): stamp = stamps / name stamp.parent.mkdir(parents=True, exist_ok=True) @@ -293,6 +299,13 @@ def activation_environment(project_root: Path) -> dict[str, str]: # directory also holds activation_inputs_dir, the input-mtime stamps # `scripts/_hermes-python` compares against to decide staleness. env["__HERMES_ACTIVATED"] = str(runtime_facts_path(project_root)) + # The suite's interpreter (pm.testenv): an isolated side environment, so it + # never appears on PYTHONPATH/PATH above. scripts/run_tests.sh reads it. + from pm.testenv import testenv_python + + test_python = testenv_python(project_root) + if test_python is not None: + env["__HERMES_TEST_PYTHON"] = str(test_python) return env diff --git a/pm/operations.py b/pm/operations.py index 3307759796..14c4a5fdd0 100644 --- a/pm/operations.py +++ b/pm/operations.py @@ -5,7 +5,7 @@ Only the private environment engine knows how to obtain or invoke uv. """ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence import hashlib import json import os @@ -189,9 +189,69 @@ def ensure_environment( ) -> Path: """Make an isolated dependency set current, then atomically select it. + An optional tool entrypoint is validated before publication, not afterwards. + """ + root = _environment_root(name, root) + requirements = _requirements(requirements) + if executable is not None: + _tool(Path("unused/python"), executable) # Validate before any write. + + def build(generation: Path) -> Path: + (generation / "pyproject.toml").write_text( + '[project]\nname = "hermes-side-environment"\nversion = "0"\n' + 'requires-python = ">=3.11"\ndependencies = ' + + json.dumps(requirements) + '\n[tool.uv]\npackage = false\n', encoding="utf-8", + ) + previous = _selection(root) + if previous: + seed = root / previous["generation"] / "uv.lock" + if seed.is_file(): + shutil.copyfile(seed, generation / "uv.lock") + return build_environment(source=generation, out=generation / "venv", frozen=False, + explicit=explicit, timeout=timeout) + + return _ensure_generation(name, root, {"requirements": requirements}, build, + record={"requirements": requirements}, explicit=explicit, + executable=executable) + + +def ensure_project_environment( + name: str, project: Path, *, extras: Sequence[str] = (), groups: Sequence[str] = (), + root: Path | None = None, explicit: bool = False, timeout: int = 1800, +) -> Path: + """Make an isolated environment of a project's LOCKED dependencies current. + + The project itself is not installed, and nothing reaches PM facts or the + selected application generation: this serves side environments such as the + test suite's. The identity covers the lock and manifest bytes, so any edit + that can change the resolved set selects a fresh generation. + """ + root = _environment_root(name, root) + project = Path(project).absolute() + extras, groups = sorted(set(extras)), sorted(set(groups)) + manifests = {} + for manifest in ("pyproject.toml", "uv.lock"): + try: + manifests[manifest] = hashlib.sha256((project / manifest).read_bytes()).hexdigest() + except FileNotFoundError as exc: + raise InstallError("venv", f"locked project environment needs {project / manifest}") from exc + + def build(generation: Path) -> Path: + return build_environment(source=project, out=generation / "venv", extras=extras, groups=groups, + no_install_project=True, frozen=True, explicit=explicit, timeout=timeout) + + return _ensure_generation(name, root, {"manifests": manifests, "extras": extras, "groups": groups}, + build, record={"extras": extras, "groups": groups}, explicit=explicit) + + +def _ensure_generation( + name: str, root: Path, inputs: dict, build: Callable[[Path], Path], *, record: dict, explicit: bool, + executable: str | None = None, +) -> Path: + """Select the generation whose inputs match, building one only when none does. + Build at the final path: Windows launchers and scripts embed that path. The prior generation survives both successful replacement and failed builds. - An optional tool entrypoint is validated before publication, not afterwards. """ from hermes_cli.runtime_state import _lock from pm.install import _refuse_lazy, lazy_installs_allowed @@ -199,20 +259,16 @@ def ensure_environment( from pm import paths from pm.store import current_target - root = _environment_root(name, root) - requirements = _requirements(requirements) - if executable is not None: - _tool(Path("unused/python"), executable) # Validate before any write. lock = Lockfile(paths.lockfile_path()) target = current_target() - inputs = {"requirements": requirements, "python": lock.version("python"), "target": target, + inputs = {**inputs, "python": lock.version("python"), "target": target, "artifacts": [item["sha256"] for item in lock.artifacts("python", target)]} identity = hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest() def current() -> Path | None: - record = _selection(root) + selected = _selection(root) python = environment_python(name, root=root) - if (record.get("inputs") == identity and python is not None + if (selected.get("inputs") == identity and python is not None and (executable is None or _tool(python, executable) is not None)): return python return None @@ -231,22 +287,10 @@ def ensure_environment( generation = root / f"gen-{uuid.uuid4().hex}" generation.mkdir() try: - (generation / "pyproject.toml").write_text( - '[project]\nname = "hermes-side-environment"\nversion = "0"\n' - 'requires-python = ">=3.11"\ndependencies = ' - + json.dumps(requirements) + '\n[tool.uv]\npackage = false\n', encoding="utf-8", - ) - previous = _selection(root) - if previous: - seed = root / previous["generation"] / "uv.lock" - if seed.is_file(): - shutil.copyfile(seed, generation / "uv.lock") - python = build_environment(source=generation, out=generation / "venv", frozen=False, - explicit=explicit, timeout=timeout) + python = build(generation) if executable is not None and _tool(python, executable) is None: raise InstallError(name, f"installed requirements do not provide {executable!r}") - _write(root / "active.json", {"generation": generation.name, "inputs": identity, - "requirements": requirements}) + _write(root / "active.json", {"generation": generation.name, "inputs": identity, **record}) except BaseException: shutil.rmtree(generation, ignore_errors=True) raise diff --git a/pm/testenv.py b/pm/testenv.py new file mode 100644 index 0000000000..7e66e7b979 --- /dev/null +++ b/pm/testenv.py @@ -0,0 +1,45 @@ +"""The isolated environment a checkout's test suite runs under. + +Activation builds it beside the checkout's install state, and CI builds it the +same way. It carries the locked project dependencies plus the ``dev`` and +``test`` groups. It is a side environment +(``ensure_project_environment``), never the selected application generation: +test-only dependencies must not enter PM facts or anything that ships. +""" +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from pm.environments import install_state_dir + +NAME = "test-environment" +# The default developer environment includes the app's normal features. +# CI lanes pass their wider provider matrix explicitly. +DEFAULT_TEST_EXTRAS = ("all",) +GROUPS = ("dev", "test") + + +def testenv_root(project_root: Path) -> Path: + return install_state_dir(project_root) / NAME + + +def ensure_testenv(project_root: Path, extras: Sequence[str] | None = None) -> Path: + """Build the test environment unless its locked inputs are unchanged; return its python.""" + from pm import ensure_project_environment + + chosen = sorted(set(DEFAULT_TEST_EXTRAS if extras is None else extras)) + return ensure_project_environment(NAME, project_root, extras=chosen, groups=GROUPS, + root=testenv_root(project_root), explicit=True) + + +def testenv_python(project_root: Path) -> Path | None: + """The selected test interpreter, read without acquiring tools or writing state.""" + from pm.operations import environment_python + + return environment_python(NAME, root=testenv_root(project_root)) + + +def parse_extras(value: str) -> list[str] | None: + """An empty value selects default extras; commas and spaces separate overrides.""" + return value.replace(",", " ").split() or None diff --git a/pm/worker_operations.py b/pm/worker_operations.py index 61e1a84bd0..24fc355e04 100644 --- a/pm/worker_operations.py +++ b/pm/worker_operations.py @@ -30,6 +30,7 @@ OPERATIONS = { "build_environment": Operation("pm.operations", ("uv",), "policy"), "lock_project": Operation("pm.operations", ("uv",), "policy"), "ensure_environment": Operation("pm.operations", ("uv",), "policy"), + "ensure_project_environment": Operation("pm.operations", ("uv",), "policy"), "ensure_python_tool": Operation("pm.operations", ("uv",), "policy"), "check_project_lock": Operation("pm.build_operations", ("uv",), "policy"), "export_requirements": Operation("pm.build_operations", ("uv",), "policy"), diff --git a/pyproject.toml b/pyproject.toml index adb27a5fb7..111075a123 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -241,18 +241,6 @@ daytona = ["daytona==0.155.0"] vercel = ["vercel==0.7.2"] hindsight = ["hindsight-client==0.6.1"] google-meet = ["playwright==1.62.0", "websockets==15.0.1"] -dev = [ - "debugpy==1.8.20", - "pytest==9.1.1", - "pytest-asyncio==1.3.0", - "mcp==2.0.0", - "httpx2==2.7.0", - "starlette==1.3.1", - "ty==0.0.82", - "ruff==0.15.10", - "setuptools==83.0.0", -] # starlette: CVE-2026-48710; setuptools: 83 (torch >=2.13 requires setuptools 83) - messaging = [ "python-telegram-bot[webhooks]==22.8", "discord.py[voice]==2.7.1", @@ -531,7 +519,18 @@ all = [ ] [dependency-groups] -# Build tooling is not a runtime extra: payloads select --all-extras. +# Build and test tooling never ships in runtime payloads. +dev = [ + "debugpy==1.8.20", + "pytest==9.1.1", + "pytest-asyncio==1.3.0", + "mcp==2.0.0", + "httpx2==2.7.0", + "starlette==1.3.1", + "ty==0.0.82", + "ruff==0.15.10", + "setuptools==83.0.0", +] # starlette: CVE-2026-48710; setuptools: 83 (torch >=2.13 requires setuptools 83) # --only-group supplies Pillow without installing the Hermes application. icon-build = ["Pillow==12.3.0", "resvg-py==0.4.0"] # Native launcher acceptance is required in test environments, not payloads. @@ -579,6 +578,8 @@ kittentts = "python_version < '3.13'" piper = "(platform_machine != 'ARM64' or sys_platform != 'win32') and (platform_machine != 'x86_64' or sys_platform != 'darwin')" [tool.uv] +# uv otherwise enables the dev group by default, including in payload builds. +default-groups = [] # 3.11 is an install bridge for pre-PM updaters, never a runtime, so the lock # covers 3.14 only. Without this the lock must resolve every supported version, # where extras that only ever coexist on 3.14 (kittentts vs neutts) conflict. diff --git a/scripts/_activation.sh b/scripts/_activation.sh new file mode 100644 index 0000000000..6763917ad0 --- /dev/null +++ b/scripts/_activation.sh @@ -0,0 +1,50 @@ +# Sourced, never executed: the activation staleness check shared by +# scripts/_hermes-python (the shebang prologue) and scripts/run_tests.sh. +# +# pm records, beside the installed-state file named by __HERMES_ACTIVATED, one +# stamp per dependency input carrying the exact mtime that input had when the +# install was last verified against it (pm.environments.record_activation_inputs). +# Any input whose mtime DIFFERS from its stamp (newer or older, since a branch +# switch can move it either way) means the inherited environment may not match +# its inputs. `-nt`/`-ot` are bash builtins, so the check costs no process +# spawn, and every successful activation re-records, so one re-activation +# settles it. Missing stamps or inputs, a dangling sentinel or a literal "1" +# all read as stale. + +# hermes_activation_current REPO: status 0 when the inherited environment +# matches REPO's current inputs. +hermes_activation_current() { + local repo="$1" sentinel stamps stamp input stamped=0 + [ -n "${__HERMES_ACTIVATED:-}" ] && [ -e "${__HERMES_ACTIVATED}" ] || return 1 + # activate.ps1 records a Windows path; Git Bash accepts it with slashes. + sentinel="${__HERMES_ACTIVATED//\\//}" + stamps="${sentinel%/*}/inputs" + [ -f "$stamps/.project-root" ] || return 1 + local owner="$repo" recorded + recorded="$(< "$stamps/.project-root")" + if command -v cygpath >/dev/null 2>&1; then + owner="$(cygpath -am "$repo")" || return 1 + recorded="${recorded//\\//}" + [ "${owner,,}" = "${recorded,,}" ] || return 1 + else + [ "$(cd "$owner" && pwd -P)" = "$recorded" ] || return 1 + fi + for stamp in "$stamps"/* "$stamps"/*/*; do + [ -f "$stamp" ] || continue + stamped=1 + input="$repo/${stamp#"$stamps"/}" + if [ ! -e "$input" ] || [ "$input" -nt "$stamp" ] || [ "$input" -ot "$stamp" ]; then + return 1 + fi + done + [ "$stamped" = 1 ] +} + +# hermes_ensure_activated REPO: source REPO/activate unless the inherited +# environment is current. `--` passes no options: `source` without arguments +# would hand activate the CALLER's positional parameters. +hermes_ensure_activated() { + hermes_activation_current "$1" && return 0 + # shellcheck source=/dev/null + . "$1/activate" -- +} diff --git a/scripts/_hermes-python b/scripts/_hermes-python index 625dc048d5..45091f68bf 100755 --- a/scripts/_hermes-python +++ b/scripts/_hermes-python @@ -6,8 +6,9 @@ # # The kernel appends the invoking script, so `$1` is the Python file to run # and `$@` its arguments. Activates when there is no environment to inherit, or -# when the inherited one predates its inputs; then execs the interpreter on the -# same file, so tracebacks and __file__ point at the real script. +# when the inherited one no longer matches its inputs (scripts/_activation.sh); +# then execs the interpreter on the same file, so tracebacks and __file__ point +# at the real script. set -u target=${1:-} @@ -24,35 +25,8 @@ if [ ! -f "$_here/activate" ]; then exit 1 fi -# pm records, beside the installed-state file named by __HERMES_ACTIVATED, one -# stamp per dependency input carrying the exact mtime that input had when the -# install was last verified against it (pm.environments.record_activation_inputs). -# Any input whose mtime DIFFERS from its stamp — newer or older, since a branch -# switch can move it either way — means the inherited environment may not match -# its inputs. `-nt`/`-ot` are bash builtins, so this costs no process spawn, and -# every successful activation re-records, so one re-activation settles it. No -# stamps (an install from before stamps existed), a missing input, a dangling -# sentinel or a legacy literal "1" all activate. -_needs_activation=1 -if [ -n "${__HERMES_ACTIVATED:-}" ] && [ -e "${__HERMES_ACTIVATED}" ]; then - _stamps="${__HERMES_ACTIVATED%/*}/inputs" - _needs_activation=0 - _stamped=0 - for _stamp in "$_stamps"/* "$_stamps"/*/*; do - [ -f "$_stamp" ] || continue - _stamped=1 - _input="$_here/${_stamp#"$_stamps"/}" - if [ ! -e "$_input" ] || [ "$_input" -nt "$_stamp" ] || [ "$_input" -ot "$_stamp" ]; then - _needs_activation=1 - break - fi - done - [ "$_stamped" = 1 ] || _needs_activation=1 -fi - -if [ "$_needs_activation" = 1 ]; then - # shellcheck source=/dev/null - . "$_here/activate" || exit 1 -fi +# shellcheck source=scripts/_activation.sh +. "$_here/scripts/_activation.sh" +hermes_ensure_activated "$_here" || exit 1 exec python3 "$target" "$@" diff --git a/scripts/ci/setup_toolchain.py b/scripts/ci/setup_toolchain.py index 8f7a4e5d77..d4c515168a 100644 --- a/scripts/ci/setup_toolchain.py +++ b/scripts/ci/setup_toolchain.py @@ -183,28 +183,30 @@ def install(args) -> None: def dependencies(args) -> None: - if args.extras is None: + if args.extras is None and not args.test_environment: return import tomllib from pm.environments import selected_venv - from pm import build_environment, check_project_lock, sync_venv + from pm import check_project_lock, sync_venv from pm.paths import repo_root project = repo_root() metadata = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8-sig")) - unknown = set(args.extras) - metadata["project"]["optional-dependencies"].keys() + extras = args.extras or [] + unknown = set(extras) - metadata["project"].get("optional-dependencies", {}).keys() if unknown: raise ValueError(f"unknown project extras: {sorted(unknown)}") # Frozen sync must not turn a stale project lock into a green job. check_project_lock(project, explicit=True) - if "dev" in args.extras: - # Test-only groups must not enter PM facts or a shipped generation. - venv = args.home.resolve() / "test-environment" - build_environment(source=project, out=venv, extras=args.extras, - groups=["test"], no_install_project=True, explicit=True) + if args.test_environment: + from pm.testenv import ensure_testenv + + # CI and activation select the same side environment, never PM facts. + python = ensure_testenv(project, extras) + venv = python.parent.parent else: - sync_venv(args.extras, explicit=True, plugin_dirs=[]) + sync_venv(extras, explicit=True, plugin_dirs=[]) venv = selected_venv(project) bindir = venv / ("Scripts" if os.name == "nt" else "bin") python = bindir / ("python.exe" if os.name == "nt" else "python") @@ -226,10 +228,11 @@ def main() -> None: parser.add_argument("--toolchain", choices=["python", "node", "all"], default="python") parser.add_argument("--home", type=Path, required=True) parser.add_argument("--extras", type=parse_extras, default="") + parser.add_argument("--test-environment", action="store_true") parser.add_argument("--packages", type=parse_package_list, default=[], help="extra PM tools beyond the toolchain roots (e.g. ffmpeg)") args = parser.parse_args() - if args.toolchain == "node" and args.extras is not None: - parser.error("extras require the python or all toolchain") + if args.toolchain == "node" and (args.extras is not None or args.test_environment): + parser.error("Python dependencies require the python or all toolchain") os.environ["HERMES_HOME"] = str(args.home.resolve()) os.environ["HERMES_RUNTIME_DIR"] = str(args.home.resolve() / "tools") phases[args.phase](args) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index f1456e7bfb..927957f95b 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -11,7 +11,7 @@ # * Env vars blanked (conftest.py also does this, but this # is belt-and-suspenders for anyone running pytest outside our # conftest path — e.g. on a single file) -# * Proper venv activation (probes .venv, venv, then ~/.hermes/...) +# * The activated checkout's test environment (activates when needed) # # Usage: # scripts/run_tests.sh # full suite @@ -38,64 +38,45 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # ── Locate python ─────────────────────────────────────────────────────────── -# Probe local venvs first; fall back to the Nix devShell's editable venv -# (HERMES_PYTHON is exported by the devShell hook and ships [dev] extras: -# pytest, pytest-asyncio, pytest-timeout, ruff, ty). +# The suite runs under the activated checkout's isolated test environment +# (pm.testenv: `activate` builds it beside the checkout's install state, and CI +# activates the same way). An inherited activation is re-checked against its +# inputs (scripts/_activation.sh) and re-sourced when stale, so a branch switch +# or lock edit never runs the suite against the previous dependency set. # -# A candidate must have pytest INSTALLED, not merely exist. The release venv -# at ~/.hermes/hermes-agent/venv has bin/activate but no pytest, so an -# existence-only probe selected it in checkouts/worktrees without a local -# .venv — every file then died with "No module named pytest" and the run -# reported "0 tests passed" (which reads green at a glance even though the -# exit code is 1). Skip such a venv and keep probing instead. -VENV="" -VENV_PYTHON="" -SKIPPED_VENVS="" -for candidate in "$REPO_ROOT/.venv" "$REPO_ROOT/venv" "$HOME/.hermes/hermes-agent/venv"; do - if [ -f "$candidate/bin/activate" ]; then - if "$candidate/bin/python" -c 'import pytest' 2>/dev/null; then - VENV="$candidate" - VENV_PYTHON="$candidate/bin/python" - break - fi - SKIPPED_VENVS="$SKIPPED_VENVS $candidate" - fi - # Native Windows venv layout: python.exe and activate live under - # Scripts/, and there is no bin/. Anyone running this script from - # Git Bash / MSYS with a `python -m venv`- or uv-created venv hits - # this branch — without it the canonical runner refuses to start. - if [ -f "$candidate/Scripts/activate" ]; then - if "$candidate/Scripts/python.exe" -c 'import pytest' 2>/dev/null; then - VENV="$candidate" - VENV_PYTHON="$candidate/Scripts/python.exe" - break - fi - SKIPPED_VENVS="$SKIPPED_VENVS $candidate" - fi -done - -if [ -n "$SKIPPED_VENVS" ]; then - for skipped in $SKIPPED_VENVS; do - echo "▶ skipping venv without pytest: $skipped" >&2 - done -fi - -if [ -n "$VENV" ]; then - PYTHON="$VENV_PYTHON" -elif [ -n "${HERMES_PYTHON:-}" ] && [ -x "$HERMES_PYTHON" ] \ - && "$HERMES_PYTHON" -c 'import pytest' 2>/dev/null; then - # Guard with an import check: HERMES_PYTHON may point at the RELEASE - # venv (no pytest) when inherited from a wrapped `hermes` binary rather - # than the devShell hook. +# Without an activation, an explicit HERMES_PYTHON that has pytest is honored: +# the Nix devShell's editable venv and CI's minimal installer lanes provide +# one on purpose. The import check matters: a wrapped `hermes` binary exports +# HERMES_PYTHON pointing at a release venv without pytest. +_has_pytest() { [ -n "$1" ] && [ -x "$1" ] && "$1" -c 'import pytest' 2>/dev/null; } +# shellcheck source=scripts/_activation.sh +. "$SCRIPT_DIR/_activation.sh" +if [ -z "${__HERMES_ACTIVATED:-}" ] && _has_pytest "${HERMES_PYTHON:-}"; then PYTHON="$HERMES_PYTHON" - echo "▶ no local venv — using Nix dev venv via HERMES_PYTHON: $PYTHON" + echo "▶ not activated — using HERMES_PYTHON: $PYTHON" else - echo "error: no virtualenv with pytest found in $REPO_ROOT/.venv or $REPO_ROOT/venv," >&2 - echo " and HERMES_PYTHON is not a python with pytest (enter the Nix devShell or create a venv)" >&2 - if [ -n "$SKIPPED_VENVS" ]; then - echo " (skipped for missing pytest:$SKIPPED_VENVS — install dev extras there, or create $REPO_ROOT/.venv)" >&2 + test_stamp="${__HERMES_ACTIVATED:-}" + test_stamp="${test_stamp//\\//}" + if ! hermes_activation_current "$REPO_ROOT" || + [ ! -f "${test_stamp%/*}/inputs/.test-environment" ] || + ! _has_pytest "${__HERMES_TEST_PYTHON:-}"; then + echo "▶ activating $REPO_ROOT (environment missing or stale)" >&2 + # activate is written for interactive shells, not errexit/nounset. + set +euo pipefail + # shellcheck source=/dev/null + . "$REPO_ROOT/activate" -- + activated=$? + set -euo pipefail + if [ "$activated" != 0 ]; then + echo "error: activation failed (see above)" >&2 + exit 1 + fi + fi + PYTHON="${__HERMES_TEST_PYTHON:-}" + if ! _has_pytest "$PYTHON"; then + echo "error: activation provided no test interpreter with pytest (__HERMES_TEST_PYTHON=${PYTHON:-unset})" >&2 + exit 1 fi - exit 1 fi diff --git a/setup-hermes.ps1 b/setup-hermes.ps1 index 481b261bd2..9ca6c5f249 100644 --- a/setup-hermes.ps1 +++ b/setup-hermes.ps1 @@ -10,7 +10,9 @@ # 3. Point you at `.\activate.ps1` - the venv-style way to put the pm env # (PATH + tool vars) into your current session. # ============================================================================ -param([switch]$RuntimeOnly) +# Setup and activation both prepare the isolated test interpreter; installers +# invoke pm.cli directly and do not select it. -TestExtras overrides coverage. +param([switch]$RuntimeOnly, [string]$TestExtras = '') $ErrorActionPreference = 'Stop' Write-Host '' @@ -84,7 +86,7 @@ try { if ($LASTEXITCODE -ne 0) { throw 'bootstrap Python installation failed' } $bootPy = (& $uv python find --managed-python $pyVersion) -join "`n" if ($LASTEXITCODE -ne 0 -or -not $bootPy) { throw 'bootstrap Python lookup failed' } - & $bootPy.Trim() -m pm.cli install $(if ($RuntimeOnly) { '--trust-recorded' }) + & $bootPy.Trim() -m pm.cli install $(if ($RuntimeOnly) { '--trust-recorded' }) "--test-environment=$TestExtras" if ($LASTEXITCODE -ne 0) { throw 'pm install failed - see output above.' } } finally { Pop-Location diff --git a/setup-hermes.sh b/setup-hermes.sh index a9ee6b9572..f1fbaf00ee 100755 --- a/setup-hermes.sh +++ b/setup-hermes.sh @@ -15,14 +15,17 @@ set -e -# Activation needs only provisioning, not user-facing installation side effects. +# Setup and activation prepare the isolated test environment. Installers call +# pm.cli directly and never select it. runtime_only=false -case "${1:-}" in - --runtime-only) runtime_only=true ;; - '') ;; - *) printf 'Unknown setup option: %s\n' "$1" >&2; exit 2 ;; -esac - +test_environment="--test-environment" +for option in "$@"; do + case "$option" in + --runtime-only) runtime_only=true ;; + --test-environment|--test-environment=*) test_environment="$option" ;; + *) printf 'Unknown setup option: %s\n' "$option" >&2; exit 2 ;; + esac +done # Colors GREEN='\033[0;32m' YELLOW='\033[0;33m' @@ -164,7 +167,11 @@ echo -e "${CYAN}→${NC} (first run on a fresh checkout can take 1-5 minutes)" "$uv" python install --no-bin --no-registry "$py_version" boot_py="$("$uv" python find --managed-python "$py_version")" boot_py="${boot_py%$'\r'}" -if ! "$boot_py" -m pm.cli install ${runtime_only:+--trust-recorded}; then +# Activation trusts the recorded tool digest; a direct setup re-checks it +# (setup-hermes.ps1 draws the same line). +pm_args=("$test_environment") +[ "$runtime_only" = true ] && pm_args+=(--trust-recorded) +if ! "$boot_py" -m pm.cli install "${pm_args[@]}"; then echo -e "${RED}✗${NC} pm install failed — see output above." exit 1 fi diff --git a/skills/autonomous-ai-agents/hermes-agent/references/contributor-guide.md b/skills/autonomous-ai-agents/hermes-agent/references/contributor-guide.md index 240b3ef8ab..0b308973e3 100644 --- a/skills/autonomous-ai-agents/hermes-agent/references/contributor-guide.md +++ b/skills/autonomous-ai-agents/hermes-agent/references/contributor-guide.md @@ -97,7 +97,7 @@ scripts/run_tests.sh -v --tb=long # pass-through pytest flags - Tests auto-redirect `HERMES_HOME` to temp dirs — never touch real `~/.hermes/`. - Prepare Python through the PM developer workflow before building a test environment. -- Run `python -m pm.build_env --source . --out .venv --extra dev --group test`. +- Run `python -m pm.build_env --source . --out .venv --group dev --group test`. The output must not exist. Stop its processes and intentionally remove only that disposable environment before regeneration. - The runner probes repository `.venv`, `venv`, and the standard source-install diff --git a/skills/autonomous-ai-agents/hermes-agent/references/windows-quirks.md b/skills/autonomous-ai-agents/hermes-agent/references/windows-quirks.md index 0f0e1ddd0c..b94595ed82 100644 --- a/skills/autonomous-ai-agents/hermes-agent/references/windows-quirks.md +++ b/skills/autonomous-ai-agents/hermes-agent/references/windows-quirks.md @@ -35,7 +35,7 @@ Prepare the checkout through PM first. With its Python 3.14, build an independen test environment at a fresh path: ```powershell -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test ``` The output must not exist. Before regeneration, stop its processes and explicitly diff --git a/skills/software-development/python-debugpy/SKILL.md b/skills/software-development/python-debugpy/SKILL.md index 66d0a9f304..d37e80a4c1 100644 --- a/skills/software-development/python-debugpy/SKILL.md +++ b/skills/software-development/python-debugpy/SKILL.md @@ -154,7 +154,7 @@ prepared checkout's Python: ```bash source ./activate -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test .venv/bin/python -c "import debugpy; print(debugpy.__file__)" ``` diff --git a/tests/gateway/conftest.py b/tests/gateway/conftest.py index 6f3665d81b..237da596da 100644 --- a/tests/gateway/conftest.py +++ b/tests/gateway/conftest.py @@ -518,8 +518,8 @@ def pytest_configure(config): # Concurrent subprocesses all hit pytest_configure simultaneously; # without a lock they'd all find no cache and all run the scan. # - # NOTE: filelock is NOT in CI's dependency closure (`uv sync --extra all - # --extra dev ...` does not pull it), so on CI the _NoLock fallback is + # NOTE: filelock is NOT in CI's app and dev/test dependency closure, + # so on CI the _NoLock fallback is # what actually runs. Correctness therefore cannot depend on the lock: # the cache write must be atomic and the read must tolerate a # not-yet-visible cache. Before the atomic-write fix, a reader could diff --git a/tests/hermes_cli/test_update_launch_completion.py b/tests/hermes_cli/test_update_launch_completion.py index 0e96f2b7c1..b02c3d64f3 100644 --- a/tests/hermes_cli/test_update_launch_completion.py +++ b/tests/hermes_cli/test_update_launch_completion.py @@ -192,7 +192,7 @@ def test_failed_launch_keeps_previous_completion_and_retries(tmp_path, monkeypat monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) fact = runtime_facts_path(root) fact.parent.mkdir(parents=True) - previous = '{"packages":{"venv":{"stamp":"previous","extras":["all","dev"]}}}' + previous = '{"packages":{"venv":{"stamp":"previous","extras":["all","anthropic"]}}}' fact.write_text(previous) monkeypatch.setattr(pm, "venv_is_current", lambda **kw: False) calls = [] diff --git a/tests/pm/test_activation_runtime.py b/tests/pm/test_activation_runtime.py index 3271b03dba..8e5b1078c5 100644 --- a/tests/pm/test_activation_runtime.py +++ b/tests/pm/test_activation_runtime.py @@ -32,7 +32,7 @@ def _sync_checkout(tmp_path: Path): with (root / "calls.jsonl").open("a", encoding="utf-8") as stream: stream.write(json.dumps(record) + "\\n") assert all(value is None for value in record["python_env"].values()), record - assert sys.argv[1:] == ["runtime-only"], record + assert sys.argv[1:] == ["runtime-only", "--trust-recorded", "--test-environment"], record print("setup progress") if (root / "fail").exists(): sys.exit(42) @@ -63,9 +63,9 @@ def _sync_checkout(tmp_path: Path): # Activation's contract with setup is the runtime-only switch; setup # itself maps that to PM's --trust-recorded install. (root / "setup-hermes.sh").write_text( - 'test "$#" = 1 && test "$1" = --runtime-only || exit 2\n' + 'test "$#" = 2 && test "$1" = --runtime-only && test "$2" = --test-environment || exit 2\n' f'cd {shlex.quote(str(root))} || exit 3\n' - f'exec {shlex.quote(str(python))} sync.py runtime-only\n', encoding="utf-8", + f'exec {shlex.quote(str(python))} sync.py runtime-only --trust-recorded --test-environment\n', encoding="utf-8", ) (root / "setup-hermes.ps1").write_text( "param([switch]$RuntimeOnly)\n" @@ -75,14 +75,14 @@ def _sync_checkout(tmp_path: Path): "argv=[Environment]::GetCommandLineArgs()}\n" "$record | ConvertTo-Json -Compress | Add-Content -LiteralPath \"$PSScriptRoot\\ps-calls.jsonl\"\n" "Set-Location -LiteralPath $PSScriptRoot\n" - f"& '{python}' sync.py runtime-only\nexit $LASTEXITCODE\n", encoding="utf-8", + f"& '{python}' sync.py runtime-only --trust-recorded --test-environment\nexit $LASTEXITCODE\n", encoding="utf-8", ) return root, env def _assert_syncs(root: Path): calls = [json.loads(line) for line in (root / "calls.jsonl").read_text(encoding="utf-8").splitlines()] - assert calls == [{"argv": ["runtime-only"], "python_env": { + assert calls == [{"argv": ["runtime-only", "--trust-recorded", "--test-environment"], "python_env": { "PYTHONHOME": None, "PYTHONPATH": None, "VIRTUAL_ENV": None, }}] * 3 assert (root / "builds").read_text(encoding="utf-8").splitlines() == ["first", "second"] diff --git a/tests/pm/test_activation_setup.py b/tests/pm/test_activation_setup.py index 56ed340bfa..8bb0202924 100644 --- a/tests/pm/test_activation_setup.py +++ b/tests/pm/test_activation_setup.py @@ -85,10 +85,13 @@ def test_activation_real_setup_pm_lifecycle(tmp_path, served): wheels = tmp_path / "wheels" wheels.mkdir() _wheel(wheels, "activation_dep", "1.0") + _wheel(wheels, "dev_fixture", "1.0") + _wheel(wheels, "test_fixture", "1.0") (core / "pyproject.toml").write_text( '[project]\nname="activation-proof"\nversion="1"\nrequires-python=">=3.11"\n' 'dependencies=["activation-dep==1.0"]\n[project.optional-dependencies]\nall=[]\n' - '[tool.uv]\npackage=false\nno-index=true\n' + '[dependency-groups]\ndev=["dev-fixture==1.0"]\ntest=["test-fixture==1.0"]\n' + '[tool.uv]\npackage=false\nno-index=true\ndefault-groups=[]\n' f'find-links=[{json.dumps(wheels.as_posix())}]\n', encoding="utf-8", ) locked = subprocess.run( @@ -160,7 +163,7 @@ def test_activation_real_setup_pm_lifecycle(tmp_path, served): script = ''' prior_path=$PATH prior_pythonpath=${PYTHONPATH-} -source "$1/activate" +source "$1/activate" --test-extras=all status=$? if [ "$status" != 0 ]; then test "$PATH" = "$prior_path" || exit 91 @@ -170,6 +173,8 @@ if [ "$status" != 0 ]; then exit "$status" fi python3 -c 'import activation_dep, json, os; print(json.dumps({"version": activation_dep.__version__, "module": activation_dep.__file__, "pythonpath": os.environ["PYTHONPATH"]}))' || exit 94 +"$__HERMES_TEST_PYTHON" -c 'import dev_fixture, test_fixture' || exit 97 +python3 -c 'import importlib.util; assert importlib.util.find_spec("dev_fixture") is None' || exit 98 deactivate test "$PATH" = "$prior_path" || exit 95 test "${PYTHONPATH-}" = "$prior_pythonpath" || exit 96 @@ -191,7 +196,8 @@ test "${PYTHONPATH-}" = "$prior_pythonpath" || exit 96 return [line for line in calls.read_text().splitlines() if line.split()[0] == name] def app_syncs(): - return [line for line in operations("sync") if "--frozen --all-packages" in line] + return [line for line in operations("sync") + if "--frozen --all-packages" in line and "--group dev" not in line] cold = activate() first = selection() diff --git a/tests/pm/test_install_default_closure.py b/tests/pm/test_install_default_closure.py index 54b73d7327..d1feeeb982 100644 --- a/tests/pm/test_install_default_closure.py +++ b/tests/pm/test_install_default_closure.py @@ -80,7 +80,7 @@ def test_trust_recorded_skips_the_byte_check_and_still_syncs(install_spy): @pytest.mark.parametrize("kwargs, message", [ ({"names": ["ripgrep"], "extra": [], "target": None, "tools_only": False, "trust_recorded": True}, "--trust-recorded"), - ({"names": None, "extra": ["dev"], "target": None, "tools_only": False, "trust_recorded": True}, + ({"names": None, "extra": ["anthropic"], "target": None, "tools_only": False, "trust_recorded": True}, "--trust-recorded"), ({"names": None, "extra": [], "target": "linux-x64", "tools_only": False, "trust_recorded": True}, "--target"), diff --git a/tests/pm/test_runtime_entrypoints.py b/tests/pm/test_runtime_entrypoints.py index faa1af868c..606431424e 100644 --- a/tests/pm/test_runtime_entrypoints.py +++ b/tests/pm/test_runtime_entrypoints.py @@ -33,24 +33,25 @@ def test_ci_dependency_phase_uses_isolated_runtime(tmp_path, monkeypatch): project = tmp_path / "source" project.mkdir() - (project / "pyproject.toml").write_text('[project]\nname="ci-proof"\nversion="1"\n[project.optional-dependencies]\ndev=[]\n') + (project / "pyproject.toml").write_text('[project]\nname="ci-proof"\nversion="1"\n' + '[dependency-groups]\ndev=[]\ntest=[]\n') monkeypatch.setattr(paths, "repo_root", lambda: project) monkeypatch.setattr(client, "is_runtime", lambda: False) calls = [] def request(operation, arguments, **kwargs): calls.append((operation, arguments)) - return str(tmp_path / "test-environment/bin/python") if operation == "build_environment" else None + return str(tmp_path / "test-environment/bin/python") if operation == "ensure_project_environment" else None monkeypatch.setattr(client, "_request", request) monkeypatch.setattr(setup_toolchain, "python3_alias", lambda _: None) monkeypatch.setattr(setup_toolchain, "file_commands", lambda *args: None) monkeypatch.setattr(setup_toolchain, "add_path", lambda *args: None) - setup_toolchain.dependencies(SimpleNamespace(home=tmp_path, extras=["dev"], toolchain="all")) - assert [operation for operation, _ in calls] == ["check_project_lock", "build_environment"] + setup_toolchain.dependencies(SimpleNamespace(home=tmp_path, extras=[], test_environment=True, toolchain="all")) + assert [operation for operation, _ in calls] == ["check_project_lock", "ensure_project_environment"] assert calls[0][1]["source"] == str(project) - assert calls[1][1]["extras"] == ["dev"] - assert calls[1][1]["groups"] == ["test"] + assert calls[1][1]["extras"] == [] + assert calls[1][1]["groups"] == ["dev", "test"] assert all(arguments["explicit"] for _, arguments in calls) diff --git a/tests/scripts/test_hermes_python_prologue.py b/tests/scripts/test_hermes_python_prologue.py index ad72f12e10..bad468ed6a 100644 --- a/tests/scripts/test_hermes_python_prologue.py +++ b/tests/scripts/test_hermes_python_prologue.py @@ -67,6 +67,7 @@ def checkout(tmp_path: Path) -> Path: (root / "shim").mkdir() (root / "state").mkdir() shutil.copy2(PROLOGUE, root / "scripts" / PROLOGUE.name) + shutil.copy2(REPO_ROOT / "scripts" / "_activation.sh", root / "scripts" / "_activation.sh") for name in ACTIVATION_INPUTS: (root / name).parent.mkdir(parents=True, exist_ok=True) (root / name).touch() @@ -94,9 +95,10 @@ def checkout(tmp_path: Path) -> Path: return root -def _record(root: Path) -> None: +def _record(root: Path, *, test_environment: bool = True) -> None: """What a successful ``pm install`` leaves behind.""" - record_activation_inputs(root / "state" / "inputs", activation_input_mtimes(root)) + record_activation_inputs(root / "state" / "inputs", activation_input_mtimes(root), root, + test_environment=test_environment) def _run(root: Path, sentinel: str | None = "{root}/state/facts.json") -> str: @@ -114,6 +116,13 @@ def _run(root: Path, sentinel: str | None = "{root}/state/facts.json") -> str: return result.stderr +def test_foreign_checkout_with_matching_input_times_reactivates(checkout: Path): + _record(checkout) + other = checkout.parent / "other" + shutil.copytree(checkout, other, copy_function=shutil.copy2) + assert "ACTIVATED" in _run(other, str(checkout / "state" / "facts.json")) + + def test_recorded_inputs_are_left_alone(checkout: Path): """A checkout rewrote every input after facts.json was last written, then a no-op sync recorded them: the environment is current. Re-syncing on every @@ -124,6 +133,15 @@ def test_recorded_inputs_are_left_alone(checkout: Path): assert "ACTIVATED" not in _run(checkout) +def test_runtime_only_install_cannot_mark_test_environment_current(checkout: Path): + marker = checkout / "state" / "inputs" / ".test-environment" + _record(checkout) + assert marker.is_file() + _record(checkout, test_environment=False) + assert not marker.exists() + assert "ACTIVATED" not in _run(checkout) # The app is current; the test env isn't. + + @pytest.mark.parametrize("moved_to", [JUST_AFTER, EARLIER], ids=["newer", "older"]) @pytest.mark.parametrize("input_name", ACTIVATION_INPUTS) def test_input_mtime_differing_from_its_stamp_reactivates(checkout: Path, input_name: str, moved_to: str): @@ -137,7 +155,5 @@ def test_input_mtime_differing_from_its_stamp_reactivates(checkout: Path, input_ @pytest.mark.parametrize("sentinel", [None, "{root}/gone", "1", "{root}/state/facts.json"], ids=["cold", "dangling", "legacy-1", "no-stamps"]) def test_unusable_sentinel_activates(checkout: Path, sentinel: str | None): - """Cold, dangling stamp, the bare ``1`` an older activate exported, or an - install from before stamps existed — each must activate rather than read - as current.""" + """Cold, dangling, or incomplete activation state must not read as current.""" assert "ACTIVATED" in _run(checkout, sentinel) diff --git a/tests/scripts/test_setup_toolchain.py b/tests/scripts/test_setup_toolchain.py index 756c0bd5a7..46389973a4 100644 --- a/tests/scripts/test_setup_toolchain.py +++ b/tests/scripts/test_setup_toolchain.py @@ -15,8 +15,8 @@ from pm.store import current_target from tests.pm._fixtures import build_worker, client, isolated_python # noqa: F401 -@pytest.mark.parametrize("extras", [[], ["dev"]], ids=["runtime", "tests"]) -def test_development_setup_keeps_test_groups_out_of_the_runtime(tmp_path, monkeypatch, extras, build_worker): +@pytest.mark.parametrize("test_environment", [False, True], ids=["runtime", "tests"]) +def test_development_setup_keeps_test_groups_out_of_the_runtime(tmp_path, monkeypatch, test_environment, build_worker): from types import SimpleNamespace import shutil @@ -30,11 +30,12 @@ def test_development_setup_keeps_test_groups_out_of_the_runtime(tmp_path, monkey wheels = tmp_path / "wheels" wheels.mkdir() _wheel(wheels, "test_only_dep", "1.0") + _wheel(wheels, "dev_only_dep", "1.0") (core / "pyproject.toml").write_text( '[project]\nname="ci-test-environment"\nversion="1"\nrequires-python=">=3.11"\n' - '[project.optional-dependencies]\ndev=[]\n' - '[dependency-groups]\ntest=["test-only-dep==1.0"]\n' - '[tool.uv]\npackage=false\nno-index=true\n' + '[project.optional-dependencies]\nall=[]\n' + '[dependency-groups]\ndev=["dev-only-dep==1.0"]\ntest=["test-only-dep==1.0"]\n' + '[tool.uv]\npackage=false\nno-index=true\ndefault-groups=[]\n' f'find-links=[{json.dumps(wheels.as_posix())}]\n', encoding="utf-8", ) lock_project(core, python=Path(sys.executable), offline=True, explicit=True) @@ -45,7 +46,7 @@ def test_development_setup_keeps_test_groups_out_of_the_runtime(tmp_path, monkey for name, file in files.items(): monkeypatch.setenv(name, str(file)) - setup_toolchain.dependencies(SimpleNamespace(extras=extras, home=home)) + setup_toolchain.dependencies(SimpleNamespace(extras=[], home=home, test_environment=test_environment)) outputs = dict(line.split("=", 1) for line in files["GITHUB_OUTPUT"].read_text(encoding="utf-8").splitlines()) result = subprocess.run( @@ -53,11 +54,30 @@ def test_development_setup_keeps_test_groups_out_of_the_runtime(tmp_path, monkey cwd=tmp_path, capture_output=True, text=True, timeout=30, ) assert result.returncode == 0, result.stderr - assert result.stdout.strip() == str("dev" in extras) + assert result.stdout.strip() == str(test_environment) + probe = subprocess.run( + [outputs["python-path"], "-I", "-c", + "import importlib.util; print(importlib.util.find_spec('dev_only_dep') is not None)"], + cwd=tmp_path, capture_output=True, text=True, timeout=30, + ) + assert probe.returncode == 0, probe.stderr + assert probe.stdout.strip() == str(test_environment) assert Path(outputs["venv"]).is_relative_to(home) assert not (core / ".venv").exists() from pm.environments import runtime_facts_path - assert runtime_facts_path(core).exists() == ("dev" not in extras) + assert runtime_facts_path(core).exists() != test_environment + if not test_environment: + from pm import build_environment + + bundle_python = build_environment(source=core, out=tmp_path / "bundle-env", + all_extras=True, no_install_project=True, explicit=True) + bundle = subprocess.run( + [bundle_python, "-I", "-c", + "import importlib.util; assert importlib.util.find_spec('dev_only_dep') is None; " + "assert importlib.util.find_spec('test_only_dep') is None"], + cwd=tmp_path, capture_output=True, text=True, timeout=30, + ) + assert bundle.returncode == 0, bundle.stderr @pytest.mark.parametrize("toolchain,names", [ diff --git a/tests/test_packaging_metadata.py b/tests/test_packaging_metadata.py index a59b8578b9..1b610a79a3 100644 --- a/tests/test_packaging_metadata.py +++ b/tests/test_packaging_metadata.py @@ -8,6 +8,17 @@ from packaging.version import Version REPO_ROOT = Path(__file__).resolve().parents[1] +def test_test_dependencies_are_group_only_in_manifest_and_lock(): + manifest = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + lock = tomllib.loads((REPO_ROOT / "uv.lock").read_text(encoding="utf-8")) + hermes = next(package for package in lock["package"] if package["name"] == manifest["project"]["name"]) + assert manifest["tool"]["uv"]["default-groups"] == [] + assert "dev" in manifest["dependency-groups"] + assert "dev" not in manifest["project"]["optional-dependencies"] + assert "dev" in hermes["dev-dependencies"] + assert "dev" not in hermes.get("optional-dependencies", {}) + + def test_core_and_optional_speech_dependencies(): project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] core = {Requirement(dep).name for dep in project["dependencies"]} @@ -32,6 +43,11 @@ def test_starlette_server_pins_and_lock_exclude_cve_2026_48710(): assert len(pins) == 1 and pins[0].operator == "==", (extra, requirement) assert Version(pins[0].version) >= floor, (extra, requirement) found.add(extra) - assert {"web", "mcp", "computer-use", "dev"} <= found + assert {"web", "mcp", "computer-use"} <= found + dev = [req for req in map(Requirement, metadata["dependency-groups"]["dev"]) + if req.name == "starlette"] + assert len(dev) == 1 + pins = list(dev[0].specifier) + assert len(pins) == 1 and pins[0].operator == "==" and Version(pins[0].version) >= floor versions = [Version(row["version"]) for row in lock["package"] if row["name"] == "starlette"] assert versions and all(version >= floor for version in versions) diff --git a/uv.lock b/uv.lock index 558c6e9520..9796699410 100644 --- a/uv.lock +++ b/uv.lock @@ -2103,17 +2103,6 @@ daytona = [ ddgs = [ { name = "ddgs" }, ] -dev = [ - { name = "debugpy" }, - { name = "httpx2" }, - { name = "mcp" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "ruff" }, - { name = "setuptools" }, - { name = "starlette" }, - { name = "ty" }, -] dingtalk = [ { name = "alibabacloud-dingtalk" }, { name = "dingtalk-stream" }, @@ -2314,6 +2303,17 @@ youtube = [ ] [package.dev-dependencies] +dev = [ + { name = "debugpy" }, + { name = "httpx2" }, + { name = "mcp" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "ruff" }, + { name = "setuptools" }, + { name = "starlette" }, + { name = "ty" }, +] icon-build = [ { name = "pillow" }, { name = "resvg-py" }, @@ -2345,7 +2345,6 @@ requires-dist = [ { name = "cryptography", marker = "python_full_version >= '3.14'", specifier = "==50.0.1" }, { name = "daytona", marker = "extra == 'daytona'", specifier = "==0.155.0" }, { name = "ddgs", marker = "extra == 'ddgs'", specifier = "==9.16.0" }, - { name = "debugpy", marker = "extra == 'dev'", specifier = "==1.8.20" }, { name = "defusedxml", marker = "extra == 'wecom'", specifier = "==0.7.1" }, { name = "dingtalk-stream", marker = "extra == 'dingtalk'", specifier = "==0.24.3" }, { name = "discord-py", extras = ["voice"], marker = "extra == 'discord'", specifier = "==2.7.1" }, @@ -2394,7 +2393,6 @@ requires-dist = [ { name = "httptools", marker = "python_full_version >= '3.14'", specifier = ">=0.6.3,<0.9" }, { name = "httpx", extras = ["socks"], marker = "python_full_version >= '3.14'", specifier = "==0.28.1" }, { name = "httpx2", marker = "extra == 'computer-use'", specifier = "==2.7.0" }, - { name = "httpx2", marker = "extra == 'dev'", specifier = "==2.7.0" }, { name = "httpx2", marker = "extra == 'mcp'", specifier = "==2.7.0" }, { name = "huggingface-hub", marker = "extra == 'trace-upload'", specifier = "==1.24.0" }, { name = "jinja2", marker = "python_full_version >= '3.14'", specifier = "==3.1.6" }, @@ -2404,7 +2402,6 @@ requires-dist = [ { name = "markdown", marker = "python_full_version >= '3.14'", specifier = "==3.10.2" }, { name = "mautrix", extras = ["encryption"], marker = "sys_platform == 'linux' and extra == 'matrix'", specifier = "==0.21.1" }, { name = "mcp", marker = "extra == 'computer-use'", specifier = "==2.0.0" }, - { name = "mcp", marker = "extra == 'dev'", specifier = "==2.0.0" }, { name = "mcp", marker = "extra == 'mcp'", specifier = "==2.0.0" }, { name = "mem0ai", marker = "(platform_machine != 'ARM64' and extra == 'mem0') or (sys_platform != 'win32' and extra == 'mem0')", specifier = "==2.0.10" }, { name = "microsoft-teams-apps", marker = "extra == 'teams'", specifier = "==2.0.13.4" }, @@ -2439,8 +2436,6 @@ requires-dist = [ { name = "pyopen-wakeword", marker = "(platform_machine != 'x86_64' and sys_platform == 'darwin' and extra == 'wake-openwakeword') or (platform_machine != 'ARM64' and sys_platform == 'win32' and extra == 'wake-openwakeword') or (sys_platform != 'darwin' and sys_platform != 'win32' and extra == 'wake-openwakeword')", specifier = "==1.1.0" }, { name = "pypinyin", marker = "extra == 'wake'", specifier = "==0.55.0" }, { name = "pypinyin", marker = "extra == 'wake-sherpa'", specifier = "==0.55.0" }, - { name = "pytest", marker = "extra == 'dev'", specifier = "==9.1.1" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = "==1.3.0" }, { name = "python-dotenv", marker = "python_full_version >= '3.14'", specifier = "==1.2.2" }, { name = "python-multipart", marker = "python_full_version >= '3.14'", specifier = ">=0.0.9,<1" }, { name = "python-multipart", marker = "extra == 'web'", specifier = "==0.0.32" }, @@ -2455,10 +2450,8 @@ requires-dist = [ { name = "requests", marker = "python_full_version >= '3.14'", specifier = "==2.33.0" }, { name = "rich", marker = "python_full_version >= '3.14'", specifier = "==14.3.3" }, { name = "ruamel-yaml", marker = "python_full_version >= '3.14'", specifier = "==0.18.16" }, - { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.10" }, { name = "sentencepiece", marker = "extra == 'wake'", specifier = "==0.2.2" }, { name = "sentencepiece", marker = "extra == 'wake-sherpa'", specifier = "==0.2.2" }, - { name = "setuptools", marker = "extra == 'dev'", specifier = "==83.0.0" }, { name = "sherpa-onnx", marker = "extra == 'wake'", specifier = "==1.13.8" }, { name = "sherpa-onnx", marker = "extra == 'wake-sherpa'", specifier = "==1.13.8" }, { name = "sherpa-onnx-core", marker = "extra == 'wake'", specifier = "==1.13.8" }, @@ -2473,14 +2466,12 @@ requires-dist = [ { name = "sounddevice", marker = "extra == 'wake'", specifier = "==0.5.5" }, { name = "soundfile", marker = "python_full_version < '3.13' and extra == 'kittentts'", specifier = "==0.14.0" }, { name = "starlette", marker = "extra == 'computer-use'", specifier = "==1.3.1" }, - { name = "starlette", marker = "extra == 'dev'", specifier = "==1.3.1" }, { name = "starlette", marker = "extra == 'mcp'", specifier = "==1.3.1" }, { name = "starlette", marker = "extra == 'web'", specifier = "==1.3.1" }, { name = "supermemory", marker = "extra == 'supermemory'", specifier = "==3.50.0" }, { name = "tenacity", marker = "python_full_version >= '3.14'", specifier = "==9.1.4" }, { name = "tomli-w", marker = "python_full_version >= '3.14'", specifier = "==1.2.0" }, { name = "truststore", marker = "python_full_version >= '3.14'", specifier = ">=0.10.4,<0.11" }, - { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.82" }, { name = "tzdata", marker = "python_full_version >= '3.14' and sys_platform == 'win32'", specifier = "==2025.3" }, { name = "urllib3", marker = "python_full_version >= '3.14'", specifier = ">=2.7.0,<3" }, { name = "uvicorn", marker = "python_full_version >= '3.14'", specifier = ">=0.31.0,<1" }, @@ -2496,9 +2487,20 @@ requires-dist = [ { name = "winrt-windows-services-store", marker = "python_full_version >= '3.14' and sys_platform == 'win32'", specifier = ">=3.2.1,<4" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["uvloop", "anthropic", "exa", "firecrawl", "parallel-web", "ddgs", "fal", "edge-tts", "neutts", "kittentts", "piper", "modal", "daytona", "vercel", "hindsight", "google-meet", "dev", "messaging", "cron", "slack", "matrix", "wecom", "tts-premium", "voice", "wake", "honcho", "telegram", "discord", "stt-whisper", "audio-io", "silk", "wake-openwakeword", "wake-sherpa", "wake-porcupine", "google-chat", "doc-extract", "trace-upload", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "langfuse", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["uvloop", "anthropic", "exa", "firecrawl", "parallel-web", "ddgs", "fal", "edge-tts", "neutts", "kittentts", "piper", "modal", "daytona", "vercel", "hindsight", "google-meet", "messaging", "cron", "slack", "matrix", "wecom", "tts-premium", "voice", "wake", "honcho", "telegram", "discord", "stt-whisper", "audio-io", "silk", "wake-openwakeword", "wake-sherpa", "wake-porcupine", "google-chat", "doc-extract", "trace-upload", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "langfuse", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] [package.metadata.requires-dev] +dev = [ + { name = "debugpy", specifier = "==1.8.20" }, + { name = "httpx2", specifier = "==2.7.0" }, + { name = "mcp", specifier = "==2.0.0" }, + { name = "pytest", specifier = "==9.1.1" }, + { name = "pytest-asyncio", specifier = "==1.3.0" }, + { name = "ruff", specifier = "==0.15.10" }, + { name = "setuptools", specifier = "==83.0.0" }, + { name = "starlette", specifier = "==1.3.1" }, + { name = "ty", specifier = "==0.0.82" }, +] icon-build = [ { name = "pillow", specifier = "==12.3.0" }, { name = "resvg-py", specifier = "==0.4.0" }, diff --git a/website/docs/developer-guide/contributing.md b/website/docs/developer-guide/contributing.md index 1a0585ff5b..6a556e4d41 100644 --- a/website/docs/developer-guide/contributing.md +++ b/website/docs/developer-guide/contributing.md @@ -76,7 +76,7 @@ architecture before building source dependencies. Build an independent interpreter for tests and editor tools: ```bash -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test ``` PM builds from the committed lock and checks dependency consistency before diff --git a/website/docs/getting-started/nix-setup.md b/website/docs/getting-started/nix-setup.md index e0637b8ede..9ef9246ed5 100644 --- a/website/docs/getting-started/nix-setup.md +++ b/website/docs/getting-started/nix-setup.md @@ -919,7 +919,7 @@ A build-time collision check prevents plugin packages from shadowing core hermes ### Dev Shell The flake provides an editable Python environment with the lock-derived interpreter -and the `dev` extra. `HERMES_PYTHON` points to its interpreter. It does not install +and the `dev` dependency group. `HERMES_PYTHON` points to its interpreter. It does not install Python dependencies into a repository-local `.venv`. The shell also provides Node.js and runtime tools. Its npm hook refreshes JS workspaces when their inputs change. diff --git a/website/docs/reference/package-management.md b/website/docs/reference/package-management.md index 6bc2a0bba9..f471ce7825 100644 --- a/website/docs/reference/package-management.md +++ b/website/docs/reference/package-management.md @@ -404,13 +404,14 @@ Managed tool names and Python extra names are different interfaces: ```bash python -m pm.cli install chromium -python -c "from pm import sync_venv; sync_venv(['dev'], explicit=True)" +python -c "from pm import sync_venv; sync_venv(['anthropic'], explicit=True)" ``` -The first command installs a tool. The second adds the declared `dev` extra +The first command installs a tool. The second adds a declared runtime extra to this installation's existing Python selection. Extras accumulate through PM -sync. `pm install dev` is not a supported command: `dev` is an extra, not a tool. -After changing extras, reactivate before starting another Python process. +sync. The `dev` and `test` dependency groups belong only to the separate test +environment, not the selected application venv. After changing extras, reactivate +before starting another Python process. For a new project dependency, edit `pyproject.toml` and regenerate `uv.lock`: @@ -425,25 +426,27 @@ workspaces, and do not install packages directly into a selected generation. ### Test and editor environments -PM's `dev` extra does not make a bare store Python suitable for the canonical -test runner. The runner clears `PYTHONPATH` and needs an interpreter with pytest -installed in its own environment. Use the contributor guide's -[independent test environment](../developer-guide/contributing.md#manual-development-and-test-environment) -with this command from the prepared checkout: +`source ./activate` (or `. .\activate.ps1` in PowerShell) and both direct +`setup-hermes` scripts prepare an isolated test interpreter from the locked +`dev` and `test` dependency groups. `scripts/run_tests.sh` uses that interpreter, +re-activating if the checkout or its dependency inputs changed. The application +venv, installers, and bundles select neither group. The developer default +covers `[all]`; to change test coverage, pass `--test-extras=anthropic` to POSIX +activation or `-TestExtras anthropic` to PowerShell; +those arguments select runtime extras *in the test interpreter only*. + +In an isolated environment where activation is unavailable (for example, a Nix +dev shell), a caller can explicitly supply `HERMES_PYTHON` with pytest, or build +an independent disposable environment: ```bash -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test ``` The output must not exist. To regenerate it, stop its processes and intentionally -remove only that disposable environment first. PM never deletes an existing -output. Then run `scripts/run_tests.sh` (through Bash on Windows). The `test` -dependency group includes native launcher tests and does not enter a packaged runtime. - -The runner checks repository `.venv`, repository `venv`, and the standard -source-install venv before using `HERMES_PYTHON` as a fallback. Read its startup -message to confirm which interpreter it selected. A worktree without a local -venv can use the independent test interpreter through that variable. +remove only that disposable environment first. Then run `scripts/run_tests.sh` +(through Bash on Windows). The test dependency group includes native launcher +tests and never enters a packaged runtime. For editor debugging, select that independent interpreter, set the working directory to this checkout, and launch `hermes` as the script. Keep its diff --git a/website/docs/user-guide/skills/bundled/software-development/software-development-python-debugpy.md b/website/docs/user-guide/skills/bundled/software-development/software-development-python-debugpy.md index 8c41f50831..d0d8fe7b4b 100644 --- a/website/docs/user-guide/skills/bundled/software-development/software-development-python-debugpy.md +++ b/website/docs/user-guide/skills/bundled/software-development/software-development-python-debugpy.md @@ -172,7 +172,7 @@ prepared checkout's Python: ```bash source ./activate -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test .venv/bin/python -c "import debugpy; print(debugpy.__file__)" ``` diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/contributing.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/contributing.md index 73994f434b..c3a78cdb81 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/contributing.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/developer-guide/contributing.md @@ -68,7 +68,7 @@ PowerShell 开头的点和空格用于 dot-source，不能省略。 PM 必须能够启动，才能构建独立测试环境： ```bash -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test ``` 此命令使用提交的锁文件，创建新环境并检查依赖一致性。输出路径必须不存在。 diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/nix-setup.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/nix-setup.md index e527870623..4c5af5c989 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/nix-setup.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/getting-started/nix-setup.md @@ -727,7 +727,7 @@ services.hermes-agent.settings.plugins.enabled = [ ### 开发 Shell -该 flake 提供包含 `dev` extra 的可编辑 Python 环境，解释器主/次版本来自 PM 锁文件。 +该 flake 提供包含 `dev` 依赖组的可编辑 Python 环境，解释器主/次版本来自 PM 锁文件。 `HERMES_PYTHON` 指向该解释器，不会把 Python 依赖安装到仓库内的 `.venv`。 shell 还提供 Node.js 和运行时工具。npm hook 根据输入变更刷新 JS workspaces。 diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/skills/bundled/software-development/software-development-python-debugpy.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/skills/bundled/software-development/software-development-python-debugpy.md index c5f1669016..de00b40034 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/skills/bundled/software-development/software-development-python-debugpy.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/skills/bundled/software-development/software-development-python-debugpy.md @@ -168,7 +168,7 @@ Python 构建全新的调试/测试环境： ```bash source ./activate -python -m pm.build_env --source . --out .venv --extra dev --group test +python -m pm.build_env --source . --out .venv --group dev --group test .venv/bin/python -c "import debugpy; print(debugpy.__file__)" ```