From 438d10200cf7b481b05dd194dc8718ff7733fbdf Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:26:48 -0700 Subject: [PATCH] ci: run the SSH and Docker terminal backends against a real sshd and daemon The SSH backend's real-host tests skipped on every CI run (nothing set TERMINAL_SSH_HOST), and every Docker backend test mocked subprocess or the docker CLI, so no CI run had executed either backend end to end. - remote-backends.yml: starts a throwaway sshd container on 127.0.0.1 with a per-run keypair (scripts/ci/start_test_sshd.sh) and runs test_ssh_environment.py plus the new test_docker_backend_real.py against it and the runner's Docker daemon with a digest-pinned debian:12-slim. A file that skips fails the job, so the lane cannot go green over zero coverage. - New classifier lane `remote_backends` (tools/environments/, tools/terminal_tool*, the real tests, the sshd harness, run_tests.sh). ci.yaml calls the workflow only when it fires, plus every push to main; required_results makes a selected lane mandatory, like the update consumers. - The real tests read HERMES_TEST_SSH_* / HERMES_TEST_DOCKER_IMAGE (forwarded by run_tests.sh), because the hermetic conftest blanks TERMINAL_SSH_KEY as a credential before any test can read it. - New real SSH cases: exit code/stderr, cwd persistence with spaces vs per-call workdir, quoting across the ssh hop, SendEnv passthrough through a real AcceptEnv, file tools write/read on the remote. - New real Docker cases: exec + exit codes, cwd vs workdir, exported env persistence, docker_env/docker_forward_env reach the container and nothing else does, quoting, file tools write/read, /root surviving a container replacement. - The existing one-shot test is xfail(strict): TERMINAL_SSH_PERSISTENT=false is read into ssh_config["persistent"] but SSHEnvironment never consumes it, so every SSH session persists exported env. Its _cleanup also cleaned the raw task id, not the resolved env key, so one test's env leaked into the next. --- .github/actions/detect-changes/action.yml | 3 + .github/workflows/ci.yaml | 12 ++ .github/workflows/remote-backends.yml | 77 +++++++++++ scripts/ci/classify_changes.py | 18 +++ scripts/ci/required_results.py | 8 +- scripts/ci/start_test_sshd.sh | 73 ++++++++++ scripts/run_tests.sh | 8 +- tests/ci/test_classify_changes.py | 36 ++++- tests/ci/test_required_results.py | 1 + tests/tools/test_docker_backend_real.py | 155 ++++++++++++++++++++++ tests/tools/test_ssh_environment.py | 81 ++++++++++- 11 files changed, 460 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/remote-backends.yml create mode 100755 scripts/ci/start_test_sshd.sh create mode 100644 tests/tools/test_docker_backend_real.py diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index 3c17a72fcb..906d24a0e2 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -66,6 +66,9 @@ outputs: rust: description: Run `cargo test` for the Tauri bootstrap installer. value: ${{ steps.classify.outputs.rust }} + remote_backends: + description: Run the SSH and Docker terminal backends against a real sshd and Docker daemon. + value: ${{ steps.classify.outputs.remote_backends }} runs: using: composite diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index acb543f6be..d0dc83f232 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -80,6 +80,7 @@ jobs: bootstrap: ${{ steps.gate-lanes.outputs.bootstrap }} desktop_updater: ${{ steps.gate-lanes.outputs.desktop_updater }} rust: ${{ steps.gate-lanes.outputs.rust }} + remote_backends: ${{ steps.gate-lanes.outputs.remote_backends }} docker_meta: ${{ steps.gate-lanes.outputs.docker_meta }} e2e: ${{ steps.gate-lanes.outputs.e2e }} e2e_upgrade: ${{ steps.gate-lanes.outputs.e2e_upgrade }} @@ -185,6 +186,16 @@ jobs: if: needs.detect.outputs.rust == 'true' uses: ./.github/workflows/rust-tests.yml + remote-backends: + name: Remote terminal backends + needs: detect + # The SSH and Docker terminal backends against a real sshd container and the + # runner's Docker daemon. Only PRs that touch the backends / terminal tool / + # their real tests start it (and every push to main); it runs in parallel with + # the other lanes and is required whenever detect selects it. + if: needs.detect.outputs.remote_backends == 'true' + uses: ./.github/workflows/remote-backends.yml + bootstrap-installer: name: Bootstrap installer needs: detect @@ -351,6 +362,7 @@ jobs: - lint - js-tests - rust-tests + - remote-backends - bootstrap-installer - e2e-desktop - e2e-desktop-core diff --git a/.github/workflows/remote-backends.yml b/.github/workflows/remote-backends.yml new file mode 100644 index 0000000000..60740af3a2 --- /dev/null +++ b/.github/workflows/remote-backends.yml @@ -0,0 +1,77 @@ +name: Remote terminal backends + +# The SSH and Docker terminal backends against a real sshd and a real Docker +# daemon. Every other test of these backends mocks subprocess / the docker CLI; +# the gated real-backend tests in tests/tools/test_ssh_environment.py skipped on +# every CI run because nothing provided a host. +# +# This job provides one: a throwaway sshd container on 127.0.0.1 that accepts a +# keypair generated for this run only (scripts/ci/start_test_sshd.sh), plus the +# runner's own Docker daemon with a small public image. No secrets are involved +# and none may be added: ci.yaml runs PR-controlled code. +# +# ci.yaml calls this only when a PR touches the backends (the classifier's +# `remote_backends` lane) and on every push to main. + +on: + workflow_call: + +permissions: + contents: read + +concurrency: + group: remote-backends-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} + +env: + # One small image for both the sshd host and the docker backend's sandbox, pinned + # by digest so a tag move cannot change what the lane tests. + HERMES_TEST_IMAGE_REF: debian:12-slim@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595db4a9d65c06ccc587 + +jobs: + real-backends: + name: SSH + Docker backends (real sshd, real daemon) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1 + + - name: Pull the test image + uses: ./.github/actions/retry + with: + command: docker pull -q "$HERMES_TEST_IMAGE_REF" + + - name: Start a throwaway sshd + # Exports HERMES_TEST_SSH_{HOST,USER,PORT,KEY} through $GITHUB_ENV. + env: + HERMES_TEST_SSHD_IMAGE: ${{ env.HERMES_TEST_IMAGE_REF }} + run: scripts/ci/start_test_sshd.sh "$RUNNER_TEMP/hermes-test-sshd" + + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm + with: + extras: '[]' + test-environment: 'true' + prune-python-cache: true + + - name: Run the real-backend tests + env: + HERMES_TEST_DOCKER_IMAGE: ${{ env.HERMES_TEST_IMAGE_REF }} + OPENROUTER_API_KEY: "" + OPENAI_API_KEY: "" + NOUS_API_KEY: "" + run: | + set -o pipefail + scripts/run_tests.sh tests/tools/test_ssh_environment.py tests/tools/test_docker_backend_real.py \ + | tee "$RUNNER_TEMP/remote-backends.log" + # The real tests skip when their host is missing; here that would be a green + # lane over zero coverage. The runner prints a file's skip count as "s,". + if grep -E '\((.* )?[0-9]+s,' "$RUNNER_TEMP/remote-backends.log"; then + echo "::error::real-backend tests skipped: the sshd or docker image was not wired through" + exit 1 + fi + + - name: sshd log + if: failure() + run: docker logs hermes-test-sshd || true diff --git a/scripts/ci/classify_changes.py b/scripts/ci/classify_changes.py index 4799e778fa..4ded763dee 100644 --- a/scripts/ci/classify_changes.py +++ b/scripts/ci/classify_changes.py @@ -40,6 +40,10 @@ Lanes: * ``rust`` — ``cargo test`` for the Tauri bootstrap installer. ``.rs`` lives under ``apps/``, so without this lane a Rust change matched ``frontend`` and only the TypeScript matrix ran. +* ``remote_backends`` — the SSH and Docker terminal backends against a real + sshd container and the runner's Docker daemon (remote-backends.yml). Every + other test of those backends is mocked, so a change to the backends, the + terminal tool, or the real-backend tests runs it on the PR. ``docker``, ``nix`` and the E2E lanes take most of the larger-runner minutes. An ordinary product change does not start them on a pull request. Every push @@ -185,6 +189,18 @@ _FIXTURE_CONSUMER_LANES = ("python", "rust", "frontend", "desktop_updater") _RUST_PATHS = ("apps/bootstrap-installer/src-tauri/",) _RUST_FILENAMES = {"Cargo.toml", "Cargo.lock"} +# The real SSH/Docker backend lane: the backends, the terminal tool that drives +# them, the tests it runs, the sshd it starts, and the runner's env allowlist +# that carries the sshd/image coordinates into each pytest process. +_REMOTE_BACKEND_PATHS = ( + "tools/environments/", + "tools/terminal_tool", + "tests/tools/test_ssh_environment.py", + "tests/tools/test_docker_backend_real.py", + "scripts/ci/start_test_sshd.sh", + "scripts/run_tests.sh", +) + # The slow lanes and the paths that start each one on a pull request. A path # belongs here when the lane is the only CI that exercises it for real: the # suite itself, its harness, and the product code the suite exists to guard. @@ -583,6 +599,7 @@ def classify(files: list[str], run_e2e: bool = False) -> dict[str, bool]: ), "desktop_updater": any(_is_desktop_updater(f) for f in files), "rust": any(_is_rust(f) for f in files), + "remote_backends": any(f.startswith(_REMOTE_BACKEND_PATHS) for f in files), **{lane: run_e2e or on for lane, on in _slow_lanes(files).items()}, } consumers = [c for f in files for c in _SHARED_FIXTURE_CONSUMERS.get(f, ())] @@ -603,6 +620,7 @@ def classify(files: list[str], run_e2e: bool = False) -> dict[str, bool]: ret["bootstrap"] = True ret["desktop_updater"] = True ret["rust"] = True + ret["remote_backends"] = True ret.update(dict.fromkeys(_slow_lanes([]), True)) return ret diff --git a/scripts/ci/required_results.py b/scripts/ci/required_results.py index 8835c95e1b..1e6dbd014c 100644 --- a/scripts/ci/required_results.py +++ b/scripts/ci/required_results.py @@ -29,13 +29,15 @@ EXCLUDED_JOBS = frozenset((*PR_ONLY_JOBS, *DEFERRED_JOBS)) NEEDS_JSON_OUTPUT = "needs-json" -# Reusable-workflow parents of the update/E2E consumers. The actual detect -# outputs already arrive in toJSON(needs); a selected consumer cannot skip -# merely because this is a PR rather than a release. +# Reusable-workflow parents of the update/E2E consumers, and of the real +# remote-backend lane. The actual detect outputs already arrive in +# toJSON(needs); a selected consumer cannot skip merely because this is a PR +# rather than a release. _UPDATE_CONSUMER_LANES = { "tests": ("python", "e2e", "e2e_upgrade"), "tests-os": ("python", "desktop_updater", "e2e", "e2e_upgrade"), "e2e-desktop-update": ("e2e_desktop_update",), + "remote-backends": ("remote_backends",), } diff --git a/scripts/ci/start_test_sshd.sh b/scripts/ci/start_test_sshd.sh new file mode 100755 index 0000000000..aff7983457 --- /dev/null +++ b/scripts/ci/start_test_sshd.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Start a throwaway sshd container for the real SSH terminal-backend tests. +# +# scripts/ci/start_test_sshd.sh [port] +# +# Generates (once per state dir) a client keypair and a host key, starts a +# Debian sshd on 127.0.0.1: (default 2222) that accepts only that key for +# user `hermes`, waits until it answers, and prints the HERMES_TEST_SSH_* +# variables tests/tools/test_ssh_environment.py reads (appended to $GITHUB_ENV +# too when it is set). `docker rm -f hermes-test-sshd` stops it. +# +# The image is a plain Debian with GNU tar and bash, the shape of a real remote +# host (the backend's file sync uses `tar --no-overwrite-dir`). AcceptEnv lets +# the SendEnv passthrough test reach the remote shell. +set -euo pipefail + +state_dir="${1:?usage: start_test_sshd.sh [port]}" +port="${2:-2222}" +image="${HERMES_TEST_SSHD_IMAGE:-debian:12-slim}" +name="hermes-test-sshd" + +mkdir -p "$state_dir" +state_dir="$(cd "$state_dir" && pwd)" +[ -f "$state_dir/id_ed25519" ] || ssh-keygen -q -t ed25519 -N '' -C hermes-test -f "$state_dir/id_ed25519" +[ -f "$state_dir/host_ed25519" ] || ssh-keygen -q -t ed25519 -N '' -C hermes-test-host -f "$state_dir/host_ed25519" +cp "$state_dir/id_ed25519.pub" "$state_dir/authorized_keys" + +docker rm -f "$name" >/dev/null 2>&1 || true +docker run -d --name "$name" -p "127.0.0.1:${port}:22" \ + -v "$state_dir:/keys:ro" "$image" bash -c ' + set -e + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq --no-install-recommends openssh-server >/dev/null + install -m 600 /keys/host_ed25519 /etc/ssh/ssh_host_ed25519_key + install -m 644 /keys/host_ed25519.pub /etc/ssh/ssh_host_ed25519_key.pub + useradd -m -s /bin/bash hermes + usermod -p "*" hermes + install -d -m 700 -o hermes -g hermes /home/hermes/.ssh + install -m 600 -o hermes -g hermes /keys/authorized_keys /home/hermes/.ssh/authorized_keys + printf "%s\n" "HostKey /etc/ssh/ssh_host_ed25519_key" "PasswordAuthentication no" \ + "AcceptEnv HERMES_RB_*" > /etc/ssh/sshd_config.d/hermes-test.conf + mkdir -p /run/sshd + exec /usr/sbin/sshd -D -e + ' >/dev/null + +for _ in $(seq 1 120); do + if ssh -i "$state_dir/id_ed25519" -p "$port" -o BatchMode=yes -o ConnectTimeout=2 \ + -o StrictHostKeyChecking=accept-new hermes@127.0.0.1 true 2>/dev/null; then + ready=1 + break + fi + if [ "$(docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null)" != true ]; then + docker logs "$name" >&2 || true + echo "sshd container exited before accepting connections" >&2 + exit 1 + fi + sleep 1 +done +if [ -z "${ready:-}" ]; then + docker logs "$name" >&2 || true + echo "sshd did not accept the test key within 120s" >&2 + exit 1 +fi + +env_lines="HERMES_TEST_SSH_HOST=127.0.0.1 +HERMES_TEST_SSH_USER=hermes +HERMES_TEST_SSH_PORT=$port +HERMES_TEST_SSH_KEY=$state_dir/id_ed25519" +if [ -n "${GITHUB_ENV:-}" ]; then + printf '%s\n' "$env_lines" >> "$GITHUB_ENV" +fi +printf '%s\n' "$env_lines" diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 0e73010e65..36d7abffbf 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -144,6 +144,11 @@ _pf86="$(env | sed -n 's/^ProgramFiles(x86)=//p' | head -n1)" # their user profiles and the transcripts CI uploads. # * HERMES_E2E_STRICT_ACCEPTANCE makes tests/e2e/core/_pending_fixes.py's # known gaps hard failures in an acceptance run (workflow dispatch input). +# * HERMES_TEST_SSH_HOST / _USER / _PORT / _KEY and HERMES_TEST_DOCKER_IMAGE +# arm the real-backend tests (tests/tools/test_ssh_environment.py, +# tests/tools/test_docker_backend_real.py); remote-backends.yml points them at +# a throwaway sshd container and a small public image. The key is a +# per-run throwaway, never a credential. # # These are test-infrastructure knobs, not credentials — same class as the # HERMES_RUN_SLOW_PET_TESTS / HERMES_E2E_BROWSER / HERMES_RUN_E2E opt-ins @@ -158,7 +163,8 @@ for _test_var in HERMES_TEST_IMAGE HERMES_TEST_WORKERS HERMES_TEST_PATHS \ HERMES_TEST_FILE_TIMEOUT HERMES_TEST_FILE_RETRIES HERMES_TEST_SLICE \ SSL_CERT_FILE SSL_CERT_DIR HERMES_GATEWAY_LOCK_DIR HERMES_E2E_REQUIRE_TUI CI GITHUB_ACTIONS \ HERMES_E2E_WINDOWS_INSTALL HERMES_E2E_MACHINE_ROOT HERMES_E2E_PROFILES_ROOT HERMES_E2E_ARTIFACTS \ - HERMES_E2E_STRICT_ACCEPTANCE; do + HERMES_E2E_STRICT_ACCEPTANCE HERMES_TEST_SSH_HOST HERMES_TEST_SSH_USER HERMES_TEST_SSH_PORT \ + HERMES_TEST_SSH_KEY HERMES_TEST_DOCKER_IMAGE; do if [ -n "${!_test_var:-}" ]; then TEST_ENV+=("$_test_var=${!_test_var}") fi diff --git a/tests/ci/test_classify_changes.py b/tests/ci/test_classify_changes.py index 15c65e419d..0f490f2d56 100644 --- a/tests/ci/test_classify_changes.py +++ b/tests/ci/test_classify_changes.py @@ -47,12 +47,13 @@ DEFAULT = { "bootstrap": True, "desktop_updater": True, "rust": True, + "remote_backends": True, } SLOW_LANES = {"docker", "nix", "e2e", "e2e_upgrade", "e2e_desktop_core", "e2e_desktop_update"} -def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_lock=False, npm_lock=False, bootstrap=False, desktop_updater=False, rust=False, docker_meta=False, python_prod=None, nix=False, docker=None, e2e=False, e2e_upgrade=False, e2e_desktop_core=False, e2e_desktop_update=False) -> dict[str, bool]: +def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_lock=False, npm_lock=False, bootstrap=False, desktop_updater=False, rust=False, docker_meta=False, python_prod=None, nix=False, docker=None, e2e=False, e2e_upgrade=False, e2e_desktop_core=False, e2e_desktop_update=False, remote_backends=False) -> dict[str, bool]: # python_prod tracks python except for tests-only diffs; default it to # python so the majority of cases don't need to spell it out. # @@ -80,6 +81,7 @@ def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_ "bootstrap": bootstrap, "desktop_updater": desktop_updater, "rust": rust, + "remote_backends": remote_backends, } @@ -323,6 +325,22 @@ CASES = { ), # Slow lanes on a pull request: an ordinary product change starts none of # them; the suite, its harness, or the code it guards starts its own. + # Real SSH/Docker backend lane: the backends, the terminal tool, the real tests + # and their sshd harness start it; neighbouring tools code does not. + "ssh backend → remote_backends": ( + ["tools/environments/ssh.py"], _lanes(python=True, scan=True, remote_backends=True)), + "docker backend → remote_backends": ( + ["tools/environments/docker.py"], _lanes(python=True, scan=True, remote_backends=True)), + "terminal tool sibling → remote_backends": ( + ["tools/terminal_tool_backends.py"], _lanes(python=True, scan=True, remote_backends=True)), + "real docker backend test → remote_backends": ( + ["tests/tools/test_docker_backend_real.py"], + _lanes(python=True, python_prod=False, scan=True, remote_backends=True)), + "mocked docker unit test → no remote_backends": ( + ["tests/tools/test_docker_environment.py"], _lanes(python=True, python_prod=False, scan=True)), + "sshd harness → remote_backends": ( + ["scripts/ci/start_test_sshd.sh"], _lanes(python=True, remote_backends=True)), + "other tool → no remote_backends": (["tools/file_tools.py"], _lanes(python=True, scan=True)), "gateway python → no slow lane": (["gateway/run.py", "agent/x.py"], _lanes(python=True, scan=True)), "desktop renderer → no slow lane": (["apps/desktop/src/app/chat/composer.tsx"], _lanes(frontend=True)), "python e2e suite → e2e only": ( @@ -421,7 +439,8 @@ def test_every_slow_lane_path_matches_a_tracked_file(): if tracked.returncode != 0: pytest.skip("not a git checkout") files = tracked.stdout.splitlines() - tables = {**_mod._E2E_LANES, "docker": _mod._DOCKER_PATHS, "nix": _mod._NIX_LANE_PATHS} + tables = {**_mod._E2E_LANES, "docker": _mod._DOCKER_PATHS, "nix": _mod._NIX_LANE_PATHS, + "remote_backends": _mod._REMOTE_BACKEND_PATHS} dead = { (lane, prefix) for lane, prefixes in tables.items() @@ -557,6 +576,8 @@ _REAL_UPDATE_LANES = ( ("ci.yaml", "tests-os", "tests-os.yml", "e2e_upgrade"), ("ci.yaml", "e2e-desktop-update", "e2e-desktop-update.yml", "e2e_desktop_update"), ("tests-os.yml", "install-update-e2e", "windows-install-update-e2e.yml", "e2e_upgrade"), + # Not an update suite, but the same contract: the real SSH/Docker backend lane. + ("ci.yaml", "remote-backends", "remote-backends.yml", "remote_backends"), ) @@ -579,6 +600,17 @@ def test_real_update_suites_gate_the_merge_when_their_lane_fires(caller, job, ca assert not body.get("continue-on-error"), f"{called}::{name} cannot fail the merge" +def test_remote_backends_lane_runs_test_files_that_exist_and_start_it(): + """The lane's test list and its path gate must agree: a renamed test file would + otherwise drop silently out of the run (run_tests.sh skips a missing path) or stop + starting the lane on the PR that edits it.""" + text = (_REPO / ".github/workflows/remote-backends.yml").read_text(encoding="utf-8-sig") + paths = sorted(set(re.findall(r"(tests/[\w/.-]+\.py)", text))) + assert paths + assert [p for p in paths if not (_REPO / p).is_file()] == [] + assert [p for p in paths if not classify([p])["remote_backends"]] == [] + + def test_windows_venv_e2e_runs_only_test_files_that_exist(): """``run_tests.sh`` drops a missing path (or a ``::node`` selector) without a word, so a stale entry silently stops running on every wine2e push.""" diff --git a/tests/ci/test_required_results.py b/tests/ci/test_required_results.py index f7ef6b2e32..22ef268cf0 100644 --- a/tests/ci/test_required_results.py +++ b/tests/ci/test_required_results.py @@ -41,6 +41,7 @@ def test_status_policy(job, result, ordinary, excluded, release): ("e2e_upgrade", ("tests", "tests-os")), ("desktop_updater", ("tests-os",)), ("e2e_desktop_update", ("e2e-desktop-update",)), + ("remote_backends", ("remote-backends",)), ]) @pytest.mark.parametrize("result", ["skipped", "absent", "success"]) def test_selected_update_consumers_must_succeed(lane, jobs, result): diff --git a/tests/tools/test_docker_backend_real.py b/tests/tools/test_docker_backend_real.py new file mode 100644 index 0000000000..b8c1077649 --- /dev/null +++ b/tests/tools/test_docker_backend_real.py @@ -0,0 +1,155 @@ +"""The Docker terminal backend against a real daemon and a real container. + +Every other Docker backend test mocks ``subprocess`` or the docker CLI. These run +``terminal_tool`` and the file tools with ``TERMINAL_ENV=docker`` end to end, so +a broken ``docker run``/``docker exec`` argv, snapshot bootstrap, cwd tracking or +env forwarding fails here instead of on a user's machine. + +Armed by ``HERMES_TEST_DOCKER_IMAGE`` (a small public image with bash, e.g. +``debian:12-slim``); the remote-backends CI lane sets it. Skipped otherwise. Once +armed a missing daemon FAILS rather than skips: a lane that reports green over +zero coverage is the state this file exists to end. +""" + +import json +import os +import shlex +import subprocess +import uuid + +import pytest + +_IMAGE = os.getenv("HERMES_TEST_DOCKER_IMAGE", "") + +pytestmark = pytest.mark.skipif(not _IMAGE, reason="HERMES_TEST_DOCKER_IMAGE not set") + + +@pytest.fixture +def docker_task(monkeypatch): + """A fresh docker-backed task: own container, removed (not left running) afterwards.""" + monkeypatch.setenv("TERMINAL_ENV", "docker") + monkeypatch.setenv("TERMINAL_DOCKER_IMAGE", _IMAGE) + monkeypatch.setenv("TERMINAL_DOCKER_IMAGE_PINNED", "1") + # Remove the container at cleanup instead of leaving it for cross-process reuse, + # and never sweep the host's other hermes containers. + monkeypatch.setenv("TERMINAL_DOCKER_PERSIST_ACROSS_PROCESSES", "false") + monkeypatch.setenv("TERMINAL_DOCKER_ORPHAN_REAPER", "false") + monkeypatch.setenv("TERMINAL_CONTAINER_DISK", "0") + monkeypatch.setenv("TERMINAL_DOCKER_ENV", json.dumps({"HERMES_RB_STATIC": "from-config"})) + monkeypatch.setenv("TERMINAL_DOCKER_FORWARD_ENV", json.dumps(["HERMES_RB_FORWARDED"])) + monkeypatch.setenv("HERMES_RB_FORWARDED", "from-host") + monkeypatch.setenv("HERMES_RB_NOT_FORWARDED", "must-stay-on-host") + task_id = f"rb-docker-{uuid.uuid4().hex[:8]}" + yield task_id + _teardown(task_id) + + +def _env_key(task_id) -> str: + # Without a session key, persistent Docker collapses every task onto the profile's + # container key ("default" here), exactly as a CLI run does. + from tools.terminal_tool import _resolve_container_task_id + + return _resolve_container_task_id(task_id) + + +def _teardown(task_id): + from tools.environments.docker import DockerEnvironment + from tools.terminal_tool_lifecycle import cleanup_vm + + cleanup_vm(_env_key(task_id), force_remove=True) + assert DockerEnvironment.wait_for_all_teardowns(timeout=60) + + +def _run(command, task_id, **kwargs): + from tools.terminal_tool import terminal_tool + + return json.loads(terminal_tool(command, task_id=task_id, **kwargs)) + + +def _container_id(task_id) -> str | None: + """The live env's container, confirmed running by the daemon itself.""" + from tools.terminal_tool import _active_environments + + env = _active_environments.get(_env_key(task_id)) + cid = getattr(env, "_container_id", None) + if not cid: + return None + state = subprocess.run(["docker", "inspect", "-f", "{{.State.Running}}", cid], + capture_output=True, text=True, timeout=30, check=False) + return cid if state.stdout.strip() == "true" else None + + +def _container_exists(cid: str) -> bool: + return subprocess.run(["docker", "inspect", cid], capture_output=True, timeout=30, + check=False).returncode == 0 + + +def test_exec_runs_in_the_container_and_reports_exit_codes(docker_task): + r = _run("echo hello-docker; cat /etc/os-release | head -1", docker_task) + assert r["exit_code"] == 0, r + assert "hello-docker" in r["output"] + assert "Debian" in r["output"] # ran in the image, not on the host + assert _container_id(docker_task) + + failed = _run("echo before; exit 7", docker_task) + assert failed["exit_code"] == 7 + assert "before" in failed["output"] + + +def test_cwd_persists_and_workdir_does_not(docker_task): + assert _run("pwd", docker_task)["output"].strip() == "/root" + _run("mkdir -p /workspace/proj && cd /workspace/proj", docker_task) + assert _run("pwd", docker_task)["output"].strip() == "/workspace/proj" + + once = _run("pwd", docker_task, workdir="/usr") + assert once["output"].strip() == "/usr" + assert _run("pwd", docker_task)["output"].strip() == "/workspace/proj" + + +def test_exported_env_persists_between_commands(docker_task): + _run("export HERMES_RB_EXPORTED='kept value'", docker_task) + assert _run('echo "$HERMES_RB_EXPORTED"', docker_task)["output"].strip() == "kept value" + _run("unset HERMES_RB_EXPORTED", docker_task) + assert _run('echo "${HERMES_RB_EXPORTED-gone}"', docker_task)["output"].strip() == "gone" + + +def test_configured_and_forwarded_env_reach_the_container_and_nothing_else(docker_task): + r = _run('echo "s=$HERMES_RB_STATIC f=$HERMES_RB_FORWARDED n=${HERMES_RB_NOT_FORWARDED:-unset}"', + docker_task) + assert r["output"].strip() == "s=from-config f=from-host n=unset" + + +def test_quoting_survives_the_docker_exec_hop(docker_task): + payload = "it's \"quoted\" $HOME `tick` \\back" + r = _run(f"printf '%s\\n' {shlex.quote(payload)}", docker_task) + assert r["output"].rstrip("\n") == payload + + +def test_file_tools_write_and_read_inside_the_container(docker_task): + from tools.file_tools import read_file_tool, write_file_tool + + content = "first line\nsecond: ünïcödé ✓\n" + wrote = json.loads(write_file_tool("/workspace/notes.txt", content, task_id=docker_task)) + assert wrote.get("success", True) and "error" not in wrote, wrote + + # Byte-exact on disk (the tool output itself is trailing-newline-trimmed). + assert _run("cat /workspace/notes.txt; echo; wc -c < /workspace/notes.txt", docker_task)[ + "output"].splitlines()[-1].strip() == str(len(content.encode())) + assert _run("cat /workspace/notes.txt", docker_task)["output"] == content.rstrip("\n") + read = json.loads(read_file_tool("/workspace/notes.txt", task_id=docker_task)) + assert "second: ünïcödé ✓" in read["content"], read + + +def test_home_survives_a_container_replacement(docker_task): + """container_persistent (default on): /root is a host bind mount, so a file written + there is still present after the container is removed and a new one starts.""" + _run("echo durable > /root/state.txt", docker_task) + first = _container_id(docker_task) + assert first + _teardown(docker_task) + assert not _container_exists(first) + + r = _run("cat /root/state.txt", docker_task) + assert r["output"].strip() == "durable" + second = _container_id(docker_task) + assert second and second != first diff --git a/tests/tools/test_ssh_environment.py b/tests/tools/test_ssh_environment.py index 4b2e0010c8..eb6ba79ef2 100644 --- a/tests/tools/test_ssh_environment.py +++ b/tests/tools/test_ssh_environment.py @@ -2,6 +2,7 @@ import json import os +import shlex import subprocess from unittest.mock import MagicMock @@ -10,16 +11,20 @@ import pytest from tools.environments.ssh import SSHEnvironment from tools.environments import ssh as ssh_env -_SSH_HOST = os.getenv("TERMINAL_SSH_HOST", "") -_SSH_USER = os.getenv("TERMINAL_SSH_USER", "") -_SSH_PORT = int(os.getenv("TERMINAL_SSH_PORT", "22")) -_SSH_KEY = os.getenv("TERMINAL_SSH_KEY", "") +# A real sshd to run against: scripts/ci/start_test_sshd.sh starts a throwaway one +# and prints these (the remote-backends CI lane exports them). HERMES_TEST_* rather +# than the product's TERMINAL_SSH_* so the hermetic conftest, which blanks +# TERMINAL_SSH_KEY as a credential, cannot strip them before the tests read them. +_SSH_HOST = os.getenv("HERMES_TEST_SSH_HOST", "") +_SSH_USER = os.getenv("HERMES_TEST_SSH_USER", "") +_SSH_PORT = int(os.getenv("HERMES_TEST_SSH_PORT", "22")) +_SSH_KEY = os.getenv("HERMES_TEST_SSH_KEY", "") _has_ssh = bool(_SSH_HOST and _SSH_USER) requires_ssh = pytest.mark.skipif( not _has_ssh, - reason="TERMINAL_SSH_HOST / TERMINAL_SSH_USER not set", + reason="HERMES_TEST_SSH_HOST / HERMES_TEST_SSH_USER not set (scripts/ci/start_test_sshd.sh)", ) @@ -29,8 +34,12 @@ def _run(command, task_id="ssh_test", **kwargs): def _cleanup(task_id="ssh_test"): + # The env is cached under the resolved key ("default" without a session key), not + # the raw task id: cleaning the raw id was a no-op that let one test's env leak + # into the next. + from tools.terminal_tool import _resolve_container_task_id from tools.terminal_tool_lifecycle import cleanup_vm - cleanup_vm(task_id) + cleanup_vm(_resolve_container_task_id(task_id)) class TestBuildSSHCommand: @@ -333,6 +342,10 @@ class TestOneShotSSH: assert "hello" in r["output"] + @pytest.mark.xfail(strict=True, reason=( + "TERMINAL_SSH_PERSISTENT=false is a dead knob: _get_env_config() reads it into " + "ssh_config['persistent'], but SSHEnvironment never takes it since the spawn-per-call " + "rewrite, so every SSH session persists exported env through its snapshot.")) def test_state_does_not_persist(self): _run("export HERMES_ONESHOT_TEST=yes") r = _run("echo $HERMES_ONESHOT_TEST") @@ -366,3 +379,59 @@ class TestPersistentSSH: assert len(lines) == 1000 assert lines[0] == "1" assert lines[-1] == "1000" + + +@requires_ssh +class TestRealSSHBackend: + """terminal_tool and the file tools over a real sshd, with the default (persistent) config.""" + + @pytest.fixture(autouse=True) + def _setup(self, monkeypatch): + _setup_ssh_env(monkeypatch, persistent=True) + yield + _cleanup() + + def test_exit_code_and_stderr_come_back(self): + r = _run("echo out; echo err >&2; exit 3") + assert r["exit_code"] == 3 + assert "out" in r["output"] and "err" in r["output"] + + def test_cwd_persists_and_workdir_does_not(self): + home = _run("pwd")["output"].strip() + assert home.startswith("/") + _run("mkdir -p 'proj/with space' && cd 'proj/with space'") + assert _run("pwd")["output"].strip() == f"{home}/proj/with space" + assert _run("pwd", workdir="/usr")["output"].strip() == "/usr" + assert _run("pwd")["output"].strip() == f"{home}/proj/with space" + + def test_quoting_survives_the_ssh_hop(self): + # One quoting layer for bash -c, one for the remote login shell sshd runs it under. + payload = "it's \"quoted\" $HOME `tick` \\back; echo injected" + r = _run(f"printf '%s\\n' {shlex.quote(payload)}") + assert r["output"].rstrip("\n") == payload + + def test_passthrough_env_reaches_remote_via_sendenv(self, monkeypatch): + """#14091 end to end: the name rides SendEnv, the value the ssh client env, and the + test sshd's ``AcceptEnv HERMES_RB_*`` hands it to the remote shell.""" + from tools import env_passthrough + + monkeypatch.setenv("HERMES_RB_PASS", "through-sendenv") + monkeypatch.setenv("HERMES_RB_PRIVATE", "stays-local") + env_passthrough.register_env_passthrough(["HERMES_RB_PASS"]) + try: + r = _run('echo "p=$HERMES_RB_PASS q=${HERMES_RB_PRIVATE:-unset}"') + finally: + env_passthrough.clear_env_passthrough() + assert r["output"].strip() == "p=through-sendenv q=unset" + + def test_file_tools_write_and_read_on_the_remote(self): + from tools.file_tools import read_file_tool, write_file_tool + + home = _run("pwd")["output"].strip() + path = f"{home}/notes/remote.txt" + content = "alpha\nbeta: ünïcödé ✓\n" + wrote = json.loads(write_file_tool(path, content, task_id="ssh_test")) + assert "error" not in wrote, wrote + assert _run(f"wc -c < {shlex.quote(path)}")["output"].strip() == str(len(content.encode())) + read = json.loads(read_file_tool(path, task_id="ssh_test")) + assert "beta: ünïcödé ✓" in read["content"], read