From 2d15a72b5120488d262ccc7672d65aec4d58f613 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 16 Sep 2026 13:01:26 +1000 Subject: [PATCH] fix(docker): carry deploy-injected Nous routing overrides into every profile .env Hosted deploys set HERMES_PORTAL_BASE_URL and NOUS_INFERENCE_BASE_URL only in the container environment and run the gateway with GATEWAY_MULTIPLEX_PROFILES=true. Since #108319 / #111809 both are resolved through the profile secret scope, which is built from /.env and never falls back to os.environ, so on every routed turn the override is absent: the Portal allowlist heals the URL to production, the staging refresh token is POSTed to portal.nousresearch.com, the Portal answers invalid_grant, and the Nous login is quarantined ~10s after boot ("No access token found for Nous Portal login"). Observed live on hermes-agent-stg-gg-probe-test-0062: 7 rebootstrap/quarantine cycles in one night, auth.json 5223 -> 714 bytes each time. stage2 now syncs both variables from the container env into $HERMES_HOME/.env and every profiles/*/.env (created 0600 hermes-owned when missing), replacing a stale line rather than adding a second assignment, skipping files that already carry the value, refusing symlinked paths, and degrading to a warning on a read-only volume. Files are untouched when the variable is not set. Three invariant tests run the block under `set -eu` with sh; also exercised under busybox sh. --- docker/stage2-hook.sh | 61 +++++++++ .../test_stage2_hook_nous_routing_env.py | 126 ++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 tests/tools/test_stage2_hook_nous_routing_env.py diff --git a/docker/stage2-hook.sh b/docker/stage2-hook.sh index 54b50bd2e4..dbdf9c7d7d 100755 --- a/docker/stage2-hook.sh +++ b/docker/stage2-hook.sh @@ -514,6 +514,67 @@ elif ! grep -q '^API_SERVER_KEY=..*' "$HERMES_HOME/.env" 2>/dev/null; then fi fi +# --- Sync deploy-injected Nous routing overrides into every profile .env --- +# Hosted deploys point an instance at a non-production Portal / inference +# host with HERMES_PORTAL_BASE_URL and NOUS_INFERENCE_BASE_URL in the +# container environment, and run the gateway with GATEWAY_MULTIPLEX_PROFILES. +# Under multiplex, hermes_cli.auth_nous resolves both through the profile +# secret scope (agent.secret_scope.get_secret, #108319 / #111809), which is +# built from /.env and never falls back to os.environ. A value that +# lives only in the process env is therefore invisible on every routed turn: +# the Portal allowlist then heals the URL to production, the staging refresh +# token is POSTed to portal.nousresearch.com, the Portal answers +# invalid_grant, and the login is quarantined ("No access token found for +# Nous Portal login") within seconds of every boot. The two commits that +# scoped these reads assign the deploy this duty: "Deployments that set +# HERMES_PORTAL_BASE_URL only in the process env must carry it in each served +# profile's .env". The container value wins over a stale line (the platform +# is the authority on where this instance routes); operators who never set +# the variable are untouched. Idempotent: an already-correct line is left +# alone so the volume is not rewritten every boot. +sync_routing_override() { + _name="$1" + _value="$2" + _file="$3" + if refuse_symlinked_path "sync $_name" "$_file"; then + return 0 + fi + if grep -qxF -- "$_name=$_value" "$_file" 2>/dev/null; then + return 0 + fi + if [ ! -f "$_file" ]; then + # Owner-only from the first instant, hermes-owned (same shape as the + # API_SERVER_KEY bootstrap above). + if ! (umask 077 && as_hermes touch "$_file") 2>/dev/null; then + echo "[stage2] Warning: could not create $_file — $_name will not reach this profile's secret scope" + return 0 + fi + fi + # Rewrite in place (redirect keeps inode, owner and mode) so a stale value + # is replaced, not shadowed by a second assignment. `grep -v` exits 1 when + # nothing remains (new or single-line file) — that is not an error here. + # Guarded: a read-only volume degrades to a warning, never a boot abort + # under `set -e`. + if _rewritten=$( { grep -v -- "^$_name=" "$_file" 2>/dev/null || true; } ; printf '%s=%s\n' "$_name" "$_value") \ + && printf '%s\n' "$_rewritten" > "$_file" 2>/dev/null; then + echo "[stage2] Synced $_name from the container environment into $_file" + else + echo "[stage2] Warning: could not write $_name to $_file (read-only volume?) — routed turns will fall back to the production Portal" + fi + unset _rewritten +} + +for _routing_name in HERMES_PORTAL_BASE_URL NOUS_INFERENCE_BASE_URL; do + eval "_routing_value=\${$_routing_name:-}" + [ -n "$_routing_value" ] || continue + sync_routing_override "$_routing_name" "$_routing_value" "$HERMES_HOME/.env" + for _profile_dir in "$HERMES_HOME"/profiles/*/; do + [ -d "$_profile_dir" ] || continue + sync_routing_override "$_routing_name" "$_routing_value" "${_profile_dir}.env" + done +done +unset _routing_name _routing_value _profile_dir + # .env holds API keys and secrets — restrict to owner-only access. Applied # unconditionally (not only on first-seed) so a host-mounted .env that was # created with a permissive umask gets tightened on every container start. diff --git a/tests/tools/test_stage2_hook_nous_routing_env.py b/tests/tools/test_stage2_hook_nous_routing_env.py new file mode 100644 index 0000000000..e0a80b606e --- /dev/null +++ b/tests/tools/test_stage2_hook_nous_routing_env.py @@ -0,0 +1,126 @@ +"""Regression tests for the stage2 Nous routing-override sync. + +Hosted deploys carry ``HERMES_PORTAL_BASE_URL`` / ``NOUS_INFERENCE_BASE_URL`` only in the +container environment. Under ``GATEWAY_MULTIPLEX_PROFILES`` both are resolved through the +profile secret scope (#108319 / #111809), which is built from ``/.env`` and never +falls back to ``os.environ`` — so the staging Portal URL was dropped, the refresh token went +to the production Portal, and the login was quarantined on every boot. stage2 must carry the +container value into every served profile's ``.env``. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +STAGE2_HOOK = REPO_ROOT / "docker" / "stage2-hook.sh" + +PORTAL = "https://portal.staging-nousresearch.com" +INFERENCE = "https://stg-inference-api.nousresearch.com/v1" + + +@pytest.fixture(scope="module") +def stage2_text() -> str: + if not STAGE2_HOOK.exists(): + pytest.skip("docker/stage2-hook.sh not present in this checkout") + return STAGE2_HOOK.read_text() + + +def _sync_block(text: str) -> str: + start = text.index("# --- Sync deploy-injected Nous routing overrides") + end = text.index("# .env holds API keys and secrets", start) + return text[start:end] + + +def _path_guard_functions(text: str) -> str: + start = text.index("path_has_symlink_component() {") + end = text.index("\n\nchown_hermes_tree() {", start) + return text[start:end] + + +def _run_sync(stage2_text: str, home: Path, env: dict[str, str | None]) -> subprocess.CompletedProcess[str]: + if shutil.which("sh") is None: + pytest.skip("sh not available") + env_setup = "".join( + f"unset {k}\n" if v is None else f"{k}='{v}'\n" for k, v in env.items() + ) + script = ( + "set -eu\n" # production runs the hook under set -eu + f"{env_setup}" + f'HERMES_HOME="{home}"\n' + 'as_hermes() { "$@"; }\n' + f"{_path_guard_functions(stage2_text)}\n" + f"{_sync_block(stage2_text)}\n" + ) + return subprocess.run(["sh", "-c", script], capture_output=True, text=True, timeout=30) + + +def _lines(path: Path, name: str) -> list[str]: + return [ln for ln in path.read_text().splitlines() if ln.startswith(f"{name}=")] + + +def test_container_value_reaches_home_and_every_profile_env(stage2_text: str, tmp_path: Path) -> None: + """Both overrides land in $HERMES_HOME/.env and each profiles/*/.env (created when missing).""" + home = tmp_path / "home" + (home / "profiles" / "work").mkdir(parents=True) + (home / "profiles" / "ops").mkdir() + (home / ".env").write_text("API_SERVER_KEY=abc\n") + (home / "profiles" / "ops" / ".env").write_text("SLACK_BOT_TOKEN=xoxb-x\n") + + result = _run_sync( + stage2_text, home, {"HERMES_PORTAL_BASE_URL": PORTAL, "NOUS_INFERENCE_BASE_URL": INFERENCE} + ) + + assert result.returncode == 0, result.stderr + for env_file in (home / ".env", home / "profiles" / "work" / ".env", home / "profiles" / "ops" / ".env"): + assert _lines(env_file, "HERMES_PORTAL_BASE_URL") == [f"HERMES_PORTAL_BASE_URL={PORTAL}"], env_file + assert _lines(env_file, "NOUS_INFERENCE_BASE_URL") == [f"NOUS_INFERENCE_BASE_URL={INFERENCE}"], env_file + # Existing unrelated secrets survive the rewrite. + assert "API_SERVER_KEY=abc" in (home / ".env").read_text() + assert "SLACK_BOT_TOKEN=xoxb-x" in (home / "profiles" / "ops" / ".env").read_text() + created = home / "profiles" / "work" / ".env" + assert (created.stat().st_mode & 0o777) == 0o600 + + +def test_stale_value_replaced_and_correct_value_left_alone(stage2_text: str, tmp_path: Path) -> None: + """The container value wins over a stale line (one assignment, not two); a matching line is not rewritten.""" + home = tmp_path / "home" + home.mkdir() + env_file = home / ".env" + env_file.write_text("HERMES_PORTAL_BASE_URL=https://portal.nousresearch.com\nOTHER=1\n") + + first = _run_sync(stage2_text, home, {"HERMES_PORTAL_BASE_URL": PORTAL, "NOUS_INFERENCE_BASE_URL": None}) + assert first.returncode == 0, first.stderr + assert _lines(env_file, "HERMES_PORTAL_BASE_URL") == [f"HERMES_PORTAL_BASE_URL={PORTAL}"] + assert "OTHER=1" in env_file.read_text() + assert "Synced HERMES_PORTAL_BASE_URL" in first.stdout + + before = env_file.stat().st_mtime_ns + os.utime(env_file, ns=(before - 5_000_000_000, before - 5_000_000_000)) + stamped = env_file.stat().st_mtime_ns + second = _run_sync(stage2_text, home, {"HERMES_PORTAL_BASE_URL": PORTAL, "NOUS_INFERENCE_BASE_URL": None}) + assert second.returncode == 0, second.stderr + assert env_file.stat().st_mtime_ns == stamped, "an already-correct line must not rewrite the volume" + assert "Synced" not in second.stdout + + +def test_unset_override_touches_nothing_and_symlinked_env_is_refused(stage2_text: str, tmp_path: Path) -> None: + """No container value → no .env is created or edited; a symlinked .env is never written through.""" + home = tmp_path / "home" + (home / "profiles" / "work").mkdir(parents=True) + result = _run_sync(stage2_text, home, {"HERMES_PORTAL_BASE_URL": None, "NOUS_INFERENCE_BASE_URL": None}) + assert result.returncode == 0, result.stderr + assert not (home / ".env").exists() + assert not (home / "profiles" / "work" / ".env").exists() + + outside = tmp_path / "outside.env" + outside.write_text("KEEP=1\n") + (home / ".env").symlink_to(outside) + result = _run_sync(stage2_text, home, {"HERMES_PORTAL_BASE_URL": PORTAL, "NOUS_INFERENCE_BASE_URL": None}) + assert result.returncode == 0, result.stderr + assert outside.read_text() == "KEEP=1\n" + assert "refusing sync HERMES_PORTAL_BASE_URL" in result.stdout