From a10bbf95bb416f2b83f60f84832328d4eeec69c3 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Wed, 16 Sep 2026 04:18:43 -0700 Subject: [PATCH] feat(gateway): gateway.multiplex_profiles defaults to on, gated by a boot-time serve guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DEFAULT_CONFIG now ships gateway.multiplex_profiles: true. GatewayConfig keeps an UNSET flag as None so the boot can tell "the operator chose" from "the default applies"; every reader tests truthiness, so an undecided flag never multiplexes by accident. hermes_cli/gateway_multiplex_mode.py settles the unset default once per boot (called from load_gateway_config_for_runner and `gateway run --config`): the same preflight `hermes gateway migrate --multiplex` runs — default profile, >= 2 profiles, no secondary running its own gateway (live pid or installed unit), no duplicate-credential / port-binder blocker, migratable host. A refusal is a logged warning naming the blocker and the migrate one-liner; the gateway comes up standalone exactly as before. Explicit values (config.yaml, GATEWAY_MULTIPLEX_PROFILES) pass through verbatim; `--standalone` already pins false. Other processes stop guessing the verdict from the merged default: named_profile_served_by_running _multiplexer, the enroll warning, the dashboard listener guard, the cron-fire port resolver, container boot and the migration plan (_read_multiplex_flag) read the live gateway's served_profiles record first and the EXPLICIT flag second — so a per-profile fleet with the flag unset still reads as "not yet multiplexed" and the fold proceeds. Docs: multi-profile-gateways.md, multiplexing-gateway.md, hermes_cli/AGENTS.md. --- gateway/config.py | 20 ++- gateway/run.py | 21 ++- hermes_cli/AGENTS.md | 6 + hermes_cli/config_defaults.py | 17 ++- hermes_cli/container_boot.py | 4 +- hermes_cli/gateway.py | 21 +-- hermes_cli/gateway_enroll.py | 23 +-- hermes_cli/gateway_migrate.py | 16 +- hermes_cli/gateway_multiplex_mode.py | 138 ++++++++++++++++++ hermes_cli/web_routers/messaging.py | 13 +- hermes_cli/web_server_cron.py | 9 +- tests/gateway/test_multiplex_phase0.py | 8 +- tests/hermes_cli/test_config.py | 2 +- .../hermes_cli/test_gateway_multiplex_mode.py | 137 +++++++++++++++++ .../developer-guide/multiplexing-gateway.md | 26 +++- .../docs/user-guide/multi-profile-gateways.md | 56 +++++-- 16 files changed, 417 insertions(+), 100 deletions(-) create mode 100644 hermes_cli/gateway_multiplex_mode.py create mode 100644 tests/hermes_cli/test_gateway_multiplex_mode.py diff --git a/gateway/config.py b/gateway/config.py index 2437f539b9..e8ecc94dec 100644 --- a/gateway/config.py +++ b/gateway/config.py @@ -548,9 +548,14 @@ class GatewayConfig: group_sessions_per_user: bool = True # Isolate group sessions per participant when user IDs exist thread_sessions_per_user: bool = False # False = threads shared across participants max_concurrent_sessions: Optional[int] = None # Positive int caps simultaneous active sessions - # Opt-in: the default profile's gateway serves every profile on the host (profiles stamped into - # session keys, per-profile adapters/credentials). - multiplex_profiles: bool = False + # The default profile's gateway serves every profile on the host (profiles stamped into session + # keys, per-profile adapters/credentials). On by default (DEFAULT_CONFIG), but UNSET here is + # ``None``: a request the gateway settles at boot, not a verdict. ``hermes_cli.gateway_multiplex_mode + # .resolve_multiplex_mode`` runs the migration preflight (default profile, >= 2 profiles, no + # secondary running its own gateway, no blocker, migratable host) and only then writes True/False. + # An explicit value (config.yaml, GATEWAY_MULTIPLEX_PROFILES, a constructor argument) is honoured + # verbatim. Every reader tests truthiness, so an unresolved ``None`` never multiplexes by accident. + multiplex_profiles: Optional[bool] = None # Public HTTPS endpoint for scoped RoomLink calls (an API key alone must never advertise a # route); HERMES_ROOM_LINK_URL overrides. room_link_url: Optional[str] = None @@ -693,9 +698,10 @@ class GatewayConfig: systemd_watchdog_seconds = coerce_systemd_watchdog_seconds( pick("systemd_watchdog_seconds"), key_label("systemd_watchdog_seconds") ) - # env > config.yaml > False: a recognized GATEWAY_MULTIPLEX_PROFILES wins (hosted deployments - # stamp it on the container); blank/unrecognized falls through to the top-level VALUE when - # not None, else ``gateway.multiplex_profiles``. + # env > config.yaml > unset: a recognized GATEWAY_MULTIPLEX_PROFILES wins (hosted deployments + # stamp it on the container); blank/unrecognized falls through to the top-level VALUE when not + # None, else ``gateway.multiplex_profiles``. Nothing set stays ``None`` so the boot-time guard + # (``resolve_multiplex_mode``) can tell "the operator chose" from "the default applies". multiplex_profiles = data.get("multiplex_profiles") if multiplex_profiles is None: multiplex_profiles = nested_gateway.get("multiplex_profiles") @@ -721,7 +727,7 @@ class GatewayConfig: **{name: _coerce_bool(data.get(name), default) for name, default in _TOPLEVEL_BOOL_DEFAULTS.items()}, stt_enabled=_coerce_bool(stt_setting("stt_enabled", "enabled"), True), stt_echo_transcripts=_coerce_bool(stt_setting("stt_echo_transcripts", "echo_transcripts"), True), - multiplex_profiles=_coerce_bool(multiplex_profiles, False), + multiplex_profiles=None if multiplex_profiles is None else _coerce_bool(multiplex_profiles, True), room_link_url=room_link_url if isinstance(room_link_url, str) else None, systemd_watchdog_seconds=systemd_watchdog_seconds, loop_watchdog=_coerce_bool(pick("loop_watchdog"), True), diff --git a/gateway/run.py b/gateway/run.py index a072dac7dd..067f084131 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1775,15 +1775,19 @@ async def _async_profile_runtime_scope(profile_home: "Path"): def load_gateway_config_for_runner() -> "GatewayConfig": - """Load gateway config for the process-level GatewayRunner. Multiplexed: reload under the default - profile's ``_profile_runtime_scope`` so platform tokens in its ``.env`` resolve via the secret - scope; unscoped ``_getenv`` falls to ``os.environ``, which often lacks a token living only under + """Load gateway config for the process-level GatewayRunner. An UNSET ``multiplex_profiles`` is + settled first by ``resolve_multiplex_mode`` (the default is on; the boot guard keeps a fleet that + still runs per-profile gateways standalone). Multiplexed: reload under the default profile's + ``_profile_runtime_scope`` so platform tokens in its ``.env`` resolve via the secret scope; + unscoped ``_getenv`` falls to ``os.environ``, which often lacks a token living only under ``profiles//.env``. Off -> identical to ``load_gateway_config()``. See #64674. """ + from hermes_cli.gateway_multiplex_mode import log_multiplex_decision, resolve_multiplex_mode cfg = load_gateway_config() - if not getattr(cfg, "multiplex_profiles", False): + log_multiplex_decision(resolve_multiplex_mode(cfg)) + if not cfg.multiplex_profiles: return cfg try: home = get_hermes_home() @@ -1791,10 +1795,12 @@ def load_gateway_config_for_runner() -> "GatewayConfig": return cfg try: with _profile_runtime_scope(Path(home)): - return load_gateway_config() + scoped = load_gateway_config() except Exception: logger.debug("multiplex default-scope config reload failed; using unscoped load", exc_info=True) return cfg + scoped.multiplex_profiles = cfg.multiplex_profiles # the verdict above, not a second unset flag + return scoped async def _discover_gateway_mcp_tools(config: object) -> None: @@ -3401,6 +3407,8 @@ class GatewayRunner( # With multiplex_profiles on, load under the default profile secret scope so bot tokens in its # .env resolve as secondary profiles' do; explicit config= injection (tests) is left untouched. # See #64674. + # An injected config (tests, ``gateway run --config``) is taken verbatim: an unset flag there + # stays None (= standalone); only the loaded path runs the boot-time default-on guard. self.config = config if config is not None else load_gateway_config_for_runner() # Multiplexer flag flips agent.secret_scope.get_secret() to fail-closed on unscoped credential # reads, so a missed migration crashes loudly instead of leaking a cross-profile value. @@ -5459,6 +5467,9 @@ def main(): import yaml with open(args.config, encoding="utf-8") as f: config = GatewayConfig.from_dict(yaml.safe_load(f) or {}) + # Same boot-time verdict the loaded config gets when the file leaves the flag unset. + from hermes_cli.gateway_multiplex_mode import log_multiplex_decision, resolve_multiplex_mode + log_multiplex_decision(resolve_multiplex_mode(config)) # start_gateway() completes teardown before returning/raising SystemExit; force-exit after so a # wedged non-daemon worker can't block Py_FinalizeEx's join. SystemExit caught so EVERY path exits. diff --git a/hermes_cli/AGENTS.md b/hermes_cli/AGENTS.md index 087e5a207e..291e5eabc9 100644 --- a/hermes_cli/AGENTS.md +++ b/hermes_cli/AGENTS.md @@ -160,6 +160,12 @@ Enumeration is a pure read: never `mkdir` a profile home from a served path (`Se cron all go through `mkdir_under_hermes_home` / `_ensure_cron_dir`, which refuse a deleted or missing named profile, #94590). Process-global per-profile slots (MCP discovery in `mcp_startup.py`, tool registry overlays) key on `hermes_constants.hermes_home_key()`, never a single flag. +`gateway.multiplex_profiles` defaults to **on**, but `GatewayConfig` keeps an unset flag `None` and +`gateway_multiplex_mode.resolve_multiplex_mode` settles it once per boot (called from +`load_gateway_config_for_runner`): default profile, >= 2 profiles, no standalone secondary gateway, +no preflight blocker, migratable host → `True`; else `False` + a logged reason. Explicit values pass +through. CLI/dashboard readers use `default_gateway_multiplexes` (live `served_profiles` record, then +the explicit flag) — never the merged default, which would guess a verdict only the gateway makes. Migration from per-profile gateways: `hermes_cli/gateway_migrate.py` (`hermes gateway migrate --multiplex|--standalone`, table-driven `_PREFLIGHT_CHECKS`, manifest `/gateway_migration.json`); `update_cmd_fleet._verify_fleet_after_update` calls `maybe_auto_migrate_after_update` on the success diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index c56f1ab658..3acddad8fe 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1979,13 +1979,16 @@ DEFAULT_CONFIG = { "write_sessions_json": True, # One gateway for every profile on this host: the DEFAULT profile's gateway also connects # each named profile's bots (their own .env / config.yaml, per-profile secret scope) and - # stamps the profile into session keys. Flip with `hermes gateway migrate --multiplex` - # (records a rollback manifest; `--standalone` undoes it) or `hermes config set - # gateway.multiplex_profiles true` + `hermes gateway restart`. GATEWAY_MULTIPLEX_PROFILES - # in the environment overrides. Two profiles configuring the same bot token cannot be - # served together — the duplicate adapter is parked; `hermes profile create --clone` - # therefore leaves messaging channels behind unless --clone-channels is passed. - "multiplex_profiles": False, + # stamps the profile into session keys. On by default. An UNSET key is a request, not a + # verdict: at boot the default gateway runs the migration preflight and stays standalone + # (logging why) when a secondary still runs its own gateway or a blocker exists — an + # explicit `true` (config or GATEWAY_MULTIPLEX_PROFILES) is honoured as before, an explicit + # `false` keeps per-profile gateways for good. `hermes gateway migrate --multiplex` folds a + # per-profile fleet (records a rollback manifest; `--standalone` undoes it and pins false). + # Two profiles configuring the same bot token cannot be served together — the duplicate + # adapter is parked; `hermes profile create --clone` therefore leaves messaging channels + # behind unless --clone-channels is passed. + "multiplex_profiles": True, # May `hermes update` fold this install onto a multiplexed default gateway by itself? # True (the default) keeps today's behaviour: a multi-profile install whose secondaries run # their own gateways is migrated automatically after an update when nothing blocks it. diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 9d189aa919..602cedc374 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -79,10 +79,12 @@ def reconcile_profile_gateways( actions: list[ReconcileAction] = [] # Under a multiplexing root gateway named slots are still registered but must not boot from # their persisted run intent, or they would become additional multiplex owners. + # Explicit opt-in only: the unset default (on) is refused on s6 hosts by the gateway's own boot + # guard (per-profile gateways are s6 slots the preflight cannot fold), so the slots keep booting. from gateway.config import load_gateway_config from utils import is_truthy_value try: - multiplex_profiles = load_gateway_config().multiplex_profiles + multiplex_profiles = load_gateway_config().multiplex_profiles is True except Exception: log.warning("Unable to load gateway configuration during container boot; using the " "GATEWAY_MULTIPLEX_PROFILES override if set.", exc_info=True) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 195cc3cd0b..19c0e7c7ea 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -4451,23 +4451,10 @@ def named_profile_served_by_running_multiplexer(profile_name: str | None = None) if recorded is not None: return normalize_profile_name(suffix) in {normalize_profile_name(p) for p in recorded} - from gateway.config import _env_multiplex_profiles_override - cfg_path = default_root / "config.yaml" - cfg = {} - if cfg_path.exists(): - from hermes_cli.config import read_user_config_raw - cfg = read_user_config_raw(cfg_path) - - env_multiplex = _env_multiplex_profiles_override() - if env_multiplex is False: - return False - if env_multiplex is not True: - if not cfg_path.exists(): - return False - if not (cfg.get("multiplex_profiles") or (cfg.get("gateway", {}) or {}).get("multiplex_profiles")): - return False - - return True # a multiplexing default gateway serves every named profile + # No record (older gateway): only an EXPLICIT opt-in counts. The unset default is settled by + # the gateway at boot (it may have stayed standalone); a CLI process must not guess it on. + from hermes_cli.gateway_multiplex_mode import explicit_multiplex_flag + return explicit_multiplex_flag(default_root) is True # a multiplexer serves every named profile except Exception: logger.debug("Multiplexer-serving probe failed", exc_info=True) return False diff --git a/hermes_cli/gateway_enroll.py b/hermes_cli/gateway_enroll.py index f72baed210..7ff31fb488 100644 --- a/hermes_cli/gateway_enroll.py +++ b/hermes_cli/gateway_enroll.py @@ -224,24 +224,13 @@ def _warn_if_secondary_multiplex_profile() -> bool: except ValueError: return False # default profile or custom layout — not a secondary - # Multiplex precedence mirrors gateway.config: recognized env override wins, else a RAW read - # of the DEFAULT root's config.yaml (the active profile's load_gateway_config() is the wrong - # owner and runs the full enablement pass, whose log output has no place in enroll output). - from gateway.config import _env_multiplex_profiles_override - env_multiplex = _env_multiplex_profiles_override() - if env_multiplex is False: + # The LIVE default gateway's served record, else the operator's explicit flag (env override + # wins, then a RAW read of the DEFAULT root's config.yaml — the active profile's + # load_gateway_config() is the wrong owner and runs the full enablement pass, whose log output + # has no place in enroll output). An unset flag is settled by the gateway at boot, not here. + from hermes_cli.gateway_multiplex_mode import default_gateway_multiplexes + if not default_gateway_multiplexes(default_root): return False - if env_multiplex is not True: - cfg_path = default_root / "config.yaml" - if not cfg_path.exists(): - return False - from hermes_cli.config import read_user_config_raw - cfg = read_user_config_raw(cfg_path) or {} - if not bool( - cfg.get("multiplex_profiles") - or (cfg.get("gateway", {}) or {}).get("multiplex_profiles") - ): - return False print( " ⚠ This profile is a SECONDARY profile of a multiplexed gateway.\n" diff --git a/hermes_cli/gateway_migrate.py b/hermes_cli/gateway_migrate.py index 6dca0ba71b..52596df0bc 100644 --- a/hermes_cli/gateway_migrate.py +++ b/hermes_cli/gateway_migrate.py @@ -250,17 +250,11 @@ def _spawn_detached_gateway(home: Path) -> bool: def _read_multiplex_flag(default_home: Path) -> bool: - from gateway.config import _env_multiplex_profiles_override - env = _env_multiplex_profiles_override() - if env is not None: - return env - cfg_path = default_home / "config.yaml" - if not cfg_path.exists(): - return False - from hermes_cli.config import read_user_config_raw - cfg = read_user_config_raw(cfg_path) or {} - gateway_section = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {} - return bool(cfg.get("multiplex_profiles") or gateway_section.get("multiplex_profiles")) + """The operator's EXPLICIT opt-in only. The unset default (on) is settled by the default gateway at + boot and refused while a secondary runs its own gateway — exactly the fleet this command folds — + so the plan reads it as "not yet multiplexed" and the migration proceeds.""" + from hermes_cli.gateway_multiplex_mode import explicit_multiplex_flag + return explicit_multiplex_flag(default_home) is True def _write_multiplex_flag(default_home: Path, value: bool) -> None: diff --git a/hermes_cli/gateway_multiplex_mode.py b/hermes_cli/gateway_multiplex_mode.py new file mode 100644 index 0000000000..b07ad0409c --- /dev/null +++ b/hermes_cli/gateway_multiplex_mode.py @@ -0,0 +1,138 @@ +"""Boot-time verdict for an UNSET ``gateway.multiplex_profiles`` (the default is on). + +``GatewayConfig.from_dict`` leaves the flag ``None`` when neither config.yaml nor +``GATEWAY_MULTIPLEX_PROFILES`` set it. Turning the default on must not make a default gateway +double-bind a fleet that still runs per-profile gateways (two pollers on one bot token, port +fights), so the implicit default is a *request*: the gateway runs the same preflight +``hermes gateway migrate --multiplex`` runs and multiplexes only when the fold would have been +safe. An explicit value is never second-guessed — ``true`` multiplexes (the operator or the +migration chose it), ``false`` keeps per-profile gateways for good (``--standalone`` pins it). + +The refusal is logged, never fatal: the gateway comes up standalone exactly as before the +default flipped, and the log names the blocker plus ``hermes gateway migrate --multiplex``. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +SINGLE_PROFILE_REASON = "only one profile exists (nothing to multiplex)" + + +def explicit_multiplex_flag(default_home: Path) -> Optional[bool]: + """The operator's explicit choice for the DEFAULT profile's gateway: a recognized + ``GATEWAY_MULTIPLEX_PROFILES``, else ``gateway.multiplex_profiles`` (or the top-level alias) as + written in its config.yaml; ``None`` when neither is set. Raw read on purpose: the callers are + other processes (``hermes -p X ...`` has X's config loaded) asking about the default's file.""" + from gateway.config import _bool_token, _env_multiplex_profiles_override + env = _env_multiplex_profiles_override() + if env is not None: + return env + cfg_path = Path(default_home) / "config.yaml" + if not cfg_path.exists(): + return None + from hermes_cli.config import read_user_config_raw + cfg = read_user_config_raw(cfg_path) or {} + gateway_section = cfg.get("gateway") if isinstance(cfg.get("gateway"), dict) else {} + value = cfg.get("multiplex_profiles") + if value is None: + value = gateway_section.get("multiplex_profiles") + if value is None: + return None + if isinstance(value, str): + parsed = _bool_token(value) + return True if parsed is None else parsed + return bool(value) + + +def default_gateway_multiplexes(default_home: Optional[Path] = None) -> bool: + """Does the default profile's gateway serve every profile? For CLI/dashboard processes: the LIVE + gateway's ``served_profiles`` record when one runs (it settled the unset default itself), else the + explicit flag, else False — an unset flag is decided by the gateway at boot, never guessed here.""" + from hermes_constants import get_default_hermes_root + from hermes_cli.gateway_multiplex_served import recorded_served_profiles + root = Path(default_home) if default_home is not None else get_default_hermes_root() + recorded = recorded_served_profiles(root) + if recorded is not None: + return bool(recorded) + return bool(explicit_multiplex_flag(root)) + + +@dataclass(frozen=True) +class MultiplexDecision: + enabled: bool + # "config" (config.yaml / env override — explicit), "default" (implicit default applied), + # "guard" (implicit default refused; ``reason`` names the blocker). + source: str + reason: str = "" + + +def implicit_multiplex_blocker() -> Optional[str]: + """Why THIS process must not multiplex on the implicit default, or None when it may. + + Mirrors what makes ``hermes gateway migrate --multiplex`` refuse or leave a per-profile gateway + in place: a named-profile gateway serves only itself; hosts whose per-profile gateways the + preflight cannot see (s6 slots, Windows scheduled tasks) stay standalone; a secondary that still + runs its own gateway (live process or installed service) or a preflight blocker (duplicate bot + credential, port binder without a ``/p//`` ingress) keeps the default standalone. + """ + from hermes_cli.profiles import get_active_profile_name, profiles_to_serve + active = get_active_profile_name() or "default" + if active != "default": + return (f"this is profile '{active}'s own gateway; only the default profile's gateway " + f"multiplexes (hermes gateway migrate --multiplex folds the fleet onto it)") + # Cheap and first: a single-profile install has nothing to multiplex, and the fail-closed secret + # scope the multiplexer arms buys it nothing. (Also keeps every embedded/test runner off the + # service-manager probes below.) Create a second profile and restart to start serving it. + if len(profiles_to_serve(multiplex=True)) < 2: + return SINGLE_PROFILE_REASON + from hermes_cli.gateway_migrate import MIGRATE_COMMAND, _host_supports_migration, build_migration_plan + host_reason = _host_supports_migration() + if host_reason: + return host_reason + plan = build_migration_plan() + if plan.standalone_secondaries: + owned = ", ".join( + f"'{p.name}' ({'pid ' + str(p.pid) if p.pid else p.service_label()})" + for p in plan.standalone_secondaries) + return (f"profile(s) {owned} still run their own gateway; fold them with `{MIGRATE_COMMAND}` " + f"or pin gateway.multiplex_profiles: false to keep per-profile gateways") + if plan.blocked: + return "; ".join(plan.blockers) + return None + + +def resolve_multiplex_mode(config) -> MultiplexDecision: + """Settle ``config.multiplex_profiles`` for one gateway boot; the config is updated in place.""" + current = getattr(config, "multiplex_profiles", None) + if current is not None: + return MultiplexDecision(bool(current), "config") + try: + blocker = implicit_multiplex_blocker() + except Exception as exc: # a broken preflight must not take the gateway down with it + logger.warning("Multiplex preflight failed; starting standalone: %s", exc, exc_info=True) + blocker = f"preflight failed ({exc})" + decision = (MultiplexDecision(False, "guard", blocker) if blocker + else MultiplexDecision(True, "default", "gateway.multiplex_profiles unset; default applies")) + config.multiplex_profiles = decision.enabled + return decision + + +def log_multiplex_decision(decision: MultiplexDecision) -> None: + if decision.source == "config" and not decision.enabled: + logger.info("gateway.multiplex_profiles is false: serving this profile only " + "(hermes gateway migrate --multiplex folds every profile onto the default gateway).") + elif decision.source == "guard" and decision.reason == SINGLE_PROFILE_REASON: + logger.info("Single-profile install: gateway.multiplex_profiles unset, serving the default profile only.") + elif decision.source == "guard": + logger.warning( + "gateway.multiplex_profiles is unset (default: on) but this gateway stays standalone: %s. " + "It serves the default profile only; set gateway.multiplex_profiles explicitly to silence this.", + decision.reason) + elif decision.source == "default": + logger.info("Serving every profile on this host (gateway.multiplex_profiles unset; default on).") diff --git a/hermes_cli/web_routers/messaging.py b/hermes_cli/web_routers/messaging.py index d9a731d3a8..4e98867730 100644 --- a/hermes_cli/web_routers/messaging.py +++ b/hermes_cli/web_routers/messaging.py @@ -807,7 +807,7 @@ def _multiplex_port_binding_conflict(platform_id: str, requested_profile: Option enable a second one. Every other inbound-port platform (Twilio, LINE, Teams, ...) IS allowed on a secondary: the gateway serves it on the shared listener at ``/p//``. """ - from gateway.config import SHARED_LISTENER_MIRROR_PLATFORMS, load_gateway_config + from gateway.config import SHARED_LISTENER_MIRROR_PLATFORMS if platform_id not in SHARED_LISTENER_MIRROR_PLATFORMS: return None @@ -825,11 +825,12 @@ def _multiplex_port_binding_conflict(platform_id: str, requested_profile: Option if target in ("default", "custom"): return None - # The flag that matters is the one the shared gateway reads at startup: the DEFAULT - # profile's config (plus the process-wide GATEWAY_MULTIPLEX_PROFILES override). - with _config_profile_scope("default"): - if not load_gateway_config().multiplex_profiles: - return None + # The flag that matters is the one the shared gateway settled at startup: its served record when + # it runs, else the DEFAULT profile's explicit config (plus the process-wide + # GATEWAY_MULTIPLEX_PROFILES override). An unset flag is decided by the gateway, not guessed here. + from hermes_cli.gateway_multiplex_mode import default_gateway_multiplexes + if not default_gateway_multiplexes(): + return None return ( f"Cannot enable '{platform_id}' on profile '{target}': gateway.multiplex_profiles is on and the " diff --git a/hermes_cli/web_server_cron.py b/hermes_cli/web_server_cron.py index 63e301a67b..83267d27ad 100644 --- a/hermes_cli/web_server_cron.py +++ b/hermes_cli/web_server_cron.py @@ -319,11 +319,10 @@ def _gateway_fire_endpoint(profile: str, home: Path) -> str: import os as _os multiplex = False try: - from gateway.config import _env_multiplex_profiles_override - multiplex = bool(cfg_get(load_config(), "gateway", "multiplex_profiles", default=False)) - env_flag = _env_multiplex_profiles_override() - if env_flag is not None: - multiplex = env_flag + # The live default gateway's own record, else the explicit flag — never the merged default: + # an unset gateway.multiplex_profiles is settled by the gateway at boot, not by this process. + from hermes_cli.gateway_multiplex_mode import default_gateway_multiplexes + multiplex = default_gateway_multiplexes() except Exception: _log.debug("cron fire: multiplex detection failed; assuming single-profile", exc_info=True) diff --git a/tests/gateway/test_multiplex_phase0.py b/tests/gateway/test_multiplex_phase0.py index f228961e8c..d311a04fba 100644 --- a/tests/gateway/test_multiplex_phase0.py +++ b/tests/gateway/test_multiplex_phase0.py @@ -97,8 +97,12 @@ class TestMultiplexConfigFlag: hk.join(timeout=5) assert captured["default_profile"] == "rex" - def test_default_is_false(self): - assert GatewayConfig().multiplex_profiles is False + def test_unset_is_undecided_and_reads_as_off(self): + """The default (on) is applied by the boot guard, not the dataclass: an unset flag stays + ``None`` so the guard can tell it from an explicit choice, and every reader treats it as off.""" + assert GatewayConfig().multiplex_profiles is None + assert not GatewayConfig().multiplex_profiles + assert GatewayConfig.from_dict({}).multiplex_profiles is None def test_from_dict_top_level(self): diff --git a/tests/hermes_cli/test_config.py b/tests/hermes_cli/test_config.py index 96bc612e8b..c83c5e5372 100644 --- a/tests/hermes_cli/test_config.py +++ b/tests/hermes_cli/test_config.py @@ -1932,7 +1932,7 @@ def test_gateway_multiplex_keys_are_recognized_config_keys(): key' although gateway/config.py reads it; the key (and profile_routes) live in DEFAULT_CONFIG.""" from hermes_cli.config import _validate_config_key from hermes_cli.config_defaults import DEFAULT_CONFIG - assert DEFAULT_CONFIG["gateway"]["multiplex_profiles"] is False + assert DEFAULT_CONFIG["gateway"]["multiplex_profiles"] is True assert DEFAULT_CONFIG["gateway"]["auto_multiplex_migration"] is True assert "auto_migrate" not in DEFAULT_CONFIG["gateway"] assert _validate_config_key("gateway.multiplex_profiles") == (True, None) diff --git a/tests/hermes_cli/test_gateway_multiplex_mode.py b/tests/hermes_cli/test_gateway_multiplex_mode.py new file mode 100644 index 0000000000..0afb4f898a --- /dev/null +++ b/tests/hermes_cli/test_gateway_multiplex_mode.py @@ -0,0 +1,137 @@ +"""``gateway.multiplex_profiles`` defaults to ON, but an UNSET flag is settled at boot by +``hermes_cli.gateway_multiplex_mode.resolve_multiplex_mode`` — the same preflight +``hermes gateway migrate --multiplex`` runs — so flipping the default can never make a default +gateway double-bind a fleet that still runs per-profile gateways. + +The service layer is faked through ``gateway_migrate``'s ``_installed_services`` / ``_live_gateway_pid`` +seams (the shape ``test_gateway_migrate_multiplex.py`` uses). +""" + +from __future__ import annotations + +import json +import os +import shutil +from pathlib import Path + +import pytest + +import hermes_constants +from gateway.config import GatewayConfig, load_gateway_config +from hermes_cli import gateway_migrate as gm +from hermes_cli import gateway_multiplex_mode as mode +from hermes_cli.config_defaults import DEFAULT_CONFIG + + +@pytest.fixture +def fleet(tmp_path, monkeypatch): + """default + coder + ops with distinct bot tokens; nothing runs a gateway unless a test says so.""" + root = tmp_path / "hermes" + for sub in ("profiles/coder", "profiles/ops"): + (root / sub).mkdir(parents=True) + (root / "config.yaml").write_text("model:\n default: x\n", encoding="utf-8") + (root / ".env").write_text("TELEGRAM_BOT_TOKEN=111111:default-token\n", encoding="utf-8") + (root / "profiles/coder/.env").write_text("TELEGRAM_BOT_TOKEN=222222:coder-token\n", encoding="utf-8") + (root / "profiles/ops/.env").write_text("DISCORD_BOT_TOKEN=ops-discord-333333\n", encoding="utf-8") + monkeypatch.setenv("HERMES_HOME", str(root)) + monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False) + for name in ("TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "API_SERVER_KEY", "WEBHOOK_ENABLED"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(hermes_constants, "_default_hermes_root_memo", None) + services: dict[str, list] = {} + pids: dict[str, int] = {} + monkeypatch.setattr(gm, "_installed_services", lambda home: services.get(_name(home), [])) + monkeypatch.setattr(gm, "_live_gateway_pid", lambda home: pids.get(_name(home))) + monkeypatch.setattr(gm, "_host_supports_migration", lambda: None) + return root, services, pids + + +def _name(home: Path) -> str: + return hermes_constants.profile_name_for_home(home) or "default" + + +def test_default_config_is_on_but_the_loader_leaves_an_unset_flag_undecided(): + """DEFAULT_CONFIG says on; GatewayConfig keeps "unset" distinguishable from "chosen" so the boot + guard can tell them apart (an explicit value must survive verbatim).""" + assert DEFAULT_CONFIG["gateway"]["multiplex_profiles"] is True + assert GatewayConfig.from_dict({}).multiplex_profiles is None + assert not GatewayConfig.from_dict({}).multiplex_profiles # readers treat undecided as off + assert GatewayConfig.from_dict({"gateway": {"multiplex_profiles": False}}).multiplex_profiles is False + assert GatewayConfig.from_dict({"multiplex_profiles": "true"}).multiplex_profiles is True + + +def test_unset_flag_multiplexes_a_quiet_fleet_and_stays_standalone_beside_a_live_secondary(fleet): + root, services, pids = fleet + decision = mode.resolve_multiplex_mode(cfg := load_gateway_config()) + assert decision == mode.MultiplexDecision(True, "default", decision.reason) + assert cfg.multiplex_profiles is True + + pids["coder"] = 4101 # coder still runs its own gateway: a multiplexer would double-poll its bot + decision = mode.resolve_multiplex_mode(cfg := load_gateway_config()) + assert not decision.enabled and decision.source == "guard" + assert "'coder'" in decision.reason and gm.MIGRATE_COMMAND in decision.reason + assert cfg.multiplex_profiles is False + # The CLI side agrees: no live multiplexer record and no explicit opt-in means coder is NOT served. + from hermes_cli.gateway import named_profile_served_by_running_multiplexer + assert named_profile_served_by_running_multiplexer("coder") is False + assert mode.default_gateway_multiplexes(root) is False + + pids.clear() + services["ops"] = [("systemd", False)] # an installed unit counts the same as a live pid + assert mode.resolve_multiplex_mode(load_gateway_config()).source == "guard" + + +def test_preflight_blocker_and_single_profile_keep_the_unset_default_standalone(fleet): + root, _services, _pids = fleet + # coder reuses the default's bot token -> the migration preflight's duplicate-credential blocker. + (root / "profiles/coder/.env").write_text("TELEGRAM_BOT_TOKEN=111111:default-token\n", encoding="utf-8") + decision = mode.resolve_multiplex_mode(load_gateway_config()) + assert decision.source == "guard" and "profile_routes" in decision.reason + + for sub in ("profiles/coder", "profiles/ops"): + shutil.rmtree(root / sub) + decision = mode.resolve_multiplex_mode(load_gateway_config()) + assert decision == mode.MultiplexDecision(False, "guard", mode.SINGLE_PROFILE_REASON) + + +def test_explicit_flag_is_never_second_guessed(fleet, monkeypatch): + root, _services, pids = fleet + pids["coder"] = 4101 + (root / "config.yaml").write_text("gateway:\n multiplex_profiles: true\n", encoding="utf-8") + cfg = load_gateway_config() + assert mode.resolve_multiplex_mode(cfg) == mode.MultiplexDecision(True, "config") + assert cfg.multiplex_profiles is True + assert mode.explicit_multiplex_flag(root) is True + + monkeypatch.setenv("GATEWAY_MULTIPLEX_PROFILES", "false") + pids.clear() + cfg = load_gateway_config() + assert mode.resolve_multiplex_mode(cfg) == mode.MultiplexDecision(False, "config") + assert cfg.multiplex_profiles is False + assert mode.explicit_multiplex_flag(root) is False + + +def test_migration_plan_treats_the_unset_default_as_not_yet_multiplexed(fleet): + """The fleet the boot guard refuses is exactly the one ``hermes gateway migrate --multiplex`` folds: + an unset flag must not read as "already multiplexed" or the migration would short-circuit.""" + root, services, pids = fleet + pids.update({"coder": 4101, "ops": 4102}) + services.update({"coder": [("systemd", False)], "ops": [("systemd", False)]}) + plan = gm.build_migration_plan() + assert plan.multiplex_flag_on is False and not plan.already_multiplexed + assert plan.eligible_for_migration() + + +def test_live_record_outranks_the_raw_flag_for_other_processes(fleet, monkeypatch): + """A CLI process asks the LIVE default gateway (which settled the unset default itself) before + reading config; a gateway that stayed standalone recorded an empty served set.""" + root, _services, _pids = fleet + import gateway.status as status + monkeypatch.setattr(status, "_read_process_cmdline", lambda pid: "hermes gateway run") + (root / "gateway.pid").write_text(json.dumps({"pid": os.getpid(), "hermes_home": str(root)})) + record = {"pid": os.getpid(), "hermes_home": str(root), "gateway_state": "running", "served_profiles": []} + (root / "gateway_state.json").write_text(json.dumps(record)) + assert mode.default_gateway_multiplexes(root) is False + record["served_profiles"] = ["default", "coder", "ops"] + (root / "gateway_state.json").write_text(json.dumps(record)) + assert mode.default_gateway_multiplexes(root) is True diff --git a/website/docs/developer-guide/multiplexing-gateway.md b/website/docs/developer-guide/multiplexing-gateway.md index 7f9ac7db94..613f4891f9 100644 --- a/website/docs/developer-guide/multiplexing-gateway.md +++ b/website/docs/developer-guide/multiplexing-gateway.md @@ -5,9 +5,13 @@ description: "Design of the one-gateway-for-all-profiles mode: scope composition # Multiplexing Gateway -One gateway process can serve every profile in the install. The mode is opt-in -(`gateway.multiplex_profiles`, default `false`), and everything it changes -reverts the moment the flag is off. This document is the design rationale +One gateway process can serve every profile in the install. The mode is on by +default (`gateway.multiplex_profiles`, default `true`), and everything it +changes reverts the moment the flag is off. An *unset* flag is settled at boot +by `hermes_cli/gateway_multiplex_mode.py::resolve_multiplex_mode`, which runs +the `hermes gateway migrate` preflight and keeps the gateway standalone when a +secondary still runs its own gateway, a blocker exists, or the host cannot be +migrated (see "The mode flag"). This document is the design rationale referenced from `agent/secret_scope.py` ("Workstream A"): what is isolated per profile, the mechanism that isolates it, and what deliberately stays process-global. @@ -28,8 +32,20 @@ is documented as a known limitation at the end of this document. ## The mode flag -- Config: `gateway.multiplex_profiles: true` (also accepted at top level). - Parsed in `gateway/config.py` with precedence env > config > default. +- Config: `gateway.multiplex_profiles` (also accepted at top level). Parsed in + `gateway/config.py` with precedence env > config > unset. `GatewayConfig` + keeps an unset flag as `None` (readers test truthiness, so it reads as off); + `load_gateway_config_for_runner` then calls `resolve_multiplex_mode`, which + writes the boot verdict — `True` on a quiet multi-profile default install, + `False` with a logged reason otherwise. Explicit values pass through + verbatim; a config injected into `GatewayRunner(config=...)` is not resolved. +- Other processes read the LIVE gateway's `served_profiles` record first and + the explicit flag second (`gateway_multiplex_mode.default_gateway_multiplexes` + / `explicit_multiplex_flag`), never the merged default: `named_profile_served_ + by_running_multiplexer`, the enroll warning, the dashboard's listener guard, + the cron-fire port resolver, container boot, and the migration plan + (`_read_multiplex_flag`, so an unset default reads as "not yet multiplexed" + and the fold proceeds). - Env override: `GATEWAY_MULTIPLEX_PROFILES` accepts explicit truthy/falsy tokens only; a blank or unrecognized value returns "no override" so an empty deployment secret cannot shadow a config opt-in. diff --git a/website/docs/user-guide/multi-profile-gateways.md b/website/docs/user-guide/multi-profile-gateways.md index a7bbf5f5df..b0bcc4ea96 100644 --- a/website/docs/user-guide/multi-profile-gateways.md +++ b/website/docs/user-guide/multi-profile-gateways.md @@ -63,15 +63,35 @@ automatically on crash and on user login. ## Alternative: one gateway for all profiles (multiplexing) -The model above runs **one process per profile**. That is the default and is -the right choice for most setups. But on a host with many profiles — or a -container deployment where one process per profile is operationally heavy — you -can instead run a **single multiplexing gateway**: the default profile's gateway -becomes the sole inbound process and serves messages for *every* profile on the -box. +The model above runs **one process per profile**. The alternative is a +**single multiplexing gateway**: the default profile's gateway becomes the sole +inbound process and serves messages for *every* profile on the box. -This is **opt-in** and **off by default**. When it's off, nothing on this page -changes — every behavior below is inert. +Multiplexing is **on by default** (`gateway.multiplex_profiles` defaults to +`true`), with one safety rule: an *unset* flag is a request the default gateway +settles at boot, never a verdict. Each start it runs the same preflight as +[`hermes gateway migrate --multiplex`](#migrating-from-per-profile-gateways) and +multiplexes only when the fold would have been safe — the default profile, two +or more profiles, no secondary still running its own gateway (live process or +installed service), no duplicate bot credential, no port-binding platform +without a `/p//` ingress, and a host the migration understands (not an +s6 container or Windows Scheduled Tasks). Otherwise it comes up exactly as +before — serving the default profile only — and logs the blocker plus the +`hermes gateway migrate --multiplex` one-liner. Nothing is changed on disk. + +An **explicit** value is never second-guessed: + +- `gateway.multiplex_profiles: true` (what the migration writes) multiplexes + regardless of the preflight — you, or the migration, made the call. +- `gateway.multiplex_profiles: false` (what `--standalone` restores) keeps + per-profile gateways for good. When it's off, nothing on this page changes — + every behavior below is inert. +- `GATEWAY_MULTIPLEX_PROFILES` in the process environment overrides both. + +Other processes (`hermes -p gateway start`, the dashboard, `hermes gateway +migrate`) never guess how an unset flag was settled: they read the running +default gateway's `served_profiles` record, and fall back to the explicit flag +only when no gateway runs. ### When to prefer multiplexing @@ -84,13 +104,15 @@ Stick with one-process-per-profile when you want hard process-level isolation between profiles (separate memory footprints, independent crash domains, the ability to restart one profile without touching the others). -### How to opt in +### Pinning the flag -Set the flag on the **default profile** (it owns the multiplexer) and restart -its gateway: +With the flag unset, the default gateway decides at each boot (above). To pin +it, set it on the **default profile** (it owns the multiplexer) and restart its +gateway — `true` forces multiplexing even where the boot preflight would have +held back, `false` opts out durably: ```bash -hermes config set gateway.multiplex_profiles true +hermes config set gateway.multiplex_profiles true # or false hermes gateway restart ``` @@ -102,7 +124,7 @@ gateway: ``` (The flag is also accepted as a top-level `multiplex_profiles: true` for -convenience.) On the next start the default gateway enumerates every profile, +convenience.) When multiplexing, the default gateway enumerates every profile, brings up each profile's enabled platforms under that profile's own credentials, and routes each inbound message to the profile it belongs to. Each turn resolves the routed profile's config, skills, memory, SOUL, **and provider @@ -818,9 +840,11 @@ grep -H 'TELEGRAM_BOT_TOKEN\|DISCORD_BOT_TOKEN' \ ## Migrating from per-profile gateways If your profiles each run their own gateway today (one systemd unit or launchd -agent per profile), you can fold them into a single multiplexed default gateway -with one command — and roll back with another. Standalone per-profile gateways -remain fully supported; this is an optional migration, not a removal. +agent per profile), the default gateway's boot preflight keeps it standalone +(the unset default never double-binds a running fleet). Fold them into a single +multiplexed default gateway with one command — and roll back with another. +Standalone per-profile gateways remain fully supported; this is an optional +migration, not a removal. ```bash hermes gateway migrate --multiplex --dry-run # print the plan and any blockers; changes nothing