Independent review found the previous commit could make the unbounded-growth symptom it fixes PERMANENT, and that its argv narrowing re-opened #92401 inside a single install. - other_generations_for_path() counts only LIVE generations. A retired one is already write-fenced (StateDbReplacedError, close-time checkpoint disabled) and leaves the registry only when its holder releases -- which a gateway handle does not do before shutdown. One inode replacement (repair swap, backup restore, snapshot) therefore skipped auto-VACUUM for that path for the whole process lifetime. - _argv_scoped_to_other_home is ranked evidence now. argv[0] is the SHARED install binary for every profile on a host, so it is neutral, never proof of a hold; the process's own --hermes-home / HERMES_HOME= / --profile / -p selection decides which home it serves, and a token naming ANOTHER profile's store is tested before any own-prefix check. argv[0] also stops dismissing a holder of a store whose home is not part of an install layout (a custom HERMES_HOME is served BY the binary under ~/.hermes). - install_root comes from hermes_constants.named_profile_home, not basename(parent) == "profiles": an arbitrary <X>/profiles/<n>/ tree no longer promotes all of <X> to "ours". - The launch profile keeps its gateway.sessions_dir override when its own store is pruned; every other served profile prunes under its own <home>/sessions. Pruning under the wrong dir orphaned transcripts forever. - glob.escape on the request_dump_<id>_* sweep (pre-existing). Tests: the retired-generation test asserted the starvation mechanism; it is replaced by the invariant (a live sibling defers VACUUM) plus a red-on-base test that a retired, write-fenced generation does not. New argv cases cover the shared binary with -p other, another profile's store token, and a non-Hermes <X>/profiles/ tree; the housekeeping fixture now asserts the unpinned store still resolves inside the sandbox before yielding.
230 lines
9.6 KiB
Python
230 lines
9.6 KiB
Python
"""Housekeeping chores that read a profile's home, config or credentials run under the OWNING
|
|
profile's runtime scope on a multiplexed gateway.
|
|
|
|
The housekeeping thread has no turn on the stack, so nothing bound a profile for it: under
|
|
``gateway.multiplex_profiles`` the skills-sync pulls resolved Nous credentials through the
|
|
fail-closed reader and logged ``no profile secret scope on a multiplexed call`` four times per
|
|
hourly tick, per chore, while the launch profile's home/credentials leaked into every served
|
|
profile's pull. The MCP config reconciler already iterated the served profiles under
|
|
``_profile_runtime_scope``; the sync/curator ticks now ride the same iteration.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import gateway.run as gateway_run
|
|
|
|
|
|
class _Ticks:
|
|
"""Stop event that lets the housekeeping loop run exactly ``n`` ticks with no sleeping."""
|
|
|
|
def __init__(self, n: int):
|
|
self.n, self.left = 0, n
|
|
|
|
def is_set(self):
|
|
return self.n >= self.left
|
|
|
|
def wait(self, timeout=None):
|
|
self.n += 1
|
|
return True
|
|
|
|
|
|
def _profile(home: Path, base_url: str) -> None:
|
|
home.mkdir(parents=True, exist_ok=True)
|
|
(home / "config.yaml").write_text("model:\n provider: nous\n", encoding="utf-8")
|
|
(home / ".env").write_text(f"NOUS_INFERENCE_BASE_URL={base_url}\n", encoding="utf-8")
|
|
(home / "auth.json").write_text(json.dumps({"version": 1, "providers": {"nous": {
|
|
"access_token": "x.y.z", "refresh_token": "r", "expires_at": 0,
|
|
"portal_base_url": "https://portal.nousresearch.com", "client_id": "c"}}}), encoding="utf-8")
|
|
|
|
|
|
@pytest.fixture
|
|
def two_homes(tmp_path, monkeypatch):
|
|
"""Launch home A (= multiplex ``default``) and served named profile B under ``A/profiles/b``."""
|
|
fake_home = tmp_path / "home"
|
|
a = fake_home / ".hermes"
|
|
b = a / "profiles" / "b"
|
|
_profile(a, "https://a.example/v1")
|
|
_profile(b, "https://b.example/v1")
|
|
monkeypatch.setattr(Path, "home", classmethod(lambda cls: fake_home))
|
|
monkeypatch.setenv("HERMES_HOME", str(a))
|
|
monkeypatch.delenv("NOUS_INFERENCE_BASE_URL", raising=False)
|
|
# The hermetic conftest pins ``hermes_state.DEFAULT_DB_PATH`` at one sandbox store whenever
|
|
# hermes_state is already imported, and that pin WINS over ``get_hermes_home()`` inside
|
|
# ``_default_db_path()`` — exactly the per-profile resolution these tests exist to prove.
|
|
# Restore the import-time sentinel so an argless ``acquire()`` resolves through the scope.
|
|
import hermes_state
|
|
monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", hermes_state._IMPORT_DEFAULT_DB_PATH)
|
|
# Disabling the hermetic pin is only safe while the sentinel still resolves INSIDE the sandbox:
|
|
# a resolution that escaped to the real home would have these tests writing the live store.
|
|
resolved = Path(hermes_state._default_db_path())
|
|
assert resolved.is_relative_to(tmp_path), f"unpinned store escaped the sandbox: {resolved}"
|
|
return a, b
|
|
|
|
|
|
def _record_credential_chores(monkeypatch):
|
|
"""Replace the three credential-reading chores with recorders of (home, Nous override) they see."""
|
|
import agent.curator as curator
|
|
import tools.skills_sync_client as ssc
|
|
import tools.skills_sync_client_org as sso
|
|
from hermes_cli.auth_nous import _nous_inference_env_override
|
|
from hermes_constants import get_hermes_home
|
|
|
|
seen: dict = {"sync": [], "org": [], "curator": []}
|
|
|
|
def _rec(key):
|
|
return lambda *a, **k: seen[key].append((get_hermes_home().name, _nous_inference_env_override()))
|
|
|
|
monkeypatch.setattr(ssc, "maybe_pull_skills", _rec("sync"))
|
|
monkeypatch.setattr(sso, "maybe_pull_org_skills", _rec("org"))
|
|
monkeypatch.setattr(curator, "maybe_run_curator", _rec("curator"))
|
|
return seen
|
|
|
|
|
|
def _run_60_ticks(runner):
|
|
gateway_run._start_gateway_housekeeping(_Ticks(60), interval=0, runner=runner)
|
|
|
|
|
|
def test_multiplexed_sync_ticks_run_once_per_profile_in_its_own_scope(two_homes, monkeypatch, caplog):
|
|
"""Under multiplex every credential-reading chore visits each served profile inside ITS scope:
|
|
A's tick reads A's override, B's reads B's (B never sees A's), and no fail-closed credential
|
|
read fires the ``no profile secret scope`` warning. The ambient home is untouched afterwards."""
|
|
from agent.secret_scope import set_multiplex_active
|
|
from hermes_constants import get_hermes_home
|
|
|
|
a, b = two_homes
|
|
seen = _record_credential_chores(monkeypatch)
|
|
set_multiplex_active(True)
|
|
try:
|
|
with caplog.at_level(logging.WARNING):
|
|
_run_60_ticks(SimpleNamespace(config=SimpleNamespace(multiplex_profiles=True)))
|
|
finally:
|
|
set_multiplex_active(False)
|
|
|
|
expected = [(a.name, "https://a.example/v1"), (b.name, "https://b.example/v1")]
|
|
assert seen == {"sync": expected, "org": expected, "curator": expected}
|
|
assert not [r for r in caplog.records if "no profile secret scope" in r.getMessage()]
|
|
assert get_hermes_home() == a
|
|
|
|
|
|
def test_multiplexed_auto_archive_tick_sweeps_every_served_profile_store(two_homes, monkeypatch):
|
|
"""The auto-archive sweep reaches each served profile's OWN state.db.
|
|
|
|
``acquire()`` resolves through ``get_hermes_home()``, so an unscoped tick archived the
|
|
launch profile's store only — and `hermes serve`/the dashboard defer to the gateway for
|
|
every profile it owns, so a served secondary would have had no archiver at all.
|
|
"""
|
|
from agent.secret_scope import set_multiplex_active
|
|
from hermes_state import SessionDB
|
|
|
|
a, b = two_homes
|
|
swept: list = []
|
|
monkeypatch.setattr(
|
|
SessionDB, "maybe_auto_archive", lambda self, **kw: swept.append(Path(self.db_path)))
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda *args, **kwargs: {"sessions": {"auto_archive": True, "min_interval_hours": 0}})
|
|
|
|
set_multiplex_active(True)
|
|
try:
|
|
_run_60_ticks(SimpleNamespace(config=SimpleNamespace(multiplex_profiles=True)))
|
|
finally:
|
|
set_multiplex_active(False)
|
|
|
|
assert swept == [a / "state.db", b / "state.db"]
|
|
|
|
|
|
def test_multiplexed_maintenance_tick_prunes_every_served_profile_store(two_homes, monkeypatch):
|
|
"""Prune/VACUUM reaches each served profile's OWN state.db, under its OWN ``sessions:`` config.
|
|
|
|
Prune and VACUUM ran once in the gateway constructor against a handle pinned to the launch
|
|
home, so a multiplexed secondary's store was never pruned or vacuumed by anybody — it grew
|
|
without bound while the launch profile's ``retention_days`` decided whether it happened at all.
|
|
Real stores, real config files: nothing here is patched.
|
|
"""
|
|
from agent.secret_scope import set_multiplex_active
|
|
from hermes_state import SessionDB
|
|
|
|
homes = two_homes
|
|
for home in homes:
|
|
(home / "config.yaml").write_text(
|
|
"model:\n provider: nous\n"
|
|
"sessions:\n"
|
|
" auto_prune: true\n"
|
|
" retention_days: 0\n"
|
|
" min_interval_hours: 0\n"
|
|
" vacuum_after_prune: false\n",
|
|
encoding="utf-8")
|
|
db = SessionDB(db_path=home / "state.db")
|
|
db.create_session("old", "cli")
|
|
db.end_session("old", "done")
|
|
db.close()
|
|
|
|
set_multiplex_active(True)
|
|
try:
|
|
_run_60_ticks(SimpleNamespace(config=SimpleNamespace(multiplex_profiles=True)))
|
|
finally:
|
|
set_multiplex_active(False)
|
|
|
|
for home in homes:
|
|
db = SessionDB(db_path=home / "state.db")
|
|
try:
|
|
assert db.get_session("old") is None, f"{home.name}'s store was never pruned"
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def test_prune_unlinks_transcripts_under_the_configured_sessions_dir(two_homes, tmp_path):
|
|
"""``gateway.sessions_dir`` governs the LAUNCH profile's transcripts; others use their own home.
|
|
|
|
Hardcoding ``<home>/sessions`` made the prune unlink under a directory nothing writes to, so an
|
|
override left every pruned session's ``.json``/``.jsonl``/``request_dump_*`` orphaned forever.
|
|
"""
|
|
from agent.secret_scope import set_multiplex_active
|
|
from hermes_state import SessionDB
|
|
|
|
a, b = two_homes
|
|
override = tmp_path / "custom-transcripts"
|
|
override.mkdir()
|
|
for home, transcripts in ((a, override), (b, b / "sessions")):
|
|
(home / "config.yaml").write_text(
|
|
"model:\n provider: nous\n"
|
|
"sessions:\n"
|
|
" auto_prune: true\n"
|
|
" retention_days: 0\n"
|
|
" min_interval_hours: 0\n"
|
|
" vacuum_after_prune: false\n",
|
|
encoding="utf-8")
|
|
db = SessionDB(db_path=home / "state.db")
|
|
db.create_session("old", "cli")
|
|
db.end_session("old", "done")
|
|
db.close()
|
|
transcripts.mkdir(parents=True, exist_ok=True)
|
|
(transcripts / "old.jsonl").write_text("{}\n", encoding="utf-8")
|
|
|
|
set_multiplex_active(True)
|
|
try:
|
|
_run_60_ticks(SimpleNamespace(config=SimpleNamespace(
|
|
multiplex_profiles=True, sessions_dir=override)))
|
|
finally:
|
|
set_multiplex_active(False)
|
|
|
|
assert not (override / "old.jsonl").exists(), "launch profile's configured transcript survived"
|
|
assert not (b / "sessions" / "old.jsonl").exists(), "profile b's transcript survived"
|
|
|
|
|
|
def test_single_profile_sync_ticks_run_once_against_the_process_home(two_homes, monkeypatch):
|
|
"""Control: a single-profile gateway (multiplex off) still runs each chore exactly once against
|
|
the process home — the named profile directory on disk is not visited."""
|
|
a, _b = two_homes
|
|
seen = _record_credential_chores(monkeypatch)
|
|
|
|
_run_60_ticks(SimpleNamespace(config=SimpleNamespace(multiplex_profiles=False)))
|
|
|
|
assert {k: [h for h, _ in v] for k, v in seen.items()} == {
|
|
"sync": [a.name], "org": [a.name], "curator": [a.name]}
|