diff --git a/hermes_state_wal.py b/hermes_state_wal.py index fe3bf0b85f..7b228302d9 100644 --- a/hermes_state_wal.py +++ b/hermes_state_wal.py @@ -8,6 +8,7 @@ from __future__ import annotations import contextlib import functools import logging +import os import sqlite3 import sys import threading @@ -172,6 +173,74 @@ class WalUnsupportedError(sqlite3.OperationalError): opener). Subclasses ``OperationalError`` so DB-init handlers still catch it.""" +# Docker Desktop / OrbStack / Podman expose host bind mounts to the guest as ``fuse.virtiofs`` or ``9p``. WAL's +# -shm file must be coherent shared memory for every opener; across the VM boundary it is not, and under write +# pressure the failure is SILENT (zero-filled pages), so the reactive ``_WAL_INCOMPAT_MARKERS`` fallback in +# ``_enable_wal`` never gets a signal — WAL must be refused before the pragma. Port of openclaw/openclaw#120597. +# Detection is Linux ``/proc/self/mountinfo`` only (statvfs carries no fstype); elsewhere, or when the table is +# unreadable, the answer is False and behaviour is unchanged. Nothing else (ext4/btrfs/xfs/zfs/tmpfs/overlay/nfs) +# is ever flagged here — those keep the existing reactive paths. +_CROSS_VM_FSTYPES = frozenset({"virtiofs", "fuse.virtiofs", "9p", "9p2000", "9p2000.l", "9p2000.u"}) +_cross_vm_fs_cache: Dict[str, bool] = {} # per DB directory; kanban_db.connect() opens per operation +_cross_vm_fs_cache_lock = threading.Lock() +_cross_vm_warned_paths: set[str] = set() +_cross_vm_warned_lock = threading.Lock() + + +def _mountinfo_fstype(directory: str, mountinfo_path: str = "/proc/self/mountinfo") -> str: + """fstype of the longest mount point that is a prefix of ``directory`` (``""`` if unreadable / no match).""" + try: + with open(mountinfo_path, "r", encoding="utf-8", errors="replace") as fh: + lines = fh.read().splitlines() + except OSError: + return "" + best_len, best_fstype = -1, "" + for line in lines: + # proc(5): [optional...] - + fields, _, tail = line.partition(" - ") + parts = fields.split() + if len(parts) < 5 or not tail: + continue + mount_point = parts[4] + if "\\" in mount_point: # octal escapes (\040 = space) + mount_point = mount_point.encode("latin-1", "ignore").decode("unicode_escape") + if directory == mount_point or directory.startswith(mount_point.rstrip("/") + "/"): + if len(mount_point) > best_len: + best_len, best_fstype = len(mount_point), tail.split()[0] + return best_fstype.lower() + + +def _detect_cross_vm_fs(directory: str, mountinfo_path: str = "/proc/self/mountinfo") -> bool: + """True only when ``directory`` sits on a virtiofs/9p mount per ``mountinfo_path``.""" + if sys.platform != "linux": + return False + return _mountinfo_fstype(directory, mountinfo_path) in _CROSS_VM_FSTYPES + + +def _path_on_cross_vm_fs(path: str) -> bool: + """True when ``path`` resides on a virtiofs/9p (cross-VM) filesystem; cached per resolved directory.""" + try: + directory = os.path.dirname(os.path.realpath(path)) or "/" + except (OSError, ValueError): + return False + with _cross_vm_fs_cache_lock: + cached = _cross_vm_fs_cache.get(directory) + if cached is None: + cached = _detect_cross_vm_fs(directory) + with _cross_vm_fs_cache_lock: + _cross_vm_fs_cache[directory] = cached + return cached + + +def _connection_db_file(conn: sqlite3.Connection) -> str: + """Filesystem path of the ``main`` database, ``""`` for in-memory / unknown.""" + try: + row = conn.execute("PRAGMA database_list").fetchone() + return str(row[2]) if row and row[2] else "" + except (sqlite3.OperationalError, IndexError, TypeError): + return "" + + def _verify_configured_delete(actual: str) -> str: """Raise unless SQLite reported ``delete`` for an explicit operator request.""" if actual != "delete": @@ -233,6 +302,15 @@ def apply_wal_with_fallback(conn: sqlite3.Connection, *, db_label: str = "state. "concurrent openers); cannot guarantee WAL") _log_once("wal_probe_unknown", db_label) return "wal" + # Cross-VM bind mount (virtiofs/9p): WAL corrupts silently there, so refuse to ENABLE it. On-disk WAL + # databases were returned above (never live-downgrade); a 0-page / DELETE file just stays DELETE. + db_file = _connection_db_file(conn) + if db_file and _path_on_cross_vm_fs(db_file): + if require_wal: + raise WalUnsupportedError("journal_mode=WAL refused: database is on a cross-VM filesystem (virtiofs/9p) " + "where WAL shared-memory silently corrupts") + _log_once("cross_vm_fs", db_label) + return _set_journal_mode_no_wait(conn, "DELETE") or "delete" return _enable_wal(conn, db_label, require_wal, current_mode) @@ -435,6 +513,13 @@ _ONCE_LOGS = { "%s: could not verify the on-disk journal mode (database is locked / busy); not issuing a journal-mode " "set-pragma while another connection may hold the file (it could unlink the -wal/-shm sidecars that " "connection still uses). Leaving the file untouched; this connection inherits the on-disk mode. This message fires once per process per database."), + "cross_vm_fs": (_cross_vm_warned_lock, "_cross_vm_warned_paths", logging.WARNING, + # The DB keeps working in DELETE mode; the loss is concurrency, so WARNING with the actionable fix. + "%s: database directory is on a cross-VM filesystem (virtiofs/9p — typical for Docker Desktop / OrbStack / " + "Podman host bind mounts). SQLite WAL shared-memory is not coherent across the VM boundary and can silently " + "corrupt the database, so journal_mode=DELETE is used instead. To restore WAL concurrency, move the database " + "onto a native volume (e.g. a named Docker volume) instead of a host bind mount. This message fires once per " + "process per database."), } diff --git a/tests/hermes_state/test_cross_vm_fs_wal_refusal.py b/tests/hermes_state/test_cross_vm_fs_wal_refusal.py new file mode 100644 index 0000000000..53a1f95e32 --- /dev/null +++ b/tests/hermes_state/test_cross_vm_fs_wal_refusal.py @@ -0,0 +1,96 @@ +"""Cross-VM filesystem (virtiofs/9p) WAL refusal — port of openclaw#120597. + +WAL over a VM-boundary filesystem (Docker Desktop / OrbStack / Podman host bind mounts) corrupts silently, so +``apply_wal_with_fallback`` must refuse to ENABLE WAL when the DB lives on such a mount — before the pragma — +while never live-downgrading an on-disk WAL database and never flagging an ordinary filesystem. +""" + +import sqlite3 + +import pytest + +import hermes_state_wal +from hermes_state_wal import WalUnsupportedError, _detect_cross_vm_fs, apply_wal_with_fallback + + +def _mountinfo(tmp_path, lines): + p = tmp_path / "mountinfo" + p.write_text("\n".join(lines) + "\n") + return str(p) + + +# Realistic mountinfo rows (id parent major:minor root mountpoint opts ... - fstype source superopts) +ROOT_EXT4 = "25 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw" +BIND_VIRTIOFS = "612 25 0:53 / /data rw,relatime shared:300 - fuse.virtiofs mount0 rw" +BIND_9P = "613 25 0:54 / /mnt/host rw,relatime - 9p host0 rw,trans=virtio" +NESTED_EXT4 = "614 612 8:2 / /data/native rw,relatime - ext4 /dev/sdb1 rw" +SPACE_VIRTIOFS = "615 25 0:55 / /mnt/my\\040share rw,relatime - virtiofs share rw" + + +class TestDetectCrossVmFs: + @pytest.mark.parametrize("path,expected", [ + ("/data/agent", True), # fuse.virtiofs bind mount + ("/mnt/host/db", True), # 9p bind mount + ("/mnt/my share/db", True), # octal-escaped mount point + ("/home/user/.hermes", False), # ext4 root + ("/data/native/db", False), # ext4 mounted over the virtiofs tree — longest prefix wins + ("/datastore", False), # sibling path sharing a prefix string, not a mount prefix + ]) + def test_only_virtiofs_and_9p_mounts_are_flagged(self, tmp_path, path, expected): + mi = _mountinfo(tmp_path, [ROOT_EXT4, BIND_VIRTIOFS, BIND_9P, NESTED_EXT4, SPACE_VIRTIOFS]) + assert _detect_cross_vm_fs(path, mountinfo_path=mi) is expected + + @pytest.mark.parametrize("fstype", [ + "ext4", "xfs", "btrfs", "zfs", "tmpfs", "overlay", "nfs", "nfs4", "cifs", "fuse.sshfs", "apfs", "f2fs", + ]) + def test_ordinary_filesystems_never_flagged(self, tmp_path, fstype): + # A false positive here would put every session on DELETE mode — the class bug this pins absent. + mi = _mountinfo(tmp_path, [f"25 1 8:1 / / rw,relatime shared:1 - {fstype} /dev/sda1 rw"]) + assert _detect_cross_vm_fs("/home/user/.hermes", mountinfo_path=mi) is False + + def test_missing_mountinfo_conservative_false(self, tmp_path): + assert _detect_cross_vm_fs("/data", mountinfo_path=str(tmp_path / "nope")) is False + + +class TestWalRefusalOnCrossVmFs: + @pytest.fixture(autouse=True) + def _isolate(self, monkeypatch): + # Pin the WAL-reset vulnerability gate OFF: on builds bundling a vulnerable SQLite (3.50.4 on CI) + # apply_wal_with_fallback returns via _apply_delete_for_wal_reset_bug before the cross-VM check. + monkeypatch.setattr(hermes_state_wal, "is_sqlite_wal_reset_vulnerable", lambda *a, **k: False) + monkeypatch.setattr(hermes_state_wal, "resolve_journal_mode", lambda: "wal") + hermes_state_wal._cross_vm_warned_paths.clear() + + def test_fresh_db_on_cross_vm_fs_gets_delete_and_without_detection_gets_wal(self, tmp_path, monkeypatch): + monkeypatch.setattr(hermes_state_wal, "_path_on_cross_vm_fs", lambda p: True) + conn = sqlite3.connect(str(tmp_path / "a.db")) + assert apply_wal_with_fallback(conn, db_label="a.db") == "delete" + conn.close() + # Sabotage guard: same environment, detection off -> WAL is enabled, so the refusal above did the work. + monkeypatch.setattr(hermes_state_wal, "_path_on_cross_vm_fs", lambda p: False) + conn = sqlite3.connect(str(tmp_path / "b.db")) + mode = apply_wal_with_fallback(conn, db_label="b.db") + conn.close() + if mode != "wal": + pytest.skip("environment refuses WAL for unrelated reasons") + + def test_require_wal_raises_on_cross_vm_fs(self, tmp_path, monkeypatch): + monkeypatch.setattr(hermes_state_wal, "_path_on_cross_vm_fs", lambda p: True) + conn = sqlite3.connect(str(tmp_path / "state.db")) + with pytest.raises(WalUnsupportedError, match=r"cross-VM"): + apply_wal_with_fallback(conn, db_label="state.db", require_wal=True) + conn.close() + + def test_on_disk_wal_db_is_never_downgraded(self, tmp_path, monkeypatch): + db = tmp_path / "already-wal.db" + seed = sqlite3.connect(str(db)) + if str(seed.execute("PRAGMA journal_mode=WAL").fetchone()[0]).lower() != "wal": + seed.close() + pytest.skip("environment refuses WAL") + seed.execute("CREATE TABLE t (x)") + seed.commit() + seed.close() + monkeypatch.setattr(hermes_state_wal, "_path_on_cross_vm_fs", lambda p: True) + conn = sqlite3.connect(str(db)) + assert apply_wal_with_fallback(conn, db_label=str(db)) == "wal" + conn.close()