fix(files): one live-DB refusal source; hold the lock through /api/files/read

Gate r2 Low cleanups (house rule: no aliases/shims):
- Drop the is_live_database_file alias; its point-in-time caveat now lives on
  has_live_connection.
- _refuse_live_database reuses offline_file_access's message (via _serve_offline),
  so a download 409 on state.db-shm names the main database like the read path;
  the verb is "serve" so it fits read/download/stream.
- /api/files/read reads whole files in-process, so _read_base64_file now holds
  offline_file_access through close (409 on a live DB; OSError stays 500). Only
  the streamed FileResponse routes keep the point-in-time check.
- That check takes the global _live_lock, which other threads hold across
  whole-file reads, so fs_download and the managed stream routes run it via
  asyncio.to_thread instead of stalling the event loop.
- _managed_readable_file docstring no longer claims a size cap;
  _read_file_reference returns (early, text) instead of a str|Expansion union
  sniffed with isinstance.

Co-authored-by: Benjamin PERRY <benjaminperry6@yahoo.fr>

agent/context_references.py

@@ -302,22 +302,25 @@ def _expand_path_reference(ref: ContextReference, cwd: Path, *, allowed_root: Pa
         # between a check and a later open otherwise), but release it before token
         # counting/formatting: the registry lock blocks every tracked connect/close.
         with offline_file_access(path, what="preview context reference"):
-            raw = _read_file_reference(ref, path, max_inline_tokens)
+            early, text = _read_file_reference(ref, path, max_inline_tokens)
     except LiveConnectionError:
         return None, _on_disk_reference_block(
             ref, path, descriptor="live SQLite database file",
             reason="not previewed: raw access would cancel SQLite's POSIX locks.",
             guidance="Do not open this file directly while its database connection is live.",
         )
-    return raw if isinstance(raw, tuple) else _format_file_reference(ref, path, raw, max_inline_tokens)
+    return early or _format_file_reference(ref, path, text, max_inline_tokens)
 
 
-def _read_file_reference(ref: ContextReference, path: Path, max_inline_tokens: int | None) -> str | Expansion:
+def _read_file_reference(
-    """Raw file I/O for an @file ref: the text to inline, or an early refusal block."""
+    ref: ContextReference, path: Path, max_inline_tokens: int | None,
+) -> tuple[Expansion | None, str]:
+    """Raw file I/O for an @file ref: ``(early, text)`` where ``early`` is a refusal block
+    (then ``text`` is empty) or ``None`` with the text to inline."""
     if _is_binary_file(path):
         # A bare "not supported" warning was a dead end (the model gave up); the file IS
         # on disk where the agent's tools run, so hand it an actionable block instead.
-        return None, _binary_reference_block(ref, path)
+        return (None, _binary_reference_block(ref, path)), ""
     if ref.line_start is not None:
         # A ranged ref wants a slice, not the file: stream to the window so a GB-scale
         # file serves :1-5 without being materialized. Lines are read in bounded pieces
@@ -356,7 +359,7 @@ def _read_file_reference(ref: ContextReference, path: Path, max_inline_tokens: i
                     break
                 total_chars += len(line)
                 if char_budget is not None and total_chars > char_budget:
-                    return None, _oversized_text_reference_block(ref, path, total_chars // CHARS_PER_TOKEN)
+                    return (None, _oversized_text_reference_block(ref, path, total_chars // CHARS_PER_TOKEN)), ""
                 parts.append(line)
         text = "".join(parts)
     else:
@@ -364,9 +367,9 @@ def _read_file_reference(ref: ContextReference, path: Path, max_inline_tokens: i
         # file past that byte ceiling is certainly oversized; refuse without reading it.
         size = path.stat().st_size
         if max_inline_tokens is not None and size > max_inline_tokens * CHARS_PER_TOKEN:
-            return None, _oversized_text_reference_block(ref, path, size // CHARS_PER_TOKEN)
+            return (None, _oversized_text_reference_block(ref, path, size // CHARS_PER_TOKEN)), ""
         text = path.read_text(encoding="utf-8-sig")
-    return text
+    return None, text
 
 
 def _format_file_reference(ref: ContextReference, path: Path, text: str, max_inline_tokens: int | None) -> Expansion:

hermes_cli/sqlite_safe_read.py

@@ -90,19 +90,14 @@ def _live_main_key(key: str) -> Optional[str]:
 
 
 def has_live_connection(path: Path | str) -> bool:
-    """Whether this process holds a connection to *path* (or to the database it is a sidecar of)."""
+    """Whether this process holds a connection to *path* (or to the database it is a sidecar of).
+
+    Point-in-time answer: a raw open/close right after it returns ``False`` can still race a
+    new connection. Hold :func:`offline_file_access` across the I/O whenever possible."""
     with _live_lock:
         return _live_main_key(_key(path)) is not None
 
 
-def is_live_database_file(path: Path | str) -> bool:
-    """Whether a raw open/close of *path* would cancel a live connection's POSIX locks.
-
-    Point-in-time answer for callers that cannot hold ``_live_lock`` across their I/O
-    (e.g. a streamed download: the lock must never be held across an await/yield)."""
-    return has_live_connection(path)
-
-
 class _TrackingMixin:
     """Untrack-on-close behaviour, mixable into any Connection subclass.
 

hermes_cli/web_routers/files.py

@@ -25,7 +25,7 @@ from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
 from fastapi.responses import FileResponse
 
 from hermes_cli._subprocess_compat import windows_hide_flags
-from hermes_cli.sqlite_safe_read import LiveConnectionError, is_live_database_file, offline_file_access
+from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access
 from hermes_cli.web_deps import late
 from hermes_cli.web_server_files import (
     _fs_path, _managed_file_entry, _managed_response_meta, _resolve_managed_path,
@@ -175,18 +175,26 @@ def _fs_regular_file(path: Path) -> tuple[Path, os.stat_result]:
     return target, st
 
 
+@contextlib.contextmanager
+def _serve_offline(target: Path):
+    """Hold ``offline_file_access`` for serving ``target``; 409 while a SQLite connection to it is live."""
+    try:
+        with offline_file_access(target, what="serve"):
+            yield
+    except LiveConnectionError as exc:
+        raise HTTPException(status_code=409, detail=str(exc)) from exc
+
+
 def _refuse_live_database(target: Path) -> None:
-    """409 before streaming ``target`` while this process has a live SQLite connection to it.
+    """Point-in-time 409 before streaming ``target`` while a SQLite connection to it is live.
 
     ``FileResponse`` opens and closes the file in-process, and that raw close cancels the
     connection's POSIX locks. The registry lock can't be held across a streamed response
-    (never across an await/yield), so this is a point-in-time admission check."""
+    (never across an await/yield), so streamed routes only get this admission check. It
-    if is_live_database_file(target):
+    takes the global registry lock, which other threads hold across whole-file reads, so
-        raise HTTPException(
+    call it via ``asyncio.to_thread``."""
-            status_code=409,
+    with _serve_offline(target):
-            detail=f"Refusing to download {target}: a SQLite connection to its database is "
+        pass
-            "open in this process, and raw file access would cancel its POSIX advisory locks.",
-        )
 
 
 def _fs_read_bytes(target: Path, limit: Optional[int] = None) -> bytes:
@@ -263,8 +271,10 @@ def _media_serve_roots() -> list[Path]:
 
 
 def _read_base64_file(path: Path) -> str:
-    """Read and encode a bounded file from a worker thread."""
+    """Read and encode a bounded file from a worker thread; 409 while a SQLite connection to it is live."""
-    return base64.b64encode(path.read_bytes()).decode("ascii")
+    # Keep admission through close; a raw close cancels this process's SQLite locks.
+    with _serve_offline(path):
+        return base64.b64encode(path.read_bytes()).decode("ascii")
 
 
 @router.get("/api/media")
@@ -408,8 +418,9 @@ async def list_managed_files(request: Request, path: Optional[str] = None):
 
 def _managed_readable_file(request: Request, path: str) -> tuple[Any, Path, str, int, str]:
     """Resolve + guard a managed file for reading: existence, regular file,
-    sensitive-path denylist, size cap. Returns (policy, target, display_path,
+    sensitive-path denylist. Returns (policy, target, display_path, max_bytes,
-    size, mime_type)."""
+    mime_type). Callers own the live-SQLite 409 (held through the read for
+    /api/files/read, point-in-time for streamed responses)."""
     from hermes_cli.web_server import _MANAGED_FILE_MAX_BYTES
     policy, target, display_path = _resolve_managed_path(path, request)
     if not target.exists():
@@ -418,7 +429,6 @@ def _managed_readable_file(request: Request, path: str) -> tuple[Any, Path, str,
         raise HTTPException(status_code=400, detail="Path is not a file")
     if _is_sensitive_path(target):
         raise HTTPException(status_code=403, detail="Access to sensitive files is not allowed")
-    _refuse_live_database(target)
     mime_type = mimetypes.guess_type(target.name)[0] or "application/octet-stream"
     return policy, target, display_path, _MANAGED_FILE_MAX_BYTES, mime_type
 
@@ -449,7 +459,7 @@ async def read_managed_file(request: Request, path: str):
     }
 
 
-def _managed_file_response(
+async def _managed_file_response(
     request: Request,
     path: str,
     *,
@@ -461,6 +471,7 @@ def _managed_file_response(
     if media_only and target.suffix.lower() not in _STREAMABLE_MEDIA_EXTENSIONS:
         raise HTTPException(status_code=415, detail="Unsupported media type")
     _managed_file_size(target, max_bytes)
+    await asyncio.to_thread(_refuse_live_database, target)
     return FileResponse(
         path=str(target),
         media_type=mime_type,
@@ -482,7 +493,7 @@ async def download_managed_file(request: Request, path: str):
     """
     fetch_destination = request.headers.get("sec-fetch-dest", "").lower()
     is_media_subresource = fetch_destination in {"audio", "video"}
-    return _managed_file_response(
+    return await _managed_file_response(
         request,
         path,
         content_disposition_type="inline" if is_media_subresource else "attachment",
@@ -497,7 +508,7 @@ async def stream_managed_file(request: Request, path: str):
     media pipeline may reject an attachment response as an ``<audio>``/
     ``<video>`` source. Same auth, size cap, sensitive guard and MIME detection
     as download."""
-    return _managed_file_response(request, path, content_disposition_type="inline", media_only=True)
+    return await _managed_file_response(request, path, content_disposition_type="inline", media_only=True)
 
 
 def _managed_write_target(path: str, request: Request, overwrite: bool):
@@ -755,7 +766,7 @@ async def fs_download(
     path: str, profile: Optional[str] = None, session_id: Optional[str] = None,
 ):
     target, _st = _fs_regular_file(await _fs_download_path(path, profile, session_id))
-    _refuse_live_database(target)
+    await asyncio.to_thread(_refuse_live_database, target)
     return FileResponse(
         path=str(target),
         media_type=_fs_mime_type(target),

tests/agent/test_live_db_preview_locks.py

@@ -61,15 +61,14 @@ def test_preview_preserves_live_database_locks(tmp_path, route, target_kind):
         assert rival_locked(), "second process must be excluded before the preview"
 
         if route == "desktop":
-            with pytest.raises(HTTPException) as refused:
+            # FileResponse opens/closes in-process too, so a download must be refused as well,
-                asyncio.run(fs_read_text(str(target)))
+            # with the same (sidecar-aware) refusal text as the read.
-            assert refused.value.status_code == 409
+            for route_fn in (fs_read_text, fs_download):
-            if target_kind == "shm":
+                with pytest.raises(HTTPException) as refused:
-                assert "main database" in refused.value.detail
+                    asyncio.run(route_fn(str(target)))
-            # FileResponse opens/closes in-process too, so a download must be refused as well.
+                assert refused.value.status_code == 409
-            with pytest.raises(HTTPException) as refused:
+                if target_kind == "shm":
-                asyncio.run(fs_download(str(target)))
+                    assert "main database" in refused.value.detail
-            assert refused.value.status_code == 409
             assert asyncio.run(fs_read_text(str(text)))["text"] == "ordinary readable text"
         else:
             ref = parse_context_references(f"@{route}:{target}")[0]