fix(local-runtime): read lazy tensors on demand on integrated GPUs too

llama.cpp's --lazy-mode auto resolves to off on any device without mmap support, which covers every integrated GPU (b11370, #28160). On a DGX Spark, Flash Next's 26.8 GiB per-layer embedding table therefore loaded in full (27.2 GiB resident) while Hermes priced it as left on disk. --lazy-mode on keeps it out (0.4 GiB resident) at the same decode speed and within 2% of the prefill speed on GB10. The preset now passes lazy-mode = on wherever Hermes prices the table as on disk, so the engine's per-device default never decides residency. AMD and Intel integrated GPUs, where reading on demand halved prefill, keep the engine default and are priced with the table loaded.
2026-10-05 22:06:54 -04:00
parent feaa55d9bb
commit 827f4ff1d1
6 changed files with 85 additions and 20 deletions
--- a/hermes_cli/local_runtime/catalog.py
+++ b/hermes_cli/local_runtime/catalog.py
@@ -18,7 +18,8 @@ from pathlib import PurePosixPath

 from hermes_cli.local_runtime.context_policy import (
    FLOOR, RUNTIME_OVERHEAD_BYTES, TARGET_WINDOW, LaunchPlan, plan_launch)
-from hermes_cli.local_runtime.estimator import HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal
+from hermes_cli.local_runtime.estimator import (
+    HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal, as_loaded)
 from hermes_cli.local_runtime.gguf import model_id_from_stem
 from hermes_platform.host.products import is_nvidia_n1x_pci_id

@@ -131,11 +132,12 @@ class CatalogEntry:
        return ModelProfile(
            name=variant.model_id, weights_bytes=variant.weights_bytes, embd_table_bytes=0,
            n_ctx_train=self.n_ctx_train, layers=layers, swa_window=self.swa_window, moe=self.moe,
-            n_vocab=self.n_vocab, kv_scale=1.2 if self.mtp_capable else 1.0)
+            n_vocab=self.n_vocab, kv_scale=1.2 if self.mtp_capable else 1.0,
+            lazy_bytes=variant.lazy_bytes)

    def launch_plan(self, variant: QuantVariant, budget: HardwareBudget) -> LaunchPlan:
        # Optional external drafts may use spare memory after download, never reduce this grant.
-        return plan_launch(self.profile(variant), budget, mtp_capable=self.mtp_capable,
+        return plan_launch(as_loaded(self.profile(variant), budget), budget, mtp_capable=self.mtp_capable,
                           fixed_overhead=RUNTIME_OVERHEAD_BYTES + self.companion_bytes)

    def download_files(self, variant: QuantVariant) -> tuple:
--- a/hermes_cli/local_runtime/estimator.py
+++ b/hermes_cli/local_runtime/estimator.py
@@ -6,7 +6,7 @@ ground truth after it. Unknown shapes round UP (never underestimate memory).

 from __future__ import annotations

-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from enum import Enum

 from hermes_cli.local_runtime.gguf import GGUFHeader
@@ -55,7 +55,9 @@ class ModelProfile:
    construct profiles directly."""

    name: str
-    weights_bytes: int          # weights the engine loads; tensors it reads from disk on demand are not
+    # Weights the engine loads when it reads lazy_bytes from disk on demand; as_loaded() adds them
+    # back on a machine where it reads them up front.
+    weights_bytes: int
    embd_table_bytes: int
    n_ctx_train: int
    layers: list[tuple[LayerKind, int]]   # (kind, kv_bytes_per_token_f16); SWA capped, recurrent ignored
@@ -70,6 +72,9 @@ class ModelProfile:
    kv_scale: float = 1.0
    # block index -> FFN weight bytes (from the tensor table); empty when unknown.
    ffn_block_bytes: dict[int, int] = field(default_factory=dict)
+    # Bytes of architecture-marked tensors (gguf._LAZY_READ_TENSORS) the engine can read from disk
+    # on demand instead of loading.
+    lazy_bytes: int = 0

    @property
    def per_token_kv_f16(self) -> int:
@@ -94,6 +99,18 @@ class HardwareBudget:
    gpu_name: str = ""          # display name; legacy fallback for performance estimates
    platform: str = ""          # sys.platform of the machine being priced
    gpu_pci_id: int | None = None  # nvidia-smi's packed PCI device/vendor ID
+    # The engine reads lazy tensors from disk here. llama.cpp's own default does so everywhere
+    # except integrated GPUs (b11370 #28160); Hermes passes --lazy-mode on to NVIDIA's, so only
+    # AMD/Intel integrated GPUs load them up front.
+    lazy_reads: bool = True
+
+
+def as_loaded(profile: ModelProfile, budget: HardwareBudget) -> ModelProfile:
+    """The profile as this machine loads it: on-demand tensors count as weights where the engine
+    reads them up front."""
+    if profile.lazy_bytes and not budget.lazy_reads:
+        return replace(profile, weights_bytes=profile.weights_bytes + profile.lazy_bytes, lazy_bytes=0)
+    return profile


 def profile_from_gguf(header: GGUFHeader) -> ModelProfile:
@@ -139,7 +156,7 @@ def profile_from_gguf(header: GGUFHeader) -> ModelProfile:
        embd_table_bytes=header.embd_table_bytes,
        n_ctx_train=header.n_ctx_train, layers=layers, swa_window=header.sliding_window,
        moe=header.expert_count > 0, architecture=header.architecture, n_vocab=header.n_vocab,
-        ffn_block_bytes=dict(header.ffn_block_bytes))
+        ffn_block_bytes=dict(header.ffn_block_bytes), lazy_bytes=header.lazy_bytes)


 def kv_dtype_factor(flash_attention: bool) -> float:
--- a/hermes_cli/local_runtime/growth.py
+++ b/hermes_cli/local_runtime/growth.py
@@ -73,7 +73,7 @@ def maybe_grow_window(model_id: str, *, base_url: str, session_tokens: int,
    from hermes_cli.local_runtime.bootstrap import (
        _launch_budget, get_supervisor, refresh_local_runtime, staged_models)
    from hermes_cli.local_runtime.context_policy import growth_decision
-    from hermes_cli.local_runtime.estimator import profile_from_gguf
+    from hermes_cli.local_runtime.estimator import as_loaded, profile_from_gguf
    from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header
    from hermes_cli.local_runtime.hardware import probe_budget
    from hermes_cli.local_runtime.presets import (
@@ -103,7 +103,7 @@ def maybe_grow_window(model_id: str, *, base_url: str, session_tokens: int,
        # Capacity budget, not live-free: growth executes via a server bounce, so the grown
        # instance loads onto a freed card. Live-free is distorted by the very model being grown
        # — it reads its own residency as unavailable and vetoes rungs that fit.
-        profile, budget,
+        as_loaded(profile, budget), budget,
        current_window=current_window,
        session_tokens=session_tokens,
        measured_decode_tok_s=measured_decode_tok_s,
--- a/hermes_cli/local_runtime/hardware.py
+++ b/hermes_cli/local_runtime/hardware.py
@@ -22,6 +22,7 @@ from pathlib import Path

 from hermes_cli.local_runtime.devices import GGML_DEVICE_GPU
 from hermes_cli.local_runtime.estimator import HardwareBudget
+from hermes_platform.host.facts import gpu_class

 logger = logging.getLogger(__name__)

@@ -393,11 +394,11 @@ def _unified_pool_bytes(smi_total: int, ram_total: int) -> int | None:


 def _uma_budget(base: int, total: int, *, gpu_name: str = "",
-                gpu_pci_id: int | None = None) -> HardwareBudget:
+                gpu_pci_id: int | None = None, lazy_reads: bool = True) -> HardwareBudget:
    usable = max(0, int(base * (1 - _UMA_HEADROOM_FRACTION)))
    return HardwareBudget(usable_vram_bytes=usable, total_device_bytes=total,
                          ram_available_bytes=0, uma=True, gpu_name=gpu_name, platform=sys.platform,
-                          gpu_pci_id=gpu_pci_id)
+                          gpu_pci_id=gpu_pci_id, lazy_reads=lazy_reads)


 def probe_budget(*, planning: bool = False) -> HardwareBudget:
@@ -444,8 +445,11 @@ def probe_budget(*, planning: bool = False) -> HardwareBudget:
            return HardwareBudget(usable_vram_bytes=max(0, total - margin), total_device_bytes=total,
                                  ram_available_bytes=ram_total if planning else ram_avail,
                                  uma=False, gpu_name=device["description"], platform=sys.platform)
-        # Integrated GPU, Metal, CPU, or no answer: budget from RAM as unified memory.
-        return _uma_budget(ram_total if planning else ram_avail, ram_total)
+        # Integrated GPU, Metal, CPU, or no answer: budget from RAM as unified memory. llama.cpp
+        # loads lazy tensors up front on an AMD/Intel integrated GPU, where reading them on demand
+        # halved prefill (#28160).
+        return _uma_budget(ram_total if planning else ram_avail, ram_total,
+                           lazy_reads=gpu_class() not in ("amd", "intel"))

    total, free, gpu_name, gpu_pci_id = vram
    margin = max(_MARGIN_FLOOR, int(total * _MARGIN_FRACTION))
--- a/hermes_cli/local_runtime/presets.py
+++ b/hermes_cli/local_runtime/presets.py
@@ -12,7 +12,7 @@ from pathlib import Path
 from hermes_cli.local_runtime.context_policy import (
    RUNTIME_OVERHEAD_BYTES, fit_to_free_memory, launch_args, plan_launch, ub_logits_bytes)
 from hermes_cli.local_runtime.estimator import (
-    HardwareBudget, PhysicsRefusal, ctx_bytes, footprint_bytes, profile_from_gguf)
+    HardwareBudget, PhysicsRefusal, as_loaded, ctx_bytes, footprint_bytes, profile_from_gguf)
 from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header

 logger = logging.getLogger(__name__)
@@ -112,7 +112,7 @@ def preset_for_model(gguf: Path, budget: HardwareBudget,
    model_id = model_id_from_stem(gguf.stem)
    try:
        header = read_gguf_header(gguf)
-        profile = profile_from_gguf(header)
+        profile = as_loaded(profile_from_gguf(header), budget)
    except (ValueError, OSError) as exc:
        logger.warning("preset skip %s: %s", gguf.name, exc)
        return None
@@ -136,6 +136,10 @@ def preset_for_model(gguf: Path, budget: HardwareBudget,
        profile, decision, mtp_capable=is_mtp, uma=budget.uma, mtp_prefill=plan.mtp_prefill,
        mtp_draft_depth=entry.mtp_draft_depth if entry is not None else 3))
    keys["model"] = str(gguf)
+    if profile.lazy_bytes:
+        # The price above leaves these tensors on disk; llama.cpp's own default loads them on
+        # integrated GPUs.
+        keys["lazy-mode"] = "on"
    if entry is not None and is_mtp:
        # Integrated-MTP targets sample on the backend, and so does the draft (pairing validated
        # against the vendor's published llama.cpp recipes).
@@ -172,7 +176,7 @@ def resident_footprint(gguf: Path, budget: HardwareBudget, window: int) -> int |

    model_id = model_id_from_stem(gguf.stem)
    try:
-        profile = profile_from_gguf(read_gguf_header(gguf))
+        profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget)
    except (ValueError, OSError) as exc:
        logger.debug("footprint skip %s: %s", gguf.name, exc)
        return None
@@ -196,7 +200,7 @@ def _launch_footprint(gguf: Path, budget: HardwareBudget) -> int | None:

    model_id = model_id_from_stem(gguf.stem)
    try:
-        profile = profile_from_gguf(read_gguf_header(gguf))
+        profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget)
    except (ValueError, OSError) as exc:
        logger.debug("footprint skip %s: %s", gguf.name, exc)
        return None
--- a/tests/hermes_cli/test_local_runtime_lazy_read.py
+++ b/tests/hermes_cli/test_local_runtime_lazy_read.py
@@ -2,17 +2,22 @@

 The engine's model loader leaves a tensor its architecture marks TENSOR_READ_LAZY in the file and
 reads that tensor's rows when a token needs them. Under the default ``--lazy-mode auto`` it does so
-only for a marked tensor larger than 4 GiB. Qwen3.8 Flash Next's 26.8 GiB per-layer embedding
-table is one: priced as loaded, the IQ4_XS build needs ~87 GiB and a 128 GB unified-memory machine
-refuses it; priced the way the engine loads it, the build holds ~60 GiB.
+only for a marked tensor larger than 4 GiB, and never on an integrated GPU, where it loads the
+table in full. Qwen3.8 Flash Next's 26.8 GiB per-layer embedding table is one: priced as loaded,
+the IQ4_XS build needs ~87 GiB; read on demand, it holds ~60 GiB. On a DGX Spark (GB10) the table
+took 27.2 GiB of resident memory under ``auto`` and 0.4 GiB under ``--lazy-mode on``, at the same
+decode speed and within 2% of the prefill speed, so Hermes passes ``on`` wherever it prices the
+table as left on disk. AMD/Intel integrated GPUs keep the engine's default (reading on demand
+halved their prefill, llama.cpp #28160) and pay for the table.
 """

 from __future__ import annotations

 import struct

+from hermes_cli.local_runtime import presets
 from hermes_cli.local_runtime.catalog import AssetFile, QuantVariant
-from hermes_cli.local_runtime.estimator import profile_from_gguf
+from hermes_cli.local_runtime.estimator import HardwareBudget, as_loaded, profile_from_gguf
 from hermes_cli.local_runtime.gguf import read_gguf_header

 GIB = 1 << 30
@@ -98,3 +103,36 @@ def test_a_catalog_build_is_priced_without_its_lazy_tables():

    assert build.size_bytes == 90 * GIB + (10 << 20)
    assert build.weights_bytes == build.size_bytes - 30 * GIB
+
+
+def _unified(*, lazy_reads: bool) -> HardwareBudget:
+    return HardwareBudget(usable_vram_bytes=64 * GIB, total_device_bytes=80 * GIB,
+                          ram_available_bytes=0, uma=True, lazy_reads=lazy_reads)
+
+
+def test_an_engine_that_loads_the_table_up_front_pays_for_it(tmp_path):
+    gguf = tmp_path / "flash.gguf"
+    _write_gguf(gguf, _arch("qwen4exp"), [("per_layer_token_embd.weight", OVER_4_GIB),
+                                          ("blk.0.attn_q.weight", 16 << 20)])
+    profile = profile_from_gguf(read_gguf_header(gguf))
+
+    assert as_loaded(profile, _unified(lazy_reads=True)).weights_bytes == 64 << 20
+    assert as_loaded(profile, _unified(lazy_reads=False)).weights_bytes == (64 << 20) + OVER_4_GIB * 4
+
+
+def test_the_preset_reads_the_table_on_demand_wherever_it_prices_it_that_way(tmp_path, monkeypatch):
+    """The price and the flag travel together, so the engine's own per-device default never decides
+    whether the table is resident."""
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+    gguf = tmp_path / "flash.gguf"
+    _write_gguf(gguf, _arch("qwen4exp"), [("per_layer_token_embd.weight", OVER_4_GIB),
+                                          ("blk.0.attn_q.weight", 16 << 20)])
+
+    lazy = presets.preset_for_model(gguf, _unified(lazy_reads=True), set())
+    eager = presets.preset_for_model(gguf, _unified(lazy_reads=False), set())
+
+    assert lazy.keys["lazy-mode"] == "on"
+    assert "lazy-mode" not in eager.keys
+    window = min(lazy.window, eager.window)
+    assert (presets.resident_footprint(gguf, _unified(lazy_reads=False), window)
+            - presets.resident_footprint(gguf, _unified(lazy_reads=True), window)) == OVER_4_GIB * 4