From 827f4ff1d1b613c49e037e817d7d3ccc4009ebfd Mon Sep 17 00:00:00 2001 From: emozilla Date: Mon, 5 Oct 2026 22:06:54 -0400 Subject: [PATCH] fix(local-runtime): read lazy tensors on demand on integrated GPUs too llama.cpp's --lazy-mode auto resolves to off on any device without mmap support, which covers every integrated GPU (b11370, #28160). On a DGX Spark, Flash Next's 26.8 GiB per-layer embedding table therefore loaded in full (27.2 GiB resident) while Hermes priced it as left on disk. --lazy-mode on keeps it out (0.4 GiB resident) at the same decode speed and within 2% of the prefill speed on GB10. The preset now passes lazy-mode = on wherever Hermes prices the table as on disk, so the engine's per-device default never decides residency. AMD and Intel integrated GPUs, where reading on demand halved prefill, keep the engine default and are priced with the table loaded. --- hermes_cli/local_runtime/catalog.py | 8 ++-- hermes_cli/local_runtime/estimator.py | 23 ++++++++-- hermes_cli/local_runtime/growth.py | 4 +- hermes_cli/local_runtime/hardware.py | 12 +++-- hermes_cli/local_runtime/presets.py | 12 +++-- .../test_local_runtime_lazy_read.py | 46 +++++++++++++++++-- 6 files changed, 85 insertions(+), 20 deletions(-) diff --git a/hermes_cli/local_runtime/catalog.py b/hermes_cli/local_runtime/catalog.py index a037b5793c..8e7ba7621a 100644 --- a/hermes_cli/local_runtime/catalog.py +++ b/hermes_cli/local_runtime/catalog.py @@ -18,7 +18,8 @@ from pathlib import PurePosixPath from hermes_cli.local_runtime.context_policy import ( FLOOR, RUNTIME_OVERHEAD_BYTES, TARGET_WINDOW, LaunchPlan, plan_launch) -from hermes_cli.local_runtime.estimator import HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal +from hermes_cli.local_runtime.estimator import ( + HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal, as_loaded) from hermes_cli.local_runtime.gguf import model_id_from_stem from hermes_platform.host.products import is_nvidia_n1x_pci_id @@ -131,11 +132,12 @@ class CatalogEntry: return ModelProfile( name=variant.model_id, weights_bytes=variant.weights_bytes, embd_table_bytes=0, n_ctx_train=self.n_ctx_train, layers=layers, swa_window=self.swa_window, moe=self.moe, - n_vocab=self.n_vocab, kv_scale=1.2 if self.mtp_capable else 1.0) + n_vocab=self.n_vocab, kv_scale=1.2 if self.mtp_capable else 1.0, + lazy_bytes=variant.lazy_bytes) def launch_plan(self, variant: QuantVariant, budget: HardwareBudget) -> LaunchPlan: # Optional external drafts may use spare memory after download, never reduce this grant. - return plan_launch(self.profile(variant), budget, mtp_capable=self.mtp_capable, + return plan_launch(as_loaded(self.profile(variant), budget), budget, mtp_capable=self.mtp_capable, fixed_overhead=RUNTIME_OVERHEAD_BYTES + self.companion_bytes) def download_files(self, variant: QuantVariant) -> tuple: diff --git a/hermes_cli/local_runtime/estimator.py b/hermes_cli/local_runtime/estimator.py index 7cb4d6827f..b765367501 100644 --- a/hermes_cli/local_runtime/estimator.py +++ b/hermes_cli/local_runtime/estimator.py @@ -6,7 +6,7 @@ ground truth after it. Unknown shapes round UP (never underestimate memory). from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import Enum from hermes_cli.local_runtime.gguf import GGUFHeader @@ -55,7 +55,9 @@ class ModelProfile: construct profiles directly.""" name: str - weights_bytes: int # weights the engine loads; tensors it reads from disk on demand are not + # Weights the engine loads when it reads lazy_bytes from disk on demand; as_loaded() adds them + # back on a machine where it reads them up front. + weights_bytes: int embd_table_bytes: int n_ctx_train: int layers: list[tuple[LayerKind, int]] # (kind, kv_bytes_per_token_f16); SWA capped, recurrent ignored @@ -70,6 +72,9 @@ class ModelProfile: kv_scale: float = 1.0 # block index -> FFN weight bytes (from the tensor table); empty when unknown. ffn_block_bytes: dict[int, int] = field(default_factory=dict) + # Bytes of architecture-marked tensors (gguf._LAZY_READ_TENSORS) the engine can read from disk + # on demand instead of loading. + lazy_bytes: int = 0 @property def per_token_kv_f16(self) -> int: @@ -94,6 +99,18 @@ class HardwareBudget: gpu_name: str = "" # display name; legacy fallback for performance estimates platform: str = "" # sys.platform of the machine being priced gpu_pci_id: int | None = None # nvidia-smi's packed PCI device/vendor ID + # The engine reads lazy tensors from disk here. llama.cpp's own default does so everywhere + # except integrated GPUs (b11370 #28160); Hermes passes --lazy-mode on to NVIDIA's, so only + # AMD/Intel integrated GPUs load them up front. + lazy_reads: bool = True + + +def as_loaded(profile: ModelProfile, budget: HardwareBudget) -> ModelProfile: + """The profile as this machine loads it: on-demand tensors count as weights where the engine + reads them up front.""" + if profile.lazy_bytes and not budget.lazy_reads: + return replace(profile, weights_bytes=profile.weights_bytes + profile.lazy_bytes, lazy_bytes=0) + return profile def profile_from_gguf(header: GGUFHeader) -> ModelProfile: @@ -139,7 +156,7 @@ def profile_from_gguf(header: GGUFHeader) -> ModelProfile: embd_table_bytes=header.embd_table_bytes, n_ctx_train=header.n_ctx_train, layers=layers, swa_window=header.sliding_window, moe=header.expert_count > 0, architecture=header.architecture, n_vocab=header.n_vocab, - ffn_block_bytes=dict(header.ffn_block_bytes)) + ffn_block_bytes=dict(header.ffn_block_bytes), lazy_bytes=header.lazy_bytes) def kv_dtype_factor(flash_attention: bool) -> float: diff --git a/hermes_cli/local_runtime/growth.py b/hermes_cli/local_runtime/growth.py index 572e57738d..7def041970 100644 --- a/hermes_cli/local_runtime/growth.py +++ b/hermes_cli/local_runtime/growth.py @@ -73,7 +73,7 @@ def maybe_grow_window(model_id: str, *, base_url: str, session_tokens: int, from hermes_cli.local_runtime.bootstrap import ( _launch_budget, get_supervisor, refresh_local_runtime, staged_models) from hermes_cli.local_runtime.context_policy import growth_decision - from hermes_cli.local_runtime.estimator import profile_from_gguf + from hermes_cli.local_runtime.estimator import as_loaded, profile_from_gguf from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header from hermes_cli.local_runtime.hardware import probe_budget from hermes_cli.local_runtime.presets import ( @@ -103,7 +103,7 @@ def maybe_grow_window(model_id: str, *, base_url: str, session_tokens: int, # Capacity budget, not live-free: growth executes via a server bounce, so the grown # instance loads onto a freed card. Live-free is distorted by the very model being grown # — it reads its own residency as unavailable and vetoes rungs that fit. - profile, budget, + as_loaded(profile, budget), budget, current_window=current_window, session_tokens=session_tokens, measured_decode_tok_s=measured_decode_tok_s, diff --git a/hermes_cli/local_runtime/hardware.py b/hermes_cli/local_runtime/hardware.py index 350bcd1228..6023ad5cff 100644 --- a/hermes_cli/local_runtime/hardware.py +++ b/hermes_cli/local_runtime/hardware.py @@ -22,6 +22,7 @@ from pathlib import Path from hermes_cli.local_runtime.devices import GGML_DEVICE_GPU from hermes_cli.local_runtime.estimator import HardwareBudget +from hermes_platform.host.facts import gpu_class logger = logging.getLogger(__name__) @@ -393,11 +394,11 @@ def _unified_pool_bytes(smi_total: int, ram_total: int) -> int | None: def _uma_budget(base: int, total: int, *, gpu_name: str = "", - gpu_pci_id: int | None = None) -> HardwareBudget: + gpu_pci_id: int | None = None, lazy_reads: bool = True) -> HardwareBudget: usable = max(0, int(base * (1 - _UMA_HEADROOM_FRACTION))) return HardwareBudget(usable_vram_bytes=usable, total_device_bytes=total, ram_available_bytes=0, uma=True, gpu_name=gpu_name, platform=sys.platform, - gpu_pci_id=gpu_pci_id) + gpu_pci_id=gpu_pci_id, lazy_reads=lazy_reads) def probe_budget(*, planning: bool = False) -> HardwareBudget: @@ -444,8 +445,11 @@ def probe_budget(*, planning: bool = False) -> HardwareBudget: return HardwareBudget(usable_vram_bytes=max(0, total - margin), total_device_bytes=total, ram_available_bytes=ram_total if planning else ram_avail, uma=False, gpu_name=device["description"], platform=sys.platform) - # Integrated GPU, Metal, CPU, or no answer: budget from RAM as unified memory. - return _uma_budget(ram_total if planning else ram_avail, ram_total) + # Integrated GPU, Metal, CPU, or no answer: budget from RAM as unified memory. llama.cpp + # loads lazy tensors up front on an AMD/Intel integrated GPU, where reading them on demand + # halved prefill (#28160). + return _uma_budget(ram_total if planning else ram_avail, ram_total, + lazy_reads=gpu_class() not in ("amd", "intel")) total, free, gpu_name, gpu_pci_id = vram margin = max(_MARGIN_FLOOR, int(total * _MARGIN_FRACTION)) diff --git a/hermes_cli/local_runtime/presets.py b/hermes_cli/local_runtime/presets.py index 7e50120aa4..14c85eba25 100644 --- a/hermes_cli/local_runtime/presets.py +++ b/hermes_cli/local_runtime/presets.py @@ -12,7 +12,7 @@ from pathlib import Path from hermes_cli.local_runtime.context_policy import ( RUNTIME_OVERHEAD_BYTES, fit_to_free_memory, launch_args, plan_launch, ub_logits_bytes) from hermes_cli.local_runtime.estimator import ( - HardwareBudget, PhysicsRefusal, ctx_bytes, footprint_bytes, profile_from_gguf) + HardwareBudget, PhysicsRefusal, as_loaded, ctx_bytes, footprint_bytes, profile_from_gguf) from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header logger = logging.getLogger(__name__) @@ -112,7 +112,7 @@ def preset_for_model(gguf: Path, budget: HardwareBudget, model_id = model_id_from_stem(gguf.stem) try: header = read_gguf_header(gguf) - profile = profile_from_gguf(header) + profile = as_loaded(profile_from_gguf(header), budget) except (ValueError, OSError) as exc: logger.warning("preset skip %s: %s", gguf.name, exc) return None @@ -136,6 +136,10 @@ def preset_for_model(gguf: Path, budget: HardwareBudget, profile, decision, mtp_capable=is_mtp, uma=budget.uma, mtp_prefill=plan.mtp_prefill, mtp_draft_depth=entry.mtp_draft_depth if entry is not None else 3)) keys["model"] = str(gguf) + if profile.lazy_bytes: + # The price above leaves these tensors on disk; llama.cpp's own default loads them on + # integrated GPUs. + keys["lazy-mode"] = "on" if entry is not None and is_mtp: # Integrated-MTP targets sample on the backend, and so does the draft (pairing validated # against the vendor's published llama.cpp recipes). @@ -172,7 +176,7 @@ def resident_footprint(gguf: Path, budget: HardwareBudget, window: int) -> int | model_id = model_id_from_stem(gguf.stem) try: - profile = profile_from_gguf(read_gguf_header(gguf)) + profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget) except (ValueError, OSError) as exc: logger.debug("footprint skip %s: %s", gguf.name, exc) return None @@ -196,7 +200,7 @@ def _launch_footprint(gguf: Path, budget: HardwareBudget) -> int | None: model_id = model_id_from_stem(gguf.stem) try: - profile = profile_from_gguf(read_gguf_header(gguf)) + profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget) except (ValueError, OSError) as exc: logger.debug("footprint skip %s: %s", gguf.name, exc) return None diff --git a/tests/hermes_cli/test_local_runtime_lazy_read.py b/tests/hermes_cli/test_local_runtime_lazy_read.py index 0f69bbc7db..a0c6a02be5 100644 --- a/tests/hermes_cli/test_local_runtime_lazy_read.py +++ b/tests/hermes_cli/test_local_runtime_lazy_read.py @@ -2,17 +2,22 @@ The engine's model loader leaves a tensor its architecture marks TENSOR_READ_LAZY in the file and reads that tensor's rows when a token needs them. Under the default ``--lazy-mode auto`` it does so -only for a marked tensor larger than 4 GiB. Qwen3.8 Flash Next's 26.8 GiB per-layer embedding -table is one: priced as loaded, the IQ4_XS build needs ~87 GiB and a 128 GB unified-memory machine -refuses it; priced the way the engine loads it, the build holds ~60 GiB. +only for a marked tensor larger than 4 GiB, and never on an integrated GPU, where it loads the +table in full. Qwen3.8 Flash Next's 26.8 GiB per-layer embedding table is one: priced as loaded, +the IQ4_XS build needs ~87 GiB; read on demand, it holds ~60 GiB. On a DGX Spark (GB10) the table +took 27.2 GiB of resident memory under ``auto`` and 0.4 GiB under ``--lazy-mode on``, at the same +decode speed and within 2% of the prefill speed, so Hermes passes ``on`` wherever it prices the +table as left on disk. AMD/Intel integrated GPUs keep the engine's default (reading on demand +halved their prefill, llama.cpp #28160) and pay for the table. """ from __future__ import annotations import struct +from hermes_cli.local_runtime import presets from hermes_cli.local_runtime.catalog import AssetFile, QuantVariant -from hermes_cli.local_runtime.estimator import profile_from_gguf +from hermes_cli.local_runtime.estimator import HardwareBudget, as_loaded, profile_from_gguf from hermes_cli.local_runtime.gguf import read_gguf_header GIB = 1 << 30 @@ -98,3 +103,36 @@ def test_a_catalog_build_is_priced_without_its_lazy_tables(): assert build.size_bytes == 90 * GIB + (10 << 20) assert build.weights_bytes == build.size_bytes - 30 * GIB + + +def _unified(*, lazy_reads: bool) -> HardwareBudget: + return HardwareBudget(usable_vram_bytes=64 * GIB, total_device_bytes=80 * GIB, + ram_available_bytes=0, uma=True, lazy_reads=lazy_reads) + + +def test_an_engine_that_loads_the_table_up_front_pays_for_it(tmp_path): + gguf = tmp_path / "flash.gguf" + _write_gguf(gguf, _arch("qwen4exp"), [("per_layer_token_embd.weight", OVER_4_GIB), + ("blk.0.attn_q.weight", 16 << 20)]) + profile = profile_from_gguf(read_gguf_header(gguf)) + + assert as_loaded(profile, _unified(lazy_reads=True)).weights_bytes == 64 << 20 + assert as_loaded(profile, _unified(lazy_reads=False)).weights_bytes == (64 << 20) + OVER_4_GIB * 4 + + +def test_the_preset_reads_the_table_on_demand_wherever_it_prices_it_that_way(tmp_path, monkeypatch): + """The price and the flag travel together, so the engine's own per-device default never decides + whether the table is resident.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + gguf = tmp_path / "flash.gguf" + _write_gguf(gguf, _arch("qwen4exp"), [("per_layer_token_embd.weight", OVER_4_GIB), + ("blk.0.attn_q.weight", 16 << 20)]) + + lazy = presets.preset_for_model(gguf, _unified(lazy_reads=True), set()) + eager = presets.preset_for_model(gguf, _unified(lazy_reads=False), set()) + + assert lazy.keys["lazy-mode"] == "on" + assert "lazy-mode" not in eager.keys + window = min(lazy.window, eager.window) + assert (presets.resident_footprint(gguf, _unified(lazy_reads=False), window) + - presets.resident_footprint(gguf, _unified(lazy_reads=True), window)) == OVER_4_GIB * 4