diff --git a/hermes_cli/local_runtime/gguf.py b/hermes_cli/local_runtime/gguf.py index 72e6ee04e4..3531f134ae 100644 --- a/hermes_cli/local_runtime/gguf.py +++ b/hermes_cli/local_runtime/gguf.py @@ -184,7 +184,7 @@ class GGUFHeader: return self.head_dim_k -def _split_parts(path: Path) -> "list[Path] | None": +def split_parts(path: Path) -> "list[Path] | None": """Every on-disk part of the split ``path`` belongs to, first part first; None when ``path`` is not a split member or no other part is present. @@ -203,7 +203,13 @@ def _split_parts(path: Path) -> "list[Path] | None": def _read_part(path: Path) -> GGUFHeader: """One file's own header: metadata and that file's tensor table.""" def read(f, fmt: str): - return struct.unpack(fmt, f.read(struct.calcsize(fmt))) + # A header cut short raises ValueError, which every caller already treats as "skip this + # model"; struct.error would escape them and take the whole preset pass down. + size = struct.calcsize(fmt) + data = f.read(size) + if len(data) != size: + raise ValueError(f"truncated GGUF header: {path}") + return struct.unpack(fmt, data) def read_str(f) -> str: (n,) = read(f, " GGUFHeader: part 1 alone underprices every model whose first shard is a metadata stub (the Hugging Face layout), which then makes the physics check and the residency cap admit giants the card cannot hold. Architecture metadata (block count, train context, the per-layer SWA pattern, vocab) is - taken from the first part, which is where GGUF writes it. + taken from the first part: gguf-split writes the model's metadata there only, and every later + part carries just its ``split.*`` keys. A part that has gone missing or become unreadable is skipped rather than fatal: a half-arrived split prices at what is actually on disk. Refusing it is ``staged_in(require_complete=True)``'s job — an incomplete split is never servable, it is only underpriced here.""" path = Path(path) - parts = _split_parts(path) + parts = split_parts(path) if parts is None: return _read_part(path) @@ -280,9 +287,7 @@ def read_gguf_header(path: str | Path) -> GGUFHeader: for part in parts[1:]: try: readable.append(_read_part(part)) - except (ValueError, OSError, struct.error) as exc: - # struct.error is neither ValueError nor OSError, and it is what a header cut - # mid-stream actually raises. A part that cannot be parsed is priced out, not fatal. + except (ValueError, OSError) as exc: logger.debug("split part unreadable %s: %s", part.name, exc) if len(readable) == 1: return first diff --git a/hermes_cli/web_routers/local_models.py b/hermes_cli/web_routers/local_models.py index cf380ce88e..a982b198a3 100644 --- a/hermes_cli/web_routers/local_models.py +++ b/hermes_cli/web_routers/local_models.py @@ -39,6 +39,7 @@ from agent.memory_provider import spawn_context_thread from pm.downloader import Download, DownloadPaused, Source from hermes_cli.local_runtime.endpoint import _state_endpoint +from hermes_cli.local_runtime.gguf import split_parts logger = logging.getLogger(__name__) @@ -470,9 +471,8 @@ def _loaded_models(running: Dict[str, Any]) -> "tuple[Dict[str, str], Dict[str, def _staged_row(gguf: Path) -> Dict[str, Any]: model_id = _model_id_for(gguf) - # Split models: report the whole variant's bytes, not one part's. - hit = catalog.find_entry_for_model(model_id) - size = hit[1].size_bytes if hit is not None else gguf.stat().st_size + # A split model is every part on disk; its first file can be a metadata stub of a few MB. + size = sum(p.stat().st_size for p in split_parts(gguf) or [gguf]) return {"id": model_id, "size_bytes": size, "size_label": _human_gb(size)} diff --git a/tests/hermes_cli/test_local_models_routes.py b/tests/hermes_cli/test_local_models_routes.py index 1e7d462695..78303596d2 100644 --- a/tests/hermes_cli/test_local_models_routes.py +++ b/tests/hermes_cli/test_local_models_routes.py @@ -84,6 +84,18 @@ def test_status_lists_staged_models_with_labels(client, tmp_path): assert row["size_label"].endswith("GB") +def test_status_sizes_a_split_model_by_all_its_parts(client): + """A split's first file can be a metadata stub of a few MB. The row reports the model the user + has on disk, so a side-loaded split outside the catalog is every part, not part 1.""" + from hermes_cli.local_runtime.bootstrap import models_dir + + _write_fake_gguf(models_dir() / "Side-Split-00001-of-00002.gguf", size=1024) + _write_fake_gguf(models_dir() / "Side-Split-00002-of-00002.gguf", size=8192) + rows = {m["id"]: m for m in client.get("/api/local-models/status").json()["models"]} + + assert rows["Side-Split"]["size_bytes"] == (4 + 1024) + (4 + 8192) + + def test_status_tracks_preset_spill_and_restored_window(client, tmp_path, monkeypatch): from dataclasses import replace from types import SimpleNamespace diff --git a/tests/hermes_cli/test_local_runtime_gguf_split.py b/tests/hermes_cli/test_local_runtime_gguf_split.py index ef627622ca..a21941d888 100644 --- a/tests/hermes_cli/test_local_runtime_gguf_split.py +++ b/tests/hermes_cli/test_local_runtime_gguf_split.py @@ -17,6 +17,8 @@ from __future__ import annotations import re import struct +import pytest + from hermes_cli.local_runtime.context_policy import spill_overrides from hermes_cli.local_runtime.estimator import profile_from_gguf from hermes_cli.local_runtime.gguf import read_gguf_header @@ -52,13 +54,21 @@ ARCH = { } +def _split_keys(no: int, count: int) -> dict: + """What gguf-split writes into every part after the first: its split keys and nothing else + (checked against a published Flash Next split: part 1 carries 67 keys, parts 2-3 carry 3).""" + return {"split.no": no, "split.count": count, "split.tensors.count": 0} + + def _split(tmp_path, stem: str, parts: list[dict], total: int | None = None): - """A split on disk: ``parts[i]`` is the metadata + tensor table of shard i+1.""" + """A split on disk laid out as gguf-split writes it: ``parts[i]`` is the tensor table of shard + i+1, and only shard 1 carries the model's metadata unless a spec overrides it.""" total = total or len(parts) written = [] for i, spec in enumerate(parts, start=1): path = tmp_path / f"{stem}-{i:05d}-of-{total:05d}.gguf" - _write_gguf(path, spec.get("metadata", ARCH), spec.get("tensors", [])) + default = ARCH if i == 1 else _split_keys(i - 1, total) + _write_gguf(path, spec.get("metadata", default), spec.get("tensors", [])) written.append(path) return written @@ -121,8 +131,8 @@ def test_ffn_blocks_spread_across_shards_still_place_spill(tmp_path): (f"blk.{i}.ffn_up.weight", 256), (f"blk.{i}.ffn_down.weight", 240)] parts = _split(tmp_path, "hybrid-split", [ {"metadata": hybrid, "tensors": tensors[:1]}, - {"metadata": hybrid, "tensors": tensors[1:37]}, # blocks 0-8 (4 tensors each) - {"metadata": hybrid, "tensors": tensors[37:]}, # blocks 9-11 + {"tensors": tensors[1:37]}, # blocks 0-8 (4 tensors each) + {"tensors": tensors[37:]}, # blocks 9-11 ], total=3) profile = profile_from_gguf(read_gguf_header(parts[0])) @@ -163,11 +173,33 @@ def test_residency_cap_prices_a_split_against_the_card(tmp_path, monkeypatch): mdir.mkdir() # A 48 GiB split laid out the reported way: part 1 a metadata stub, the weights in part 2. _write_gguf(mdir / "giant-00001-of-00002.gguf", ARCH, []) - _write_gguf(mdir / "giant-00002-of-00002.gguf", ARCH, [("blk.0.attn_q.weight", 48 * GIB // 4)]) + _write_gguf(mdir / "giant-00002-of-00002.gguf", _split_keys(1, 2), + [("blk.0.attn_q.weight", 48 * GIB // 4)]) # A 10 GB single-shard neighbour: a small second resident must stay possible. _write_gguf(mdir / "utility.gguf", ARCH, [("blk.0.attn_q.weight", 10 * GIB // 4)]) budget = HardwareBudget(usable_vram_bytes=64 * GIB, total_device_bytes=64 * GIB, ram_available_bytes=64 * GIB) - assert presets.admitted_residency_count(mdir, budget, 4) == 1 \ No newline at end of file + assert presets.admitted_residency_count(mdir, budget, 4) == 1 + + +def test_a_truncated_header_skips_the_model_instead_of_raising(tmp_path, monkeypatch): + """Every caller skips a model whose header raises ValueError or OSError. A header cut short + used to raise struct.error, which none of them catch, so one bad file took down the whole + preset pass instead of dropping out of it.""" + from hermes_cli.local_runtime import presets + from hermes_cli.local_runtime.estimator import HardwareBudget + + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + parts = _split(tmp_path, "cut-head", [ + {"tensors": []}, + {"tensors": [("blk.0.attn_q.weight", 16 << 20)]}, + ]) + parts[0].write_bytes(b"GGUF\x03\x00") + budget = HardwareBudget(usable_vram_bytes=64 * GIB, total_device_bytes=64 * GIB, + ram_available_bytes=64 * GIB) + + with pytest.raises(ValueError): + read_gguf_header(parts[0]) + assert presets.resident_footprint(parts[0], budget, 8192) is None \ No newline at end of file