"""The recommendation decision table — the reviewable matrix. The recommendation itself is DERIVED (catalog.recommended_entry: best quality among resident entries clearing the pleasant speed floor, else fastest resident). Spilled models stay browseable but are never automatic recommendations, so nobody hand-maintains per-hardware-class picks. The one exception is a recognized product whose manufacturer chose its default (catalog._PRODUCT_DEFAULTS), tested at the bottom of this file. This table pins the model and reason across discrete and unified memory classes so changes to the recommendation remain reviewable. These are decision pins, not change-detectors: each cell is a choice a human approved, exactly like a golden file. When a cell flips on purpose, update it in the same commit and say why. When one flips by surprise, that is the test doing its job. Budgets mirror hardware.probe_budget's planning-mode shapes (margins, UMA headroom) so the cells match what a real machine of that class resolves. """ from __future__ import annotations import pytest from hermes_cli.local_runtime.catalog import ( CATALOG, PLEASANT_FLOOR_TOK_S, predicted_decode_tok_s, recommended_entry, select_variant, ) from hermes_cli.local_runtime.estimator import HardwareBudget _GIB = 1 << 30 def _discrete(size_gb: float) -> HardwareBudget: total = int(size_gb * _GIB) margin = max(2 * _GIB, int(total * 0.09)) return HardwareBudget(usable_vram_bytes=max(0, total - margin), total_device_bytes=total, ram_available_bytes=64 * _GIB, uma=False) def _unified(size_gb: float) -> HardwareBudget: total = int(size_gb * _GIB) return HardwareBudget(usable_vram_bytes=int(total * 0.80), total_device_bytes=total, ram_available_bytes=0, uma=True) # The decision table. Cells were generated by the resolver and then # reviewed as editorial decisions: # # VRAM | discrete | unified # -----+-------------------------+------------------------ # 8 | (no recommendation) | (none fits) # 16 | (no recommendation) | (none fits) # 24 | qwen3.8-27b | (none fits) # 32 | qwen3.8-27b | qwen3.6-35b-a3b # 48 | qwen3.8-27b | qwen3.6-35b-a3b # 96 | qwen3.8-flash-next | qwen3.8-flash-next # 128 | qwen3.8-flash-next | qwen3.8-flash-next # 256 | qwen3.8-flash-next | qwen3.8-flash-next # 512 | qwen3.8-flash-next | qwen3.8-flash-next # # Reading guide for reviewers: # - Discrete <=16 GB: nothing runs resident; no automatic recommendation. # Browse remains available for explicit spill choices. # - Discrete 24-48 GB: the 27B is the flagship experience — dense reads # at ~1 TB/s clear the floor easily, so quality decides. # - 96 GB and up, discrete or unified: Flash Next fits resident. Its # IQ4_XS build loads ~60 GiB — the engine reads the 26.8 GiB per-layer # embedding table from disk on demand — so with its MTP head and # projector it holds the native 256K window in ~83 GiB on 128 GB, and # 144K on 96 GB. It is the # pick: highest quality, and its sparse decode clears the floor even at # UMA bandwidth (~28 tok/s predicted at 210 GB/s). # - Unified 32-48 GB: the dense 27B predicts ~13 tok/s at UMA bandwidth # (below the pleasant floor), so the 35B-A3B (~60 tok/s) wins. # - Unified <=24 GB: no entry passes the physics check inside the UMA # budget (spilling is impossible on UMA by construction — the pool IS # the RAM). The pane's browse flow is the path for those machines # until a small catalog entry lands (revisit when one does). DECISION_TABLE = [ (8, "discrete", None, None), (8, "unified", None, None), (16, "discrete", None, None), (16, "unified", None, None), (24, "discrete", "qwen3.8-27b", "best-quality-resident"), (24, "unified", None, None), (32, "discrete", "qwen3.8-27b", "best-quality-resident"), (32, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"), (48, "discrete", "qwen3.8-27b", "best-quality-resident"), (48, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"), (96, "discrete", "qwen3.8-flash-next", "best-quality-resident"), (96, "unified", "qwen3.8-flash-next", "best-quality-resident"), (128, "discrete", "qwen3.8-flash-next", "best-quality-resident"), (128, "unified", "qwen3.8-flash-next", "best-quality-resident"), (256, "discrete", "qwen3.8-flash-next", "best-quality-resident"), (256, "unified", "qwen3.8-flash-next", "best-quality-resident"), (512, "discrete", "qwen3.8-flash-next", "best-quality-resident"), (512, "unified", "qwen3.8-flash-next", "best-quality-resident"), ] @pytest.mark.parametrize( ("size_gb", "kind", "expected", "expected_reason"), DECISION_TABLE, ids=[f"{s}GB-{k}" for s, k, _, _ in DECISION_TABLE]) def test_recommendation_decision_table(size_gb, kind, expected, expected_reason): """Pins the pick AND its reason per cell: the reason is user-facing (the Recommended badge's tooltip), so a cell whose rationale flips without the pick flipping is still a review-worthy change.""" budget = _discrete(size_gb) if kind == "discrete" else _unified(size_gb) picked = recommended_entry(budget) if expected is None: assert picked is None else: assert picked is not None assert (picked[0].id, picked[1]) == (expected, expected_reason) @pytest.mark.parametrize("gpu_name", [ "NVIDIA RTX Spark N1X", "NVIDIA RTX Spark N1X (5120-core Blackwell RTX GPU)", "NVIDIA RTX Spark N1X (updated device description)", ]) @pytest.mark.parametrize("capacity", [24, 48, 256]) def test_measured_n1x_profile_changes_speed_eligibility_not_fit_or_quality(monkeypatch, capacity, gpu_name): import subprocess import urllib.request from hermes_cli.local_runtime import catalog def no_io(*args, **kwargs): raise AssertionError("calibration must use shipped data, not a runtime benchmark") monkeypatch.setattr(subprocess, "run", no_io) monkeypatch.setattr(urllib.request, "urlopen", no_io) # The derived rule alone; the RTX Spark's product default has its own tests below. monkeypatch.setattr(catalog, "_PRODUCT_DEFAULTS", {}) budget = _unified(capacity) budget.gpu_name = gpu_name budget.platform = "win32" entry = next(e for e in CATALOG if e.id == "qwen3.8-27b") assert predicted_decode_tok_s(entry, entry.variants[0], budget) >= PLEASANT_FLOOR_TOK_S picked = recommended_entry(budget) expected = {24: None, 48: "qwen3.8-27b", 256: "qwen3.8-flash-next"}[capacity] assert (picked[0].id if picked else None) == expected @pytest.mark.parametrize("budget_changes, entry_changes, quant, backend, spilled", [ ({"gpu_name": ""}, {}, "UD-Q4_K_M", "cuda", False), ({"gpu_name": "NVIDIA Other Device"}, {}, "UD-Q4_K_M", "cuda", False), ({"gpu_name": "NVIDIA RTX Spark N1X2"}, {}, "UD-Q4_K_M", "cuda", False), ({"gpu_name": "NVIDIA RTX Spark N1X Pro"}, {}, "UD-Q4_K_M", "cuda", False), ({"platform": "linux"}, {}, "UD-Q4_K_M", "cuda", False), ({"uma": False}, {}, "UD-Q4_K_M", "cuda", False), ({}, {}, "UD-Q4_K_M", "vulkan", False), ({}, {}, "UD-Q4_K_M", "cpu", False), ({}, {"mtp": False}, "UD-Q4_K_M", "cuda", False), ({}, {"mtp_draft_depth": 3}, "UD-Q4_K_M", "cuda", False), ({}, {}, "Q8_0", "cuda", False), ({}, {}, "UD-Q4_K_M", "cuda", True), ]) def test_unmatched_or_spilled_profiles_keep_the_existing_estimate( monkeypatch, budget_changes, entry_changes, quant, backend, spilled): from dataclasses import replace import subprocess import urllib.request budget = replace(_unified(48), gpu_name="NVIDIA RTX Spark N1X (5120-core Blackwell RTX GPU)", platform="win32") budget = replace(budget, **budget_changes) entry = replace(next(e for e in CATALOG if e.id == "qwen3.8-27b"), **entry_changes) variant = replace(entry.variants[0], quant=quant) def no_io(*args, **kwargs): raise AssertionError("speed selection must be an offline lookup and arithmetic") monkeypatch.setattr(subprocess, "run", no_io) monkeypatch.setattr(urllib.request, "urlopen", no_io) fallback = predicted_decode_tok_s(entry, variant, replace(budget, gpu_name=""), spilled=spilled, backend=backend) assert predicted_decode_tok_s(entry, variant, budget, spilled=spilled, backend=backend) == fallback # ── invariants behind the table (survive catalog changes) ── def test_every_entry_carries_the_recommendation_axes(): """quality and decode_fraction are authoring requirements: an entry without them silently loses every quality comparison (quality=0) or prices as dense (decode_fraction=1.0). An entry that is never recommended needs no quality ordering.""" for entry in CATALOG: if entry.recommend: assert entry.quality > 0, f"{entry.id} has no quality ordering" assert 0.0 < entry.decode_fraction <= 1.0, entry.id if not entry.moe: assert entry.decode_fraction == 1.0, ( f"{entry.id} is dense — it reads every weight per token") @pytest.mark.parametrize("budget", [_discrete(32), _discrete(48), _discrete(96), _unified(32), _unified(64), _unified(128)], ids=lambda b: f"{'unified' if b.uma else 'discrete'}-{b.total_device_bytes / _GIB:g}") def test_nemotron_bolt_is_an_option_never_the_recommendation(budget): """NVIDIA asked for Nemotron 3.5 Bolt as an option beside the others, not a default. It runs resident on these machines and clears the floor, and still is never picked, even alone.""" bolt = next(e for e in CATALOG if e.id == "nemotron-3.5-bolt-74b-a9b") assert not bolt.recommend choice = select_variant(bolt, budget) assert choice is not None and choice.zero_spill assert predicted_decode_tok_s(bolt, choice.variant, budget) >= PLEASANT_FLOOR_TOK_S assert recommended_entry(budget, (bolt,)) is None picked = recommended_entry(budget) assert picked is None or picked[0].id != bolt.id def test_unified_never_recommends_a_below_floor_dense_model(): """The Spark rule, as an invariant: whatever the catalog holds, a unified-memory machine must not be told to run a model whose predicted decode is below the pleasant floor while a resident alternative clears it.""" budget = _unified(128) pick = recommended_entry(budget)[0].id assert pick is not None entry = next(e for e in CATALOG if e.id == pick) choice = select_variant(entry, budget) assert choice is not None and choice.zero_spill clears = [ e for e in CATALOG if (c := select_variant(e, budget)) is not None and c.zero_spill and predicted_decode_tok_s(e, c.variant, budget) >= PLEASANT_FLOOR_TOK_S ] if clears: assert predicted_decode_tok_s(entry, choice.variant, budget) >= PLEASANT_FLOOR_TOK_S @pytest.mark.parametrize("budget", [ *(_unified(gb) for gb in (16, 24, 31.5, 32, 48, 64, 96, 128, 256)), *(_discrete(gb) for gb in (8, 12, 16, 24, 32, 48, 96)), ], ids=lambda b: f"{'unified' if b.uma else 'discrete'}-{b.total_device_bytes / _GIB:g}") def test_a_derived_recommendation_always_clears_the_floor(budget): """A machine where nothing resident reaches the floor gets no recommendation. A "32 GB" laptop reads 31.5 GB, where only the 27B fits and runs well under the floor on an iGPU or the CPU; it stays one click away in Browse.""" picked = recommended_entry(budget) if picked is None or picked[1] == "product-default": return choice = select_variant(picked[0], budget) assert choice is not None and choice.zero_spill assert predicted_decode_tok_s(picked[0], choice.variant, budget) >= PLEASANT_FLOOR_TOK_S def test_quality_decides_where_speed_permits(): """On big discrete hardware every resident entry clears the floor, so the pick must be the highest-quality fitting entry — the axis that justifies carrying an editorial field at all.""" budget = _discrete(512) pick = recommended_entry(budget)[0].id resident = [ e for e in CATALOG if e.recommend and (c := select_variant(e, budget)) is not None and c.zero_spill ] assert pick == max(resident, key=lambda e: e.quality).id # ── the RTX Spark's product default ── _N1X_PCI_ID = 0x2E1210DE # NVML packs the device ID above the 16-bit vendor ID @pytest.mark.parametrize("identity", [ {"gpu_pci_id": _N1X_PCI_ID}, {"gpu_name": "NVIDIA RTX Spark N1X (5120-core Blackwell RTX GPU)"}, ]) def test_the_rtx_spark_defaults_to_the_27b_with_flash_next_still_fitting(identity): """Its manufacturer's pick. Flash Next fits too and stays one click away; on Windows it loads once the carve-out leaves Windows enough commit to back it.""" from dataclasses import replace budget = replace(_unified(128), platform="win32", **identity) picked = recommended_entry(budget) assert (picked[0].id, picked[1]) == ("qwen3.8-27b", "product-default") flash_next = next(e for e in CATALOG if e.id == "qwen3.8-flash-next") choice = select_variant(flash_next, budget) assert choice is not None and choice.zero_spill def test_the_derived_rule_decides_on_other_platforms_backends_and_when_the_default_is_ineligible(): from dataclasses import replace spark = replace(_unified(128), platform="win32", gpu_pci_id=_N1X_PCI_ID) without_the_27b = tuple(e for e in CATALOG if e.id != "qwen3.8-27b") assert recommended_entry(spark, without_the_27b)[1] != "product-default" assert recommended_entry(replace(spark, platform="linux"))[1] != "product-default" assert recommended_entry(spark, backend="vulkan")[1] != "product-default"