From 780a8c61e4e31de87e19f1a610ca490b95f65bbe Mon Sep 17 00:00:00 2001 From: prabod Date: Wed, 12 Aug 2026 13:23:04 +1000 Subject: [PATCH 1/3] sync: Muse Glimmer converter, k-quant passthrough and mmproj bundling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Open-core sync for 0.2.2, computed by tools/sync_public.py (allow-list; tuned quant profiles stay private). CONVERTER - base-arch: new muse_glimmer mapper (HF + GGUF), including the perception tower's canonical `vision.*` naming. The HF and GGUF name tables are pinned to agree by test, so a bundle built from either source is interchangeable. - base-convert: `--kquant-passthrough` copies GGUF Q4_K/Q5_K/Q6_K super-blocks in verbatim (lossless — no dequant/repack round trip), and `--mmproj` folds a companion mmproj GGUF's vision tower into the same bundle, so a GGUF-sourced build is no longer text-only. - base-format: reader support for the above. CATALOG Two rows for basecompute/Muse-Glimmer-30B so `basert pull` resolves it: `default-kquant-dynamic` (what a bare pull gets) and `kquant-17gb`. Both carry real size + sha256; fetch verifies and bails on mismatch. The weights are published at huggingface.co/basecompute/Muse-Glimmer-30B. DELIBERATELY EXCLUDED: the header and binding mirrors. `embed_norm_eps` widens BaseRTModelConfig from 1540 to 1704 bytes. The rust-sys ABI job links the LATEST ENGINE RELEASE (v0.2.1) and compares it against the in-tree mirror, so syncing the headers before a matching engine release fails by construction — 1540 from the binary vs 1704 from source. That half has to follow the 0.2.2 engine release, not lead it. Nothing here touches the ABI: base-convert is pure Rust and links no engine. Verified in this tree: base-convert builds clean and its 23 test suites pass. --- base-convert/crates/base-arch/src/lib.rs | 132 ++ .../crates/base-arch/src/muse_glimmer.rs | 1087 +++++++++++++++++ base-convert/crates/base-convert/src/hub.rs | 8 + base-convert/crates/base-convert/src/main.rs | 984 ++++++++++++++- base-convert/crates/base-format/src/reader.rs | 18 +- base-convert/crates/base-hub/catalog.json | 20 + 6 files changed, 2233 insertions(+), 16 deletions(-) create mode 100644 base-convert/crates/base-arch/src/muse_glimmer.rs diff --git a/base-convert/crates/base-arch/src/lib.rs b/base-convert/crates/base-arch/src/lib.rs index 3fbcf0b..a2a1bc0 100644 --- a/base-convert/crates/base-arch/src/lib.rs +++ b/base-convert/crates/base-arch/src/lib.rs @@ -9,6 +9,7 @@ pub mod bert; pub mod gemma; pub mod llama; +pub mod muse_glimmer; pub mod qwen; pub mod tokenizer; pub mod whisper; @@ -38,6 +39,14 @@ pub fn source_mapper_for_gguf(arch: &str) -> Option<&'static dyn GgufMapper> { "gemma" | "gemma2" | "gemma3" => Some(&gemma::Gemma3Mapper), "gemma4" => Some(&gemma::Gemma4Mapper), "nomic-bert" => Some(&bert::NomicBertMapper), + // Muse Glimmer. `general.architecture` in the llama.cpp-produced + // GGUF is the HYPHENATED "muse-glimmer" (llama.cpp's arch registry + // spells multi-word archs with hyphens: "nomic-bert", + // "deepseek2"...), while the HF `model_type` — and therefore the + // canonical `.base` `arch` field — is the UNDERSCORED + // "muse_glimmer". Both spellings resolve here so a hand-edited or + // future exporter that emits the underscore form still converts. + "muse-glimmer" | "muse_glimmer" => Some(&muse_glimmer::MuseGlimmerGgufMapper), _ => None, } } @@ -73,6 +82,22 @@ pub trait HfMapper: Sync { fn rope_permute_heads(&self, _canonical: &str, _cfg: &ArchConfig) -> Option { None } + + /// RMS-normalize each ROW of a 2-D tensor at HF→.base conversion + /// time, returning the epsilon to use, or None to leave it alone. + /// + /// Muse Glimmer applies a weightless RMSNorm to the token embedding + /// immediately after lookup (`MuseGlimmerTextNormedEmbedding`). Since + /// a lookup returns exactly one row and the norm mixes nothing across + /// rows, it is mathematically identical to normalizing every row of + /// the embedding matrix up front — so the runtime needs no extra op. + /// (The reference keeps them separate only because its DFlash drafter + /// needs the un-normed embedding.) Safe only while the embedding is + /// NOT tied to `lm_head`, which holds for Muse Glimmer + /// (`tie_word_embeddings=false`). Default: no normalization. + fn row_rms_normalize(&self, _canonical: &str, _cfg: &ArchConfig) -> Option { + None + } } pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMapper> { @@ -107,6 +132,12 @@ pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMappe "qwen3_5" | "qwen3_5_text" | "qwen35" => Some(&qwen::Qwen35HfMapper), "qwen3_5_moe" | "qwen3_5_moe_text" | "qwen35_moe" => Some(&qwen::Qwen35MoeHfMapper), "nomic_bert" | "nomic-bert" => Some(&bert::NomicBertHfMapper), + // Muse Glimmer: dense SWA/global decoder with a perception (ViT) + // tower. The multimodal wrapper is `muse_glimmer` + // (`MuseGlimmerForConditionalGeneration`) with the text tower under + // `text_config.model_type = muse_glimmer_text`; both resolve here so + // a text-only checkpoint and the multimodal wrapper convert alike. + "muse_glimmer" | "muse_glimmer_text" => Some(&muse_glimmer::MuseGlimmerHfMapper), "gemma" | "gemma2" | "gemma3" | "gemma3_text" => Some(&gemma::Gemma3HfMapper), // gemma3n is a distinct arch (AltUp/Laurel/per-layer-FFN); the // existing local fixture historically named "gemma-4-e2b" was @@ -156,6 +187,8 @@ pub const SUPPORTED_HF_MODEL_TYPES: &[&str] = &[ "gemma4", "gemma4_text", "gemma4_unified", + "muse_glimmer", + "muse_glimmer_text", "whisper", ]; @@ -174,6 +207,57 @@ pub trait GgufMapper: Sync { /// `rope_freqs.weight` — we precompute RoPE elsewhere or recompute /// at runtime). fn map_tensor_name(&self, gguf_name: &str) -> Option; + + /// RoPE row-permutation head count for a canonical tensor on the GGUF + /// path, or None when the tensor's rows are already in the layout this + /// runtime's rope kernel expects. The INVERSE of + /// [`HfMapper::rope_permute_heads`]: that one converts HF split-half + /// rows INTO the interleaved-pair layout, this one converts GGUF + /// interleaved-pair rows BACK to split-half. + /// + /// Background. There are two rotary row layouts in the wild: + /// + /// - "interleaved pair" (Meta original / GPT-J): rope rotates the + /// element pairs `(2i, 2i+1)`. Runtime kernel: `rope_f16`. + /// - "split half" (NeoX / HF `rotate_half`): rope pairs element `i` + /// with `i + head_dim/2`. Runtime kernel: `rope_neox_f16`. + /// + /// `convert_hf_to_gguf.py::LlamaModel.permute` rewrites `q_proj` / + /// `k_proj` from split-half into interleaved-pair when it exports an + /// HF checkpoint, per head: + /// + /// ```text + /// gguf_row[h*HD + 2j + k] = hf_row[h*HD + k*HD/2 + j] + /// ``` + /// + /// so the inverse this hook drives is + /// + /// ```text + /// hf_row[h*HD + k*HD/2 + j] = gguf_row[h*HD + 2j + k] + /// ``` + /// + /// Whether a GGUF needs the inverse depends on which kernel the arch + /// runs, NOT on the source format: + /// + /// - llama/mistral run `rope_f16` (interleaved). llama.cpp permutes + /// on export, the HF path permutes at convert time, so both + /// sources agree and the GGUF needs NOTHING here. Returning + /// `Some(..)` for llama would actively break it. + /// - gemma/qwen/nomic-bert run `rope_neox_f16`, and llama.cpp does + /// NOT permute those archs on export — again nothing to do. + /// - Muse Glimmer runs `rope_neox_f16` (split-half) but its + /// llama.cpp exporter DOES apply the Llama permute. That is the + /// one combination that needs undoing, and it is why this hook + /// exists. + /// + /// The permutation moves whole ROWS (output features), never elements + /// within a row, so it can be applied to packed k-quant blocks by + /// reordering row-sized byte runs — no dequantization, fully lossless. + /// The caller must still check that each row is a whole number of + /// blocks (`in_features % block_elems == 0`). + fn rope_unpermute_heads(&self, _canonical: &str, _cfg: &ArchConfig) -> Option { + None + } } #[derive(Debug, Clone, Default, PartialEq)] @@ -313,6 +397,37 @@ pub struct ArchConfig { // The decoder half reuses the standard fields above (hidden_size / // num_hidden_layers / num_attention_heads / intermediate_size / // max_position_embeddings); the encoder half is described here. + // ── Muse Glimmer fields (zero/empty for other archs) ───────────── + /// Multiplier applied to Q AFTER the scaleless (weightless) QK-norm, + /// on top of the standard `1/sqrt(head_dim)` attention scaling + /// (`qk_scale_factor`, 3.87 on the released checkpoint). 0 = not a + /// Muse-Glimmer-style model. Distinct from `attention_scale`, which + /// REPLACES the `1/sqrt(head_dim)` term rather than scaling it. + pub qk_scale_factor: f32, + /// Scale applied to the logits BEFORE the final tanh softcap + /// (`output_multiplier`; `1/sqrt(hidden_size/256)` on the released + /// checkpoint). 0 = no multiplier. + pub output_multiplier: f32, + /// Epsilon for the post-attention / post-FFN norms, which differs + /// from `rms_norm_eps` on Muse Glimmer (1e-8 vs 1e-5). 0 = reuse + /// `rms_norm_eps` for every norm. + pub post_norm_eps: f32, + /// Epsilon for a weightless RMSNorm the RUNTIME must apply to the token + /// embedding after lookup. 0 = the norm is already folded into the + /// embedding rows (the HF path does this via `row_rms_normalize`, which is + /// exact) or the arch has no such norm. + /// + /// The GGUF path cannot fold it: the rows arrive as packed k-quant + /// super-blocks, and folding would require dequantizing — the very thing + /// passthrough exists to avoid. So a GGUF-sourced Muse Glimmer bundle sets + /// this and pays for one extra kernel per prefill instead. + pub embed_norm_eps: f32, + + /// Per-layer NoPE mask (true = layer applies NO rotary embedding). + /// Derived from `layer_rope_theta[i] == 0`. Empty = every layer + /// gets RoPE. Length = num_hidden_layers. + pub nope_layers: Vec, + /// Encoder transformer depth (`encoder_layers`). 0 = not an /// encoder-decoder model. pub encoder_layers: u32, @@ -493,6 +608,23 @@ impl ArchConfig { if self.attn_output_gate { m.insert("attn_output_gate".into(), json!(self.attn_output_gate)); } + // Muse Glimmer scales / epsilons / NoPE schedule — emitted only + // when set so other archs' headers stay unchanged. + if self.qk_scale_factor > 0.0 { + m.insert("qk_scale_factor".into(), json!(self.qk_scale_factor)); + } + if self.output_multiplier > 0.0 { + m.insert("output_multiplier".into(), json!(self.output_multiplier)); + } + if self.post_norm_eps > 0.0 { + m.insert("post_norm_eps".into(), json!(self.post_norm_eps)); + } + if !self.nope_layers.is_empty() { + m.insert("nope_layers".into(), json!(self.nope_layers)); + } + if self.embed_norm_eps > 0.0 { + m.insert("embed_norm_eps".into(), json!(self.embed_norm_eps)); + } if self.partial_rotary_factor > 0.0 { m.insert( "partial_rotary_factor".into(), diff --git a/base-convert/crates/base-arch/src/muse_glimmer.rs b/base-convert/crates/base-arch/src/muse_glimmer.rs new file mode 100644 index 0000000..de5901b --- /dev/null +++ b/base-convert/crates/base-arch/src/muse_glimmer.rs @@ -0,0 +1,1087 @@ +//! Muse Glimmer HF mapper. +//! +//! `MuseGlimmerForConditionalGeneration` (model_type `muse_glimmer`) is a +//! dense decoder with a perception (ViT) tower. The language tower nests +//! under `text_config` (model_type `muse_glimmer_text`) and is shaped like +//! Gemma 3 — four zero-centered RMSNorms per layer, SwiGLU FFN, a final +//! logit softcap — with four Muse-Glimmer-specific twists: +//! +//! 1. Attention layers alternate local(sliding, 2048) x3 → global, and +//! the GLOBAL layers are NoPE: `layer_rope_theta[i] == 0` disables +//! rotary entirely there (the reference passes +//! `position_embeddings=None` for those layers). Local layers use the +//! ordinary theta from `rope_parameters`. +//! 2. Q and K pass through a SCALELESS (weightless) RMSNorm, after which +//! Q is multiplied by `qk_scale_factor` — on top of, not instead of, +//! the usual `1/sqrt(head_dim)`. There are therefore no `q_norm` / +//! `k_norm` tensors in the checkpoint. +//! 3. Attention output is gated: `attn_out *= sigmoid(gate_proj(x))` +//! BEFORE `o_proj`, where `gate_proj` is a SEPARATE `[n_heads*head_dim, +//! hidden]` projection over the layer input (not a doubled `q_proj` +//! like Qwen3.5). +//! 4. The post-attention / post-FFN norms use `post_norm_eps` (1e-8), +//! distinct from `rms_norm_eps` (1e-5) on the other two norms. +//! +//! The token embedding carries a weightless RMSNorm applied right after +//! lookup; `row_rms_normalize` bakes it into the matrix at convert time +//! (see the trait docs for why that is exact). + +use crate::{ArchConfig, GgufMapper, HfMapper}; +use anyhow::{Context, Result}; +use base_readers::gguf::KvValue; +use std::collections::BTreeMap; + +pub struct MuseGlimmerHfMapper; +pub struct MuseGlimmerGgufMapper; + +/// Canonical names of the four per-layer norms that use Muse Glimmer's +/// zero-centered `(1 + weight)` RMSNorm formulation, AFTER the HF→canonical +/// rename in base-convert's `hf_rename`. The final `norm.weight` → +/// `final_norm.weight` is a PLAIN weighted RMSNorm in the reference +/// (`MuseGlimmerRMSNorm`, not `MuseGlimmerTextCenteredRMSNorm`) and must +/// NOT be shifted — nor may any `vision.*` LayerNorm. +const CENTERED_NORM_SUFFIXES: [&str; 4] = [ + ".input_norm.weight", + ".post_attention_norm.weight", + ".post_attn_norm.weight", + ".post_ffw_norm.weight", +]; + +impl HfMapper for MuseGlimmerHfMapper { + fn canonical_arch(&self) -> &'static str { + "muse_glimmer" + } + + /// Bake the +1 unit offset into the zero-centered per-layer RMSNorm + /// gammas so the runtime's plain `rmsnorm(x) * weight` kernel yields + /// the reference's `rmsnorm(x) * (1 + weight)`. Same treatment Gemma 3 + /// gets, but scoped to the four per-layer norms: unlike Gemma 3, Muse + /// Glimmer's FINAL norm is not zero-centered, so a blanket + /// `ends_with("norm.weight")` rule would corrupt it. + fn norm_shift(&self, canonical: &str) -> f32 { + if !canonical.starts_with("layers.") { + return 0.0; + } + if CENTERED_NORM_SUFFIXES + .iter() + .any(|s| canonical.ends_with(s)) + { + 1.0 + } else { + 0.0 + } + } + + /// Fold the weightless embedding norm into the embedding matrix. + /// Uses `rms_norm_eps`, matching `MuseGlimmerTextNormedEmbedding` + /// (constructed with `config.rms_norm_eps`). + fn row_rms_normalize(&self, canonical: &str, cfg: &ArchConfig) -> Option { + if canonical == "embed_tokens.weight" { + // Baking is only valid while the embedding is untied from the + // output head — otherwise the LM head would inherit the norm. + if cfg.tie_word_embeddings { + return None; + } + return Some(cfg.rms_norm_eps); + } + None + } + + fn config_from_hf(&self, c: &serde_json::Value) -> Result { + // The multimodal wrapper nests the language tower under + // `text_config`; a text-only checkpoint hoists it to the top level. + let tc = c.get("text_config").unwrap_or(c); + let mut config = crate::llama::hf_generic_config(tc)?; + + let f32_v = + |v: &serde_json::Value, k: &str| v.get(k).and_then(|x| x.as_f64()).map(|f| f as f32); + let u32_v = + |v: &serde_json::Value, k: &str| v.get(k).and_then(|x| x.as_u64()).map(|n| n as u32); + + // Muse Glimmer keeps RoPE under `rope_parameters`, so + // hf_generic_config's top-level `rope_theta` default (10000) is + // wrong — override it. + if let Some(rp) = tc.get("rope_parameters") { + if let Some(theta) = f32_v(rp, "rope_theta") { + config.rope_theta = theta; + } + } + + config.tie_word_embeddings = tc + .get("tie_word_embeddings") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + // Sliding-window schedule. `layer_types` is authoritative; the + // reference default is "full_attention on every 4th layer counted + // BACKWARD from the last, sliding otherwise", which we reproduce + // when the key is absent so a trimmed config still converts. + let n_layers = config.num_hidden_layers as usize; + config.sliding_window = u32_v(tc, "sliding_window").unwrap_or(0); + let layer_types: Vec = match tc.get("layer_types").and_then(|v| v.as_array()) { + Some(lt) => lt + .iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect(), + None => default_layer_types(n_layers), + }; + config.swa_layers = layer_types + .iter() + .map(|t| t == "sliding_attention") + .collect(); + + // NoPE mask: `layer_rope_theta[i] == 0` means the layer runs with + // no rotary at all. The reference default puts NoPE on exactly the + // full-attention layers, so fall back to that when the key is + // absent rather than silently giving every layer RoPE. + config.nope_layers = match tc.get("layer_rope_theta").and_then(|v| v.as_array()) { + Some(arr) => arr + .iter() + .map(|x| x.as_f64().unwrap_or(0.0) == 0.0) + .collect(), + None => layer_types.iter().map(|t| t == "full_attention").collect(), + }; + + config.logit_softcap = f32_v(tc, "final_logit_softcapping").unwrap_or(0.0); + config.qk_scale_factor = f32_v(tc, "qk_scale_factor").unwrap_or(0.0); + config.output_multiplier = f32_v(tc, "output_multiplier").unwrap_or(0.0); + config.post_norm_eps = f32_v(tc, "post_norm_eps").unwrap_or(0.0); + // Gated attention via a dedicated `self_attn.gate_proj`. + config.attn_output_gate = true; + + Ok(config) + } +} + +/// The reference's default attention schedule when a config/GGUF omits an +/// explicit one: `full_attention` on every 4th layer counted BACKWARD from +/// the last, `sliding_attention` everywhere else. Shared by the HF and +/// GGUF paths so a trimmed config and a metadata-poor GGUF land on the +/// same schedule. +fn default_layer_types(n_layers: usize) -> Vec { + (0..n_layers) + .map(|i| { + if (n_layers - 1 - i) % 4 == 0 { + "full_attention".to_string() + } else { + "sliding_attention".to_string() + } + }) + .collect() +} + +// ── GGUF path ─────────────────────────────────────────────────────── +// +// `general.architecture` is the HYPHENATED "muse-glimmer", so every +// per-arch metadata key is prefixed `muse-glimmer.` — see +// `source_mapper_for_gguf`. `canonical_arch()` returns the UNDERSCORED +// "muse_glimmer" so a GGUF-sourced and an HF-sourced bundle carry the +// identical `arch` field and select the same runtime model class. + +/// Published Muse Glimmer 30B architecture constants. llama.cpp bakes +/// most of these into its graph builder rather than exporting them, so a +/// GGUF that predates an exporter change lands here. Same treatment +/// Gemma 3's sliding-window pattern / local theta get in `gemma.rs`. +const REF_QK_SCALE_FACTOR: f32 = 3.87; +const REF_LOGIT_SOFTCAP: f32 = 20.0; +const REF_POST_NORM_EPS: f32 = 1e-8; +const REF_SLIDING_WINDOW: u32 = 2048; + +impl GgufMapper for MuseGlimmerGgufMapper { + fn canonical_arch(&self) -> &'static str { + "muse_glimmer" + } + + fn config_from_gguf(&self, m: &BTreeMap) -> Result { + // Prefer the hyphenated prefix llama.cpp actually writes; accept + // the underscored one so a hand-rolled exporter still converts. + let prefix = if m.keys().any(|k| k.starts_with("muse-glimmer.")) { + "muse-glimmer" + } else { + "muse_glimmer" + }; + let u32_req = |k: &str| { + m.get(k) + .and_then(|v| v.as_u64()) + .map(|n| n as u32) + .with_context(|| format!("missing metadata key: {k}")) + }; + let u32_key = |k: &str| m.get(k).and_then(|v| v.as_u64()).map(|n| n as u32); + let f32_key = |k: &str| m.get(k).and_then(|v| v.as_f32()); + // First key that exists wins — lets us accept more than one + // plausible spelling without a combinatorial explosion of ifs. + let f32_any = |suffixes: &[&str]| -> Option { + suffixes + .iter() + .find_map(|s| f32_key(&format!("{prefix}.{s}"))) + }; + let u32_any = |suffixes: &[&str]| -> Option { + suffixes + .iter() + .find_map(|s| u32_key(&format!("{prefix}.{s}"))) + }; + + let hidden_size = u32_req(&format!("{prefix}.embedding_length"))?; + let num_hidden_layers = u32_req(&format!("{prefix}.block_count"))?; + let num_attention_heads = u32_req(&format!("{prefix}.attention.head_count"))?; + let num_kv_heads = u32_key(&format!("{prefix}.attention.head_count_kv")) + .unwrap_or(num_attention_heads); + let intermediate_size = u32_req(&format!("{prefix}.feed_forward_length"))?; + let vocab_size = u32_key(&format!("{prefix}.vocab_size")) + .or_else(|| match m.get("tokenizer.ggml.tokens") { + Some(KvValue::Array(a)) => Some(a.len() as u32), + _ => None, + }) + .context("no vocab_size and no tokenizer.ggml.tokens")?; + let head_dim = u32_key(&format!("{prefix}.attention.key_length")) + .unwrap_or(hidden_size / num_attention_heads); + + let mut config = ArchConfig { + hidden_size, + num_hidden_layers, + num_attention_heads, + num_kv_heads, + head_dim, + intermediate_size, + vocab_size, + rope_theta: f32_key(&format!("{prefix}.rope.freq_base")).unwrap_or(500_000.0), + rope_scale: f32_key(&format!("{prefix}.rope.scaling.factor")).unwrap_or(1.0), + rms_norm_eps: f32_key(&format!("{prefix}.attention.layer_norm_rms_epsilon")) + .unwrap_or(1e-5), + // The released checkpoint keeps `lm_head` separate; the GGUF + // ships `output.weight` accordingly. + tie_word_embeddings: m + .get(&format!("{prefix}.tie_lm_head")) + .and_then(|v| match v { + KvValue::Bool(b) => Some(*b), + _ => None, + }) + .unwrap_or(false), + max_position_embeddings: u32_key(&format!("{prefix}.context_length")).unwrap_or(0), + bos_token_id: m + .get("tokenizer.ggml.bos_token_id") + .and_then(|v| v.as_u64()) + .map(|n| n as u32) + .unwrap_or(0), + eos_token_id: m + .get("tokenizer.ggml.eos_token_id") + .and_then(|v| v.as_u64()) + .map(|n| n as u32) + .unwrap_or(0), + // Gated attention via a dedicated `blk.N.attn_gate.weight`. + attn_output_gate: true, + ..ArchConfig::default() + }; + + // ── Muse-Glimmer-specific scales ──────────────────────────── + // A zero here is NOT harmless: `qk_scale_factor` multiplies the + // attention scale (muse_glimmer.cpp only applies it when > 0), so + // dropping it silently shrinks every logit by ~4x. Prefer an + // exported key; otherwise fall back to the published constant. + config.qk_scale_factor = f32_any(&["attention.qk_scale_factor", "qk_scale_factor"]) + .unwrap_or(REF_QK_SCALE_FACTOR); + config.logit_softcap = f32_any(&["final_logit_softcapping", "logit_softcap"]) + .unwrap_or(REF_LOGIT_SOFTCAP); + config.post_norm_eps = + f32_any(&["attention.post_norm_epsilon", "post_norm_eps"]).unwrap_or(REF_POST_NORM_EPS); + // llama.cpp exports HF's `output_multiplier` under its own generic + // name `logit_scale` (verified against the KV dump of + // meta-models/Muse-Glimmer-30B-GGUF: `muse-glimmer.logit_scale = + // 0.1961161345243454`); there is no `output_multiplier` key. The + // HF spelling stays in the probe list for a hand-authored GGUF. + // + // The derivation below is the fallback: the value is exactly + // 1/sqrt(hidden_size/256) on the released checkpoint (6656/256 = 26 + // → 0.19611613…), so it beats hardcoding a width-specific number. + config.output_multiplier = f32_any(&["logit_scale", "output_multiplier"]).unwrap_or_else(|| { + let d = hidden_size as f32 / 256.0; + if d > 0.0 { + 1.0 / d.sqrt() + } else { + 0.0 + } + }); + + // ── Local/global + NoPE schedules ─────────────────────────── + config.sliding_window = u32_any(&["attention.sliding_window"]).unwrap_or(REF_SLIDING_WINDOW); + let n_layers = num_hidden_layers as usize; + // A bool array under `attention.sliding_window_pattern` is the + // shape Gemma 4 uses and the natural one for this arch; fall back + // to the reference's local x3 → global cycle. + let swa_from_kv = match m.get(&format!("{prefix}.attention.sliding_window_pattern")) { + Some(KvValue::Array(arr)) if !arr.is_empty() => Some( + arr.iter() + .map(|v| match v { + KvValue::Bool(b) => *b, + other => other.as_u64().unwrap_or(0) != 0, + }) + .collect::>(), + ), + _ => None, + }; + let layer_types = default_layer_types(n_layers); + config.swa_layers = swa_from_kv + .unwrap_or_else(|| layer_types.iter().map(|t| t == "sliding_attention").collect()); + + // NoPE mask. HF encodes it as `layer_rope_theta[i] == 0`; if a + // per-layer theta array ever lands in GGUF metadata, honour it. + // Otherwise the reference default is "NoPE on exactly the + // full-attention layers", which is the complement of `swa_layers`. + let nope_from_kv = match m.get(&format!("{prefix}.rope.layer_freq_base")) { + Some(KvValue::Array(arr)) if !arr.is_empty() => Some( + arr.iter() + .map(|v| v.as_f32().unwrap_or(0.0) == 0.0) + .collect::>(), + ), + _ => None, + }; + config.nope_layers = + nope_from_kv.unwrap_or_else(|| config.swa_layers.iter().map(|s| !s).collect()); + + // llama.cpp does NOT fold Muse Glimmer's weightless embedding norm + // into `token_embd`: the GGUF rows equal the HF RAW embedding + // (verified — per-row RMS 0.0625 on both, cosine 0.997 after Q4_K + // dequant). Passthrough cannot fold it either, because the rows + // arrive as packed super-blocks and folding means dequantizing, the + // exact thing passthrough exists to avoid. So the runtime applies it. + // + // The HF path leaves this 0 — `row_rms_normalize` already baked it in + // there, which is exact and costs nothing at inference. Without this + // signal a GGUF-sourced bundle enters the stack ~16x too small and + // degenerates into single-token repetition. + config.embed_norm_eps = config.rms_norm_eps; + + // `tokenizer.ggml.eos_token_id` alone is not enough to stop this + // model. Its chat format ends an assistant turn with `<|eot|>` + // (200008), exported separately as `tokenizer.ggml.eot_token_id`, + // while `eos_token_id` (200001) is `<|end_of_text|>` and only ends + // the whole document. The HF path picks both up by merging + // generation_config.json; without the eot id here the model answers + // correctly and then keeps opening fresh turns until the token cap. + if let Some(eot) = m + .get("tokenizer.ggml.eot_token_id") + .and_then(|v| v.as_u64()) + .map(|n| n as u32) + { + if eot != 0 && eot != config.eos_token_id && !config.eos_token_ids.contains(&eot) { + config.eos_token_ids.push(eot); + } + } + + Ok(config) + } + + fn map_tensor_name(&self, n: &str) -> Option { + map_gguf_name(n) + } + + /// Muse Glimmer runs the NeoX (split-half) rope kernel — the runtime's + /// `arch_muse_glimmer()` sets `rope_standalone = "rope_neox_f16"` and + /// `muse_glimmer.cpp` passes `is_neox=true` — and the HF path leaves + /// `q_proj`/`k_proj` rows untouched (no `rope_permute_heads` impl on + /// `MuseGlimmerHfMapper`). llama.cpp's exporter, however, applies the + /// Llama interleaved-pair permute to `attn_q` / `attn_k`. The GGUF + /// path must therefore undo it, or Q and K get their trained rotary + /// frequencies assigned to the wrong dim-pairs. + /// + /// Everything else — `attn_v`, `attn_output`, `attn_gate`, the FFN — + /// is rotation-free and must NOT be touched. + fn rope_unpermute_heads(&self, canonical: &str, cfg: &ArchConfig) -> Option { + if canonical.ends_with(".self_attn.q_proj.weight") { + Some(cfg.num_attention_heads) + } else if canonical.ends_with(".self_attn.k_proj.weight") { + Some(cfg.num_kv_heads) + } else { + None + } + } +} + +/// Map a Muse Glimmer GGUF tensor name to the canonical `.base` name — +/// the SAME name `to_canonical_name` produces for the HF checkpoint, so +/// the runtime's `BaseWeightStore` alias table resolves both identically. +/// +/// Deliberately NOT `map_llama_style`: three of its rules are wrong here. +/// +/// - `attn_gate.weight` → `self_attn.gate.weight` in the Llama table, +/// but the alias table only knows `self_attn.gate_proj.weight` → +/// `attention.gate.weight` (the Muse Glimmer separate-projection +/// output gate). The Llama name would load as a null buffer. +/// - `post_attention_norm.weight` → `post_attn_norm.weight` in the +/// Llama table, which COLLIDES with `ffn_norm` → `post_attn_norm`. +/// Muse Glimmer has four distinct per-layer norms, so, exactly like +/// Gemma, `post_attention_norm` and `post_ffw_norm` stay verbatim +/// (`muse_glimmer.cpp` reads `arch.post_attn_norm_key = +/// "post_attention_norm"` / `post_ffn_norm_key = "post_ffw_norm"`) +/// and only `ffn_norm` — the PRE-FFN norm — becomes +/// `post_attn_norm.weight` (aliased to runtime `ffn_norm.weight`). +pub fn map_gguf_name(n: &str) -> Option { + match n { + "token_embd.weight" => return Some("embed_tokens.weight".into()), + "output.weight" => return Some("lm_head.weight".into()), + "output_norm.weight" => return Some("final_norm.weight".into()), + // RoPE is recomputed at runtime from `rope_theta`; Muse Glimmer + // has no partial rotary, so the divisor mask carries no + // information Gemma 4 would need preserved. + "rope_freqs.weight" => return None, + _ => {} + } + + let rest = n.strip_prefix("blk.")?; + let (layer_str, suffix) = rest.split_once('.')?; + let layer: u32 = layer_str.parse().ok()?; + let canonical = match suffix { + "attn_norm.weight" => "input_norm.weight", + // Pre-FFN norm → the canonical the alias table maps to the + // runtime's `ffn_norm.weight`. + "ffn_norm.weight" => "post_attn_norm.weight", + // Kept verbatim — read directly by name, see the doc comment. + "post_attention_norm.weight" => "post_attention_norm.weight", + "post_ffw_norm.weight" => "post_ffw_norm.weight", + "attn_q.weight" => "self_attn.q_proj.weight", + "attn_k.weight" => "self_attn.k_proj.weight", + "attn_v.weight" => "self_attn.v_proj.weight", + "attn_output.weight" => "self_attn.o_proj.weight", + // Separate `[n_heads*head_dim, hidden]` sigmoid gate over the + // layer input, applied to the attention output before o_proj. + "attn_gate.weight" => "self_attn.gate_proj.weight", + // The QK norm is WEIGHTLESS on this arch, so these should not + // exist. Map them rather than failing the conversion if an + // exporter emits all-ones tensors: `muse_glimmer.cpp` uses + // `emit_head_rmsnorm_noweight` unconditionally and never reads + // them, so they are inert ballast at worst. + "attn_q_norm.weight" => "self_attn.q_norm.weight", + "attn_k_norm.weight" => "self_attn.k_norm.weight", + "ffn_gate.weight" => "mlp.gate_proj.weight", + "ffn_up.weight" => "mlp.up_proj.weight", + "ffn_down.weight" => "mlp.down_proj.weight", + // Unknown — fail loud rather than silently dropping a weight. + _ => return None, + }; + Some(format!("layers.{layer}.{canonical}")) +} + +/// Map a Muse Glimmer perception-tower tensor name (leading `model.` +/// already stripped) to its canonical `.base` mmproj name. +/// +/// Targets the same `vision.*` vocabulary the Gemma 4 tower uses so the +/// runtime's weight lookup stays arch-agnostic. Returns `None` when no rule +/// matches; the caller then passes the name through verbatim. +/// +/// Note the tower is a plain LayerNorm ViT with SEPARATE q/k/v projections +/// that all carry biases — unlike Qwen3.5's fused, bias-carrying QKV — so +/// each maps individually. +pub fn map_mmproj_name(n: &str) -> Option { + // ── Projector (siblings of the tower) ─────────────────────────── + match n { + "vision_adapter.fc1.weight" => return Some("vision.adapter.fc1.weight".into()), + "vision_adapter.fc2.weight" => return Some("vision.adapter.fc2.weight".into()), + "vision_projection.weight" => return Some("vision.projection.weight".into()), + _ => {} + } + + let rest = n.strip_prefix("vision_tower.")?; + + // ── Patch embedder + learned position table ───────────────────── + // `patch_embedding` is a Linear over flattened + // [patch_temporal * 3 * patch^2] pixels (1176 → 1536), bias-free. + // `position_embedding_table` is a [pos_h*pos_w, dim] grid that the + // runtime bilinearly resamples to each image's patch grid. + match rest { + "patch_embedder.patch_embedding.weight" => return Some("vision.patch_embed.weight".into()), + "patch_embedder.position_embedding_table.weight" => { + return Some("vision.pos_embed.weight".into()) + } + "ln_pre.weight" => return Some("vision.ln_pre.weight".into()), + "ln_pre.bias" => return Some("vision.ln_pre.bias".into()), + "ln_post.weight" => return Some("vision.ln_post.weight".into()), + "ln_post.bias" => return Some("vision.ln_post.bias".into()), + _ => {} + } + + // ── Encoder blocks ────────────────────────────────────────────── + let layer_rest = rest.strip_prefix("layers.")?; + let (idx, suffix) = layer_rest.split_once('.')?; + idx.parse::().ok()?; + let canonical_suffix = vision_layer_suffix(suffix)?; + Some(format!("vision.layers.{idx}.{canonical_suffix}")) +} + +/// Map a perception-tower tensor name as it appears in an **mmproj GGUF** +/// (`general.type = mmproj`, `clip.projector_type = muse-glimmer`) to the +/// same canonical `vision.*` vocabulary [`map_mmproj_name`] produces from +/// the HF checkpoint. Both sources must land on identical names or the +/// runtime's weight lookup would see two different towers. +/// +/// The GGUF tower uses llama.cpp's `clip` vocabulary (`v.blk.N.*`, `v.*`, +/// `mm.N.*`) rather than HF's (`vision_tower.layers.N.*`), so this is a +/// separate table, not a pre-pass on [`map_mmproj_name`]. +/// +/// The projector is three stacked linears, named positionally in GGUF. The +/// widths pin the correspondence and are asserted by the tests below: +/// mm.0 [6144 -> 4096] adapter.fc1 (6144 = dim * spatial_merge^2) +/// mm.1 [4096 -> 4096] adapter.fc2 +/// mm.2 [4096 -> 6656] projection (6656 = text embedding_length) +/// +/// Returns `None` for anything unrecognized — the caller must fail loudly +/// rather than drop a tower weight, since a silently missing projector row +/// produces plausible-looking garbage rather than an error. +pub fn map_mmproj_gguf_name(n: &str) -> Option { + // ── Projector ─────────────────────────────────────────────────── + match n { + "mm.0.weight" => return Some("vision.adapter.fc1.weight".into()), + "mm.1.weight" => return Some("vision.adapter.fc2.weight".into()), + "mm.2.weight" => return Some("vision.projection.weight".into()), + _ => {} + } + + // ── Tower-level tensors ───────────────────────────────────────── + match n { + "v.patch_embd.weight" => return Some("vision.patch_embed.weight".into()), + "v.position_embd.weight" => return Some("vision.pos_embed.weight".into()), + "v.pre_ln.weight" => return Some("vision.ln_pre.weight".into()), + "v.pre_ln.bias" => return Some("vision.ln_pre.bias".into()), + "v.post_ln.weight" => return Some("vision.ln_post.weight".into()), + "v.post_ln.bias" => return Some("vision.ln_post.bias".into()), + _ => {} + } + + // ── Encoder blocks: v.blk... ────────────── + let rest = n.strip_prefix("v.blk.")?; + let (idx, suffix) = rest.split_once('.')?; + idx.parse::().ok()?; + let (stem, tail) = match suffix.rsplit_once('.') { + Some((stem, tail @ ("weight" | "bias"))) => (stem, tail), + _ => return None, + }; + // ln1/ln2 are pre-attention and pre-FFN respectively, matching HF's + // norm1/norm2 — NOT a post-norm arrangement. + let canonical_stem = match stem { + "ln1" => "attention_norm", + "ln2" => "ffn_norm", + "attn_q" => "attention.q", + "attn_k" => "attention.k", + "attn_v" => "attention.v", + "attn_out" => "attention.output", + "ffn_up" => "ffn.up", + "ffn_down" => "ffn.down", + _ => return None, + }; + Some(format!("vision.layers.{idx}.{canonical_stem}.{tail}")) +} + +/// Encoder-block suffix rename. `.weight` / `.bias` are carried through +/// together since every linear and norm in this tower has both (except the +/// patch embedder, handled above). +fn vision_layer_suffix(s: &str) -> Option { + let (stem, tail) = match s.rsplit_once('.') { + Some((stem, tail @ ("weight" | "bias"))) => (stem, tail), + _ => return None, + }; + let canonical_stem = match stem { + "norm1" => "attention_norm", + "norm2" => "ffn_norm", + "attn.q_proj" => "attention.q", + "attn.k_proj" => "attention.k", + "attn.v_proj" => "attention.v", + // `attn.proj` is the output projection (HF ViT naming), NOT a + // gate — the perception tower has no gating. + "attn.proj" => "attention.output", + "mlp.fc1" => "ffn.up", + "mlp.fc2" => "ffn.down", + _ => return None, + }; + Some(format!("{canonical_stem}.{tail}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Trimmed mirror of meta-models/Muse-Glimmer-30B/config.json — the + /// 4-layer form so the schedules stay readable. Layer types follow the + /// released 52-layer file: sliding x3 then full, with NoPE on the full + /// layer. + fn cfg() -> serde_json::Value { + serde_json::json!({ + "model_type": "muse_glimmer", + "image_token_id": 200092, + "video_token_id": 200091, + "out_hidden_size": 6144, + "projector_hidden_size": 4096, + "text_config": { + "model_type": "muse_glimmer_text", + "hidden_size": 6656, + "intermediate_size": 19968, + "num_hidden_layers": 4, + "num_attention_heads": 32, + "num_key_value_heads": 2, + "head_dim": 128, + "vocab_size": 202048, + "max_position_embeddings": 131072, + "rms_norm_eps": 1e-5, + "post_norm_eps": 1e-8, + "sliding_window": 2048, + "tie_word_embeddings": false, + "final_logit_softcapping": 20.0, + "qk_scale_factor": 3.87, + "output_multiplier": 0.19611613513818404, + "bos_token_id": 200000, + "eos_token_id": 200001, + "layer_types": [ + "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" + ], + "layer_rope_theta": [500000.0, 500000.0, 500000.0, 0], + "rope_parameters": { "rope_theta": 500000.0, "rope_type": "default" } + } + }) + } + + #[test] + fn reads_text_config_and_rope_parameters() { + let c = MuseGlimmerHfMapper.config_from_hf(&cfg()).unwrap(); + assert_eq!(c.hidden_size, 6656); + assert_eq!(c.num_hidden_layers, 4); + assert_eq!(c.num_kv_heads, 2); + assert_eq!(c.head_dim, 128); + assert_eq!(c.vocab_size, 202048); + // Must come from rope_parameters, NOT the 10000 default. + assert_eq!(c.rope_theta, 500_000.0); + assert!(!c.tie_word_embeddings); + } + + #[test] + fn extracts_muse_glimmer_scales() { + let c = MuseGlimmerHfMapper.config_from_hf(&cfg()).unwrap(); + assert_eq!(c.qk_scale_factor, 3.87); + assert_eq!(c.logit_softcap, 20.0); + assert_eq!(c.post_norm_eps, 1e-8); + assert!((c.output_multiplier - 0.196_116_14).abs() < 1e-7); + assert!(c.attn_output_gate); + assert_eq!(c.sliding_window, 2048); + } + + /// The local/local/local/global schedule and the NoPE mask must line + /// up: every global layer is NoPE, every sliding layer keeps RoPE. + #[test] + fn swa_and_nope_schedules_are_complementary() { + let c = MuseGlimmerHfMapper.config_from_hf(&cfg()).unwrap(); + assert_eq!(c.swa_layers, vec![true, true, true, false]); + assert_eq!(c.nope_layers, vec![false, false, false, true]); + } + + /// A config without `layer_types` / `layer_rope_theta` must reproduce + /// the reference's "every 4th layer counted backward from the last" + /// default rather than defaulting everything to full-attention+RoPE. + #[test] + fn derives_default_schedules_when_keys_absent() { + let mut v = cfg(); + let tc = v.get_mut("text_config").unwrap(); + tc.as_object_mut().unwrap().remove("layer_types"); + tc.as_object_mut().unwrap().remove("layer_rope_theta"); + tc["num_hidden_layers"] = serde_json::json!(8); + + let c = MuseGlimmerHfMapper.config_from_hf(&v).unwrap(); + // Counting backward from layer 7: 7, 3 are global/NoPE. + assert_eq!( + c.swa_layers, + vec![true, true, true, false, true, true, true, false] + ); + assert_eq!( + c.nope_layers, + vec![false, false, false, true, false, false, false, true] + ); + } + + /// Only the four zero-centered per-layer norms get the +1 offset. + /// Shifting `final_norm.weight` (a plain weighted RMSNorm here, unlike + /// Gemma 3) or a vision LayerNorm would corrupt the forward pass. + #[test] + fn norm_shift_scoped_to_centered_per_layer_norms() { + let m = MuseGlimmerHfMapper; + for s in [ + "layers.0.input_norm.weight", + "layers.7.post_attention_norm.weight", + "layers.7.post_attn_norm.weight", + "layers.51.post_ffw_norm.weight", + ] { + assert_eq!(m.norm_shift(s), 1.0, "should shift: {s}"); + } + for s in [ + "final_norm.weight", + "embed_tokens.weight", + "lm_head.weight", + "vision.layers.0.attention_norm.weight", + "vision.ln_post.weight", + "layers.0.attention.q.weight", + ] { + assert_eq!(m.norm_shift(s), 0.0, "should not shift: {s}"); + } + } + + /// The embedding norm is baked into the matrix; nothing else is, and + /// the bake is suppressed if the checkpoint ever ties the head. + #[test] + fn row_rms_normalize_targets_embedding_only() { + let m = MuseGlimmerHfMapper; + let mut c = m.config_from_hf(&cfg()).unwrap(); + assert_eq!(m.row_rms_normalize("embed_tokens.weight", &c), Some(1e-5)); + assert_eq!(m.row_rms_normalize("lm_head.weight", &c), None); + assert_eq!(m.row_rms_normalize("final_norm.weight", &c), None); + + c.tie_word_embeddings = true; + assert_eq!(m.row_rms_normalize("embed_tokens.weight", &c), None); + } + + #[test] + fn maps_perception_tower_tensors() { + let cases = [ + ( + "vision_tower.patch_embedder.patch_embedding.weight", + "vision.patch_embed.weight", + ), + ( + "vision_tower.patch_embedder.position_embedding_table.weight", + "vision.pos_embed.weight", + ), + ("vision_tower.ln_pre.weight", "vision.ln_pre.weight"), + ("vision_tower.ln_post.bias", "vision.ln_post.bias"), + ( + "vision_tower.layers.0.norm1.weight", + "vision.layers.0.attention_norm.weight", + ), + ( + "vision_tower.layers.49.norm2.bias", + "vision.layers.49.ffn_norm.bias", + ), + ( + "vision_tower.layers.3.attn.q_proj.bias", + "vision.layers.3.attention.q.bias", + ), + ( + "vision_tower.layers.3.attn.k_proj.weight", + "vision.layers.3.attention.k.weight", + ), + ( + "vision_tower.layers.3.attn.v_proj.weight", + "vision.layers.3.attention.v.weight", + ), + ( + "vision_tower.layers.3.attn.proj.weight", + "vision.layers.3.attention.output.weight", + ), + ( + "vision_tower.layers.7.mlp.fc1.weight", + "vision.layers.7.ffn.up.weight", + ), + ( + "vision_tower.layers.7.mlp.fc2.bias", + "vision.layers.7.ffn.down.bias", + ), + ("vision_adapter.fc1.weight", "vision.adapter.fc1.weight"), + ("vision_adapter.fc2.weight", "vision.adapter.fc2.weight"), + ("vision_projection.weight", "vision.projection.weight"), + ]; + for (src, want) in cases { + assert_eq!(map_mmproj_name(src).as_deref(), Some(want), "mapping {src}"); + } + } + + #[test] + fn mmproj_gguf_names_map_to_canonical() { + let cases = [ + // Tower-level. + ("v.patch_embd.weight", "vision.patch_embed.weight"), + ("v.position_embd.weight", "vision.pos_embed.weight"), + ("v.pre_ln.weight", "vision.ln_pre.weight"), + ("v.pre_ln.bias", "vision.ln_pre.bias"), + ("v.post_ln.weight", "vision.ln_post.weight"), + ("v.post_ln.bias", "vision.ln_post.bias"), + // Encoder blocks — every stem, both tails. + ("v.blk.0.ln1.weight", "vision.layers.0.attention_norm.weight"), + ("v.blk.0.ln1.bias", "vision.layers.0.attention_norm.bias"), + ("v.blk.2.ln2.weight", "vision.layers.2.ffn_norm.weight"), + ("v.blk.3.attn_q.weight", "vision.layers.3.attention.q.weight"), + ("v.blk.3.attn_k.bias", "vision.layers.3.attention.k.bias"), + ("v.blk.3.attn_v.weight", "vision.layers.3.attention.v.weight"), + ("v.blk.3.attn_out.weight", "vision.layers.3.attention.output.weight"), + ("v.blk.7.ffn_up.weight", "vision.layers.7.ffn.up.weight"), + ("v.blk.7.ffn_down.bias", "vision.layers.7.ffn.down.bias"), + // Projector. + ("mm.0.weight", "vision.adapter.fc1.weight"), + ("mm.1.weight", "vision.adapter.fc2.weight"), + ("mm.2.weight", "vision.projection.weight"), + ]; + for (src, want) in cases { + assert_eq!( + map_mmproj_gguf_name(src).as_deref(), + Some(want), + "gguf mapping {src}" + ); + } + + // Unknown names fail loudly rather than passing through: a dropped + // tower weight yields plausible garbage, not an error. + for bad in [ + "v.blk.0.attn_gate.weight", // tower has no gating + "v.blk.0.ln1.scale", // not weight/bias + "v.blk.x.ln1.weight", // non-numeric index + "mm.3.weight", // projector is exactly three linears + "blk.0.attn_q.weight", // text tensor, wrong table + ] { + assert_eq!(map_mmproj_gguf_name(bad), None, "should reject {bad}"); + } + } + + /// The HF and GGUF towers MUST land on identical canonical names — + /// otherwise a bundle converted from GGUF and one converted from + /// safetensors present different weights to the same runtime lookup. + #[test] + fn mmproj_hf_and_gguf_agree() { + let pairs = [ + ("vision_tower.layers.5.norm1.weight", "v.blk.5.ln1.weight"), + ("vision_tower.layers.5.norm2.bias", "v.blk.5.ln2.bias"), + ("vision_tower.layers.5.attn.q_proj.weight", "v.blk.5.attn_q.weight"), + ("vision_tower.layers.5.attn.k_proj.bias", "v.blk.5.attn_k.bias"), + ("vision_tower.layers.5.attn.v_proj.weight", "v.blk.5.attn_v.weight"), + ("vision_tower.layers.5.attn.proj.weight", "v.blk.5.attn_out.weight"), + ("vision_tower.layers.5.mlp.fc1.weight", "v.blk.5.ffn_up.weight"), + ("vision_tower.layers.5.mlp.fc2.bias", "v.blk.5.ffn_down.bias"), + ("vision_tower.ln_pre.weight", "v.pre_ln.weight"), + ("vision_tower.ln_post.bias", "v.post_ln.bias"), + ( + "vision_tower.patch_embedder.patch_embedding.weight", + "v.patch_embd.weight", + ), + ( + "vision_tower.patch_embedder.position_embedding_table.weight", + "v.position_embd.weight", + ), + ("vision_adapter.fc1.weight", "mm.0.weight"), + ("vision_adapter.fc2.weight", "mm.1.weight"), + ("vision_projection.weight", "mm.2.weight"), + ]; + for (hf, gguf) in pairs { + let a = map_mmproj_name(hf); + let b = map_mmproj_gguf_name(gguf); + assert!(a.is_some(), "hf mapping missing for {hf}"); + assert_eq!(a, b, "hf {hf} and gguf {gguf} must agree"); + } + } + + // ── GGUF path ─────────────────────────────────────────────────── + + /// Trimmed `muse-glimmer.*` metadata — note the HYPHEN, which is what + /// `general.architecture` (and therefore every key prefix) actually + /// carries in the llama.cpp-produced file. + fn gguf_meta() -> BTreeMap { + let mut m = BTreeMap::new(); + m.insert("muse-glimmer.embedding_length".into(), KvValue::U32(6656)); + m.insert("muse-glimmer.block_count".into(), KvValue::U32(4)); + m.insert("muse-glimmer.attention.head_count".into(), KvValue::U32(32)); + m.insert("muse-glimmer.attention.head_count_kv".into(), KvValue::U32(2)); + m.insert("muse-glimmer.feed_forward_length".into(), KvValue::U32(19968)); + m.insert("muse-glimmer.vocab_size".into(), KvValue::U32(202048)); + m.insert("muse-glimmer.attention.key_length".into(), KvValue::U32(128)); + m.insert("muse-glimmer.rope.freq_base".into(), KvValue::F32(500_000.0)); + m.insert("muse-glimmer.context_length".into(), KvValue::U32(131072)); + m.insert( + "muse-glimmer.attention.layer_norm_rms_epsilon".into(), + KvValue::F32(1e-5), + ); + m.insert("tokenizer.ggml.bos_token_id".into(), KvValue::U32(200000)); + m.insert("tokenizer.ggml.eos_token_id".into(), KvValue::U32(200001)); + m + } + + /// The hyphenated arch string must resolve, and it must resolve to a + /// mapper whose canonical arch is the UNDERSCORED form so GGUF- and + /// HF-sourced bundles carry the same `arch` header field. + #[test] + fn hyphenated_gguf_arch_maps_to_underscored_canonical() { + let m = crate::source_mapper_for_gguf("muse-glimmer") + .expect("hyphenated `muse-glimmer` must resolve"); + assert_eq!(m.canonical_arch(), "muse_glimmer"); + assert_eq!( + crate::source_mapper_for_gguf("muse_glimmer") + .map(|m| m.canonical_arch()), + Some("muse_glimmer"), + "underscored spelling accepted too" + ); + } + + #[test] + fn gguf_config_matches_hf_config() { + let g = MuseGlimmerGgufMapper.config_from_gguf(&gguf_meta()).unwrap(); + let h = MuseGlimmerHfMapper.config_from_hf(&cfg()).unwrap(); + assert_eq!(g.hidden_size, h.hidden_size); + assert_eq!(g.num_hidden_layers, h.num_hidden_layers); + assert_eq!(g.num_attention_heads, h.num_attention_heads); + assert_eq!(g.num_kv_heads, h.num_kv_heads); + assert_eq!(g.head_dim, h.head_dim); + assert_eq!(g.intermediate_size, h.intermediate_size); + assert_eq!(g.vocab_size, h.vocab_size); + assert_eq!(g.rope_theta, h.rope_theta); + assert_eq!(g.sliding_window, h.sliding_window); + assert_eq!(g.swa_layers, h.swa_layers); + assert_eq!(g.nope_layers, h.nope_layers); + assert!(g.attn_output_gate); + assert!(!g.tie_word_embeddings); + } + + /// A zero `qk_scale_factor` silently shrinks every attention logit, so + /// the mapper falls back to the published constants when llama.cpp + /// doesn't export them. `output_multiplier` is DERIVED from + /// hidden_size rather than hardcoded. + #[test] + fn gguf_falls_back_to_reference_scales() { + let g = MuseGlimmerGgufMapper.config_from_gguf(&gguf_meta()).unwrap(); + assert_eq!(g.qk_scale_factor, 3.87); + assert_eq!(g.logit_softcap, 20.0); + assert_eq!(g.post_norm_eps, 1e-8); + // 1/sqrt(6656/256) = 1/sqrt(26). + assert!((g.output_multiplier - 0.196_116_14).abs() < 1e-6); + } + + /// Exported keys must win over the fallbacks. + #[test] + fn gguf_prefers_exported_scales() { + let mut m = gguf_meta(); + m.insert( + "muse-glimmer.attention.qk_scale_factor".into(), + KvValue::F32(2.5), + ); + m.insert("muse-glimmer.output_multiplier".into(), KvValue::F32(0.5)); + m.insert( + "muse-glimmer.final_logit_softcapping".into(), + KvValue::F32(30.0), + ); + m.insert("muse-glimmer.attention.sliding_window".into(), KvValue::U32(1024)); + let g = MuseGlimmerGgufMapper.config_from_gguf(&m).unwrap(); + assert_eq!(g.qk_scale_factor, 2.5); + assert_eq!(g.output_multiplier, 0.5); + assert_eq!(g.logit_softcap, 30.0); + assert_eq!(g.sliding_window, 1024); + } + + /// Every GGUF tensor name must land on the same canonical name the HF + /// path produces — that is the whole contract between the two paths. + #[test] + fn gguf_names_match_hf_canonicals() { + let cases = [ + ("token_embd.weight", "embed_tokens.weight"), + ("output.weight", "lm_head.weight"), + ("output_norm.weight", "final_norm.weight"), + ("blk.0.attn_norm.weight", "layers.0.input_norm.weight"), + // Pre-FFN norm → the alias the runtime reads as `ffn_norm`. + ("blk.3.ffn_norm.weight", "layers.3.post_attn_norm.weight"), + // These two are read VERBATIM by muse_glimmer.cpp. + ( + "blk.7.post_attention_norm.weight", + "layers.7.post_attention_norm.weight", + ), + ("blk.7.post_ffw_norm.weight", "layers.7.post_ffw_norm.weight"), + ("blk.1.attn_q.weight", "layers.1.self_attn.q_proj.weight"), + ("blk.1.attn_k.weight", "layers.1.self_attn.k_proj.weight"), + ("blk.1.attn_v.weight", "layers.1.self_attn.v_proj.weight"), + ( + "blk.1.attn_output.weight", + "layers.1.self_attn.o_proj.weight", + ), + // NOT `self_attn.gate.weight` — see map_gguf_name's docs. + ("blk.1.attn_gate.weight", "layers.1.self_attn.gate_proj.weight"), + ("blk.1.attn_q_norm.weight", "layers.1.self_attn.q_norm.weight"), + ("blk.1.attn_k_norm.weight", "layers.1.self_attn.k_norm.weight"), + ("blk.51.ffn_gate.weight", "layers.51.mlp.gate_proj.weight"), + ("blk.51.ffn_up.weight", "layers.51.mlp.up_proj.weight"), + ("blk.51.ffn_down.weight", "layers.51.mlp.down_proj.weight"), + ]; + for (src, want) in cases { + assert_eq!(map_gguf_name(src).as_deref(), Some(want), "mapping {src}"); + } + assert_eq!(map_gguf_name("rope_freqs.weight"), None, "dropped"); + assert_eq!(map_gguf_name("blk.0.something_new.weight"), None, "unknown"); + assert_eq!(map_gguf_name("blk.abc.attn_q.weight"), None); + } + + /// The four per-layer norms must map to four DISTINCT canonical names. + /// The Llama table collapses `ffn_norm` and `post_attention_norm` onto + /// `post_attn_norm`, which would leave one of them unreadable. + #[test] + fn four_per_layer_norms_stay_distinct() { + let names: Vec = [ + "blk.0.attn_norm.weight", + "blk.0.ffn_norm.weight", + "blk.0.post_attention_norm.weight", + "blk.0.post_ffw_norm.weight", + ] + .iter() + .map(|n| map_gguf_name(n).unwrap()) + .collect(); + let mut uniq = names.clone(); + uniq.sort(); + uniq.dedup(); + assert_eq!(uniq.len(), 4, "norm canonicals collided: {names:?}"); + } + + /// Only `attn_q` / `attn_k` get the inverse rope permute, with the + /// right head count each (kv heads for K under GQA). Everything else + /// — v, o, the attention gate, the FFN — must be left alone. + #[test] + fn rope_unpermute_targets_q_and_k_only() { + let c = MuseGlimmerGgufMapper.config_from_gguf(&gguf_meta()).unwrap(); + let m = MuseGlimmerGgufMapper; + assert_eq!( + m.rope_unpermute_heads("layers.0.self_attn.q_proj.weight", &c), + Some(32) + ); + assert_eq!( + m.rope_unpermute_heads("layers.0.self_attn.k_proj.weight", &c), + Some(2) + ); + for n in [ + "layers.0.self_attn.v_proj.weight", + "layers.0.self_attn.o_proj.weight", + "layers.0.self_attn.gate_proj.weight", + "layers.0.mlp.gate_proj.weight", + "embed_tokens.weight", + "lm_head.weight", + ] { + assert_eq!(m.rope_unpermute_heads(n, &c), None, "must not permute {n}"); + } + } + + /// Llama must keep its GGUF rows untouched: it runs the INTERLEAVED + /// `rope_f16` kernel and llama.cpp already exported in that layout, so + /// a stray un-permute would corrupt it. + #[test] + fn other_gguf_archs_do_not_unpermute() { + let c = ArchConfig { + num_attention_heads: 32, + num_kv_heads: 8, + ..ArchConfig::default() + }; + for arch in ["llama", "qwen3", "gemma3", "gemma4", "nomic-bert"] { + let m = crate::source_mapper_for_gguf(arch).unwrap(); + assert_eq!( + m.rope_unpermute_heads("layers.0.self_attn.q_proj.weight", &c), + None, + "{arch} must not un-permute" + ); + } + } + + /// Language-tower tensors must not be swallowed by the vision mapper, + /// and unknown tower members fall through to verbatim pass-through. + #[test] + fn ignores_non_tower_tensors() { + for n in [ + "layers.0.self_attn.q_proj.weight", + "embed_tokens.weight", + "vision_tower.layers.0.attn.unknown.weight", + "vision_tower.layers.abc.norm1.weight", + ] { + assert_eq!(map_mmproj_name(n), None, "should not map: {n}"); + } + } +} diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 01eb736..3f45395 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -648,6 +648,14 @@ fn pull_and_convert( profile: profile_path, awq_profile: None, allow_quant_from_quant: false, + // Convert-on-pull always produces a canonical-quant bundle; + // k-quant passthrough stays an explicit `convert` opt-in. + kquant_passthrough: false, + // Convert-on-pull sources HF checkpoints, where the perception + // tower already lives in the same snapshot — the mmproj flags only + // apply to GGUF sources, which ship the tower separately. + mmproj: None, + mmproj_config: None, }; crate::cmd_convert(conv).with_context(|| format!("converting {repo}"))?; diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index b623b32..5f935c7 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -129,6 +129,55 @@ struct ConvertArgs { /// the fp16 checkpoint locally. #[arg(long)] allow_quant_from_quant: bool, + + /// GGUF sources only: copy Q4_K / Q5_K / Q6_K super-blocks into the + /// bundle VERBATIM instead of dequantizing and re-packing them. + /// + /// This is lossless — the exact block bytes land in the `.base` + /// weights blob, tagged with `layout = gguf_super`, `group_size = + /// 256` and the source `ggml_type` (12/13/14) so the runtime can + /// dispatch a native k-quant kernel. It is therefore NOT + /// "quant-from-quant" and does not need `--allow-quant-from-quant`; + /// the canonical-quant spec forbids dequant→requant precisely + /// because it compounds error, and passthrough introduces none. + /// + /// Opt-in rather than automatic: a bundle written this way needs + /// runtime kernels that read GGUF super-blocks, so flipping it on by + /// default would change what existing GGUF conversions produce. + /// Non-k-quant tensors (F32/F16 norms, Q4_0/Q8_0 weights) are + /// unaffected and take the normal path. + #[arg(long)] + kquant_passthrough: bool, + + /// GGUF sources only: fold a companion `mmproj-*.gguf` perception tower + /// into the bundle, so one `.base` carries both the text model and the + /// vision encoder. + /// + /// Without this, a GGUF-sourced bundle is TEXT-ONLY: llama.cpp ships the + /// tower as a separate `mmproj` file and the text GGUF contains none of + /// its tensors. The HF/safetensors path has no equivalent flag because + /// the tower already lives in the same checkpoint. + /// + /// The tower's tensors are renamed into the same canonical `vision.*` + /// vocabulary the safetensors path produces (asserted by + /// `mmproj_hf_and_gguf_agree`), so both sources yield interchangeable + /// bundles. Tower weights honour `--kquant-passthrough` exactly like the + /// text weights do. + /// + /// NOTE: an mmproj GGUF carries the tower geometry (`clip.vision.*`) but + /// NOT the wrapper-level multimodal settings — image/video token ids, + /// pooling kernel, soft-token count, tower RoPE theta. Those come from a + /// per-projector-type table keyed on `clip.projector_type`, or from + /// `--mmproj-config` when you have the original HF configs. Every + /// assumed value is printed at conversion time. + #[arg(long, value_name = "PATH")] + mmproj: Option, + + /// HF `config.json` (and sibling `processor_config.json`, if present) + /// to source the multimodal settings an mmproj GGUF cannot carry. + /// Overrides the built-in per-projector-type defaults. + #[arg(long, value_name = "PATH", requires = "mmproj")] + mmproj_config: Option, } #[derive(Parser, Debug)] @@ -341,7 +390,13 @@ fn cmd_convert(args: ConvertArgs) -> Result<()> { .with_context(|| format!("detecting format for {:?}", args.input))?; match fmt { - SourceFormat::Gguf => convert_gguf(&args.input, &output, &ctx), + SourceFormat::Gguf => convert_gguf( + &args.input, + &output, + &ctx, + args.mmproj.as_deref(), + args.mmproj_config.as_deref(), + ), SourceFormat::HfSafetensors => convert_hf(&args.input, &output, &ctx), SourceFormat::MlxSafetensors => convert_mlx(&args.input, &output, &ctx), } @@ -356,6 +411,8 @@ fn convert_gguf( input: &std::path::Path, output: &std::path::Path, ctx: &QuantContext, + mmproj: Option<&std::path::Path>, + mmproj_config: Option<&std::path::Path>, ) -> Result<()> { use base_arch::source_mapper_for_gguf; use base_format::{ @@ -373,6 +430,20 @@ fn convert_gguf( .ok_or_else(|| anyhow::anyhow!("GGUF missing general.architecture"))?; eprintln!(" arch: {}", arch); + // `--kquant-passthrough`: which tensors will be copied verbatim. + // Only the 256-element k-quant super-block families qualify — those + // are the layouts `Layout::GgufSuper` + `source_ggml_type` describe + // and the ones the runtime knows how to dispatch natively. + let kquant_passthrough = ctx.kquant_passthrough; + let is_kquant = |t: GgmlType| matches!(t, GgmlType::Q4K | GgmlType::Q5K | GgmlType::Q6K); + let has_kquant = gguf.tensors.iter().any(|t| is_kquant(t.ggml_type)); + if kquant_passthrough && !has_kquant { + eprintln!( + " warning: --kquant-passthrough set but no Q4_K/Q5_K/Q6_K tensors \ + in this GGUF; every tensor takes the normal convert path" + ); + } + // Per CANONICAL_QUANT_SPEC.md: profile-driven canonical-quant // requires fp16/bf16/fp32 source. A GGUF with quantized weight // tensors (Q4_0/Q5_0/Q4_K/Q8_0/...) is already-quantized; @@ -380,8 +451,16 @@ fn convert_gguf( // Reject by default; users explicitly opt in via // `--allow-quant-from-quant`. F16/BF16/F32 only-tensor GGUFs // (rare; usually only norms are non-quant) pass through silently. + // + // Tensors headed for k-quant PASSTHROUGH are exempt: they are never + // dequantized, so there is no compounded error to acknowledge. The + // guard still fires for anything else quantized in the same file + // (a stray Q4_0 / Q8_0 tensor does get dequant→requant). if ctx.profile.is_some() && !ctx.allow_quant_from_quant { for tensor in gguf.tensors.iter() { + if kquant_passthrough && is_kquant(tensor.ggml_type) { + continue; + } if !matches!( tensor.ggml_type, GgmlType::F32 | GgmlType::F16 | GgmlType::BF16 @@ -412,16 +491,24 @@ fn convert_gguf( config.vocab_size ); - let quant_scheme = match target { - TargetScheme::BaseQ2 => QuantScheme::BaseQ2, - TargetScheme::BaseQ3 => QuantScheme::BaseQ3, - TargetScheme::BaseQ4 => QuantScheme::BaseQ4, - TargetScheme::BaseQ5 => QuantScheme::BaseQ5, - TargetScheme::BaseQ6 => QuantScheme::BaseQ6, - TargetScheme::BaseQ8 => QuantScheme::BaseQ8, - TargetScheme::Bf16 => QuantScheme::Bf16, - TargetScheme::Mxfp4 => QuantScheme::Mxfp4, - TargetScheme::Nvfp4 => QuantScheme::Nvfp4, + // With passthrough active on a file that actually has k-quants, the + // bundle's dominant weight layout IS the GGUF super-block one — say + // so in the header rather than advertising a canonical scheme the + // big tensors don't use. + let quant_scheme = if kquant_passthrough && has_kquant { + QuantScheme::PassthroughGguf + } else { + match target { + TargetScheme::BaseQ2 => QuantScheme::BaseQ2, + TargetScheme::BaseQ3 => QuantScheme::BaseQ3, + TargetScheme::BaseQ4 => QuantScheme::BaseQ4, + TargetScheme::BaseQ5 => QuantScheme::BaseQ5, + TargetScheme::BaseQ6 => QuantScheme::BaseQ6, + TargetScheme::BaseQ8 => QuantScheme::BaseQ8, + TargetScheme::Bf16 => QuantScheme::Bf16, + TargetScheme::Mxfp4 => QuantScheme::Mxfp4, + TargetScheme::Nvfp4 => QuantScheme::Nvfp4, + } }; let mut header = Header { @@ -497,6 +584,8 @@ fn convert_gguf( // → GPU region. let mut dropped = 0usize; let mut kept = 0usize; + let mut passthrough_tensors = 0usize; + let mut unpermuted = 0usize; for info in gguf.tensors.iter() { let Some(canonical) = mapper.map_tensor_name(&info.name) else { dropped += 1; @@ -504,10 +593,46 @@ fn convert_gguf( }; kept += 1; - let bytes = gguf + let raw = gguf .tensor_bytes(info) .with_context(|| format!("reading {:?}", info.name))?; + // ── RoPE row layout fixup ─────────────────────────────────── + // Some exporters (llama.cpp's Llama-family converter, and the + // Muse Glimmer converter derived from it) rewrite q/k rows from + // the split-half rotary layout into the interleaved-pair one. + // Archs whose runtime kernel is NeoX/split-half need that undone. + // Done on the RAW bytes, before any dequant, so the packed and + // the dequantized paths share one implementation — and so a + // k-quant tensor can be un-permuted without unpacking it (the + // permutation only ever moves whole rows). + let unpermuted_bytes = match mapper.rope_unpermute_heads(&canonical, &config) { + Some(n_heads) => { + let out = unpermute_rope_rows(info, raw, n_heads).with_context(|| { + format!("rope row un-permute for {:?}", info.name) + })?; + unpermuted += 1; + Some(out) + } + None => None, + }; + let bytes: &[u8] = unpermuted_bytes.as_deref().unwrap_or(raw); + + // ── k-quant passthrough ───────────────────────────────────── + // Copy the super-blocks in verbatim. Must come BEFORE the + // dequant below: the canonical-quant spec forbids the + // dequantize-and-repack round trip as lossy, and passthrough is + // the lossless alternative. k-quants carry their scales and mins + // INSIDE each 256-element block, so there are no separate + // scale/bias regions — `scale_offset` / `bias_offset` stay None + // and the runtime keys off `source_ggml_type` + `Layout::GgufSuper`. + if kquant_passthrough && is_kquant(info.ggml_type) { + let (entry, data) = kquant_passthrough_entry(info, bytes, canonical); + passthrough_tensors += 1; + writer.add_tensor(TensorPayload { entry, data }); + continue; + } + let f32s = dequant_to_f32(info, bytes).with_context(|| { format!( "dequant {:?} type={}", @@ -736,6 +861,232 @@ fn convert_gguf( writer.add_tensor(TensorPayload { entry, data }); } eprintln!(" mapped: {} tensors kept, {} dropped", kept, dropped); + if unpermuted > 0 { + eprintln!( + " rope: un-permuted {} q/k tensors into split-half (NeoX) row order", + unpermuted + ); + } + if passthrough_tensors > 0 { + eprintln!( + " kquant: {} tensors passed through verbatim (no dequant/repack)", + passthrough_tensors + ); + } + + // ── Companion mmproj tower (--mmproj) ─────────────────────────── + // + // llama.cpp ships the perception tower as a SEPARATE mmproj GGUF, so + // without this a GGUF-sourced bundle is text-only. The tensors go into + // the same blob but are listed under `header.mmproj.tensors`, so a + // text-only runtime skips them. + // + // No SSM / embedding / rope-permute special cases apply here: a ViT has + // none of those. What it does have is 1-D norms AND biases (every linear + // in this tower is biased), which must not be quantized — the shape + // check below catches both. + if let Some(mmproj_path) = mmproj { + let mm = GgufFile::open(mmproj_path) + .with_context(|| format!("opening mmproj GGUF {:?}", mmproj_path))?; + match mm.metadata.get("general.type").and_then(|v| v.as_str()) { + Some("mmproj") => {} + other => bail!( + "{:?} is not an mmproj GGUF (general.type = {:?}); pass the companion \ + mmproj-*.gguf, not the text model", + mmproj_path, + other.unwrap_or("") + ), + } + + let hf_cfg = match mmproj_config { + Some(p) => { + let bytes = std::fs::read(p) + .with_context(|| format!("reading --mmproj-config {:?}", p))?; + let v: serde_json::Value = serde_json::from_slice(&bytes) + .with_context(|| format!("parsing --mmproj-config {:?}", p))?; + // Tower geometry still comes from the GGUF; this only + // supplies the wrapper-level scalars, which sit at the top + // level of an HF config.json. + Some(v) + } + None => None, + }; + let (mm_cfg, patch_expand) = mmproj_config_from_gguf(&mm, hf_cfg.as_ref())?; + + writer.set_mmproj_arch(format!("{}_mm", mapper.canonical_arch())); + writer.set_mmproj_config(mm_cfg); + + let mut mm_kept = 0usize; + let mut mm_passthrough = 0usize; + for info in mm.tensors.iter() { + // Unknown names are fatal, not skipped: a silently dropped tower + // weight produces plausible-looking output, not an error. + let canonical = base_arch::muse_glimmer::map_mmproj_gguf_name(&info.name) + .ok_or_else(|| { + anyhow::anyhow!( + "unmapped mmproj tensor {:?} — refusing to drop a tower weight", + info.name + ) + })?; + let raw = mm + .tensor_bytes(info) + .with_context(|| format!("reading mmproj {:?}", info.name))?; + + // Same routing rule as the HF mmproj path (see convert_generic): + // only a plain, group-aligned 2-D linear weight may carry a + // packed dtype. The runtime reads pos_embed / patch_embed / + // norms / biases as raw F16 via `tensor_raw_ptr`, so giving any + // of them a packed dtype breaks the vision encoder — and + // pos_embed is 2-D AND group-aligned, so a shape check alone + // does not catch it. It surfaces as + // "pos_embed missing/unsupported dtype" and a null tower. + // + // Evaluated BEFORE the passthrough branch so a k-quant tower + // tensor cannot bypass it either. + // GGUF records ne innermost-first, so a Linear lands as + // [in, out] where the HF path records [out, in] — the SAME + // bytes, described in the opposite order. The vision encoder + // sizes its GEMMs from this shape, so it has to be normalized + // to the HF convention or every tower matmul is transposed. + let mm_shape: Vec = if info.shape.len() == 2 { + vec![info.shape[1], info.shape[0]] + } else { + info.shape.clone() + }; + let logical_count: usize = info.shape.iter().map(|d| *d as usize).product(); + let must_stay_raw = info.shape.len() != 2 + || logical_count < 64 + || logical_count % 64 != 0 + || canonical.contains("pos_embed") + || canonical.contains("patch_embed") + || canonical.ends_with(".bias"); + + if !must_stay_raw && kquant_passthrough && is_kquant(info.ggml_type) { + let (entry, data) = kquant_passthrough_entry(info, raw, canonical); + mm_passthrough += 1; + mm_kept += 1; + writer.add_mmproj_tensor(TensorPayload { entry, data }); + continue; + } + + let mut f32s = dequant_to_f32(info, raw).with_context(|| { + format!( + "dequant mmproj {:?} type={}", + info.name, + ggml_type_name(info.ggml_type) + ) + })?; + + // MMPROJ_PATCH_EXPAND — widen the collapsed patch embedder back + // to the checkpoint's temporal depth. + // + // llama.cpp stores `Wa + Wb` (the temporal halves SUMMED, verified + // byte-exact against the bf16 checkpoint) at width 3*patch^2, + // where the model's own embedder is a Linear over + // temporal*3*patch^2. The preprocessor feeds `[f; f]` — the same + // frame in every slice — so writing `[Wa+Wb | 0]` reproduces + // `(Wa+Wb)·f` exactly while keeping the declared width, and hence + // the encoder's whole dispatch, identical to the safetensors + // path. Zero-padding rather than halving keeps it exact instead + // of merely close. + // + // Row-major [out][in]: each output row gets its 588 real weights + // followed by (expand-1)*588 zeros. + let mut mm_shape = mm_shape; + if patch_expand > 1 && canonical == "vision.patch_embed.weight" { + let out_w = *info.shape.last().unwrap_or(&1) as usize; + let in_w = f32s.len() / out_w.max(1); + let new_in = in_w * patch_expand as usize; + let mut wide = vec![0.0f32; out_w * new_in]; + for o in 0..out_w { + wide[o * new_in..o * new_in + in_w] + .copy_from_slice(&f32s[o * in_w..(o + 1) * in_w]); + } + f32s = wide; + mm_shape = vec![out_w as u64, new_in as u64]; + } + + let (entry, data) = if must_stay_raw { + let bytes: Vec = f32s + .iter() + .flat_map(|&f| half::f16::from_f32(f).to_le_bytes()) + .collect(); + let entry = base_format::TensorEntry { + name: canonical, + dtype: TensorDtype::F16, + shape: mm_shape.clone(), + offset: 0, + length: bytes.len() as u64, + scale_offset: None, + scale_length: None, + bias_offset: None, + bias_length: None, + awq_scale_offset: None, + awq_scale_length: None, + group_size: None, + layout: None, + residency: Some(base_format::ResidencyHint::Hot), + compute_region: ComputeRegion::Gpu, + scale_dtype: None, + symmetric: false, + flags: TensorFlags::empty(), + checksum_xxh64: None, + source_ggml_type: None, + }; + (entry, bytes) + } else { + // Tower linears take the plain target packing — no profile + // lookup. A quant profile's rules are written against text + // tensor names and would not match `vision.*` anyway. + let (packed, dtype) = pack_for_target(&f32s, target)?; + let mut data = Vec::with_capacity( + packed.packed_weights.len() + packed.scales.len() + packed.biases.len(), + ); + data.extend_from_slice(&packed.packed_weights); + let scale_off = data.len() as u64; + data.extend_from_slice(&packed.scales); + let bias_off = data.len() as u64; + data.extend_from_slice(&packed.biases); + let mut entry = base_format::TensorEntry { + name: canonical, + dtype, + shape: mm_shape.clone(), + offset: 0, + length: 0, + scale_offset: (!packed.scales.is_empty()).then_some(scale_off), + scale_length: (!packed.scales.is_empty()) + .then_some(packed.scales.len() as u64), + bias_offset: (!packed.biases.is_empty()).then_some(bias_off), + bias_length: (!packed.biases.is_empty()) + .then_some(packed.biases.len() as u64), + awq_scale_offset: None, + awq_scale_length: None, + group_size: (packed.group_size > 0).then_some(packed.group_size), + layout: None, + residency: Some(base_format::ResidencyHint::Warm), + compute_region: ComputeRegion::Accelerator, + scale_dtype: packed.scale_dtype, + symmetric: false, + flags: TensorFlags::empty(), + checksum_xxh64: None, + source_ggml_type: None, + }; + entry.length = data.len() as u64; + (entry, data) + }; + mm_kept += 1; + writer.add_mmproj_tensor(TensorPayload { entry, data }); + } + eprintln!( + " mmproj: {} tower tensors folded in{}", + mm_kept, + if mm_passthrough > 0 { + format!(" ({} passed through verbatim)", mm_passthrough) + } else { + String::new() + } + ); + } writer.finish().context("writing bundle")?; @@ -756,6 +1107,157 @@ fn convert_gguf( Ok(()) } +/// Undo llama.cpp's per-head rotary ROW permutation on a GGUF tensor, +/// operating on the raw (possibly still packed) bytes. +/// +/// `convert_hf_to_gguf.py::LlamaModel.permute` rewrites `q_proj`/`k_proj` +/// from the HF split-half layout into the interleaved-pair one: +/// +/// ```text +/// gguf_row[h*HD + 2j + k] = hf_row[h*HD + k*HD/2 + j] +/// ``` +/// +/// so this applies the inverse, `dst[h*HD + k*HD/2 + j] = src[h*HD + 2j + k]`, +/// restoring the split-half order the NeoX rope kernel expects. The two +/// fixed points per head — rows `0` and `HD-1`, the only `(j,k)` with +/// `2j+k == k*HD/2 + j` — are why a permuted and an unpermuted tensor +/// still agree on exactly those rows; anything that "matches on rows 0 and +/// HD-1 only" is this permutation. +/// +/// Why this is safe on packed k-quants: the permutation moves whole +/// ROWS (output features) and never reorders elements WITHIN a row. GGUF +/// stores `ne[0]` (the in-features axis) contiguously, so one row is a +/// contiguous byte run, and as long as the row length is a whole number +/// of quant blocks the reorder is a pure `memcpy` shuffle — bit-exact, +/// no dequantization. A row length that straddles a block boundary would +/// NOT be safe, so that is checked and rejected rather than assumed. +fn unpermute_rope_rows( + info: &base_readers::gguf::TensorInfo, + bytes: &[u8], + n_heads: u32, +) -> Result> { + let n_heads = n_heads as usize; + if n_heads == 0 { + bail!("head count is zero"); + } + // GGUF dimension order: ne[0] is the fastest-varying axis (in + // features); every later dim multiplies the row count. + let row_elems = *info + .shape + .first() + .ok_or_else(|| anyhow::anyhow!("tensor has no dimensions"))? as usize; + let total_elems: u64 = info.shape.iter().product(); + if row_elems == 0 || total_elems % row_elems as u64 != 0 { + bail!("shape {:?} is not a whole number of rows", info.shape); + } + let n_rows = (total_elems / row_elems as u64) as usize; + + let (block_elems, block_bytes) = info.ggml_type.block_geometry(); + if row_elems % block_elems != 0 { + bail!( + "row length {} is not a multiple of the {} block size {} — the rope \ + row permute cannot be applied to packed blocks that straddle rows; \ + re-convert this tensor from the fp16/bf16 checkpoint instead", + row_elems, + base_readers::gguf::ggml_type_name(info.ggml_type), + block_elems + ); + } + let row_bytes = row_elems / block_elems * block_bytes; + if bytes.len() != n_rows * row_bytes { + bail!( + "byte length {} != rows {} x row_bytes {}", + bytes.len(), + n_rows, + row_bytes + ); + } + + if n_rows % n_heads != 0 { + bail!("row count {} not divisible by head count {}", n_rows, n_heads); + } + let hd = n_rows / n_heads; + if hd % 2 != 0 { + bail!("head_dim {} is odd — rotary needs dimension pairs", hd); + } + let half = hd / 2; + + let mut out = vec![0u8; bytes.len()]; + for h in 0..n_heads { + for k in 0..2usize { + for j in 0..half { + let src = h * hd + 2 * j + k; + let dst = h * hd + k * half + j; + out[dst * row_bytes..(dst + 1) * row_bytes] + .copy_from_slice(&bytes[src * row_bytes..(src + 1) * row_bytes]); + } + } + } + Ok(out) +} + +/// Build the `.base` entry for a GGUF k-quant tensor copied through +/// verbatim. The payload is the source super-block bytes unchanged. +/// +/// A k-quant super-block is self-describing — the 256 elements, their +/// 6-bit sub-scales, the block `d`/`dmin` and (Q5_K/Q6_K) the high-bit +/// planes all live inside the block — so there are no separate scale or +/// bias regions to point at. `dtype` records the bit-width so any tool +/// that ignores `source_ggml_type` still sizes the tensor correctly, +/// while the runtime's `BaseWeightStore::get_dtype` short-circuits on +/// `source_ggml_type` and dispatches the native GGUF kernel. +fn kquant_passthrough_entry( + info: &base_readers::gguf::TensorInfo, + bytes: &[u8], + canonical: String, +) -> (base_format::TensorEntry, Vec) { + use base_format::{ComputeRegion, Layout, TensorDtype, TensorFlags}; + use base_readers::gguf::GgmlType; + + let (dtype, ggml_code) = match info.ggml_type { + GgmlType::Q4K => (TensorDtype::BaseQ4, 12u32), + GgmlType::Q5K => (TensorDtype::BaseQ5, 13u32), + GgmlType::Q6K => (TensorDtype::BaseQ6, 14u32), + // Unreachable: the caller gates on the same three types. + other => unreachable!("kquant passthrough called with {other:?}"), + }; + // Same role-based routing the re-quantizing path uses: the + // embed/lm_head pair stays hot in the GPU region, bulk layer weights + // go warm to the accelerator region. + let is_embedding = canonical == "embed_tokens.weight" || canonical == "lm_head.weight"; + let (region, residency) = if is_embedding { + (ComputeRegion::Gpu, base_format::ResidencyHint::Hot) + } else { + (ComputeRegion::Accelerator, base_format::ResidencyHint::Warm) + }; + + let data = bytes.to_vec(); + let entry = base_format::TensorEntry { + name: canonical, + dtype, + shape: info.shape.clone(), + offset: 0, + length: data.len() as u64, + // Scales/mins live inside each super-block — no side regions. + scale_offset: None, + scale_length: None, + bias_offset: None, + bias_length: None, + awq_scale_offset: None, + awq_scale_length: None, + group_size: Some(256), + layout: Some(Layout::GgufSuper), + residency: Some(residency), + compute_region: region, + scale_dtype: None, + symmetric: false, + flags: TensorFlags::empty(), + checksum_xxh64: None, + source_ggml_type: Some(ggml_code), + }; + (entry, data) +} + /// Convert from an HF safetensors directory. fn convert_hf( input: &std::path::Path, @@ -856,6 +1358,7 @@ fn convert_hf( mmproj_cfg, &|n| mapper.norm_shift(n), &|n| mapper.rope_permute_heads(n, &config_for_permute), + &|n| mapper.row_rms_normalize(n, &config_for_permute), ) } @@ -1304,6 +1807,7 @@ fn convert_mlx( mmproj_cfg, &|n| mapper.norm_shift(n), &|n| mapper.rope_permute_heads(n, &config_for_permute), + &|n| mapper.row_rms_normalize(n, &config_for_permute), ) } @@ -1331,6 +1835,264 @@ fn tokenizer_from_hf(hf: &base_readers::hf::HfDir) -> std::collections::BTreeMap m } +/// Wrapper-level multimodal settings an mmproj GGUF cannot carry, keyed on +/// `clip.projector_type`. +/// +/// An mmproj GGUF describes the TOWER (`clip.vision.*`: depth, widths, head +/// count, patch/image size, merge size) but nothing about how the text model +/// splices its output in — the image/video token ids, the pooling kernel, the +/// soft-token count and the tower's RoPE theta all live in the HF +/// `config.json` / `processor_config.json` that llama.cpp consumed and did +/// not re-emit. Without them the runtime cannot place image embeddings. +/// +/// These values were read back out of a known-good HF-sourced bundle for the +/// same checkpoint, so they are transcriptions rather than guesses — but they +/// are still per-checkpoint constants, which is why an unknown projector type +/// is a hard error and `--mmproj-config` always wins. +struct ProjectorDefaults { + image_token_id: u64, + video_token_id: Option, + vision_soft_tokens_per_image: u64, + pooling_kernel_size: Option, + rope_theta: f32, + /// Temporal depth of the checkpoint's patch embedder. The published + /// mmproj may store fewer slices (llama.cpp collapses them by summing); + /// the tower is widened back to this before it reaches the runtime. + temporal_patch_size: u64, +} + +fn projector_defaults(projector_type: &str) -> Option { + match projector_type { + // meta-models/Muse-Glimmer-30B. Verified against muse-glimmer-q4.base + // (converted from the safetensors checkpoint): image token 200092, + // one soft token per image, tower RoPE theta 10000. + "muse-glimmer" => Some(ProjectorDefaults { + image_token_id: 200092, + video_token_id: None, + vision_soft_tokens_per_image: 1, + pooling_kernel_size: None, + rope_theta: 10000.0, + temporal_patch_size: 2, + }), + _ => None, + } +} + +/// Build the mmproj config block from an mmproj GGUF's `clip.*` metadata, +/// the tower's own tensor shapes, and either `--mmproj-config` or the +/// per-projector-type table above. +/// +/// Emits the SAME schema `mmproj_config_from_hf` does — a `vision_config` +/// sub-object plus the wrapper-level scalars — because the runtime parses one +/// shape regardless of which converter path wrote the bundle. +fn mmproj_config_from_gguf( + mm: &base_readers::gguf::GgufFile, + hf_config: Option<&serde_json::Value>, +) -> Result<(std::collections::BTreeMap, u64)> { + use serde_json::json; + let meta = &mm.metadata; + let get_u64 = |k: &str| meta.get(k).and_then(|v| v.as_u64()); + let get_f32 = |k: &str| meta.get(k).and_then(|v| v.as_f32()); + + let projector_type = meta + .get("clip.projector_type") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("mmproj GGUF missing clip.projector_type"))? + .to_string(); + + // Tower geometry — straight out of clip.vision.*. + let hidden = get_u64("clip.vision.embedding_length") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.embedding_length"))?; + let layers = get_u64("clip.vision.block_count") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.block_count"))?; + let heads = get_u64("clip.vision.attention.head_count") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.attention.head_count"))?; + let ffn = get_u64("clip.vision.feed_forward_length") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.feed_forward_length"))?; + let patch = get_u64("clip.vision.patch_size") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.patch_size"))?; + let image_size = get_u64("clip.vision.image_size") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.image_size"))?; + let out_hidden = get_u64("clip.vision.projection_dim") + .ok_or_else(|| anyhow::anyhow!("mmproj missing clip.vision.projection_dim"))?; + let merge = get_u64("clip.vision.spatial_merge_size").unwrap_or(1); + let eps = get_f32("clip.vision.attention.layer_norm_epsilon").unwrap_or(1e-5); + + // Derived from tensor shapes rather than metadata, because llama.cpp + // does not record either. GGUF dims are innermost-first, so a Linear + // stored [in, out] means ne[1] is the output width. + let projector_hidden = mm + .tensor_by_name("mm.1.weight") + .and_then(|t| t.shape.get(1).copied()) + .ok_or_else(|| anyhow::anyhow!("mmproj missing mm.1.weight (projector hidden width)"))?; + // The learned position table is a [pos_h * pos_w, dim] grid the runtime + // bilinearly resamples per image. The reader wants the two sides, not the + // product, and the grid is square for every projector shipped so far. + let pos_embed_size = mm + .tensor_by_name("v.position_embd.weight") + .and_then(|t| t.shape.get(1).copied()) + .ok_or_else(|| anyhow::anyhow!("mmproj missing v.position_embd.weight"))?; + let pos_embed_side = (pos_embed_size as f64).sqrt().round() as u64; + if pos_embed_side * pos_embed_side != pos_embed_size { + bail!( + "position table has {} entries, which is not a square grid — cannot derive \ + pos_emb_height/pos_emb_width for projector {:?}", + pos_embed_size, + projector_type + ); + } + + // Wrapper-level settings: --mmproj-config wins, else the table. + let from_hf = |k: &str| hf_config.and_then(|c| c.get(k)).cloned(); + let defaults = projector_defaults(&projector_type); + if defaults.is_none() && hf_config.is_none() { + bail!( + "unknown clip.projector_type {:?}: the multimodal token ids, pooling kernel, \ + soft-token count and tower RoPE theta are not present in any mmproj GGUF. \ + Pass --mmproj-config from the HF checkpoint, or add a \ + projector_defaults() entry for this type.", + projector_type + ); + } + + let mut assumed: Vec = Vec::new(); + let mut resolve_u64 = |key: &str, fallback: Option| -> Option { + if let Some(v) = from_hf(key) { + return Some(v); + } + fallback.map(|n| { + assumed.push(format!("{key}={n}")); + json!(n) + }) + }; + + // Temporal depth of the patch embedder, derived from the weight itself. + // + // The HF checkpoint's patch embedding is a Linear over + // `temporal_patch_size * 3 * patch^2` (2*3*14*14 = 1176 for Muse + // Glimmer). llama.cpp's mmproj stores 3*patch^2 = 588 instead — NOT a + // truncation: it is the two temporal halves SUMMED. Verified against the + // bf16 checkpoint, `gguf == W[:, :588] + W[:, 588:]` to max|diff| = 0.0. + // + // That collapse is exact for still images, and only for them: the + // preprocessor fills every temporal slice with the same frame, so + // `W·[f;f] == (Wa+Wb)·f`. We therefore keep the CHECKPOINT's temporal + // depth and widen the weight back to it (MMPROJ_PATCH_EXPAND), rather + // than declaring a narrower tower — that way the encoder runs the exact + // dispatch the safetensors path already exercises. + // + // Getting the declared width out of step with the stored weight is + // silent at convert time and fatal at inference: the GEMM reads + // `n * K` past the end of a too-narrow weight, and ~49% of patch_embed + // comes back non-finite (750 of 1536 output columns), taking every + // later stage to 100% NaN. + let patch_in: u64 = mm + .tensor_by_name("v.patch_embd.weight") + .map(|pe| pe.shape.iter().rev().skip(1).product()) + .ok_or_else(|| anyhow::anyhow!("mmproj missing v.patch_embd.weight"))?; + let per_frame = 3 * patch * patch; + if per_frame == 0 || patch_in % per_frame != 0 { + bail!( + "mmproj patch embedder has {} input elements, which is not a multiple of \ + 3 * patch^2 ({}) — cannot infer temporal_patch_size for projector {:?}", + patch_in, + per_frame, + projector_type + ); + } + // The bundle always declares the checkpoint's own temporal depth, even + // when the mmproj stores the collapsed single-frame weight: the tower is + // widened back to `temporal_patch_size * 3 * patch^2` at write time (see + // MMPROJ_PATCH_EXPAND in the tensor loop). Keeping the declared width at + // the checkpoint's value means the encoder runs the exact same shape the + // safetensors path produces, instead of a second, narrower code path. + let gguf_temporal = patch_in / per_frame; + let temporal_patch_size = defaults + .as_ref() + .map(|d| d.temporal_patch_size) + .unwrap_or(gguf_temporal); + if temporal_patch_size % gguf_temporal != 0 { + bail!( + "mmproj patch embedder has {} temporal slice(s) but projector {:?} expects {} — \ + not an integer multiple, cannot widen", + gguf_temporal, + projector_type, + temporal_patch_size + ); + } + + let mut m = std::collections::BTreeMap::new(); + m.insert( + "vision_config".into(), + json!({ + "hidden_size": hidden, + "num_hidden_layers": layers, + "num_attention_heads": heads, + "intermediate_size": ffn, + "patch_size": patch, + "image_size": image_size, + // Key names below are the ones base_weight_store.cpp's + // muse_glimmer branch actually reads. They differ from the HF + // config spelling (merge_size vs spatial_merge_size, + // patch_temporal vs temporal_patch_size, pos_emb_height/width + // vs a single size, and rope_theta NESTED under + // rope_parameters). Every one of those has a default that + // happens to be right for this checkpoint, so spelling them + // the HF way looks correct in `basert inspect` and silently + // ignores whatever the file actually says. + "merge_size": merge, + "patch_temporal": temporal_patch_size, + "layer_norm_eps": eps, + "pos_emb_height": pos_embed_side, + "pos_emb_width": pos_embed_side, + "rope_parameters": { + "rope_theta": from_hf("rope_theta") + .and_then(|v| v.as_f64()) + .unwrap_or_else(|| { + let t = defaults.as_ref().map(|d| d.rope_theta).unwrap_or(10000.0); + t as f64 + }) + }, + }), + ); + m.insert("out_hidden_size".into(), json!(out_hidden)); + m.insert("projector_hidden_size".into(), json!(projector_hidden)); + + if let Some(v) = resolve_u64("image_token_id", defaults.as_ref().map(|d| d.image_token_id)) { + m.insert("image_token_id".into(), v); + } + if let Some(v) = resolve_u64( + "video_token_id", + defaults.as_ref().and_then(|d| d.video_token_id), + ) { + m.insert("video_token_id".into(), v); + } + if let Some(v) = resolve_u64( + "vision_soft_tokens_per_image", + defaults + .as_ref() + .map(|d| d.vision_soft_tokens_per_image), + ) { + m.insert("vision_soft_tokens_per_image".into(), v); + } + if let Some(v) = resolve_u64( + "pooling_kernel_size", + defaults.as_ref().and_then(|d| d.pooling_kernel_size), + ) { + m.insert("pooling_kernel_size".into(), v); + } + + if !assumed.is_empty() { + eprintln!( + " mmproj: assumed from projector_type={:?}: {}", + projector_type, + assumed.join(", ") + ); + eprintln!(" (pass --mmproj-config to take these from the HF checkpoint)"); + } + Ok((m, temporal_patch_size / gguf_temporal)) +} + /// Extract the mmproj config block from an HF directory's `config.json` /// (and `processor_config.json`, when present). Returns an empty map for /// text-only models. Captures the bits the runtime needs to drive the @@ -1360,6 +2122,12 @@ fn mmproj_config_from_hf(hf: &base_readers::hf::HfDir) -> std::collections::BTre "boa_token_id", "eoa_token_id", "vision_soft_tokens_per_image", + // Muse Glimmer: video placeholder + projector widths (the ViT's + // post-pixel-shuffle width and the adapter's hidden width, which + // live at the wrapper level rather than inside `vision_config`). + "video_token_id", + "out_hidden_size", + "projector_hidden_size", ] { if let Some(v) = cfg.get(key) { m.insert(key.into(), v.clone()); @@ -1661,6 +2429,7 @@ fn convert_generic( mmproj_config: std::collections::BTreeMap, norm_shift: &dyn Fn(&str) -> f32, rope_permute: &dyn Fn(&str) -> Option, + row_rms_normalize: &dyn Fn(&str) -> Option, ) -> Result<()> { use base_format::{ AlignmentConfig, BaseReader, BaseWriter, ComputeRegion, Header, HeaderFlags, LayerKind, @@ -1882,6 +2651,27 @@ fn convert_generic( } } + // Per-arch hook: fold a weightless post-lookup RMSNorm into the + // embedding matrix (Muse Glimmer). Exact because an embedding + // lookup returns one whole row and the norm mixes nothing across + // rows, so normalizing every row up front is the same computation + // moved to convert time. Runs on the f32 values BEFORE quantization + // so the quantizer sees the final magnitudes. + if shape.len() == 2 { + if let Some(eps) = row_rms_normalize(canonical) { + let rows = shape[0] as usize; + let cols = f32s.len() / rows.max(1); + for row in f32s.chunks_mut(cols) { + let mean_sq = + row.iter().map(|v| (*v as f64) * (*v as f64)).sum::() / cols as f64; + let inv = 1.0 / (mean_sq + eps as f64).sqrt(); + for v in row.iter_mut() { + *v = (*v as f64 * inv) as f32; + } + } + } + } + let f32s_for = || -> &[f32] { &f32s }; let is_ssm_a = canonical == "ssm.a_log" @@ -2408,8 +3198,18 @@ fn to_canonical_name(name: &str, arch: &str) -> Option { // patch_ln1/2, pos_norm) rather than `vision_tower.*`. || mm_name.starts_with("vision_embedder") || mm_name.starts_with("multi_modal_projector") + // Muse Glimmer's projector is two siblings of the tower rather than + // a `multi_modal_projector.*` subtree: `vision_adapter.{fc1,fc2}` + // (6144→4096→4096, GELU) feeding `vision_projection` (4096→6656). + || mm_name.starts_with("vision_adapter") + || mm_name.starts_with("vision_projection") { - let canonical = base_arch::gemma::map_gemma4_mmproj_name(mm_name).unwrap_or_else(|| mm_name.to_string()); + let canonical = if arch == "muse_glimmer" { + base_arch::muse_glimmer::map_mmproj_name(mm_name) + } else { + base_arch::gemma::map_gemma4_mmproj_name(mm_name) + } + .unwrap_or_else(|| mm_name.to_string()); return Some(Canonical::Mmproj(canonical)); } @@ -2564,7 +3364,9 @@ fn to_canonical_name(name: &str, arch: &str) -> Option { // Pre-arch-aware code did the Gemma rename unconditionally, // silently breaking Llama-style MLX MoE (the pre-FFN norm // ended up under a name kBasePerLayerRules doesn't know). - if arch == "gemma4" || arch == "gemma3" { + // Muse Glimmer carries the same four per-layer norms as Gemma 3/4 + // under the same HF names, so it shares the rename. + if arch == "gemma4" || arch == "gemma3" || arch == "muse_glimmer" { // Gemma 3 and Gemma 4 both have four per-layer norms // (input + post-attn + pre-FFN + post-FFN). HF and GGUF use // different names; map HF → GGUF-canonical so the runtime's @@ -3146,6 +3948,8 @@ struct QuantContext { target: TargetScheme, /// Bypass the spec's already-quantized-source rejection. allow_quant_from_quant: bool, + /// Copy GGUF Q4_K/Q5_K/Q6_K super-blocks through verbatim. + kquant_passthrough: bool, } impl QuantContext { @@ -3175,6 +3979,7 @@ impl QuantContext { awq_config: base_awq::AwqConfig::default(), target: args.target, allow_quant_from_quant: args.allow_quant_from_quant, + kquant_passthrough: args.kquant_passthrough, }) } @@ -4062,3 +4867,154 @@ mod rope_permute_tests { assert_ne!(out, input); } } + +#[cfg(test)] +mod gguf_passthrough_tests { + use super::{kquant_passthrough_entry, rope_permute_rows, unpermute_rope_rows}; + use base_readers::gguf::{GgmlType, TensorInfo}; + + /// GGUF dimension order: ne[0] is the fastest-varying (in-features) + /// axis, so `shape = [cols, rows]` for a `[rows, cols]` weight. + fn info(cols: u64, rows: u64, ty: GgmlType) -> TensorInfo { + TensorInfo { + name: "blk.0.attn_q.weight".into(), + shape: vec![cols, rows], + ggml_type: ty, + data_offset: 0, + } + } + + fn to_bytes(v: &[f32]) -> Vec { + v.iter().flat_map(|f| f.to_le_bytes()).collect() + } + + /// The un-permute must be the exact inverse of the forward permute the + /// HF path applies (and that llama.cpp applies on export). Round-trip + /// on a shape with a real head layout: 4 heads x HD=8, 3 columns. + #[test] + fn inverts_the_forward_rope_permute() { + let (n_heads, hd, cols) = (4u32, 8usize, 3usize); + let rows = n_heads as usize * hd; + let original: Vec = (0..rows * cols).map(|i| i as f32).collect(); + // What llama.cpp would have written into the GGUF. + let permuted = rope_permute_rows(&original, rows, cols, n_heads); + assert_ne!(permuted, original, "forward permute must actually move rows"); + + let out = unpermute_rope_rows( + &info(cols as u64, rows as u64, GgmlType::F32), + &to_bytes(&permuted), + n_heads, + ) + .unwrap(); + assert_eq!(out, to_bytes(&original)); + } + + /// The permutation's only fixed points are rows 0 and HD-1 of each + /// head — the experimental signature that identified it in the first + /// place (a dequantized GGUF q/k matched the HF original on exactly + /// those rows and nowhere else). + #[test] + fn fixed_points_are_first_and_last_row_of_each_head() { + let (n_heads, hd, cols) = (2u32, 8usize, 1usize); + let rows = n_heads as usize * hd; + let original: Vec = (0..rows).map(|i| i as f32).collect(); + let permuted = rope_permute_rows(&original, rows, cols, n_heads); + for h in 0..n_heads as usize { + for r in 0..hd { + let idx = h * hd + r; + let is_fixed = permuted[idx] == original[idx]; + let should_be_fixed = r == 0 || r == hd - 1; + assert_eq!( + is_fixed, should_be_fixed, + "head {h} row {r}: fixed={is_fixed}, expected {should_be_fixed}" + ); + } + } + } + + /// The whole point of doing this on raw bytes: a PACKED k-quant tensor + /// can be un-permuted without ever being dequantized, because rows are + /// contiguous whole numbers of super-blocks. 2 heads x HD=4, row = + /// 512 elements = 2 Q4_K super-blocks = 288 bytes. + #[test] + fn reorders_packed_q4k_rows_bit_exactly() { + let (n_heads, hd) = (2u32, 4usize); + let rows = n_heads as usize * hd; + let row_bytes = 2 * 144; // 512 elements / 256 per block x 144 B + // Give every row a distinct byte pattern so a mis-shuffle shows up. + let src: Vec = (0..rows) + .flat_map(|r| std::iter::repeat((r as u8) + 1).take(row_bytes)) + .collect(); + let out = unpermute_rope_rows(&info(512, rows as u64, GgmlType::Q4K), &src, n_heads) + .unwrap(); + assert_eq!(out.len(), src.len()); + // Expected destination rows: dst[h*HD + k*HD/2 + j] = src[h*HD + 2j + k]. + // Head 0 src rows 0,1,2,3 → dst 0,2,1,3. + let dst_row = |i: usize| out[i * row_bytes] - 1; + assert_eq!( + (0..rows).map(dst_row).collect::>(), + vec![0, 2, 1, 3, 4, 6, 5, 7] + ); + // Every byte within a row must be untouched (no partial-block + // rewriting, no dequant round trip). + for r in 0..rows { + let row = &out[r * row_bytes..(r + 1) * row_bytes]; + assert!(row.iter().all(|&b| b == row[0]), "row {r} was rewritten"); + } + } + + /// A row that is not a whole number of quant blocks cannot be + /// reordered in packed form — that must fail loud rather than emit + /// silently scrambled attention weights. + #[test] + fn rejects_rows_that_straddle_blocks() { + // 100 elements per row is not a multiple of Q4_K's 256. + let err = unpermute_rope_rows(&info(100, 8, GgmlType::Q4K), &[], 2).unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("not a multiple"), "unhelpful error: {msg}"); + } + + /// Passthrough entries must describe a self-contained super-block + /// tensor: no side scale/bias regions, group_size 256, the GGUF + /// layout tag, and the source ggml code the runtime dispatches on. + #[test] + fn passthrough_entry_describes_gguf_super_blocks() { + use base_format::{ComputeRegion, Layout, TensorDtype}; + for (ty, want_dtype, want_code) in [ + (GgmlType::Q4K, TensorDtype::BaseQ4, 12u32), + (GgmlType::Q5K, TensorDtype::BaseQ5, 13u32), + (GgmlType::Q6K, TensorDtype::BaseQ6, 14u32), + ] { + let bytes: Vec = (0..64u8).collect(); + let (entry, data) = kquant_passthrough_entry( + &info(256, 4, ty), + &bytes, + "layers.0.self_attn.q_proj.weight".into(), + ); + assert_eq!(data, bytes, "payload must be byte-identical to the source"); + assert_eq!(entry.dtype, want_dtype); + assert_eq!(entry.source_ggml_type, Some(want_code)); + assert_eq!(entry.layout, Some(Layout::GgufSuper)); + assert_eq!(entry.group_size, Some(256)); + assert_eq!(entry.scale_offset, None); + assert_eq!(entry.scale_length, None); + assert_eq!(entry.bias_offset, None); + assert_eq!(entry.bias_length, None); + assert_eq!(entry.scale_dtype, None); + assert_eq!(entry.length, bytes.len() as u64); + assert_eq!(entry.compute_region, ComputeRegion::Accelerator); + } + } + + /// embed/lm_head keep the hot GPU-region routing the re-quantizing + /// path gives them. + #[test] + fn passthrough_keeps_embedding_routing() { + use base_format::ComputeRegion; + for name in ["embed_tokens.weight", "lm_head.weight"] { + let (entry, _) = + kquant_passthrough_entry(&info(256, 4, GgmlType::Q6K), &[0u8; 8], name.into()); + assert_eq!(entry.compute_region, ComputeRegion::Gpu, "{name}"); + } + } +} diff --git a/base-convert/crates/base-format/src/reader.rs b/base-convert/crates/base-format/src/reader.rs index 19dabb7..6d42546 100644 --- a/base-convert/crates/base-format/src/reader.rs +++ b/base-convert/crates/base-format/src/reader.rs @@ -199,11 +199,25 @@ impl BaseReader { /// Compute the file offset where the slots section begins. One byte /// past the last tensor's end, rounded up to 8 bytes. fn slots_offset(&self) -> u64 { - let blob_end = self + // Multimodal bundles write the vision/audio tower payloads into the + // SAME blob but list them under `header.mmproj.tensors`, so scanning + // only `header.tensors` puts this offset in the middle of the tower + // data. read_slots then parses tensor bytes as a slot header and + // trips a wild length prefix (observed: a 6.3-exabyte allocation on + // a Muse Glimmer bundle). Walk both lists. + let main_end = self .header .tensors .iter() - .map(|t| self.blob_offset + t.offset + t.length) + .map(|t| self.blob_offset + t.offset + t.length); + let mmproj_end = self + .header + .mmproj + .iter() + .flat_map(|m| m.tensors.iter()) + .map(|t| self.blob_offset + t.offset + t.length); + let blob_end = main_end + .chain(mmproj_end) .max() .unwrap_or(self.blob_offset); (blob_end + 7) & !7u64 diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index e972979..d4d3ac3 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -667,6 +667,26 @@ "size": 782403584, "sha256": "6a99cc4edaf52c60da74edad11d45aea50fa105da05aed3215dc3544cab77161", "backend": "cuda" + }, + { + "id": "basecompute/Muse-Glimmer-30B", + "hf_repo": "basecompute/Muse-Glimmer-30B", + "file": "muse-glimmer-30B-kquant-dynamic.base", + "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", + "arch": "muse_glimmer", + "quant": "default-kquant-dynamic", + "size": 20952383488, + "sha256": "1eb639041f638099f6b1f7415c808cd6762b52be672d4ff5ef97829cc7669876" + }, + { + "id": "basecompute/Muse-Glimmer-30B", + "hf_repo": "basecompute/Muse-Glimmer-30B", + "file": "muse-glimmer-30B-kquant-17gb.base", + "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", + "arch": "muse_glimmer", + "quant": "kquant-17gb", + "size": 18054873088, + "sha256": "9c323d37c192e45375a936828427f50c7a303d5772465f672df478b1f8cfc1d2" } ] } From c3e8f05f5491c648b0e048a2fc1a535d33702b5c Mon Sep 17 00:00:00 2001 From: prabod Date: Wed, 12 Aug 2026 13:55:46 +1000 Subject: [PATCH 2/3] sync: catalog variants expose their bit width; 0.2.2 version macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `basert pull basecompute/Muse-Glimmer-30B` failed with no pre-converted .base for quant "q4" in this repo; it offers: 17gb, dynamic A bare pull requests "q4", and the catalog row is only served when the requested bits match. `quant_bits` scans for `q` followed by a digit, so `kquant-dynamic` yielded None — the `q` there is followed by `u` — and resolution fell through to matching by filename tag. These files are named after the upstream GGUFs, whose tags are "dynamic" and "17gb". Renamed the variants to `default-q4k-dynamic` / `q4k-17gb`, which scan to "q4". Only the catalog's variant ids change; the published artifacts are untouched. A test now asserts every catalog quant exposes its bit width. Also carries BASERT_VERSION_PATCH 2. Still no header or binding mirrors — those must follow the 0.2.2 engine release, not lead it. --- base-convert/Cargo.lock | 16 ++++++------- base-convert/Cargo.toml | 2 +- base-convert/crates/base-hub/catalog.json | 4 ++-- base-convert/crates/base-hub/src/catalog.rs | 25 +++++++++++++++++++++ include/baseRT/baseRT.h | 2 +- 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 089782f..5ab2233 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -66,7 +66,7 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "base-arch" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-format", @@ -77,7 +77,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-format", @@ -91,7 +91,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-arch", @@ -114,7 +114,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "bitflags", @@ -130,7 +130,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-format", @@ -145,7 +145,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-format", @@ -157,7 +157,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-format", @@ -171,7 +171,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index c96b05c..e69167a 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.2.1" +version = "0.2.2" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index d4d3ac3..837cce3 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -674,7 +674,7 @@ "file": "muse-glimmer-30B-kquant-dynamic.base", "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", "arch": "muse_glimmer", - "quant": "default-kquant-dynamic", + "quant": "default-q4k-dynamic", "size": 20952383488, "sha256": "1eb639041f638099f6b1f7415c808cd6762b52be672d4ff5ef97829cc7669876" }, @@ -684,7 +684,7 @@ "file": "muse-glimmer-30B-kquant-17gb.base", "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", "arch": "muse_glimmer", - "quant": "kquant-17gb", + "quant": "q4k-17gb", "size": 18054873088, "sha256": "9c323d37c192e45375a936828427f50c7a303d5772465f672df478b1f8cfc1d2" } diff --git a/base-convert/crates/base-hub/src/catalog.rs b/base-convert/crates/base-hub/src/catalog.rs index 469c5b3..1aecf50 100644 --- a/base-convert/crates/base-hub/src/catalog.rs +++ b/base-convert/crates/base-hub/src/catalog.rs @@ -307,6 +307,31 @@ mod tests { assert_eq!(e.arch.as_deref(), Some("llama")); } + /// Every catalog `quant` must expose its bit-width to `quant_bits`, because + /// `basert pull ` with no `--target` asks for "q4" and the Catalog arm + /// only serves the cataloged file when the bits match. A variant whose name + /// hides them — `kquant-dynamic`, where the `q` is followed by `u` — silently + /// falls through to picking a file by filename tag instead, which for a repo + /// named after its upstream GGUFs fails outright: + /// + /// no pre-converted .base for quant "q4" in this repo; it offers: 17gb, dynamic + /// + /// Hence `q4k`, not `kquant`. + #[test] + fn every_catalog_quant_exposes_its_bit_width() { + let cat = Catalog::bundled().unwrap(); + for e in &cat.models { + assert!( + crate::registry::quant_bits(&e.quant).is_some(), + "catalog quant {:?} (id {}) hides its bit width from quant_bits — a bare \ + `basert pull {}` cannot match it", + e.quant, + e.id, + e.id + ); + } + } + #[test] fn find_matches_exact_and_ci() { let cat = Catalog::from_json( diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index d4a0ce0..6456ede 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 2 -#define BASERT_VERSION_PATCH 1 +#define BASERT_VERSION_PATCH 2 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. From c5a25089147a01b8ebe3cd4d2bd6976c63c20776 Mon Sep 17 00:00:00 2001 From: prabod Date: Wed, 12 Aug 2026 16:50:03 +1000 Subject: [PATCH 3/3] sync: 0.2.2 version sites for the published packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version strings only — package.json, setup.py, the two Cargo manifests and BASERT_VERSION_PATCH — so the published packages carry the release they belong to. No struct or signature changes. The header and binding MIRRORS are still deliberately held back: embed_norm_eps widens BaseRTModelConfig from 1540 to 1704 bytes, and the rust-sys ABI job links the latest ENGINE RELEASE (still v0.2.1) to compare against the in-tree mirror. Syncing them before that release exists fails by construction. They follow the 0.2.2 engine release, not lead it. --- bindings/node/package.json | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT/Cargo.toml | 2 +- bindings/swift/Sources/CBaseRT/include/baseRT.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bindings/node/package.json b/bindings/node/package.json index bd9a7f7..c0b78f1 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.2.1", + "version": "0.2.2", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 031fb10..4212907 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.2.1", + version="0.2.2", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index 4f0bbde..6450a69 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.2.1" +version = "0.2.2" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 59f405a..97ea571 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.2.1" +version = "0.2.2" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index d4a0ce0..6456ede 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 2 -#define BASERT_VERSION_PATCH 1 +#define BASERT_VERSION_PATCH 2 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks.