From fecfeab44d63fb9fce3018f2fda74da920ef6776 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:57:48 -0700 Subject: [PATCH] llama: support tied Hadamard output weights A Hadamard-folded model with tied embeddings stores its latent token embedding once and uses it for both row lookup and the output projection. Until now the packer had to ship a byte-identical copy as output.weight so the output matmul went through the normal folded-weight path. This adds GGUF Hadamard contract version 2 with prism.hadamard.tied_output: - Loader: version 2 requires tied_output=true and no output.weight, and version 1 forbids the flag. The output projection gets its forward transform from the embedding's block size and signs, bound to whichever tensor the output actually uses (a separate allocation when placed on another device). - Graph verifier: rejects a latent lookup table consumed by a matmul without a registered forward transform. - Converter: accepts manifest schema 3 with tied_output=true, a tied HF config and no separate head, and writes contract version 2. Schema 3 makes older converters reject the manifest instead of ignoring the flag. Version-1 files are unchanged, and a runtime without this change refuses version 2 at load ("unsupported prism.hadamard.version: 2"). Checked on a 2B PQ2_0 band, rewritten to version 2 with the duplicate head removed (135 MB smaller): 8x512 perplexity chunks give KLD and top-1 agreement against the original at the logit-file floor on both Metal and CPU (max KLD 5.0e-5 / 4.2e-5, top-1 100%), matching the parent commit's own result on the original file. --- conversion/base.py | 21 +++++++++++++++-- docs/development/hadamard-tied-output.md | 18 +++++++++++++++ src/llama-context.cpp | 4 ++++ src/llama-model.cpp | 29 +++++++++++++++++++++++- src/llama-model.h | 1 + 5 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 docs/development/hadamard-tied-output.md diff --git a/conversion/base.py b/conversion/base.py index 3052e8a45ed2..895c00081018 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -642,7 +642,7 @@ def add_hadamard_metadata(self) -> None: manifest = json.load(f) schema_version = manifest.get("schema_version") - if schema_version not in (1, 2) or manifest.get("kind") != "hadamard-weight-fold": + if schema_version not in (1, 2, 3) or manifest.get("kind") != "hadamard-weight-fold": raise ValueError(f"Unsupported Hadamard manifest: {manifest_path}") if manifest.get("status") != "requires-matching-runtime": raise ValueError(f"Unexpected Hadamard manifest status: {manifest.get('status')!r}") @@ -738,7 +738,24 @@ def add_hadamard_metadata(self) -> None: ) weight_names.append(mapped) - self.gguf_writer.add_uint32("prism.hadamard.version", 1) + tied_output = manifest.get("tied_output", False) + if not isinstance(tied_output, bool) or (schema_version == 3) != tied_output: + raise ValueError("Hadamard schema 3 requires tied_output=true; older schemas forbid it") + if tied_output: + if inverse_weight_names != ["token_embd.weight"]: + raise ValueError("Tied Hadamard output requires one latent token embedding") + if not self.hparams.get("tie_word_embeddings", False): + raise ValueError("Tied Hadamard output requires tie_word_embeddings=true") + if "output.weight" in weight_names or any( + self.tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) == "output.weight" + for name in self.model_tensors + ): + raise ValueError("Tied Hadamard output must not carry a separate output head") + self.gguf_writer.add_bool("prism.hadamard.tied_output", True) + elif "token_embd.weight" in inverse_weight_names and self.hparams.get("tie_word_embeddings", False): + raise ValueError("A tied latent embedding requires Hadamard schema 3 and tied_output=true") + + self.gguf_writer.add_uint32("prism.hadamard.version", 2 if tied_output else 1) self.gguf_writer.add_uint32("prism.hadamard.block_size", block_size) self.gguf_writer.add_string("prism.hadamard.transform", "normalized-sylvester-walsh-hadamard") self.gguf_writer.add_string("prism.hadamard.axis", "input-last-dimension") diff --git a/docs/development/hadamard-tied-output.md b/docs/development/hadamard-tied-output.md new file mode 100644 index 000000000000..0b93463db4ee --- /dev/null +++ b/docs/development/hadamard-tied-output.md @@ -0,0 +1,18 @@ +# Shared Hadamard embedding and output weights + +A latent token embedding can serve both row lookup and the output projection. For a normalized Hadamard matrix `H` and sign vector `s`, lookup reconstructs `s * (H z)`. The output projection uses the same stored rows after transforming its input as `H (s * h)`. + +The shared-output contract requires: + +- `prism.hadamard.version = 2` +- `prism.hadamard.tied_output = true` +- `token_embd.weight` in `prism.hadamard.inverse_weight_names` +- No `output.weight` tensor or entry in `prism.hadamard.weight_names` + +The loader registers the output tensor's activation transform from the embedding's block size and signs. It binds the forward and inverse transforms to the actual output and lookup tensors, which can be separate runtime allocations when placed on different devices. Sharing on disk does not guarantee a single allocation across devices. + +Version 1 remains supported for explicit heads. A latent embedding with no explicit head requires version 2; a version-1 runtime rejects version 2 before inference. The graph verifier also rejects a latent lookup tensor consumed by a matrix multiplication without a registered forward transform. + +For conversion, a shared latent head requires `hadamard_packing.json` schema 3 with `tied_output=true`, a tied HF configuration, and no separate output head. The embedding retains its `inverse-after-lookup` record. Other folded tensors retain their existing records. Manifest schema versions and GGUF contract versions are distinct; schema 3 makes older converters reject the new contract instead of ignoring the flag. + +Plain, non-Hadamard tied embeddings do not need this metadata. Separately trained output heads must remain explicit. Consumers that only support the version-1 Hadamard contract must be updated before accepting version-2 artifacts. diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 2abeb6d0969b..0f922f6e9dfc 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -68,6 +68,10 @@ static void llama_verify_hadamard_graph( const auto it = rotations.find(node->src[0]); if (it == rotations.end()) { + if (inverses.count(node->src[0])) { + throw std::runtime_error(format( + "Hadamard-latent table '%s' is used as a head without a forward transform", node->src[0]->name)); + } continue; } const ggml_tensor * src = unwrap(node->src[1]); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 11eadbced8c3..1ac5b127447b 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1194,11 +1194,19 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { } uint32_t hadamard_version = 0; + ml.get_key("prism.hadamard.tied_output", hadamard_tied_output, false); if (ml.get_key("prism.hadamard.version", hadamard_version, false)) { - if (hadamard_version != 1) { + if (hadamard_version != 1 && hadamard_version != 2) { throw std::runtime_error(format("unsupported prism.hadamard.version: %u", hadamard_version)); } + if ((hadamard_version == 2) != hadamard_tied_output) { + throw std::runtime_error("prism.hadamard version 2 requires tied_output=true; version 1 forbids it"); + } + if (hadamard_tied_output && ml.get_weight("output.weight")) { + throw std::runtime_error("prism.hadamard.tied_output requires output.weight to be absent"); + } + uint32_t block_size = 0; std::string transform; std::string axis; @@ -1334,6 +1342,19 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { } } + if (hadamard_tied_output) { + if (hadamard_version != 2) { + throw std::runtime_error("prism.hadamard.tied_output requires version 2"); + } + const auto it = hadamard_inverse_blocks.find("token_embd.weight"); + if (it == hadamard_inverse_blocks.end()) { + throw std::runtime_error("prism.hadamard.tied_output requires a latent token embedding"); + } + hadamard_weight_blocks.emplace("token_embd.weight", it->second); + } else if (hadamard_inverse_blocks.count("token_embd.weight") && !ml.get_weight("output.weight")) { + throw std::runtime_error("a tied Hadamard output requires version 2 and tied_output=true"); + } + // get general kv ml.get_key(LLM_KV_GENERAL_NAME, name, false); @@ -1973,6 +1994,12 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { const std::string & weight_name = entry.first; const uint32_t block_size = entry.second; const ggml_tensor * weight = get_tensor(weight_name.c_str()); + if (hadamard_tied_output && weight_name == "token_embd.weight") { + weight = target == &hadamard_rotations ? output : tok_embd; + if (!weight || strcmp(weight->name, "token_embd.weight") != 0) { + throw std::runtime_error("prism.hadamard.tied_output is not bound to the token embedding"); + } + } if (weight == nullptr) { throw std::runtime_error(format("prism.hadamard weight not found: %s", weight_name.c_str())); } diff --git a/src/llama-model.h b/src/llama-model.h index d73c7341f28c..b8e46ba7eddb 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -707,6 +707,7 @@ struct llama_model { std::unordered_map hadamard_inverse_blocks; std::map> hadamard_sign_data; bool hadamard_gdn_v_grouped = false; + bool hadamard_tied_output = false; llama_hadamard_rotations hadamard_rotations; llama_hadamard_rotations hadamard_inverses;