From 3e8921f1449663cc580decb746ffebb76f0cada0 Mon Sep 17 00:00:00 2001 From: Simba Zhang Date: Thu, 24 Sep 2026 18:09:15 -0700 Subject: [PATCH 1/2] feat: warn at startup when --gpu-layers puts MoE layers on the CPU CPU-resident MoE layers run the quantized expert matmuls on one core (~0.4 tok/s prefill on Gemma 4 26B-A4B, see #176). Point users at --stream-experts instead. Co-Authored-By: Claude Opus 5.5 --- Sources/SwiftLM/Server.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Sources/SwiftLM/Server.swift b/Sources/SwiftLM/Server.swift index 4e0e185..17ab82a 100644 --- a/Sources/SwiftLM/Server.swift +++ b/Sources/SwiftLM/Server.swift @@ -776,9 +776,11 @@ struct MLXServer: AsyncParsableCommand { } var partitionPlan: PartitionPlan? + var modelIsMoE = false if let modelDir = modelDirectory { let profile = mainModelProfile ?? ModelProfiler.profile(modelDirectory: modelDir, modelId: modelId) if let profile = profile { + modelIsMoE = profile.isMoE let system = ModelProfiler.systemProfile() let contextSize = self.ctxSize ?? 4096 let plan = ModelProfiler.plan(model: profile, system: system, contextSize: contextSize, draftWeightBytes: draftFootprintBytes) @@ -1127,6 +1129,12 @@ struct MLXServer: AsyncParsableCommand { let total = partitionPlan?.totalLayers ?? actual let cpuCount = total - actual print("[SwiftLM] 🔀 Layer split active: \(actual) GPU / \(cpuCount) CPU") + if modelIsMoE && cpuCount > 0 { + // CPU-resident MoE layers run the quantized expert matmuls on a + // single core (~0.4 tok/s prefill on Gemma 4 26B-A4B, see #176). + print("[SwiftLM] ⚠️ \(cpuCount) MoE layers will run on the CPU, which is very slow (expect well under 1 tok/s).") + print("[SwiftLM] For MoE models that don't fit in GPU memory, prefer --stream-experts over --gpu-layers.") + } // Update the partition plan to reflect actual split partitionPlan?.gpuLayers = actual } else { From 60c0bf43faaefb5a9ba7292358ab0e69238a0615 Mon Sep 17 00:00:00 2001 From: Simba Zhang Date: Thu, 24 Sep 2026 18:09:29 -0700 Subject: [PATCH 2/2] docs: mark --gpu-layers timeout fixed, keep MoE CPU speed caveat Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 494a659..c0b5764 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,9 @@ TurboKV barely changes speed on this model. Only 16 of its 64 layers use full at > ⚠️ **`--turbo-kv` precision on M5 (not reproduced on M6):** on an Apple M5, Qwen3.8-27B-4bit with `--turbo-kv` gets exact long-range lookups wrong from somewhere between 2K and 5K prompt tokens. Asked how many numbered lines a prompt has, it answers "1,000" or "14" instead of 315 / 500 / 700. Without `--turbo-kv` it answers correctly, and on the M6 both modes are 24/24 correct from 2K to 11.8K tokens with the same prompt. The likely cause is a GPU-family-dependent path in TurboKV's dequant or attention kernels. Until it's fixed, avoid `--turbo-kv` on M5 when exact recall matters. Tracked in [#175](https://github.com/SharpAI/SwiftLM/issues/175). > -> ⚠️ **Known issues:** `--gpu-layers N` (CPU/GPU layer partitioning) hits a Metal GPU timeout on the first request, on both M5 and M6 (repro: `--model mlx-community/gemma-4-26b-a4b-it-4bit --gpu-layers 23`). Tracked in [#176](https://github.com/SharpAI/SwiftLM/issues/176); the fix is [SharpAI/mlx-swift#17](https://github.com/SharpAI/mlx-swift/pull/17). Even once it's fixed, CPU-resident MoE layers are very slow (~0.4 tok/s prefill), so on a 32 GB Mac try `--stream-experts` first (Qwen3.6-35B-A3B: 13.2 tok/s decode, 7.7 GB GPU). QAT-quantized Gemma 4 MTP assistants (`…-qat-assistant-4bit`) fail with `unhandledKeys pre_projection/post_projection`; use `gemma-4-26B-A4B-it-assistant-bf16`. +> ⚠️ **Known issues:** QAT-quantized Gemma 4 MTP assistants (`…-qat-assistant-4bit`) fail with `unhandledKeys pre_projection/post_projection`; use `gemma-4-26B-A4B-it-assistant-bf16`. +> +> ℹ️ **`--gpu-layers N` with MoE models:** the Metal GPU timeout on the first request ([#176](https://github.com/SharpAI/SwiftLM/issues/176)) is fixed in #177. CPU-resident MoE layers still run on a single core and are very slow (Gemma 4 26B-A4B with `--gpu-layers 23`: ~0.4 tok/s prefill, ~3 s per decoded token on both M5 and M6), and SwiftLM now warns about this at startup. Treat `--gpu-layers` as a way to avoid running out of memory, not as a speed trade-off. On a 32 GB Mac try `--stream-experts` first (Qwen3.6-35B-A3B: 13.2 tok/s decode, 7.7 GB GPU). Reproduce: