perf(qwen27 cuda): reduce repeated-prompt TTFT

build.zig

@@ -424,6 +424,7 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
         .{ .name = "tf-grid-sync-bench", .path = "zig/tests/grid_sync_bench.zig", .about = "A GPU-wide barrier in one persistent dispatch against dependent relaunches" },
         .{ .name = "tf-weight-read-check", .path = "zig/tests/weight_read_check.zig", .about = "Check native file reads and failed-read cleanup", .c_source = "zig/tests/pread_fault.c" },
         .{ .name = "tf-glm-run", .path = "zig/tests/glm_run.zig", .about = "GLM-5.3-Flash greedy replies at each draft depth against depth 0 and reference tokens" },
+        .{ .name = "tf-glm-vision", .path = "zig/tests/glm_vision.zig", .about = "GLM-5.3-Flash's image tower on an image or a reference dump's patches, compared with the dump" },
         .{ .name = "tf-glm-attn-probe", .path = "zig/tests/glm_attn_probe.zig", .about = "GLM-5.3-Flash's dense latent attention kernels on given inputs (scores, probabilities, outputs)" },
         .{ .name = "tf-moe-bench", .path = "zig/tests/moe_bench.zig", .about = "Routed-expert prompt kernels from .metal files on a skewed routing: chained speed, bits against the first" },
         .{ .name = "tf-mm-bench", .path = "zig/tests/mm_bench.zig", .about = "Prompt matmul kernels from .metal files: chained speed at one shape, bits against the first, error against fp64" },
@@ -558,6 +559,8 @@ fn metalEngineModules(b: *std.Build, target: std.Build.ResolvedTarget, optimize:
     metal.linkFramework("Metal", .{});
     metal.linkFramework("Foundation", .{});
     metal.linkFramework("IOSurface", .{}); // the Neural Engine's prompt share passes IOSurfaces
+    metal.linkFramework("ImageIO", .{}); // GLM-5.3-Flash's image input: decoding (families/glm/image.zig)
+    metal.linkFramework("CoreGraphics", .{});
     metal.linkSystemLibrary("objc", .{});
     const lanes = b.createModule(.{ .root_source_file = b.path("zig/src/core/lanes/lanes.zig"), .target = target, .optimize = optimize, .link_libc = true });
     const fabric = b.createModule(.{ .root_source_file = b.path("zig/src/fabric/fabric.zig"), .target = target, .optimize = optimize, .link_libc = true });

docs/api.md

@@ -67,7 +67,9 @@ chat fields that shape the prompt (`tools`, `reasoning_effort`, `chat_template_k
 `tokens`, and `token_strs` with `return_token_strs: true`. The IDs are the ones the chat route runs for the same
 request, images expanded. `/detokenize` takes `tokens` and returns `prompt`.
 With `--vision`, supported Qwen3.5/3.8 dense checkpoints accept `image_url` content parts alongside text in user
-messages and in tool results (`role: "tool"`), such as an agent's screenshots.
+messages and in tool results (`role: "tool"`), such as an agent's screenshots. The native server does the same for
+GLM-5.3-Flash on Metal, with data URLs, and public HTTPS URLs under `--vision-urls`
+([image input](vision.md#the-native-server)).
 See [image input](vision.md) for data URLs, public image URLs, limits and cache behavior.
 Unsupported image input, audio, video and non-text output requests receive HTTP 400.
 

docs/vision.md

@@ -129,6 +129,34 @@ This prevents identical image-placeholder token IDs from reusing another image's
 Multi-turn image conversations work when the request includes the original image content parts, but image-prefix reuse and persisted image KV are not implemented.
 For Qwen, each image request carries its own multimodal rotary positions and continuation offset, including during concurrent lane rounds. GLM uses its native KDA/NoPE attention state.
 
+## The native server
+
+`tensorfold-native serve` takes `--vision`, `--vision-max-images` (default 4) and `--vision-image-tokens` (default
+4,096) for engines that read images; GLM-5.3-Flash on one Mac reads them so far, and other native engines refuse the
+flag at startup.
+
+- Images arrive as `image_url` data URLs in user messages and tool results. With `--vision-urls` (it needs
+  `--vision`), a part may name a public HTTPS URL instead, fetched as 0.6.6 fetched it: port 443 without credentials
+  or a fragment; every address the host resolves to public (no private, loopback, link-local, carrier-grade NAT,
+  documentation, multicast or metadata address, and for IPv6 global unicast only, without 6to4 or Teredo); TLS
+  verified for the host against the system's certificates, on the address that was checked; up to 3 redirects,
+  each checked again; a declared `image/jpeg`, `image/png` or `image/webp`, uncompressed; 10 MB an image and 20 MB
+  a request across data URLs and downloads; 10 s a download and 30 s a request's downloads. The refusals are
+  0.6.6's words; a TLS failure, like any other failed connection, is "image download failed or timed out". Host
+  names that aren't ASCII and IPv6 address literals are refused. Without the flag, an HTTPS URL is refused with the
+  flag's name.
+- PNG and JPEG are decoded by the platform's decoder (ImageIO on macOS). WebP is refused by name until a decoder
+  exists; other formats are refused.
+- The limits and the processing are 0.6.6's: EXIF orientation, alpha composited over white, the GLM5-Next processor's
+  sizing and PIL's bicubic resampler. Six drawn PNG fixtures give patches bit-equal to 0.6.6 with mlx-vlm
+  (`tools/zig/glm_vision_fixtures.py`). JPEG patches differ slightly, because ImageIO and libjpeg-turbo decode
+  differently.
+- Images are counted from the request's image parts. A user or tool message whose text spells the model's image
+  markers (`<|image|>`, `<|begin_of_image|>`, `<|end_of_image|>`) is tokenized as ordinary text, so only the chat
+  template's own markers, one wrapper for each image part, become image tokens.
+- The prompt cache keys each image's placeholder rows by the image's content, so a kept state resumes the same image
+  and never another one under the same placeholders. Image conversations reuse their history like text ones.
+
 ## Verification
 
 Compare image requests with `draft: true` and `draft: false` at identical sampling settings and seed, then compare concurrent requests with their solo results.

tools/zig/glm_vision_fixtures.py

@@ -0,0 +1,165 @@
+"""GLM-5.3-Flash image input against the Python frontend, on a fixed image set (#565's gate).
+
+    python -I tools/zig/glm_vision_fixtures.py --model DIR --out DIR [--tower zig-out/bin/tf-glm-vision]
+
+Draws the image set (each case a preprocessing path: padding only, an antialiased downscale, an upscale, alpha over
+white, a palette with a transparent entry, gray with alpha, EXIF orientation, baseline JPEG), then for each image
+writes what TensorFold 0.6.6's --vision gives (vision/images.py's decode: EXIF orientation, alpha over white; then
+mlx-vlm's Glm5NextImageProcessor and tower, the code 0.6.6 serves):
+  patches.f32   (grid_h*grid_w, 1176) normalized patches, temporal frames duplicated
+  embed.f32 / block0.f32 / blocks.f32 / features.f32   the tower's stages, features the rows that replace <|image|>
+and meta.json. With --tower it then runs tf-glm-vision on each image: the native patches compared value by value,
+the native tower's rows by cosine (bf16 against bf16).
+
+Needs TensorFold 0.6.6 with its vision extra (mlx-vlm 0.7.x) and Pillow. Loads only model.visual.* (about 1 GB).
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import mlx.core as mx
+import numpy as np
+from PIL import Image, ImageDraw
+
+
+def draw_shapes(size: tuple[int, int]) -> Image.Image:
+    w, h = size
+    im = Image.new("RGB", size, (245, 240, 230))
+    d = ImageDraw.Draw(im)
+    d.ellipse((w * 0.08, h * 0.25, w * 0.42, h * 0.7), fill=(210, 40, 40))
+    d.rectangle((w * 0.55, h * 0.25, w * 0.9, h * 0.7), fill=(40, 70, 200))
+    for x in range(int(w * 0.1), int(w * 0.9), 6):
+        d.line((x, h * 0.8, x, h * 0.92), fill=(20, 20, 20), width=2)
+    d.text((w * 0.4, h * 0.06), "GLM 7", fill=(0, 0, 0))
+    return im
+
+
+def fixtures(folder: Path) -> list[Path]:
+    folder.mkdir(parents=True, exist_ok=True)
+    out = []
+
+    def save(name: str, im: Image.Image, **kw) -> None:
+        p = folder / name
+        im.save(p, **kw)
+        out.append(p)
+
+    save("shapes.png", draw_shapes((517, 389)))                                     # padding only
+    big = draw_shapes((2000, 1500))
+    px = big.load()
+    for y in range(0, 1500, 3):                                                      # fine detail for the antialias
+        for x in range(0, 2000, 7):
+            px[x, y] = ((x * 3) % 256, (y * 5) % 256, (x + y) % 256)
+    save("large.png", big)                                                           # antialiased downscale
+    save("tiny.png", draw_shapes((20, 15)))                                          # upscale to the token floor
+    logo = Image.new("RGBA", (96, 96), (0, 0, 0, 0))
+    d = ImageDraw.Draw(logo)
+    d.rounded_rectangle((4, 4, 92, 92), 16, fill=(30, 30, 40, 255))
+    d.polygon([(26, 28), (70, 28), (30, 68), (70, 68), (70, 74), (24, 74), (64, 34), (26, 34)], fill=(255, 255, 255, 230))
+    save("logo.png", logo)                                                           # RGBA upscale, alpha over white
+    pal = draw_shapes((517, 389)).quantize(32)
+    pal.info["transparency"] = 0
+    save("palette.png", pal, transparency=0)                                         # palette, a transparent entry
+    la = Image.new("LA", (240, 180), (128, 0))
+    d = ImageDraw.Draw(la)
+    for r in range(80, 0, -8):
+        d.ellipse((120 - r, 90 - r, 120 + r, 90 + r), fill=(255 - 3 * r, 255 - 2 * r))
+    save("gray-alpha.png", la)                                                       # gray with alpha
+    rot = draw_shapes((300, 200))
+    exif = Image.Exif()
+    exif[0x0112] = 6                                                                 # rotate 90 degrees clockwise
+    save("rotated.jpg", rot, quality=90, exif=exif.tobytes())                        # EXIF orientation
+    save("shapes.jpg", draw_shapes((517, 389)), quality=85)                          # baseline JPEG
+    return out
+
+
+def reference(model: Path, image_path: Path, out: Path, tower, processor, max_tokens: int) -> dict:
+    from tensorfold.vision.images import DEFAULT_LIMITS, _decode
+
+    out.mkdir(parents=True, exist_ok=True)
+    image = _decode(image_path.read_bytes(), "auto", DEFAULT_LIMITS, DEFAULT_LIMITS.max_total_pixels).to_pil()
+    done = processor([image], return_tensors="np", min_image_tokens=min(16, max_tokens), max_image_tokens=max_tokens)
+    patches = np.asarray(done["pixel_values"], dtype=np.float32)
+    grid = np.asarray(done["image_grid_thw"], dtype=np.int64)
+    dtype = tower.patch_embed.proj.weight.dtype
+    x = mx.array(patches).astype(dtype)
+    g = mx.array(grid, dtype=mx.int32)
+    h = tower.patch_embed(x)
+    embed = h
+    pos = tower._rotary_embeddings(g)
+    cu = mx.array([0, int(h.shape[0])])
+    block0 = None
+    for i, block in enumerate(tower.blocks):
+        h = block(h, cu, pos)
+        if i == 0:
+            block0 = h
+    blocks = h
+    features = tower(x, g)
+    mx.eval(embed, block0, blocks, features)
+    for name, a in (("patches", patches), ("embed", embed), ("block0", block0), ("blocks", blocks), ("features", features)):
+        np.asarray(a.astype(mx.float32) if isinstance(a, mx.array) else a, dtype=np.float32).tofile(out / f"{name}.f32")
+    meta = {"image": str(image_path), "size": list(image.size), "mode": Image.open(image_path).mode, "grid_thw": grid[0].tolist(),
+            "max_tokens": max_tokens, "tokens": int(features.shape[0]), "patch_width": int(patches.shape[1]), "dtype": str(dtype)}
+    (out / "meta.json").write_text(json.dumps(meta, indent=1) + "\n")
+    return meta
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--model", required=True, help="the GLM-5.3-Flash checkpoint folder")
+    ap.add_argument("--out", required=True, help="where the images and their references go")
+    ap.add_argument("--max-tokens", type=int, default=4096, help="the visual-token budget an image is sized to")
+    ap.add_argument("--tower", help="tf-glm-vision: also compare the native patches and tower with each reference")
+    args = ap.parse_args()
+
+    from mlx_vlm.models.glm5_next.config import VisionConfig
+    from mlx_vlm.models.glm5_next.processing import Glm5NextImageProcessor
+    from mlx_vlm.models.glm5_next.vision import VisionModel
+
+    model, out = Path(args.model), Path(args.out)
+    config = json.loads((model / "config.json").read_text())
+    proc_cfg = json.loads((model / "processor_config.json").read_text())["image_processor"]
+    proc_cfg.pop("image_processor_type", None)
+    processor = Glm5NextImageProcessor(**proc_cfg)
+    tower = VisionModel(VisionConfig.from_dict(config["vision_config"]))
+    index = json.loads((model / "model.safetensors.index.json").read_text())["weight_map"]
+    weights = {}
+    for shard in sorted({f for k, f in index.items() if k.startswith("model.visual.")}):
+        for k, v in mx.load(str(model / shard)).items():
+            if k.startswith("model.visual."):
+                weights[k[len("model.visual."):]] = v
+    tower.load_weights(list(tower.sanitize(weights).items()), strict=True)
+    mx.eval(tower.parameters())
+
+    failures = 0
+    for image in fixtures(out / "images"):
+        ref = out / image.name.replace(".", "-")
+        meta = reference(model, image, ref, tower, processor, args.max_tokens)
+        line = f"{image.name:15} {meta['size'][0]}x{meta['size'][1]} {meta['mode']:5} grid {meta['grid_thw']} {meta['tokens']} tokens"
+        if args.tower:
+            run = subprocess.run([args.tower, str(model), str(ref), "features", str(image)], capture_output=True, text=True)
+            text = run.stdout + run.stderr
+            p = re.search(r"patches: (\d+) of (\d+) values differ", text)
+            c = re.search(r"cosine mean ([0-9.]+) worst ([0-9.]+)", text)
+            if run.returncode != 0 or not p or not c:
+                failures += 1
+                line += f"  FAIL: {text.strip().splitlines()[-1] if text.strip() else run.returncode}"
+            else:
+                differ = int(p.group(1))
+                exact = differ == 0
+                # PNG patches must be bit-exact; JPEG patches differ by decoder (ImageIO against libjpeg-turbo)
+                failures += (not exact) and image.suffix == ".png"
+                line += f"  patches {'equal' if exact else f'{differ} of {p.group(2)} differ'}, rows' cosine {c.group(1)} (worst {c.group(2)})"
+        print(line)
+    print("PASS" if failures == 0 else f"FAIL: {failures}")
+    return 0 if failures == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())