Native video input: GLM-5.3-Flash reads video_url parts (part of #565, on #614)

build.zig

@@ -12,6 +12,7 @@ const counters_build = @import("zig/build/metal_counters.zig");
 const exl3_build = @import("zig/build/exl3.zig");
 const exlr8_build = @import("zig/build/exlr8.zig");
 const golden_build = @import("zig/build/golden.zig");
+const video_build = @import("zig/build/video.zig");
 
 comptime {
     const required = std.mem.trim(u8, @embedFile(".zig-version"), "\r\n");
@@ -167,6 +168,7 @@ pub fn build(b: *std.Build) void {
     }
     hip_build.steps(b, target, test_step);
     exlr8_build.hostTests(b, target, test_step);
+    video_build.steps(b, target);
     const lanes_cpu = b.createModule(.{ .root_source_file = b.path("zig/src/core/lanes/lanes.zig"), .target = b.graph.host, .optimize = .debug, .link_libc = true });
     const disk_cpu = b.addTest(.{ .root_module = b.createModule(.{
         .root_source_file = b.path("zig/src/glm_disk_tests.zig"),
@@ -411,6 +413,8 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
         .{ .name = "tf-grid-sync-bench", .path = "zig/tests/grid_sync_bench.zig", .about = "A GPU-wide barrier in one persistent dispatch against dependent relaunches" },
         .{ .name = "tf-weight-read-check", .path = "zig/tests/weight_read_check.zig", .about = "Check native file reads and failed-read cleanup", .c_source = "zig/tests/pread_fault.c" },
         .{ .name = "tf-glm-run", .path = "zig/tests/glm_run.zig", .about = "GLM-5.3-Flash greedy replies at each draft depth against depth 0 and reference tokens" },
+        .{ .name = "tf-glm-vision", .path = "zig/tests/glm_vision.zig", .about = "GLM-5.3-Flash's image tower on an image or a reference dump's patches, compared with the dump" },
+        .{ .name = "tf-glm-video", .path = "zig/tests/glm_video.zig", .about = "GLM-5.3-Flash's video preparation on a clip (libtfvideo), written for tools/zig/glm_video_fixtures.py" },
         .{ .name = "tf-glm-attn-probe", .path = "zig/tests/glm_attn_probe.zig", .about = "GLM-5.3-Flash's dense latent attention kernels on given inputs (scores, probabilities, outputs)" },
         .{ .name = "tf-moe-bench", .path = "zig/tests/moe_bench.zig", .about = "Routed-expert prompt kernels from .metal files on a skewed routing: chained speed, bits against the first" },
         .{ .name = "tf-mm-bench", .path = "zig/tests/mm_bench.zig", .about = "Prompt matmul kernels from .metal files: chained speed at one shape, bits against the first, error against fp64" },
@@ -422,7 +426,7 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
             .target = target,
             .optimize = optimize,
             .link_libc = true,
-            .imports = &.{ .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine }, .{ .name = "cluster", .module = cluster }, .{ .name = "kernel_sources", .module = sources }, .{ .name = "native_engines", .module = engines }, exlr8_build.cliImport(b, target, optimize, engine) },
+            .imports = &.{ .{ .name = "metal", .module = metal }, .{ .name = "tensorfold", .module = engine }, .{ .name = "cluster", .module = cluster }, .{ .name = "kernel_sources", .module = sources }, .{ .name = "native_engines", .module = engines }, .{ .name = "engine_api", .module = api }, exlr8_build.cliImport(b, target, optimize, engine) },
         });
         if (p.c_source) |file| {
             mod.addCSourceFile(.{ .file = b.path(file), .flags = &.{"-std=c11"} });
@@ -549,6 +553,8 @@ fn metalEngineModules(b: *std.Build, target: std.Build.ResolvedTarget, optimize:
     metal.linkFramework("Metal", .{});
     metal.linkFramework("Foundation", .{});
     metal.linkFramework("IOSurface", .{}); // the Neural Engine's prompt share passes IOSurfaces
+    metal.linkFramework("ImageIO", .{}); // GLM-5.3-Flash's image input: decoding (families/glm/image.zig)
+    metal.linkFramework("CoreGraphics", .{});
     metal.linkSystemLibrary("objc", .{});
     const lanes = b.createModule(.{ .root_source_file = b.path("zig/src/core/lanes/lanes.zig"), .target = target, .optimize = optimize, .link_libc = true });
     const fabric = b.createModule(.{ .root_source_file = b.path("zig/src/fabric/fabric.zig"), .target = target, .optimize = optimize, .link_libc = true });

docs/api.md

@@ -66,9 +66,12 @@ take a string `prompt` (`add_special_tokens`, default false) or a list of token
 chat fields that shape the prompt (`tools`, `reasoning_effort`, `chat_template_kwargs`) and `add_generation_prompt`
 (default true). It returns `count`, `max_model_len` (the context window; null when none is set on MLX) and
 `tokens`, and `token_strs` with `return_token_strs: true`. The IDs are the ones the chat route runs for the same
-request, images expanded. `/detokenize` takes `tokens` and returns `prompt`.
+request, images and videos expanded. `/detokenize` takes `tokens` and returns `prompt`.
 In the Python 0.6.6 engine, `--vision` lets supported Qwen3.5/3.8 dense checkpoints accept `image_url` content parts
-alongside text in user messages and in tool results (`role: "tool"`), such as an agent's screenshots.
+alongside text in user messages and in tool results (`role: "tool"`), such as an agent's screenshots. The native
+server reads images for GLM-5.3-Flash on Metal, with data URLs, and public HTTPS URLs under `--vision-urls`
+([image input](vision.md#the-native-server)); it also takes `video_url` parts in user messages
+([video](vision.md#video-on-the-native-server)).
 See [image input](vision.md) for data URLs, public image URLs, limits and cache behavior.
 The native binary refuses `--vision` and answers image parts with HTTP 400.
 Unsupported image input, audio, video and non-text output requests receive HTTP 400.

docs/vision.md

@@ -142,6 +142,63 @@ This prevents identical image-placeholder token IDs from reusing another image's
 Multi-turn image conversations work when the request includes the original image content parts, but image-prefix reuse and persisted image KV are not implemented.
 For Qwen, each image request carries its own multimodal rotary positions and continuation offset, including during concurrent lane rounds. GLM uses its native KDA/NoPE attention state.
 
+## The native server
+
+`tensorfold-native serve` takes `--vision`, `--vision-max-images` (default 4) and `--vision-image-tokens` (default
+4,096) for engines that read images; GLM-5.3-Flash on one Mac reads them so far, and other native engines refuse the
+flag at startup.
+
+- Images arrive as `image_url` data URLs in user messages and tool results. With `--vision-urls` (it needs
+  `--vision`), a part may name a public HTTPS URL instead, fetched as 0.6.6 fetched it: port 443 without credentials
+  or a fragment; every address the host resolves to public (no private, loopback, link-local, carrier-grade NAT,
+  documentation, multicast or metadata address, and for IPv6 global unicast only, without 6to4 or Teredo); TLS
+  verified for the host against the system's certificates, on the address that was checked; up to 3 redirects,
+  each checked again; a declared `image/jpeg`, `image/png` or `image/webp`, uncompressed; 10 MB an image and 20 MB
+  a request across data URLs and downloads; 10 s a download and 30 s a request's downloads. The refusals are
+  0.6.6's words; a TLS failure, like any other failed connection, is "image download failed or timed out". Host
+  names that aren't ASCII and IPv6 address literals are refused. Without the flag, an HTTPS URL is refused with the
+  flag's name.
+- PNG and JPEG are decoded by the platform's decoder (ImageIO on macOS). WebP is refused by name until a decoder
+  exists; other formats are refused.
+- The limits and the processing are 0.6.6's: EXIF orientation, alpha composited over white, the GLM5-Next processor's
+  sizing and PIL's bicubic resampler. Six drawn PNG fixtures give patches bit-equal to 0.6.6 with mlx-vlm
+  (`tools/zig/glm_vision_fixtures.py`). JPEG patches differ slightly, because ImageIO and libjpeg-turbo decode
+  differently.
+- Images are counted from the request's image parts. A user or tool message whose text spells the model's image
+  markers (`<|image|>`, `<|begin_of_image|>`, `<|end_of_image|>`) is tokenized as ordinary text, so only the chat
+  template's own markers, one wrapper for each image part, become image tokens.
+- The prompt cache keys each image's placeholder rows by the image's content, so a kept state resumes the same image
+  and never another one under the same placeholders. Image conversations reuse their history like text ones.
+
+### Video on the native server
+
+With `--vision`, GLM-5.3-Flash also reads `video_url` parts in user messages, as transformers 5.19's
+`Glm5NextProcessor` prepares them:
+- **Sampling:** 2 frames a second, deduplicated and padded to an even count, at most 256 frames.
+- **Frame pairs:** each pair of frames is one image to the tower.
+- **Prompt:** the template's `<|begin_of_video|><|video|><|end_of_video|>` holds one
+  `<|begin_of_image|>…<|end_of_image|>T.T seconds` block a pair.
+- **Budget:** a request's videos share `--vision-video-tokens` visual tokens (default 16,384; the processor's
+  `max_image_tokens`). The frames are sized under it and resized with torch's antialiased uint8 bicubic.
+
+Video needs `libtfvideo`, an optional library over FFmpeg 9.0.2 (the release PyAV 19.0.1 ships). The server loads it
+when the first video arrives and refuses videos by name without it:
+
+```sh
+tools/zig/build_ffmpeg.sh ~/ffmpeg-9.0.2        # static FFmpeg: H.264, HEVC, VP8, VP9, MPEG-4; MP4/MOV, MKV/WebM
+zig build video -Dffmpeg=$HOME/ffmpeg-9.0.2     # zig-out/native/lib/libtfvideo.dylib, beside the server
+```
+
+- **Limits (0.6.6's):** 2 videos a request, 16 MB each and 20 MB together, an hour of footage, 8192 pixels a side.
+  Data URLs work, and with `--vision-urls` public HTTPS URLs too (MP4, WebM, QuickTime or Matroska; 20 s a download,
+  120 s a request).
+- **Too short:** a clip too short for one sampled frame (under half a second) is refused, as the processor samples
+  nothing.
+- **Frame counts:** a container without one (Matroska, WebM) is counted by its video packets. transformers' PyAV loader
+  can't read these at all.
+- **Checks:** `tools/zig/glm_video_fixtures.py` checks the patches bit for bit against the processor on frames PyAV
+  decodes, over 13 clips. `tools/zig/video_served.py` checks a served model's prompt ids against the processor's.
+
 ## Verification
 
 Compare image requests with `draft: true` and `draft: false` at identical sampling settings and seed, then compare concurrent requests with their solo results.

tools/zig/build_ffmpeg.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# FFmpeg 9.0.2 (PyAV 19.0.1's) for libtfvideo, static PIC, PyAV's decoders only: build_ffmpeg.sh PREFIX [SOURCE_DIR]
+set -euo pipefail
+version=9.0.2
+sha256=8c3850283eb25fa026482078a04051e0be17347b09ef81a0849bec15a96e002e
+prefix="$(mkdir -p "$1" && cd "$1" && pwd)"
+work="${2:-$prefix/src}"
+mkdir -p "$work"
+cd "$work"
+if [ ! -d "ffmpeg-$version" ]; then
+  [ -f "ffmpeg-$version.tar.xz" ] || curl -sSfLO "https://ffmpeg.org/releases/ffmpeg-$version.tar.xz"
+  echo "$sha256  ffmpeg-$version.tar.xz" | shasum -a 256 -c -
+  tar xf "ffmpeg-$version.tar.xz"
+fi
+cd "ffmpeg-$version"
+./configure --prefix="$prefix" --enable-static --disable-shared --enable-pic \
+  --disable-programs --disable-doc --disable-network --disable-autodetect --disable-everything \
+  --disable-avdevice --disable-avfilter --disable-swresample \
+  --enable-avcodec --enable-avformat --enable-swscale \
+  --enable-decoder=h264,hevc,vp8,vp9,mpeg4 \
+  --enable-parser=h264,hevc,vp8,vp9,mpeg4video \
+  --enable-demuxer=mov,matroska \
+  --enable-bsf=vp9_superframe_split
+make -j"$(getconf _NPROCESSORS_ONLN)"
+make install
+echo "FFmpeg $version in $prefix"

tools/zig/glm_video_fixtures.py

@@ -0,0 +1,154 @@
+"""GLM-5.3-Flash video input against transformers' processor, on a fixed clip set (#565's video gate)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from fractions import Fraction
+from pathlib import Path
+
+import av
+import numpy as np
+
+
+def scene(i: int, w: int, h: int) -> np.ndarray:
+    y, x = np.mgrid[0:h, 0:w]
+    f = np.empty((h, w, 3), np.uint8)
+    f[..., 0] = (x * 255 // max(w - 1, 1)).astype(np.uint8)
+    f[..., 1] = ((y + 3 * i) % 256).astype(np.uint8)
+    f[..., 2] = ((x + y + 7 * i) % 200 + 30).astype(np.uint8)
+    s = max(8, min(w, h) // 4)
+    cx, cy = (5 + 4 * i) % max(w - s, 1), (h - s) // 2
+    f[cy:cy + s, cx:cx + s] = (250, 240, 20)
+    return f
+
+
+CLIPS = [  # name, container, codec, pixel format, size, rate, frames, options
+    ("h264.mp4", "mp4", "libx264", "yuv420p", (320, 180), 25, 80, {}),
+    ("h264-odd.mp4", "mp4", "libx264", "yuv420p", (318, 182), 30, 45, {}),
+    ("h264-full.mp4", "mp4", "libx264", "yuvj420p", (256, 144), 24, 30, {}),
+    ("h264-709.mov", "mov", "libx264", "yuv420p", (640, 360), Fraction(30000, 1001), 60, {"bt709": "1"}),
+    ("h264-444.mp4", "mp4", "libx264", "yuv444p", (200, 120), 25, 25, {}),
+    ("h264-hd.mp4", "mp4", "libx264", "yuv420p", (1280, 720), 30, 90, {}),
+    ("hevc.mp4", "mp4", "libx265", "yuv420p", (320, 240), 25, 50, {"x265-params": "log-level=error"}),
+    ("hevc10.mov", "mov", "libx265", "yuv420p10le", (320, 240), 25, 26, {"x265-params": "log-level=error"}),
+    ("vp9.webm", "webm", "libvpx-vp9", "yuv420p", (320, 180), 25, 50, {}),
+    ("vp8.mkv", "matroska", "libvpx", "yuv420p", (240, 160), 20, 40, {}),
+    ("mpeg4.mp4", "mp4", "mpeg4", "yuv420p", (352, 288), 25, 25, {}),
+    ("short.mp4", "mp4", "libx264", "yuv420p", (160, 120), 25, 10, {}),
+    ("long.mp4", "mp4", "libx264", "yuv420p", (64, 48), 10, 1500, {}),
+]
+
+
+def make(folder: Path) -> None:
+    folder.mkdir(parents=True, exist_ok=True)
+    for name, fmt, codec, pix, (w, h), rate, n, opts in CLIPS:
+        opts = dict(opts)
+        bt709 = opts.pop("bt709", None)
+        c = av.open(str(folder / name), "w", format=fmt)
+        s = c.add_stream(codec, rate=rate, options=opts)
+        s.width, s.height, s.pix_fmt = w, h, pix
+        if bt709:
+            s.codec_context.colorspace = s.codec_context.color_primaries = s.codec_context.color_trc = 1
+        for i in range(n):
+            for p in s.encode(av.VideoFrame.from_ndarray(scene(i, w, h), format="rgb24").reformat(format=pix)):
+                c.mux(p)
+        for p in s.encode():
+            c.mux(p)
+        c.close()
+
+
+def load(path: Path, vp, max_frames: int):
+    """read_video_pyav, with a counted frame total where the container keeps none."""
+    from transformers.video_utils import VideoMetadata
+
+    with av.open(str(path)) as c:
+        st = c.streams.video[0]
+        total = int(st.frames)
+        if total == 0:
+            total = sum(1 for p in c.demux(st) if p.size)
+            c.seek(0)
+        fps = st.average_rate
+        meta = VideoMetadata(total_num_frames=total, fps=float(fps), duration=float(total / fps) if fps else 0.0,
+                             video_backend="pyav", height=st.height, width=st.width)
+        vp.max_frames = max_frames
+        indices = vp.sample_frames(meta)
+        if len(indices) == 0:
+            return None, meta
+        frames = []
+        c.seek(0)
+        for i, frame in enumerate(c.decode(video=0)):
+            if i > indices[-1]:
+                break
+            if i in indices:
+                frames.append(frame)
+        meta.frames_indices = indices
+        return np.stack([f.to_ndarray(format="rgb24") for f in frames]), meta
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--clips", required=True)
+    ap.add_argument("--native", default="zig-out/bin/tf-glm-video")
+    ap.add_argument("--max-tokens", type=int, default=16384)
+    ap.add_argument("--max-frames", type=int, default=256)
+    ap.add_argument("--make", action="store_true")
+    args = ap.parse_args()
+    from transformers import AutoProcessor
+
+    clips = Path(args.clips)
+    if args.make:
+        make(clips)
+    proc = AutoProcessor.from_pretrained(args.model)
+    vp = proc.video_processor
+    failures = 0
+    for path in sorted(p for p in clips.iterdir() if p.suffix in {".mp4", ".mov", ".webm", ".mkv"}):
+        frames, meta = load(path, vp, args.max_frames)
+        out = clips / "out" / path.stem
+        out.mkdir(parents=True, exist_ok=True)
+        run = subprocess.run([args.native, args.model, str(path), str(out), str(args.max_tokens), str(args.max_frames)],
+                             capture_output=True, text=True)
+        if frames is None:
+            ok = run.returncode != 0 and "VideoTooShort" in run.stderr
+            failures += not ok
+            print(f"{'ok  ' if ok else 'FAIL'} {path.name}: the processor samples no frames; native {'refuses' if run.returncode else 'answers'}")
+            continue
+        if run.returncode != 0:
+            failures += 1
+            print(f"FAIL {path.name}: tf-glm-video: {run.stderr.strip()[-300:]}")
+            continue
+        native = json.loads((out / "native.json").read_text())
+        msgs = [{"role": "user", "content": [{"type": "video"}, {"type": "text", "text": "What moves?"}]}]
+        text = proc.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
+        enc = proc(text=[text], videos=[frames], video_metadata=[meta], do_sample_frames=False,
+                   max_image_tokens=args.max_tokens, return_tensors="np")
+        ref = enc["pixel_values_videos"].astype(np.float32)
+        grid = [int(v) for v in enc["video_grid_thw"][0]]
+        mine = np.fromfile(out / "native_patches.f32", dtype=np.float32).reshape(-1, ref.shape[1]) if ref.ndim == 2 else None
+        decoded = sorted(set(int(i) for i in meta.frames_indices))
+        prompt = proc.tokenizer.decode(enc["input_ids"][0])
+        texts = re.findall(r"<\|end_of_image\|>([^<]*)", prompt)
+        same = mine is not None and mine.shape == ref.shape and np.array_equal(mine.view(np.uint32), ref.view(np.uint32))
+        diff = 0 if mine is None or mine.shape != ref.shape else int((mine.view(np.uint32) != ref.view(np.uint32)).sum())
+        checks = {
+            "indices": native["indices"] == decoded,
+            "grid": native["grid"] == grid,
+            "patches": same,
+            "texts": native["texts"] == texts,
+        }
+        ok = all(checks.values())
+        failures += not ok
+        print(f"{'ok  ' if ok else 'FAIL'} {path.name}: frames {meta.total_num_frames}{' (counted)' if native['counted'] else ''} at {float(meta.fps):.3f}, "
+              f"{len(decoded)} decoded, grid {grid}, {ref.size} values {'bit-equal' if same else f'({diff} differ)'}; "
+              f"times {texts[0]}..{texts[-1]}; {native['ms']} ms"
+              + ("" if ok else f" {[k for k, v in checks.items() if not v]} native {native['indices'][:8]} {native['grid']} {native['texts'][:3]}"))
+    print("PASS" if failures == 0 else f"FAIL: {failures}")
+    return 0 if failures == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())