From 616fccaac431f90a47250ad357c77c5123f215d4 Mon Sep 17 00:00:00 2001 From: philip-pentatonic <63354535+philip-pentatonic@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:01:38 +0000 Subject: [PATCH 01/19] Lane core: requests that arrive together share their prompt passes The lane host admits every queued request at once: a lone one still goes to the backend's driver, the rest to the core's addStreams (lanes/admit.zig). It runs the backend's optional prefill_many when there is more than one, draws each stream's first token, then asks for every stream's first drafts in one request, as multi.py's _fill_batch admits a batch. addStream is its one-stream case, with the same calls in the same order; a backend without prefill_many prefills each stream as before. --- zig/src/core/lane_host.zig | 69 ++++++++++++++++++++------ zig/src/core/lanes/admit.zig | 90 ++++++++++++++++++++++++++++++++++ zig/src/core/lanes/backend.zig | 2 + zig/src/core/lanes/engine.zig | 66 +++++-------------------- zig/src/native/cuda.zig | 6 +++ 5 files changed, 166 insertions(+), 67 deletions(-) create mode 100644 zig/src/core/lanes/admit.zig diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index dc4e4db113..27dedfdc74 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -319,18 +319,60 @@ pub const LaneHost = struct { h.mutex.unlock(h.io); } - /// Prefills the next queued request into a free lane; false when none waits or no lane is free. - fn admitOne(h: *LaneHost) bool { + /// Every queued request the lanes have room for: a lone one to the backend's driver, the rest prefilled together. + fn admitAll(h: *LaneHost) void { + var batch: std.ArrayList(*Job) = .empty; + defer batch.deinit(h.gpa); + while (true) { + const job = switch (h.prepare()) { + .none => break, + .dropped => continue, + .job => |j| j, + }; + if (batch.items.len == 0 and h.loneFits(job)) { + _ = h.runLone(job, job.began); + continue; + } + batch.append(h.gpa, job) catch { + _ = h.drop(job, "out of memory"); + continue; + }; + } + if (batch.items.len == 0) return; + const streams = h.gpa.alloc(*lanes.Stream, batch.items.len) catch return h.dropAll(batch.items, "out of memory"); + defer h.gpa.free(streams); + const errs = h.gpa.alloc(?anyerror, batch.items.len) catch return h.dropAll(batch.items, "out of memory"); + defer h.gpa.free(errs); + for (streams, batch.items) |*s, job| s.* = &job.stream; + h.core.addStreams(streams, errs) catch |e| return h.dropAll(batch.items, h.words(e)); + for (batch.items, errs) |job, err| { + if (err) |e| { + _ = if (e == error.Cancelled) h.cancel(job) else h.drop(job, h.words(e)); + continue; + } + h.prefilled(job, job.began); + if (h.deliver(job)) h.remove(job); + } + } + + fn dropAll(h: *LaneHost, jobs: []const *Job, message: []const u8) void { + for (jobs) |job| _ = h.drop(job, message); + } + + const Prepared = union(enum) { none, dropped, job: *Job }; + + /// The oldest queued request admitted with its stream made, when the lanes have room. + fn prepare(h: *LaneHost) Prepared { h.lock(); if (h.queued.items.len == 0 or h.admitted.items.len >= h.info_.lanes) { h.unlock(); - return false; + return .none; } const job = h.queued.orderedRemove(0); h.admitted.append(h.gpa, job) catch { h.unlock(); h.finish(job, .failed, "out of memory"); - return true; + return .dropped; }; h.unlock(); const r = job.request; @@ -342,7 +384,10 @@ pub const LaneHost = struct { job.marks = l.marks; reuse = .{ .saved = if (l.entry) |e| e.saved else null, .at = if (l.entry) |e| e.at else 0, .marks = l.marks, .hook = .{ .ptr = job, .at = Job.kept } }; } else |_| {}; - job.proposer = lanes.SuffixLookup.init(h.gpa, .{ .min_match = h.min_match }) catch return h.drop(job, "the drafter could not start"); + job.proposer = lanes.SuffixLookup.init(h.gpa, .{ .min_match = h.min_match }) catch { + _ = h.drop(job, "the drafter could not start"); + return .dropped; + }; job.stream = lanes.Stream.init(h.gpa, .{ .id = "request", .prompt = r.prompt, @@ -362,16 +407,12 @@ pub const LaneHost = struct { .reuse = reuse, }) catch { job.proposer.deinit(); - return h.drop(job, "out of memory"); + _ = h.drop(job, "out of memory"); + return .dropped; }; job.started = true; - const began = std.Io.Clock.awake.now(h.io).toNanoseconds(); - job.began = began; - if (h.loneFits(job)) return h.runLone(job, began); - h.core.addStream(&job.stream) catch |e| return if (e == error.Cancelled) h.cancel(job) else h.drop(job, h.words(e)); - h.prefilled(job, began); - if (h.deliver(job)) h.remove(job); - return true; + job.began = std.Io.Clock.awake.now(h.io).toNanoseconds(); + return .{ .job = job }; } fn prefilled(h: *LaneHost, job: *Job, began: i96) void { @@ -480,7 +521,7 @@ pub const LaneHost = struct { fn run(h: *LaneHost) void { while (true) { h.takeCancels(); - while (h.admitOne()) {} + h.admitAll(); h.noteLive(); h.lock(); if (h.closing) { diff --git a/zig/src/core/lanes/admit.zig b/zig/src/core/lanes/admit.zig new file mode 100644 index 0000000000..37ec717b84 --- /dev/null +++ b/zig/src/core/lanes/admit.zig @@ -0,0 +1,90 @@ +//! Admitting streams: prompt passes (shared, as multi.py's _fill_batch), first tokens, then first drafts. +const std = @import("std"); +const be = @import("backend.zig"); +const ev = @import("events.zig"); +const win = @import("windows.zig"); +const trail = @import("trail.zig"); +const Engine = @import("engine.zig").Engine; +const Stream = @import("stream.zig").Stream; +const LogRow = @import("logprob.zig").Row; +const Feed = be.Feed; +const f = ev.f; +const str = trail.str; +const int = trail.int; + +/// Prompt passes (shared when the backend has them), first tokens, every stream's first drafts; failures in `errs`. +pub fn streams(e: *Engine, list: []const *Stream, errs: []?anyerror) !void { + _ = e.arena.reset(.retain_capacity); + const a = e.arena.allocator(); + @memset(errs, null); + for (list) |s| try trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }); + if (list.len > 1 and e.backend.vtable.prefill_many != null) { + try e.backend.vtable.prefill_many.?(e.backend.ptr, list, errs); + } else for (list, errs) |s, *err| e.backend.prefill(s) catch |x| { + err.* = x; + }; + const feeds = try a.alloc(Feed, list.len); + const drawn = try a.alloc(u64, list.len); + const asked = try a.alloc(?u32, list.len); + var requests: std.ArrayList(be.DraftRequest) = .empty; + for (list, errs, feeds, drawn, asked) |s, *err, *feed, *handle, *ask| { + ask.* = null; + if (err.*) |x| { + if (x == error.Cancelled) e.backend.release(s); // the host finishes a cancelled stream without the core + continue; + } + if (s.isCancelled()) { // cancelled in its last chunk: no first token + e.backend.release(s); + err.* = error.Cancelled; + continue; + } + s.context.shrinkRetainingCapacity(s.prompt_len); + s.rows.clearRetainingCapacity(); + s.pending = null; + s.cache_len = s.prompt_len; + const position: u64 = s.prompt_len; + handle.* = e.backend.first(s, position) catch |x| { + err.* = x; // the host discards it, as after a failed prompt pass + continue; + }; + feed.* = .{ .handle = handle.* }; + if (try e.forcedNext(s)) |t| feed.* = .{ .value = t }; + if (e.cfg.family_mtp and s.drafts) { + // the head reads the prompt's last row and the first token, and drafts the one after it + const d: u32 = @intCast(try e.rule.depth(win.who(s))); + ask.* = d; + try requests.append(a, .{ .stream = s, .follow = &.{}, .first = feed.*, .rows = null, .start = s.prompt_len, .position = position + 1, .depth = d }); + } else if (e.cfg.pipelined and s.logprobs == null) { + try e.queueNext(s, feed.*); + } + } + if (requests.items.len > 0) try e.backend.draft(requests.items); + for (list, errs, feeds, drawn, asked) |s, err, feed, handle, ask| { + if (err != null) continue; + const position: u64 = s.prompt_len; + if (ask) |d| { + s.dropHeld(e.gpa); + s.next = .{ .count = d }; + if (e.backend.vtable.tree) |tree| if (try tree(e.backend.ptr, s, e.gpa)) |held| { + s.next = held; // a tree head's first drafts as host tokens, as every later round's + }; + } + const value = try e.readFeed(feed); + if (ask) |d| try trail.event(e, &.{ f("ev", str("draft")), f("stream", str(s.id)), f("depth", int(d)), f("position", int(position + 1)), f("follow", .{ .u32s = &.{value} }), f("rows", .null) }); + if (e.log != null) { + const first = if (feed == .handle) value else try e.backend.read(handle); + try trail.event(e, &.{ f("ev", str("first")), f("stream", str(s.id)), f("position", int(position)), f("drawn", int(first)), f("token", int(value)) }); + } + var first_row: [1]LogRow = undefined; + if (s.logprobs != null) first_row[0] = (try e.backend.firstRow(s)).forToken(value); + _ = try s.commit(e.gpa, &.{value}, if (s.logprobs != null) &first_row else &.{}); + s.pending = value; + try trail.resolve(e); + if (s.finished) { + try trail.finish(e, s); + e.release(s); + continue; + } + try e.live.append(e.gpa, s); + } +} diff --git a/zig/src/core/lanes/backend.zig b/zig/src/core/lanes/backend.zig index a10ced8675..c50ca6b10d 100644 --- a/zig/src/core/lanes/backend.zig +++ b/zig/src/core/lanes/backend.zig @@ -65,6 +65,8 @@ pub const Backend = struct { pub const VTable = struct { /// Prefill the stream's prompt into its caches (and its draft head's). prefill: *const fn (ptr: *anyopaque, s: *Stream) anyerror!void, + /// Prefill several streams in shared passes, each with `prefill`'s bits; `errs[i]` for a failure. + prefill_many: ?*const fn (ptr: *anyopaque, streams: []const *Stream, errs: []?anyerror) anyerror!void = null, /// Draw the first token from the prompt's last row at `position`; a handle to it. first: *const fn (ptr: *anyopaque, s: *Stream, position: u64) anyerror!u64, /// Feed one token and queue the draw of the next at `position` (a one-token round); a handle to it. diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index b752d84e85..f2ad1fd57f 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -12,6 +12,7 @@ const shape = @import("shape.zig"); const plan_lanes = @import("plan_lanes.zig"); const fill = @import("fill.zig"); const trail = @import("trail.zig"); +const admit = @import("admit.zig"); const Stream = sm.Stream; const Feed = be.Feed; const LogRow = @import("logprob.zig").Row; @@ -78,55 +79,14 @@ pub const Engine = struct { /// Prefill a stream, draw its first token and ask for its first drafts; it takes part from the next round. pub fn addStream(e: *Engine, s: *Stream) !void { - _ = e.arena.reset(.retain_capacity); - try trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }); - e.backend.prefill(s) catch |err| { - if (err == error.Cancelled) e.backend.release(s); // the host finishes a cancelled stream without the core - return err; - }; - if (s.isCancelled()) { // cancelled in its last chunk: no first token - e.backend.release(s); - return error.Cancelled; - } - s.context.shrinkRetainingCapacity(s.prompt_len); - s.rows.clearRetainingCapacity(); - s.pending = null; - s.cache_len = s.prompt_len; - const position: u64 = s.prompt_len; - const drawn = try e.backend.first(s, position); - var feed: Feed = .{ .handle = drawn }; - if (try e.forcedNext(s)) |t| feed = .{ .value = t }; - var asked: ?u32 = null; - if (e.cfg.family_mtp and s.drafts) { - // the head reads the prompt's last row and the first token, and drafts the one after it - const d: u32 = @intCast(try e.rule.depth(win.who(s))); - asked = d; - try e.backend.draft(&.{.{ .stream = s, .follow = &.{}, .first = feed, .rows = null, .start = s.prompt_len, .position = position + 1, .depth = d }}); - s.dropHeld(e.gpa); - s.next = .{ .count = d }; - if (e.backend.vtable.tree) |tree| if (try tree(e.backend.ptr, s, e.gpa)) |held| { - s.next = held; // a tree head's first drafts as host tokens, as every later round's - }; - } else if (e.cfg.pipelined and s.logprobs == null) { - try e.queueNext(s, feed); - } - const value = try e.readFeed(feed); - if (asked) |d| try trail.event(e, &.{ f("ev", str("draft")), f("stream", str(s.id)), f("depth", int(d)), f("position", int(position + 1)), f("follow", .{ .u32s = &.{value} }), f("rows", .null) }); - if (e.log != null) { - const first = if (feed == .handle) value else try e.backend.read(drawn); - try trail.event(e, &.{ f("ev", str("first")), f("stream", str(s.id)), f("position", int(position)), f("drawn", int(first)), f("token", int(value)) }); - } - var first_row: [1]LogRow = undefined; - if (s.logprobs != null) first_row[0] = (try e.backend.firstRow(s)).forToken(value); - _ = try s.commit(e.gpa, &.{value}, if (s.logprobs != null) &first_row else &.{}); - s.pending = value; - try trail.resolve(e); - if (s.finished) { - try trail.finish(e, s); - e.release(s); - return; - } - try e.live.append(e.gpa, s); + var errs = [_]?anyerror{null}; + try e.addStreams(&.{s}, &errs); + if (errs[0]) |err| return err; + } + + /// addStream for several streams, their prompt passes shared (admit.zig). + pub fn addStreams(e: *Engine, streams: []const *Stream, errs: []?anyerror) !void { + return admit.streams(e, streams, errs); } /// A stream another driver prefilled and drew `token` for: drafts asked, the token committed, rounds from here. @@ -568,7 +528,7 @@ pub const Engine = struct { } /// Feed a token and queue the draw of the next at the new cache length. - fn queueNext(e: *Engine, s: *Stream, feed: Feed) !void { + pub fn queueNext(e: *Engine, s: *Stream, feed: Feed) !void { s.cache_len += 1; const h = try e.backend.queue(s, feed, s.cache_len); s.inflight = h; @@ -579,20 +539,20 @@ pub const Engine = struct { } /// The thinking budget's or a forced fix's token at the next position instead of the draw. - fn forcedNext(e: *Engine, s: *Stream) !?u32 { + pub fn forcedNext(e: *Engine, s: *Stream) !?u32 { if (s.popForce()) |t| return t; if (s.cutsNext()) return try s.startClose(e.gpa); return null; } - fn readFeed(e: *Engine, feed: Feed) !u32 { + pub fn readFeed(e: *Engine, feed: Feed) !u32 { return switch (feed) { .handle => |h| try e.backend.read(h), .value => |v| v, }; } - fn release(e: *Engine, s: *Stream) void { + pub fn release(e: *Engine, s: *Stream) void { s.dropRoundState(e.gpa); e.backend.release(s); } diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 668bd82c4b..296aa21409 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -248,6 +248,12 @@ const Host = struct { return x.inner.vtable.prefill(x.inner.ptr, s); } }.f, + .prefill_many = if (v.prefill_many != null) struct { + fn f(p: *anyopaque, streams: []const *lanes.Stream, errs: []?anyerror) anyerror!void { + const x = bind(p); + return x.inner.vtable.prefill_many.?(x.inner.ptr, streams, errs); + } + }.f else null, .first = struct { fn f(p: *anyopaque, s: *lanes.Stream, position: u64) anyerror!u64 { const x = bind(p); From 82d87215b015bf132808c9cffa07abfea11c87fe Mon Sep 17 00:00:00 2001 From: philip-pentatonic <63354535+philip-pentatonic@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:01:38 +0000 Subject: [PATCH 02/19] Nemotron: the checkpoint Source copies to a Target, so other CUDA families share it Source needed Nemotron's kernel handles only for the driver, the stream and whether the card is discrete; it now takes those as a Target (Target.of(ops) for Nemotron's own loader). cuda.zig exports the source module and the torch ops for the 27B's CUDA engine. --- zig/src/families/nemotron/cuda.zig | 4 ++++ zig/src/families/nemotron/cuda_source.zig | 26 ++++++++++++++++------ zig/src/families/nemotron/cuda_weights.zig | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/zig/src/families/nemotron/cuda.zig b/zig/src/families/nemotron/cuda.zig index 7177181a4d..5bb9a91c96 100644 --- a/zig/src/families/nemotron/cuda.zig +++ b/zig/src/families/nemotron/cuda.zig @@ -26,6 +26,10 @@ pub const train = @import("cuda_train.zig"); pub const train_ops = @import("cuda_train_ops.zig"); pub const sites = @import("cuda_sites.zig"); pub const slide_dims = @import("slide_dims.zig"); +/// Checkpoint bytes to the GPU with direct reads in flight: shared with the other CUDA families. +pub const source = @import("cuda_source.zig"); +/// The torch-op replacements (argmax, topk, casts): shared with the other CUDA families. +pub const torch_ops = @import("cuda_torch_ops.zig"); test { _ = @import("config.zig"); diff --git a/zig/src/families/nemotron/cuda_source.zig b/zig/src/families/nemotron/cuda_source.zig index ffc4ba2eec..dcf89a40d4 100644 --- a/zig/src/families/nemotron/cuda_source.zig +++ b/zig/src/families/nemotron/cuda_source.zig @@ -33,9 +33,20 @@ const Job = struct { } }; +/// Where a Source's copies go: the driver, the stream, and whether the card has its own memory. +pub const Target = struct { + d: *const cuda.Driver, + s: cuda.Stream, + discrete: bool, + + pub fn of(ops: kern.Ops) Target { + return .{ .d = ops.k.d, .s = ops.s, .discrete = ops.k.discrete }; + } +}; + pub const Source = struct { gpa: std.mem.Allocator, - ops: kern.Ops, + ops: Target, files: std.ArrayList(Mapped) = .empty, slots: [slot_count]dio.Buffer = undefined, pinned: [slot_count]?cuda.HostBuffer = @splat(null), // a discrete card's slots, page-locked: copies run at the link's speed @@ -48,21 +59,21 @@ pub const Source = struct { next: u64 = 0, /// Page-aligned slots; on a discrete card page-locked, each with an event its copies record. - pub fn init(gpa: std.mem.Allocator, ops: kern.Ops) !Source { + pub fn init(gpa: std.mem.Allocator, ops: Target) !Source { var s: Source = .{ .gpa = gpa, .ops = ops }; var made: usize = 0; errdefer s.freeSlots(made); while (made < slot_count) : (made += 1) { - if (!ops.k.discrete) { + if (!ops.discrete) { s.slots[made] = try gpa.alignedAlloc(u8, .fromByteUnits(dio.alignment), slot_bytes); continue; } - var h = try cuda.HostBuffer.alloc(ops.k.d, slot_bytes); + var h = try cuda.HostBuffer.alloc(ops.d, slot_bytes); if (@intFromPtr(h.bytes.ptr) % dio.alignment != 0) { h.free(); return error.UnalignedPinnedSlot; } - s.copied[made] = cuda.Event.init(ops.k.d, false) catch |err| { + s.copied[made] = cuda.Event.init(ops.d, false) catch |err| { h.free(); return err; }; @@ -126,7 +137,8 @@ pub const Source = struct { pub fn upload(s: *Source, dst: u64, bytes: []const u8) !void { const at = s.locate(bytes) orelse { try s.flush(); - return s.ops.upload(dst, bytes); + if (bytes.len == 0) return; + return s.ops.d.check(s.ops.d.api.cuMemcpyHtoDAsync_v2(dst, bytes.ptr, bytes.len, s.ops.s.handle), "cuMemcpyHtoDAsync"); }; var done: usize = 0; while (done < bytes.len) { @@ -212,7 +224,7 @@ pub const Source = struct { /// Page-locked slots record their copy event; pageable slots wait for the driver to stage their bytes. fn finish(s: *Source, k: usize, p: Pending, got: []u8) !void { if (got.len == 0) return; - try s.ops.k.d.check(s.ops.k.d.api.cuMemcpyHtoDAsync_v2(p.dst, got.ptr, got.len, s.ops.s.handle), "cuMemcpyHtoDAsync"); + try s.ops.d.check(s.ops.d.api.cuMemcpyHtoDAsync_v2(p.dst, got.ptr, got.len, s.ops.s.handle), "cuMemcpyHtoDAsync"); if (s.copied[k]) |e| return e.record(s.ops.s); try s.ops.s.synchronize(); } diff --git a/zig/src/families/nemotron/cuda_weights.zig b/zig/src/families/nemotron/cuda_weights.zig index e3a6f64385..0319ba41e9 100644 --- a/zig/src/families/nemotron/cuda_weights.zig +++ b/zig/src/families/nemotron/cuda_weights.zig @@ -352,7 +352,7 @@ pub fn load(gpa: std.mem.Allocator, io: std.Io, ops: kern.Ops, dir: []const u8, if (c.group_size != 64 or c.bits != 4) return error.UnsupportedQuantization; var w: Weights = .{ .gpa = gpa, .config = c }; errdefer w.deinit(); - var src = try Source.init(gpa, ops); + var src = try Source.init(gpa, .of(ops)); defer src.deinit(); var L: Loader = .{ .gpa = gpa, .ops = ops, .w = &w, .src = &src, .scratch = try cuda.DeviceBuffer.alloc(ops.k.d, 1 << 20) }; defer { From 99bcc5b29805e29a4baf2cd4beade0f7fe1a5fe8 Mon Sep 17 00:00:00 2001 From: philip-pentatonic <63354535+philip-pentatonic@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:01:38 +0000 Subject: [PATCH 03/19] Qwen3.8-27B on CUDA: the qwen27 family's engine for NVIDIA GPUs, every stream drafted A qwen3_5 checkpoint with the 27B's shapes opens zig/src/families/qwen27/cuda in tensorfold-native, beside the family's Metal engine. It reads the family's config.zig and refuses other shapes (cuda/shape.zig). The forward is the Python 0.6.6 engine's: its CUDA kernels (qmm_group, qmm_prefill at prompt tile 9, the GDN tree, replay and prompt chain, prefill attention) built from zig/kernels/cuda, and its Triton glue from a set captured on the chip (TENSORFOLD_CUDA_KERNELS). Verify rounds take every stream's window together, up to 16 streams of 16 rows, as multi_tree_forward lays them out, and commit only each stream's accepted path. DFlash2 (--drafter, else the pulled z-lab/Qwen3.8-27B-DFlash2) drafts every stream in one batch: each stream's kept taps absorbed together, every block in one forward, a best-first tree per stream, or a copied continuation where the context repeats. The lane core verifies the trees and trims them by each node's chance against a cost curve timed at load. Prompts that arrive together share one prefill pass (prefill_rows). Checked against captures of the Python engine (zig/tests/cuda/qwen27), on an RTX PRO 6000 and on one DGX Spark (GB10): serial replies equal token for token (a 4722-token prompt among them); 54 verify windows (chains, trees, shared rounds of 2 to 16 streams and up to 256 rows, every alignment of the packed inputs) equal row for row in their logits' bytes; drafted replies and each stream's rounds equal Python's; served replies of 1, 4 and 16 concurrent requests, greedy and sampled, equal the same request alone and its "draft": false decode. --- zig/build/cuda.zig | 35 +- zig/kernels/cuda/gdn_prefill.cu | 150 +++++ zig/kernels/cuda/prefill_attention.cu | 3 + zig/kernels/cuda/qmm_group.cu | 7 + zig/kernels/cuda/qmm_prefill.cu | 4 + zig/kernels/cuda/qwen_ops.cu | 68 +++ zig/src/cli/qwen27_cuda.zig | 459 +++++++++++++++ zig/src/cuda/kernels.zig | 3 + zig/src/families/qwen27/cuda.zig | 33 ++ zig/src/families/qwen27/cuda/draft.zig | 423 ++++++++++++++ zig/src/families/qwen27/cuda/draft_load.zig | 274 +++++++++ zig/src/families/qwen27/cuda/draft_policy.zig | 122 ++++ zig/src/families/qwen27/cuda/drafts.zig | 103 ++++ zig/src/families/qwen27/cuda/engine.zig | 118 ++++ zig/src/families/qwen27/cuda/forward.zig | 436 ++++++++++++++ zig/src/families/qwen27/cuda/kernels.zig | 365 ++++++++++++ zig/src/families/qwen27/cuda/lanes.zig | 549 ++++++++++++++++++ zig/src/families/qwen27/cuda/lone.zig | 181 ++++++ zig/src/families/qwen27/cuda/native.zig | 111 ++++ zig/src/families/qwen27/cuda/round.zig | 230 ++++++++ zig/src/families/qwen27/cuda/shape.zig | 102 ++++ zig/src/families/qwen27/cuda/state.zig | 304 ++++++++++ zig/src/families/qwen27/cuda/tree.zig | 181 ++++++ zig/src/families/qwen27/cuda/triton.zig | 141 +++++ zig/src/families/qwen27/cuda/weights.zig | 252 ++++++++ zig/src/native/cuda.zig | 17 +- zig/tests/cuda/qwen27/capture.py | 208 +++++++ zig/tests/cuda/qwen27/capture_draft.py | 74 +++ zig/tests/cuda/qwen27/capture_multi.py | 108 ++++ zig/tests/cuda/qwen27/capture_windows.py | 143 +++++ 30 files changed, 5187 insertions(+), 17 deletions(-) create mode 100644 zig/kernels/cuda/gdn_prefill.cu create mode 100644 zig/kernels/cuda/qwen_ops.cu create mode 100644 zig/src/cli/qwen27_cuda.zig create mode 100644 zig/src/families/qwen27/cuda.zig create mode 100644 zig/src/families/qwen27/cuda/draft.zig create mode 100644 zig/src/families/qwen27/cuda/draft_load.zig create mode 100644 zig/src/families/qwen27/cuda/draft_policy.zig create mode 100644 zig/src/families/qwen27/cuda/drafts.zig create mode 100644 zig/src/families/qwen27/cuda/engine.zig create mode 100644 zig/src/families/qwen27/cuda/forward.zig create mode 100644 zig/src/families/qwen27/cuda/kernels.zig create mode 100644 zig/src/families/qwen27/cuda/lanes.zig create mode 100644 zig/src/families/qwen27/cuda/lone.zig create mode 100644 zig/src/families/qwen27/cuda/native.zig create mode 100644 zig/src/families/qwen27/cuda/round.zig create mode 100644 zig/src/families/qwen27/cuda/shape.zig create mode 100644 zig/src/families/qwen27/cuda/state.zig create mode 100644 zig/src/families/qwen27/cuda/tree.zig create mode 100644 zig/src/families/qwen27/cuda/triton.zig create mode 100644 zig/src/families/qwen27/cuda/weights.zig create mode 100644 zig/tests/cuda/qwen27/capture.py create mode 100644 zig/tests/cuda/qwen27/capture_draft.py create mode 100644 zig/tests/cuda/qwen27/capture_multi.py create mode 100644 zig/tests/cuda/qwen27/capture_windows.py diff --git a/zig/build/cuda.zig b/zig/build/cuda.zig index 5602af1007..676bda8c88 100644 --- a/zig/build/cuda.zig +++ b/zig/build/cuda.zig @@ -13,6 +13,7 @@ const glue = &[_][]const u8{ "-O3", "--fmad=false", "--ftz=false", "--prec-div=t const kernels = [_]Kernel{ .{ .name = "gdn", .flags = &.{ "-O3", "--fmad=false" } }, // cuda/kernels/gdn.py, tensorfold_gdn_v2 + .{ .name = "gdn_prefill", .flags = &.{ "-O3", "--fmad=false" } }, // the same extension's prompt chain .{ .name = "probe", .flags = &.{"-O3"} }, .{ .name = "qmm_group", .flags = &.{"-O3"} }, // cuda/kernels/qmm.py, tensorfold_qmm_v5 .{ .name = "qmm_prefill", .flags = &.{"-O3"} }, @@ -25,6 +26,7 @@ const kernels = [_]Kernel{ .{ .name = "affine4_pack", .flags = &.{"-O3"} }, // ours: the MLX affine-4 repack into qlinear's tiles .{ .name = "lane_gemv", .flags = &.{"-O3"} }, // ours: qmm_group's arithmetic, a column tile's K slices in one CTA .{ .name = "sample", .flags = &.{ "-O3", "--fmad=false", "--ftz=false" } }, // ours: the Metal engine's keyed draws + .{ .name = "qwen_ops", .flags = &.{"-O3"} }, // ours: the 27B's batched state copies, taps, drafter norms and rotary .{ .name = "nemotron_norms", .flags = glue }, // ours, each with a host reference (glue_ref.zig): Nemotron's glue .{ .name = "nemotron_route", .flags = glue }, .{ .name = "nemotron_mamba", .flags = glue }, @@ -39,6 +41,7 @@ const kernels = [_]Kernel{ .{ .name = "torch_indexing", .src = "torch_ops/indexing", .flags = torch_ops }, .{ .name = "torch_movement", .src = "torch_ops/movement", .flags = torch_ops }, .{ .name = "torch_nemotron_constants", .src = "torch_ops/nemotron_constants", .flags = torch_ops }, + .{ .name = "torch_norm_rotary", .src = "torch_ops/norm_rotary", .flags = torch_ops }, // F.rms_norm of 5120 columns }; /// torch.utils.cpp_extension's own nvcc flags (torch 2.13): C++20 and which half/bf16 operators the headers define. @@ -69,7 +72,7 @@ fn runtime(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builti return cuda; } -fn family(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, draft_ids: *std.Build.Module) struct { core: *std.Build.Module, lanes: *std.Build.Module, nemotron: *std.Build.Module, tokenizer: *std.Build.Module, heat: *std.Build.Module } { +fn family(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, draft_ids: *std.Build.Module) struct { core: *std.Build.Module, lanes: *std.Build.Module, nemotron: *std.Build.Module, qwen27: *std.Build.Module, tokenizer: *std.Build.Module, heat: *std.Build.Module } { const tokenizer = b.createModule(.{ .root_source_file = b.path("zig/src/core/tokenizer/tokenizer.zig"), .target = target, .optimize = optimize, .link_libc = true }); const core = b.createModule(.{ .root_source_file = b.path("zig/src/core/root.zig"), .target = target, .optimize = optimize, .link_libc = true }); core.addImport("tokenizer", tokenizer); @@ -81,7 +84,13 @@ fn family(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin nemotron.addImport("lanes", lanes); nemotron.addImport("heat", heat); nemotron.addImport("nemotron_draft_ids", draft_ids); - return .{ .core = core, .lanes = lanes, .nemotron = nemotron, .tokenizer = tokenizer, .heat = heat }; + // the 27B's CUDA engine, beside its Metal one in the family's folder (it reads the family's config.zig) + const qwen27 = b.createModule(.{ .root_source_file = b.path("zig/src/families/qwen27/cuda.zig"), .target = target, .optimize = optimize, .link_libc = true }); + qwen27.addImport("cuda", cuda); + qwen27.addImport("core", core); + qwen27.addImport("lanes", lanes); + qwen27.addImport("nemotron", nemotron); // the checkpoint Source and torch ops the CUDA families share + return .{ .core = core, .lanes = lanes, .nemotron = nemotron, .qwen27 = qwen27, .tokenizer = tokenizer, .heat = heat }; } /// Linux targets: fatbins (-Dnvcc builds them, -Dfatbins embeds prebuilt ones), `tensorfold` and `tf-cuda-test`. @@ -113,7 +122,7 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu cli.addImport("core", mods.core); cli.addImport("lanes", mods.lanes); cli.addImport("nemotron", mods.nemotron); - const native = engines(b, target, optimize, cuda, mods.lanes, mods.nemotron); + const native = engines(b, target, optimize, cuda, mods.lanes, mods.nemotron, mods.qwen27); cli.addOptions("build_options", build_options); cli.addImport("checkpoint_cli", checkpointCli(b, target, optimize, strip, native.engines)); b.installArtifact(b.addExecutable(.{ .name = "tensorfold", .root_module = cli })); @@ -122,18 +131,22 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu runner.addImport("lanes", mods.lanes); runner.addImport("nemotron", mods.nemotron); b.installArtifact(b.addExecutable(.{ .name = "tf-cuda-test", .root_module = runner })); - nativeServer(b, target, optimize, cuda, mods.lanes, mods.nemotron, mods.tokenizer, build_options, true).root_module.strip = strip; + const qwen_cli = b.createModule(.{ .root_source_file = b.path("zig/src/cli/qwen27_cuda.zig"), .target = target, .optimize = optimize, .link_libc = true }); + qwen_cli.addImport("cuda", cuda); + qwen_cli.addImport("qwen27", mods.qwen27); + b.installArtifact(b.addExecutable(.{ .name = "tf-qwen27-cuda", .root_module = qwen_cli })); + nativeServer(b, target, optimize, cuda, mods.lanes, mods.nemotron, mods.qwen27, mods.tokenizer, build_options, true).root_module.strip = strip; } /// The CUDA engines a native server opens (native/cuda.zig), over the given runtime and families. -fn engines(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, lanes: *std.Build.Module, nemotron: *std.Build.Module) struct { api: *std.Build.Module, engines: *std.Build.Module } { +fn engines(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, lanes: *std.Build.Module, nemotron: *std.Build.Module, qwen27: *std.Build.Module) struct { api: *std.Build.Module, engines: *std.Build.Module } { const api = b.createModule(.{ .root_source_file = b.path("zig/src/core/engine_api.zig"), .target = target, .optimize = optimize, .link_libc = true, .imports = &.{.{ .name = "lanes", .module = lanes }} }); const mod = b.createModule(.{ .root_source_file = b.path("zig/src/native/cuda.zig"), .target = target, .optimize = optimize, .link_libc = true, - .imports = &.{ .{ .name = "cuda", .module = cuda }, .{ .name = "engine_api", .module = api }, .{ .name = "lanes", .module = lanes }, .{ .name = "nemotron", .module = nemotron } }, + .imports = &.{ .{ .name = "cuda", .module = cuda }, .{ .name = "engine_api", .module = api }, .{ .name = "lanes", .module = lanes }, .{ .name = "nemotron", .module = nemotron }, .{ .name = "qwen27", .module = qwen27 } }, }); return .{ .api = api, .engines = mod }; } @@ -144,8 +157,8 @@ fn checkpointCli(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std. } /// `zig build native`: tensorfold-native with the CUDA engines into zig-out/native/bin, as the Metal build makes it. -fn nativeServer(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, lanes: *std.Build.Module, nemotron: *std.Build.Module, tokenizer: *std.Build.Module, build_options: *std.Build.Step.Options, install_native: bool) *std.Build.Step.Compile { - const m = engines(b, target, optimize, cuda, lanes, nemotron); +fn nativeServer(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, lanes: *std.Build.Module, nemotron: *std.Build.Module, qwen27: *std.Build.Module, tokenizer: *std.Build.Module, build_options: *std.Build.Step.Options, install_native: bool) *std.Build.Step.Compile { + const m = engines(b, target, optimize, cuda, lanes, nemotron, qwen27); // the HTTP side keeps its safety checks; the engine below it runs at `optimize` (the tokenizer is the family's) const template = b.createModule(.{ .root_source_file = b.path("zig/src/core/template/template.zig"), .target = target, .optimize = .ReleaseSafe, .link_libc = true }); const exe = b.addExecutable(.{ .name = "tensorfold-native", .root_module = b.createModule(.{ @@ -172,7 +185,7 @@ pub fn hostTests(b: *std.Build, draft_ids: *std.Build.Module, build_options: *st const host = b.graph.host; const cuda = runtime(b, host, .debug, &.{}); const mods = family(b, host, .debug, cuda, draft_ids); - const native_modules = engines(b, host, .debug, cuda, mods.lanes, mods.nemotron); + const native_modules = engines(b, host, .debug, cuda, mods.lanes, mods.nemotron, mods.qwen27); const native = native_modules.engines; // the HTTP server's unit tests (zig/src/server/root.zig), which the macOS build runs, on Linux hosts too const template = b.createModule(.{ .root_source_file = b.path("zig/src/core/template/template.zig"), .target = host, .optimize = .debug, .link_libc = true }); @@ -186,7 +199,7 @@ pub fn hostTests(b: *std.Build, draft_ids: *std.Build.Module, build_options: *st const server_test = b.addRunArtifact(b.addTest(.{ .root_module = server_tests })); step.dependOn(&server_test.step); b.step("test-server-cpu", "The HTTP server's unit tests (routes, templates, tool parsing) without a GPU").dependOn(&server_test.step); - for ([_]*std.Build.Module{ cuda, mods.core, mods.lanes, mods.nemotron, mods.heat, native, stagger(b, host, .debug) }) |m| step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = m })).step); + for ([_]*std.Build.Module{ cuda, mods.core, mods.lanes, mods.nemotron, mods.qwen27, mods.heat, native, stagger(b, host, .debug) }) |m| step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = m })).step); const cli = b.createModule(.{ .root_source_file = b.path("zig/src/cli/cuda_main.zig"), .target = host, .optimize = .debug, .link_libc = true }); cli.addImport("cuda", cuda); cli.addImport("core", mods.core); @@ -222,7 +235,7 @@ pub fn distServer(b: *std.Build, target: std.Build.ResolvedTarget, draft_ids: *s }; const cuda = runtime(b, target, .fast, if (prebuilt != null) &images else &.{}); const mods = family(b, target, .fast, cuda, draft_ids); - return nativeServer(b, target, .fast, cuda, mods.lanes, mods.nemotron, mods.tokenizer, build_options, false); + return nativeServer(b, target, .fast, cuda, mods.lanes, mods.nemotron, mods.qwen27, mods.tokenizer, build_options, false); } /// Validate the complete named input set, including images unused by today's server. diff --git a/zig/kernels/cuda/gdn_prefill.cu b/zig/kernels/cuda/gdn_prefill.cu new file mode 100644 index 0000000000..a892ad52d7 --- /dev/null +++ b/zig/kernels/cuda/gdn_prefill.cu @@ -0,0 +1,150 @@ +// Device code of src/tensorfold/cuda/kernels/gdn_prefill.cu (lines 1-145, comments and ATen includes dropped). + +#include +#include +#include + +namespace tf_gdn_prefill { + +constexpr int DK = 128; +constexpr int DV = 128; + +__device__ __forceinline__ float4 widen(uint2 w) { + const __nv_bfloat162 a = *reinterpret_cast(&w.x); + const __nv_bfloat162 b = *reinterpret_cast(&w.y); + return make_float4(__low2float(a), __high2float(a), __low2float(b), __high2float(b)); +} + +__device__ __forceinline__ float4 widen(float4 w) { return w; } + +template +struct Prefetch { + using Raw = typename std::conditional::value, float4, uint2>::type; + static constexpr int KQ = STEPS * (DK / 4) / THREADS, VV = STEPS * ROWS / 8 / THREADS; + static_assert(KQ * THREADS == STEPS * (DK / 4) && VV * THREADS * 8 == STEPS * ROWS, "whole slots a thread"); + Raw kr[KQ], qr[KQ]; + uint4 vr[VV]; + float gr, br; + + __device__ __forceinline__ void fetch(const QK* q, const QK* k, const __nv_bfloat16* v, const float* g, + const float* beta, int t0, int W, int hk, int hv, int head, int key_head, + int col0) { +#pragma unroll + for (int it = 0; it < KQ; ++it) { + const int i = threadIdx.x + it * THREADS, st = i / (DK / 4), c = i % (DK / 4); + const size_t at = (static_cast(min(t0 + st, W - 1)) * hk + key_head) * DK + 4 * c; + kr[it] = *reinterpret_cast(k + at); + qr[it] = *reinterpret_cast(q + at); + } +#pragma unroll + for (int it = 0; it < VV; ++it) { + const int i = threadIdx.x + it * THREADS, st = i / (ROWS / 8), c = i % (ROWS / 8); + vr[it] = *reinterpret_cast(v + (static_cast(min(t0 + st, W - 1)) * hv + head) * DV + + col0 + 8 * c); + } + if (threadIdx.x < STEPS) { + const size_t at = static_cast(min(t0 + static_cast(threadIdx.x), W - 1)) * hv + head; + gr = g[at]; + br = beta[at]; + } + } + + __device__ __forceinline__ void store(float4 (*ks)[DK / 4], float4 (*qs)[DK / 4], float* gs, float* bs, + __nv_bfloat16 (*vs)[ROWS]) const { +#pragma unroll + for (int it = 0; it < KQ; ++it) { + const int i = threadIdx.x + it * THREADS; + ks[i / (DK / 4)][i % (DK / 4)] = widen(kr[it]); + qs[i / (DK / 4)][i % (DK / 4)] = widen(qr[it]); + } +#pragma unroll + for (int it = 0; it < VV; ++it) { + const int i = threadIdx.x + it * THREADS; + *reinterpret_cast(&vs[i / (ROWS / 8)][8 * (i % (ROWS / 8))]) = vr[it]; + } + if (threadIdx.x < STEPS) { + gs[threadIdx.x] = gr; + bs[threadIdx.x] = br; + } + } +}; + +template +__global__ void __launch_bounds__(2 * ROWS) chain_kernel( + const QK* __restrict__ q, const QK* __restrict__ k, const __nv_bfloat16* __restrict__ v, + const float* __restrict__ g, const float* __restrict__ beta, const float* __restrict__ state, + float* __restrict__ last, __nv_bfloat16* __restrict__ y, int W, int hk, int hv) { + __shared__ float4 ks[STEPS][DK / 4], qs[STEPS][DK / 4]; + __shared__ float gs[STEPS], bs[STEPS]; + __shared__ __align__(16) __nv_bfloat16 vs[STEPS][ROWS]; + const int head = blockIdx.x, row = blockIdx.y * ROWS + (threadIdx.x >> 1), half = threadIdx.x & 1; + const int key_head = head / (hv / hk); + float s[64]; + const float* s0 = state + (static_cast(head) * DV + row) * DK; +#pragma unroll + for (int j = 0; j < 16; ++j) { + const float4 t = *reinterpret_cast(s0 + 8 * j + 4 * half); + s[4 * j] = t.x; s[4 * j + 1] = t.y; s[4 * j + 2] = t.z; s[4 * j + 3] = t.w; + } + Prefetch pf; // each stage's loads in flight during the one before + pf.fetch(q, k, v, g, beta, 0, W, hk, hv, head, key_head, blockIdx.y * ROWS); + for (int t0 = 0; t0 < W; t0 += STEPS) { + const int n = min(STEPS, W - t0); + __syncthreads(); + pf.store(ks, qs, gs, bs, vs); + __syncthreads(); + if (t0 + STEPS < W) pf.fetch(q, k, v, g, beta, t0 + STEPS, W, hk, hv, head, key_head, blockIdx.y * ROWS); + for (int tt = 0; tt < n; ++tt) { + const float gt = gs[tt], bt = bs[tt]; + const size_t vat = (static_cast(t0 + tt) * hv + head) * DV + row; + const float vt = __bfloat162float(vs[tt][threadIdx.x >> 1]); + float m[4] = {0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (int j = 0; j < 16; ++j) { + const float4 kk = ks[tt][2 * j + half]; + s[4 * j] = s[4 * j] * gt; + s[4 * j + 1] = s[4 * j + 1] * gt; + s[4 * j + 2] = s[4 * j + 2] * gt; + s[4 * j + 3] = s[4 * j + 3] * gt; + m[0] = __fmaf_rn(s[4 * j], kk.x, m[0]); + m[1] = __fmaf_rn(s[4 * j + 1], kk.y, m[1]); + m[2] = __fmaf_rn(s[4 * j + 2], kk.z, m[2]); + m[3] = __fmaf_rn(s[4 * j + 3], kk.w, m[3]); + } + float mem = (m[0] + m[1]) + (m[2] + m[3]); + mem = mem + __shfl_xor_sync(0xffffffffu, mem, 1); + const float delta = (vt - mem) * bt; + float o[4] = {0.0f, 0.0f, 0.0f, 0.0f}; +#pragma unroll + for (int j = 0; j < 16; ++j) { + const float4 kk = ks[tt][2 * j + half], qq = qs[tt][2 * j + half]; + s[4 * j] = __fmaf_rn(kk.x, delta, s[4 * j]); + s[4 * j + 1] = __fmaf_rn(kk.y, delta, s[4 * j + 1]); + s[4 * j + 2] = __fmaf_rn(kk.z, delta, s[4 * j + 2]); + s[4 * j + 3] = __fmaf_rn(kk.w, delta, s[4 * j + 3]); + o[0] = __fmaf_rn(s[4 * j], qq.x, o[0]); + o[1] = __fmaf_rn(s[4 * j + 1], qq.y, o[1]); + o[2] = __fmaf_rn(s[4 * j + 2], qq.z, o[2]); + o[3] = __fmaf_rn(s[4 * j + 3], qq.w, o[3]); + } + float out = (o[0] + o[1]) + (o[2] + o[3]); + out = out + __shfl_xor_sync(0xffffffffu, out, 1); + if (half == 0) y[vat] = __float2bfloat16_rn(out); + } + } + float* s1 = last + (static_cast(head) * DV + row) * DK; +#pragma unroll + for (int j = 0; j < 16; ++j) + *reinterpret_cast(s1 + 8 * j + 4 * half) = make_float4(s[4 * j], s[4 * j + 1], s[4 * j + 2], + s[4 * j + 3]); +} + +} // namespace tf_gdn_prefill + +// The instantiations gdn_prefill_cuda launches for bf16 keys: 128 value rows a block, or 64 when heads < SMs. +template __global__ void tf_gdn_prefill::chain_kernel<__nv_bfloat16, 128, 32>(const __nv_bfloat16*, + const __nv_bfloat16*, const __nv_bfloat16*, const float*, const float*, const float*, float*, __nv_bfloat16*, int, + int, int); +template __global__ void tf_gdn_prefill::chain_kernel<__nv_bfloat16, 64, 32>(const __nv_bfloat16*, + const __nv_bfloat16*, const __nv_bfloat16*, const float*, const float*, const float*, float*, __nv_bfloat16*, int, + int, int); diff --git a/zig/kernels/cuda/prefill_attention.cu b/zig/kernels/cuda/prefill_attention.cu index 122f404495..cd9a012fc3 100644 --- a/zig/kernels/cuda/prefill_attention.cu +++ b/zig/kernels/cuda/prefill_attention.cu @@ -243,3 +243,6 @@ pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restric // Head dim 128, eight warps, eight query heads a block, eight staging slots (Nemotron's 16 heads a KV head). template __global__ void tf_prefill_attention::pattn_kernel<128, 8, 8, 8>(const __nv_bfloat16*, const __nv_bfloat16*, const __nv_bfloat16*, __nv_bfloat16*, int, int, int, int, int, float); +// Head dim 256, two query heads a block, two staging slots beside the staged queries (Qwen3.8's 6 heads a KV head). +template __global__ void tf_prefill_attention::pattn_kernel<256, 8, 2, 2>(const __nv_bfloat16*, + const __nv_bfloat16*, const __nv_bfloat16*, __nv_bfloat16*, int, int, int, int, int, float); diff --git a/zig/kernels/cuda/qmm_group.cu b/zig/kernels/cuda/qmm_group.cu index f3f65e184f..9c97ddfa71 100644 --- a/zig/kernels/cuda/qmm_group.cu +++ b/zig/kernels/cuda/qmm_group.cu @@ -339,3 +339,10 @@ __global__ void __launch_bounds__(WM * WN * 32) group_kernel( // The instantiations Nemotron's windows launch on sm_121 (tile 2: rows <= 16, bf16 out). template __global__ void tf_qmm_group::group_kernel<64, 16, 64, 1, 4, 8, false, false, false, false>( const __nv_bfloat16*, const float*, const __grid_constant__ tf_qmm_group::Parts, int, int, int, int, int); +// Tiles 3-5 (rows <= 32, <= 64, more), which a GB10 takes for wider verify rounds; tiles never change bits. +template __global__ void tf_qmm_group::group_kernel<64, 32, 64, 1, 4, 4, false, false, false, false>( + const __nv_bfloat16*, const float*, const __grid_constant__ tf_qmm_group::Parts, int, int, int, int, int); +template __global__ void tf_qmm_group::group_kernel<64, 64, 64, 1, 4, 4, false, false, false, false>( + const __nv_bfloat16*, const float*, const __grid_constant__ tf_qmm_group::Parts, int, int, int, int, int); +template __global__ void tf_qmm_group::group_kernel<64, 64, 128, 2, 4, 3, false, false, false, false>( + const __nv_bfloat16*, const float*, const __grid_constant__ tf_qmm_group::Parts, int, int, int, int, int); diff --git a/zig/kernels/cuda/qmm_prefill.cu b/zig/kernels/cuda/qmm_prefill.cu index 1610b22704..b90d3eb2af 100644 --- a/zig/kernels/cuda/qmm_prefill.cu +++ b/zig/kernels/cuda/qmm_prefill.cu @@ -151,3 +151,7 @@ __global__ void __launch_bounds__(WM * WN * 32) prefill_kernel( template __global__ void tf_qmm_prefill::prefill_kernel<64, 128, 128, 2, 4, 3, false>( const __nv_bfloat16*, const uint32_t*, const __nv_bfloat16*, const __nv_bfloat16*, void*, int, int, int, int, int, int); +// prompt_tile 9 (the 27B's: 128x128 on 2x2 warps, 2 stages, two blocks an SM on a GB10), bf16 out. +template __global__ void tf_qmm_prefill::prefill_kernel<64, 128, 128, 2, 2, 2, false>( + const __nv_bfloat16*, const uint32_t*, const __nv_bfloat16*, const __nv_bfloat16*, void*, int, int, int, + int, int, int); diff --git a/zig/kernels/cuda/qwen_ops.cu b/zig/kernels/cuda/qwen_ops.cu new file mode 100644 index 0000000000..778cc1f16e --- /dev/null +++ b/zig/kernels/cuda/qwen_ops.cu @@ -0,0 +1,68 @@ +// The 27B's own small kernels: batched state copies, the drafter's taps and its fp32 RMSNorm. +#include +#include + +struct TfCopy { + unsigned long long dst, src, bytes; // 16-byte aligned pointers, bytes a multiple of 16 +}; + +// One block an item, its threads striding through the item's 16-byte words. +extern "C" __global__ void __launch_bounds__(256) tf_copy_batch(const TfCopy* __restrict__ items) { + const TfCopy it = items[blockIdx.x]; + const uint4* src = reinterpret_cast(it.src); + uint4* dst = reinterpret_cast(it.dst); + const unsigned long long n = it.bytes / 16; + for (unsigned long long i = threadIdx.x + static_cast(blockIdx.y) * blockDim.x; i < n; + i += static_cast(blockDim.x) * gridDim.y) + dst[i] = src[i]; +} + +// forward.tree_forward's taps: (x.float() + pending.float()).to(bfloat16), row r into columns [col, col + D) of OUT. +extern "C" __global__ void __launch_bounds__(256) tf_tap(const __nv_bfloat16* __restrict__ x, + const __nv_bfloat16* __restrict__ pending, + __nv_bfloat16* __restrict__ out, int D, int out_stride, int col) { + const long long row = blockIdx.x; + for (int i = threadIdx.x + blockIdx.y * blockDim.x; i < D; i += blockDim.x * gridDim.y) + out[row * out_stride + col + i] = + __float2bfloat16_rn(__bfloat162float(x[row * D + i]) + __bfloat162float(pending[row * D + i])); +} + +// The drafter's RMSNorm in fp32: y = bf16(x * rsqrt(mean(x^2) + eps) * w), a 256-thread block a row. +extern "C" __global__ void __launch_bounds__(256) tf_rms_norm(const __nv_bfloat16* __restrict__ x, int x_stride, + const __nv_bfloat16* __restrict__ w, + __nv_bfloat16* __restrict__ y, int D, float eps) { + __shared__ float part[8]; + const long long row = blockIdx.x; + const __nv_bfloat16* xr = x + row * x_stride; + float s = 0.0f; + for (int i = threadIdx.x; i < D; i += 256) { + const float v = __bfloat162float(xr[i]); + s += v * v; + } + for (int off = 16; off > 0; off >>= 1) s += __shfl_xor_sync(0xFFFFFFFFu, s, off); + if ((threadIdx.x & 31) == 0) part[threadIdx.x >> 5] = s; + __syncthreads(); + float total = 0.0f; + for (int k = 0; k < 8; ++k) total += part[k]; + const float inv = rsqrtf(total / D + eps); + for (int i = threadIdx.x; i < D; i += 256) + y[row * D + i] = __float2bfloat16_rn(__bfloat162float(xr[i]) * inv * __bfloat162float(w[i])); +} + +// The drafter's inverse frequencies as dflash2 makes them on the GPU: 1 / theta ** (2i / dim), fp32 powf then a divide. +extern "C" __global__ void tf_inv_freq(float* __restrict__ inv, float theta, int half, int dim) { + const int i = threadIdx.x; + if (i < half) inv[i] = 1.0f / powf(theta, static_cast(i) * 2.0f / static_cast(dim)); +} + +// dflash2._rotary and add_taps_streams' tables: cos and sin of float(position) * inv_freq, fp32, a block a row. +extern "C" __global__ void tf_rotary(const int* __restrict__ pos, const float* __restrict__ inv, + float* __restrict__ cos_out, float* __restrict__ sin_out, int half) { + const int row = blockIdx.x; + const float p = static_cast(pos[row]); + for (int i = threadIdx.x; i < half; i += blockDim.x) { + const float phase = p * inv[i]; + cos_out[row * half + i] = cosf(phase); + sin_out[row * half + i] = sinf(phase); + } +} diff --git a/zig/src/cli/qwen27_cuda.zig b/zig/src/cli/qwen27_cuda.zig new file mode 100644 index 0000000000..a92416ce7b --- /dev/null +++ b/zig/src/cli/qwen27_cuda.zig @@ -0,0 +1,459 @@ +//! tf-qwen27-cuda: the 27B's Zig CUDA engine against the Python engine's capture (prompts.json, results.json). + +const std = @import("std"); +const cuda = @import("cuda"); +const qwen = @import("qwen27"); + +const usage = + \\usage: tf-qwen27-cuda run MODEL --kernels DIR --prompts prompts.json [--expect results.json] + \\ [--max-tokens N] [--context N] [--device N] + \\ Greedy, drafts off: each prompt's tokens and their sha against the capture's. + \\ tf-qwen27-cuda windows MODEL --kernels DIR --prompts windows.json + \\ capture_windows.py's chains and shared rounds: each row's logits against the capture's sha256. + \\ tf-qwen27-cuda draft MODEL --kernels DIR --drafter DIR --prompts prompts.json [--expect draft.json] + \\ draft_decode with DFlash2, greedy: tokens (equal to serial) and rounds against capture_draft.py's. + \\ tf-qwen27-cuda rounds MODEL --kernels DIR --prompts windows.json [--max-tokens ROUNDS] + \\ Greedy shared rounds of one row a stream over every prompt, timed. + \\ tf-qwen27-cuda multi MODEL --kernels DIR --drafter DIR --prompts multi.json + \\ capture_multi.py's drafted shared rounds: each round's windows and every reply against the capture's. + \\ +; + +const Options = struct { + model: []const u8, + kernels: []const u8 = "", + prompts: []const u8 = "", + expect: ?[]const u8 = null, + max_tokens: usize = 128, + context: usize = 16384, + device: c_int = 0, + dump: ?[]const u8 = null, // windows: a case's per-norm rows into this folder + cases: ?[]const u8 = null, // windows: only these cases, comma separated + drafter: ?[]const u8 = null, // draft: the DFlash2 checkpoint +}; + +pub fn main(init: std.process.Init) !u8 { + const gpa = init.gpa; + const arena = init.arena.allocator(); + const args = try init.minimal.args.toSlice(arena); + const windows = args.len >= 2 and std.mem.eql(u8, args[1], "windows"); + const rounds = args.len >= 2 and std.mem.eql(u8, args[1], "rounds"); + const drafting = args.len >= 2 and std.mem.eql(u8, args[1], "draft"); + const multi = args.len >= 2 and std.mem.eql(u8, args[1], "multi"); + if (args.len < 3 or !(std.mem.eql(u8, args[1], "run") or windows or rounds or drafting or multi)) { + std.debug.print("{s}", .{usage}); + return 2; + } + var o: Options = .{ .model = args[2] }; + var i: usize = 3; + while (i + 1 < args.len) : (i += 2) { + const k = args[i]; + const v = args[i + 1]; + if (std.mem.eql(u8, k, "--kernels")) o.kernels = v else if (std.mem.eql(u8, k, "--prompts")) o.prompts = v else if (std.mem.eql(u8, k, "--expect")) o.expect = v else if (std.mem.eql(u8, k, "--max-tokens")) o.max_tokens = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--context")) o.context = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--device")) o.device = try std.fmt.parseInt(c_int, v, 10) else if (std.mem.eql(u8, k, "--dump")) o.dump = v else if (std.mem.eql(u8, k, "--cases")) o.cases = v else if (std.mem.eql(u8, k, "--drafter")) o.drafter = v else { + std.debug.print("unknown option {s}\n{s}", .{ k, usage }); + return 2; + } + } + if (o.kernels.len == 0 or o.prompts.len == 0) { + std.debug.print("{s}", .{usage}); + return 2; + } + const prompts = try readJson(arena, init.io, o.prompts); + const expect = if (o.expect) |p| try readJson(arena, init.io, p) else null; + var driver = try cuda.Driver.open(); + defer driver.close(); + var ctx = try cuda.Context.init(&driver, o.device); + defer ctx.deinit(); + const clock = std.Io.Clock.awake; + var t0 = clock.now(init.io).toNanoseconds(); + const e = try qwen.Engine.init(gpa, init.io, &ctx, o.model, o.kernels, .{ .context = o.context }); + defer e.deinit(); + std.debug.print("loaded in {d:.1}s, weights {d:.2} GiB\n", .{ @as(f64, @floatFromInt(clock.now(init.io).toNanoseconds() - t0)) / 1e9, @as(f64, @floatFromInt(e.w.bytes())) / (1 << 30) }); + if (windows) return checkWindows(gpa, arena, init.io, e, prompts, o.dump, o.cases); + if (rounds) return benchRounds(gpa, arena, init.io, e, prompts, o.max_tokens); + if (drafting) return runDrafted(gpa, arena, init.io, e, o, prompts, expect); + if (multi) return runMulti(gpa, arena, init.io, e, o, prompts); + var seq = try e.sequence(o.context); + defer seq.deinit(); + var mismatches: usize = 0; + var it = prompts.object.iterator(); + while (it.next()) |entry| { + const name = entry.key_ptr.*; + const ids = try tokenList(arena, entry.value_ptr.*); + var out: std.ArrayList(u32) = .empty; + defer out.deinit(gpa); + t0 = clock.now(init.io).toNanoseconds(); + try e.generate(&seq, ids, o.max_tokens, &out); + const seconds = @as(f64, @floatFromInt(clock.now(init.io).toNanoseconds() - t0)) / 1e9; + const sha = try sha12(arena, out.items); + var verdict: []const u8 = ""; + if (expect) |x| blk: { + const want_v = (x.object.get("results") orelse break :blk).object.get(name) orelse break :blk; + const want = try tokenList(arena, want_v.object.get("tokens").?); + const same = std.mem.eql(u32, want, out.items); + if (!same) mismatches += 1; + var first: usize = 0; + while (first < @min(want.len, out.items.len) and want[first] == out.items[first]) first += 1; + verdict = if (same) " MATCH" else try std.fmt.allocPrint(arena, " DIFFER at token {d} (python sha {s})", .{ first, try sha12(arena, want) }); + } + std.debug.print("{s}: prompt {d}, {d} tokens sha {s} in {d:.2}s{s}\n", .{ name, ids.len, out.items.len, sha, seconds, verdict }); + } + return if (mismatches == 0) 0 else 1; +} + +fn readJson(a: std.mem.Allocator, io: std.Io, path: []const u8) !std.json.Value { + const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, a, .limited(1 << 30)); + return (try std.json.parseFromSliceLeaky(std.json.Value, a, bytes, .{})); +} + +fn tokenList(a: std.mem.Allocator, v: std.json.Value) ![]u32 { + if (v != .array) return error.BadPromptFile; + const out = try a.alloc(u32, v.array.items.len); + for (v.array.items, out) |x, *t| t.* = @intCast(x.integer); + return out; +} + +/// The capture's sha12: sha256 of json.dumps(tokens), "[1, 2, 3]", its first 12 hex digits. +fn sha12(a: std.mem.Allocator, tokens: []const u32) ![]const u8 { + var text: std.ArrayList(u8) = .empty; + try text.append(a, '['); + for (tokens, 0..) |t, i| { + if (i > 0) try text.appendSlice(a, ", "); + try text.print(a, "{d}", .{t}); + } + try text.append(a, ']'); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(text.items, &digest, .{}); + const hex = std.fmt.bytesToHex(digest, .lower); + return a.dupe(u8, hex[0..12]); +} + +/// Each case's streams prefilled afresh, its round run, every row's logits hashed; shared cases commit and go on. +fn checkWindows(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.Engine, doc: std.json.Value, dump: ?[]const u8, only: ?[]const u8) !u8 { + const root = doc.object; + const prompts = root.get("prompts").?.array.items; + const cases = root.get("cases").?.array.items; + var bad: usize = 0; + const row_bytes = qwen.config.vocab * 2; + const host = try gpa.alloc(u8, row_bytes); + defer gpa.free(host); + for (cases, 0..) |case, ci| { + if (only) |list| { + var it = std.mem.splitScalar(u8, list, ','); + const listed = while (it.next()) |x| { + if (std.fmt.parseInt(usize, x, 10) catch null == ci) break true; + } else false; + if (!listed) continue; + } + const streams = case.object.get("streams").?.array.items; + std.debug.print("case {d}: {s}, {d} streams\n", .{ ci, case.object.get("kind").?.string, streams.len }); + var seqs: [qwen.state.max_streams]qwen.state.Seq = undefined; + var parts: [qwen.state.max_streams]qwen.Part = undefined; + var made: usize = 0; + defer for (seqs[0..made]) |*sq| sq.deinit(); + for (streams, 0..) |sv, k| { + const sid: usize = @intCast(sv.object.get("stream").?.integer); + const ids = try tokenList(a, prompts[sid]); + seqs[k] = try e.sequence(4096); + made += 1; + try e.prefill(&seqs[k], ids); + parts[k] = .{ .seq = &seqs[k], .tokens = try tokenList(a, sv.object.get("tokens").?) }; + if (sv.object.get("parents")) |pv| { + const ps = try a.alloc(i32, pv.array.items.len); + for (pv.array.items, ps) |x, *y| y.* = @intCast(x.integer); + parts[k].parents = ps; + } + } + const n = streams.len; + var f = e.forward(); + if (dump) |dir| f.dump = .{ .io = io, .dir = dir }; + try f.round(parts[0..n]); + f.dump = null; + bad += try compareRows(&f, host, case.object.get("rows").?.array.items, ci, "rows"); + if (case.object.get("paths")) |paths_v| { + var paths: [qwen.state.max_streams][]const u32 = undefined; + var next: [qwen.state.max_streams]u32 = undefined; + for (paths_v.array.items, 0..) |pv, k| paths[k] = try tokenList(a, pv); + try f.commit(parts[0..n], paths[0..n]); + for (case.object.get("next").?.array.items, 0..) |nv, k| { + next[k] = @intCast(nv.integer); + parts[k].tokens = next[k .. k + 1]; + parts[k].parents = null; + } + try f.round(parts[0..n]); + bad += try compareRows(&f, host, case.object.get("after").?.array.items, ci, "after"); + } + if (case.object.get("keep")) |keep_v| { + var kept: [qwen.state.max_streams]usize = undefined; + var next: [qwen.state.max_streams]u32 = undefined; + const prefix = [_]u32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + var paths: [qwen.state.max_streams][]const u32 = undefined; + for (keep_v.array.items, 0..) |kv, k| { + kept[k] = @intCast(kv.integer); + paths[k] = prefix[0..kept[k]]; + } + try f.commit(parts[0..n], paths[0..n]); + const reply_rows = doc.object.get("replies").?.array.items; + for (streams, 0..) |sv, k| { + const sid: usize = @intCast(sv.object.get("stream").?.integer); + next[k] = @intCast(reply_rows[sid].array.items[kept[k]].integer); + parts[k].tokens = next[k .. k + 1]; + } + try f.round(parts[0..n]); + bad += try compareRows(&f, host, case.object.get("after").?.array.items, ci, "after"); + } + } + std.debug.print("{d} cases, {d} rows differ\n", .{ cases.len, bad }); + return if (bad == 0) 0 else 1; +} + +fn compareRows(f: *qwen.Forward, host: []u8, want: []const std.json.Value, case: usize, what: []const u8) !usize { + var bad: usize = 0; + for (want, 0..) |rv, r| { + try f.ops.download(host, f.s.logits + r * host.len); + try f.ops.s.synchronize(); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(host, &digest, .{}); + const hex = std.fmt.bytesToHex(digest, .lower); + if (!std.mem.eql(u8, &hex, rv.object.get("sha").?.string)) { + std.debug.print("case {d} {s} row {d}: logits differ\n", .{ case, what, r }); + bad += 1; + } + } + return bad; +} + +/// Every prompt prefilled, then greedy shared rounds of one row a stream; ms a round and tokens a second. +fn benchRounds(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.Engine, doc: std.json.Value, count: usize) !u8 { + { + // the served engine's cost curve: the forward's ms by a round's rows + var lanes_ = try qwen.Lanes.init(gpa, e, qwen.state.max_streams); + defer lanes_.deinit(); + try lanes_.calibrate(io, qwen.state.max_streams); + for (lanes_.costs[0..lanes_.cost_count]) |cst| std.debug.print("{d} rows: {d:.2} ms\n", .{ cst.width, cst.ms }); + } + const prompts = doc.object.get("prompts").?.array.items; + const n = @min(prompts.len, qwen.state.max_streams); + var seqs: [qwen.state.max_streams]qwen.state.Seq = undefined; + var parts: [qwen.state.max_streams]qwen.Part = undefined; + var next: [qwen.state.max_streams]u32 = undefined; + var ones: [qwen.state.max_streams][]const u32 = @splat(&.{0}); + var made: usize = 0; + defer for (seqs[0..made]) |*sq| sq.deinit(); + var f = e.forward(); + for (prompts[0..n], 0..) |pv, k| { + seqs[k] = try e.sequence(4096); + made += 1; + try e.prefill(&seqs[k], try tokenList(a, pv)); + try f.head(&seqs[k]); + var pick: [1]u32 = undefined; + try f.picks(1, &pick); + next[k] = pick[0]; + } + const clock = std.Io.Clock.awake; + const t0 = clock.now(io).toNanoseconds(); + for (0..count) |_| { + for (0..n) |k| parts[k] = .{ .seq = &seqs[k], .tokens = next[k .. k + 1] }; + try f.round(parts[0..n]); + try f.picks(n, next[0..n]); + try f.commit(parts[0..n], ones[0..n]); + } + try f.ops.s.synchronize(); + const ms = @as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t0)) / 1e6; + std.debug.print("{d} streams, {d} rounds: {d:.2} ms a round, {d:.1} tok/s\n", .{ n, count, ms / @as(f64, @floatFromInt(count)), @as(f64, @floatFromInt(n * count)) / ms * 1e3 }); + return 0; +} + +/// Each prompt drafted with DFlash2 as capture_draft.py runs it: tokens, rounds and accepted drafts. +fn runDrafted(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.Engine, o: Options, prompts: std.json.Value, expect: ?std.json.Value) !u8 { + const clock = std.Io.Clock.awake; + var t0 = clock.now(io).toNanoseconds(); + const f0 = e.forward(); + const d = try qwen.draft.DFlash2.init(gpa, io, f0.ops, f0.t, &e.w, o.drafter orelse return error.NoDrafter, o.model); + defer d.deinit(); + std.debug.print("drafter loaded in {d:.1}s\n", .{@as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t0)) / 1e9}); + var ctx = try qwen.draft.Context.init(f0.ops.k.d); + defer ctx.deinit(); + var seq = try e.sequence(o.context); + defer seq.deinit(); + var bad: usize = 0; + var it = prompts.object.iterator(); + while (it.next()) |entry| { + const name = entry.key_ptr.*; + const ids = try tokenList(a, entry.value_ptr.*); + var out: std.ArrayList(u32) = .empty; + defer out.deinit(gpa); + var stats: qwen.lone.Stats = .{}; + t0 = clock.now(io).toNanoseconds(); + try qwen.lone.prefill(e, &seq, &ctx, d, ids); + var f = e.forward(); + try f.head(&seq); + var pick: [1]u32 = undefined; + try f.picks(1, &pick); + const t1 = clock.now(io).toNanoseconds(); + try qwen.lone.decode(e, &seq, &ctx, d, ids, pick[0], o.max_tokens, 12, &out, &stats); + const secs = @as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t1)) / 1e9; + var verdict: []const u8 = ""; + if (expect) |x| blk: { + const want_v = (x.object.get("results") orelse break :blk).object.get(name) orelse break :blk; + const want = try tokenList(a, want_v.object.get("tokens").?); + const same = std.mem.eql(u32, want, out.items); + if (!same) bad += 1; + verdict = try std.fmt.allocPrint(a, " {s}; python rounds {d} accepted {d}", .{ if (same) "MATCH" else "DIFFER", want_v.object.get("rounds").?.integer, want_v.object.get("accepted").?.integer }); + } + std.debug.print("{s}: {d} tokens sha {s}, rounds {d} accepted {d}, {d:.1} tok/s{s}\n", .{ name, out.items.len, try sha12(a, out.items), stats.rounds, stats.accepted, @as(f64, @floatFromInt(out.items.len)) / secs, verdict }); + } + return if (bad == 0) 0 else 1; +} + +/// One stream of the multi check: its state, drafting, context (prompt and reply) and reply. +const MultiStream = struct { + seq: qwen.state.Seq, + d: qwen.drafts.Stream, + context: std.ArrayList(u32) = .empty, + out: std.ArrayList(u32) = .empty, + done: bool = false, + rounds: usize = 0, + accepted: usize = 0, +}; + +/// MultiDecoder's rounds as capture_multi.py runs them (greedy, no costs: whole trees, blocks of 16 rows). +fn runMulti(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.Engine, o: Options, doc: std.json.Value) !u8 { + const clock = std.Io.Clock.awake; + const f0 = e.forward(); + const d = try qwen.draft.DFlash2.init(gpa, io, f0.ops, f0.t, &e.w, o.drafter orelse return error.NoDrafter, o.model); + defer d.deinit(); + const prompts = doc.object.get("prompts").?.array.items; + const count: usize = @intCast(doc.object.get("max_tokens").?.integer); + const want_rounds = doc.object.get("rounds").?.array.items; + const S = prompts.len; + const ms = try a.alloc(MultiStream, S); + var made: usize = 0; + defer for (ms[0..made]) |*m| { + m.seq.deinit(); + m.d.deinit(); + m.context.deinit(gpa); + m.out.deinit(gpa); + }; + for (prompts, ms) |pv, *m| { + const ids = try tokenList(a, pv); + m.* = .{ .seq = try e.sequence(o.context), .d = qwen.drafts.Stream.init(gpa, try qwen.draft.Context.init(f0.ops.k.d)) }; + made += 1; + try qwen.lone.prefill(e, &m.seq, &m.d.ctx, d, ids); + var f = e.forward(); + try f.head(&m.seq); + var pick: [1]u32 = undefined; + try f.picks(1, &pick); + try m.context.appendSlice(gpa, ids); + try m.context.append(gpa, pick[0]); + try m.out.append(gpa, pick[0]); + } + var differ: usize = 0; + var round: usize = 0; + const t0 = clock.now(io).toNanoseconds(); + var tokens: [qwen.state.round_rows]u32 = undefined; + var parents: [qwen.state.round_rows]i32 = undefined; + var picks: [qwen.state.round_rows]u32 = undefined; + var path_rows: [qwen.state.round_rows]u32 = undefined; + var kept_rows: [qwen.state.round_rows]u32 = undefined; + while (true) : (round += 1) { + var live: [qwen.state.max_streams]usize = undefined; + var n: usize = 0; + for (ms, 0..) |m, k| if (!m.done) { + live[n] = k; + n += 1; + }; + if (n == 0) break; + var asks: [qwen.state.max_streams]qwen.drafts.Ask = undefined; + for (live[0..n], 0..) |k, j| asks[j] = .{ .x = &ms[k].d, .context = ms[k].context.items, .sampling = null }; + try qwen.drafts.propose(d, asks[0..n]); + // the windows: the pending token, then the drafts (tree parents shifted under it) + var parts: [qwen.state.max_streams]qwen.Part = undefined; + var starts: [qwen.state.max_streams + 1]usize = undefined; + var total: usize = 0; + const want = if (round < want_rounds.len) want_rounds[round].array.items else &.{}; + for (live[0..n], 0..) |k, j| { + const m = &ms[k]; + starts[j] = total; + tokens[total] = m.out.items[m.out.items.len - 1]; + parents[total] = -1; + for (m.d.tokens.items, m.d.parents.items, 1..) |t, p, r| { + tokens[total + r] = t; + parents[total + r] = if (p < 0) 0 else p + 1; + } + const rows = 1 + m.d.tokens.items.len; + parts[j] = .{ .seq = &m.seq, .tokens = tokens[total..][0..rows], .parents = parents[total..][0..rows] }; + if (j < want.len) { + const w = want[j].object; + const wt = try tokenList(a, w.get("tokens").?); + const wp = w.get("parents").?.array.items; + var same = @as(usize, @intCast(w.get("stream").?.integer)) == k and std.mem.eql(u32, wt, tokens[total..][0..rows]) and wp.len == rows; + if (same) for (wp, parents[total..][0..rows]) |x, y| { + if (x.integer != y) same = false; + }; + if (!same) { + differ += 1; + if (differ == 1) std.debug.print("round {d} stream {d}: first window that differs from Python's ({s} {d} rows, python {s} {d})\n", .{ round, k, @tagName(m.d.kind), rows, w.get("mode").?.string, wt.len }); + } + } else differ += 1; + total += rows; + } + starts[n] = total; + var f = e.forward(); + f.taps = e.scratch.taps; + try f.round(parts[0..n]); + try f.picks(total, &picks); + var paths: [qwen.state.max_streams][]const u32 = undefined; + var kept: [qwen.state.max_streams]qwen.drafts.Kept = undefined; + for (live[0..n], 0..) |k, j| { + const m = &ms[k]; + const base = starts[j]; + const rows = starts[j + 1] - base; + const room = count - m.out.items.len; + // accept: children equal to their parent's draw, within the room, none past an end token + var len: usize = 1; + path_rows[base] = 0; + var terminal = picks[base]; + while (len < room and !e.w.isEos(terminal)) { + const child = for (1..rows) |r| { + if (parents[base + r] == @as(i32, @intCast(path_rows[base + len - 1])) and tokens[base + r] == terminal) break r; + } else break; + path_rows[base + len] = @intCast(child); + len += 1; + terminal = picks[base + child]; + } + paths[j] = path_rows[base..][0..len]; + for (paths[j], kept_rows[base..][0..len]) |r, *g| g.* = @intCast(base + r); + kept[j] = .{ .x = &m.d, .rows = kept_rows[base..][0..len] }; + for (paths[j][1..]) |r| { + try m.out.append(gpa, tokens[base + r]); + try m.context.append(gpa, tokens[base + r]); + } + try m.out.append(gpa, terminal); + try m.context.append(gpa, terminal); + m.rounds += 1; + m.accepted += len - 1; + } + try f.commit(parts[0..n], paths[0..n]); + try qwen.drafts.absorb(d, f.ops, e.scratch.taps, e.scratch.copy_items, kept[0..n]); + for (live[0..n]) |k| { + const m = &ms[k]; + if (m.out.items.len >= count or e.w.isEos(m.out.items[m.out.items.len - 1])) m.done = true; + } + } + const secs = @as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t0)) / 1e9; + if (round != want_rounds.len) differ += 1; + var bad: usize = 0; + var produced: usize = 0; + const results = doc.object.get("results").?.array.items; + for (ms, results, 0..) |m, rv, k| { + const want = try tokenList(a, rv.object.get("tokens").?); + const same = std.mem.eql(u32, want, m.out.items); + if (!same or m.rounds != rv.object.get("rounds").?.integer) bad += 1; + produced += m.out.items.len; + std.debug.print("stream {d}: {d} tokens sha {s} rounds {d} accepted {d} {s}; python rounds {d} accepted {d}\n", .{ k, m.out.items.len, try sha12(a, m.out.items), m.rounds, m.accepted, if (same) "MATCH" else "DIFFER", rv.object.get("rounds").?.integer, rv.object.get("accepted").?.integer }); + } + // replies and every stream's rounds must match; a tree may order its nodes otherwise where drafts round otherwise + std.debug.print("{d} rounds (python {d}), {d} windows differ from Python's, {d} replies differ; {d:.1} tok/s\n", .{ round, want_rounds.len, differ, bad, @as(f64, @floatFromInt(produced)) / secs }); + return if (bad == 0 and round == want_rounds.len) 0 else 1; +} diff --git a/zig/src/cuda/kernels.zig b/zig/src/cuda/kernels.zig index 8c0a384051..ac6c026282 100644 --- a/zig/src/cuda/kernels.zig +++ b/zig/src/cuda/kernels.zig @@ -13,6 +13,7 @@ fn Blob(comptime import_name: []const u8) type { pub const available = options.with_kernels; pub const gdn: []const u8 = if (available) &Blob("fatbin_gdn").bytes else &.{}; +pub const gdn_prefill: []const u8 = if (available) &Blob("fatbin_gdn_prefill").bytes else &.{}; pub const probe: []const u8 = if (available) &Blob("fatbin_probe").bytes else &.{}; pub const qmm_group: []const u8 = if (available) &Blob("fatbin_qmm_group").bytes else &.{}; pub const qmm_prefill: []const u8 = if (available) &Blob("fatbin_qmm_prefill").bytes else &.{}; @@ -24,6 +25,7 @@ pub const scan_rows: []const u8 = if (available) &Blob("fatbin_scan_rows").bytes pub const nemotron_ops: []const u8 = if (available) &Blob("fatbin_nemotron_ops").bytes else &.{}; pub const lane_gemv: []const u8 = if (available) &Blob("fatbin_lane_gemv").bytes else &.{}; pub const affine4_pack: []const u8 = if (available) &Blob("fatbin_affine4_pack").bytes else &.{}; +pub const qwen_ops: []const u8 = if (available) &Blob("fatbin_qwen_ops").bytes else &.{}; pub const sample: []const u8 = if (available) &Blob("fatbin_sample").bytes else &.{}; pub const fp8_lane: []const u8 = if (available) &Blob("fatbin_fp8_lane").bytes else &.{}; pub const torch_argmax: []const u8 = if (available) &Blob("fatbin_torch_argmax").bytes else &.{}; @@ -39,6 +41,7 @@ pub const nemotron_attention: []const u8 = if (available) &Blob("fatbin_nemotron pub const nemotron_keyed: []const u8 = if (available) &Blob("fatbin_nemotron_keyed").bytes else &.{}; pub const train: []const u8 = if (available) &Blob("fatbin_train").bytes else &.{}; pub const train_mixers: []const u8 = if (available) &Blob("fatbin_train_mixers").bytes else &.{}; +pub const torch_norm_rotary: []const u8 = if (available) &Blob("fatbin_torch_norm_rotary").bytes else &.{}; /// Symbols in the gdn image as cuobjdump lists them for the built fatbin (named namespace tf_gdn). pub const gdn_symbols = struct { diff --git a/zig/src/families/qwen27/cuda.zig b/zig/src/families/qwen27/cuda.zig new file mode 100644 index 0000000000..3532c69cde --- /dev/null +++ b/zig/src/families/qwen27/cuda.zig @@ -0,0 +1,33 @@ +//! Qwen3.8-27B on CUDA in Zig: the Python 0.6.6 engine's kernels and layouts, beside the family's Metal engine. + +pub const config = @import("cuda/shape.zig"); +pub const kernels = @import("cuda/kernels.zig"); +pub const triton = @import("cuda/triton.zig"); +pub const weights = @import("cuda/weights.zig"); +pub const state = @import("cuda/state.zig"); +pub const Forward = @import("cuda/forward.zig").Forward; +pub const Part = @import("cuda/forward.zig").Part; +pub const engine = @import("cuda/engine.zig"); +pub const Engine = engine.Engine; +pub const Lanes = @import("cuda/lanes.zig").Cuda; +pub const native = @import("cuda/native.zig"); +pub const draft = @import("cuda/draft.zig"); +pub const lone = @import("cuda/lone.zig"); +pub const drafts = @import("cuda/drafts.zig"); + +test { + _ = config; + _ = kernels; + _ = triton; + _ = weights; + _ = state; + _ = engine; + _ = @import("cuda/forward.zig"); + _ = @import("cuda/lanes.zig"); + _ = @import("cuda/tree.zig"); + _ = @import("cuda/draft_load.zig"); + _ = @import("cuda/draft_policy.zig"); + _ = draft; + _ = lone; + _ = native; +} diff --git a/zig/src/families/qwen27/cuda/draft.zig b/zig/src/families/qwen27/cuda/draft.zig new file mode 100644 index 0000000000..587554ef05 --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft.zig @@ -0,0 +1,423 @@ +//! DFlash2 on CUDA as dflash2's fast path runs it: a context from the target's taps, a masked block, the host tree. + +const std = @import("std"); +const cuda = @import("cuda"); +const lanes = @import("lanes"); +const aot = cuda.aot; +const c = @import("shape.zig"); +const kern = @import("kernels.zig"); +const tri = @import("triton.zig"); +const st = @import("state.zig"); +const wts = @import("weights.zig"); +const dl = @import("draft_load.zig"); +const policy = @import("draft_policy.zig"); + +const bf = 2; +const H = dl.hidden; +const D = dl.head_dim; +pub const block = 16; // masked rows a round drafts at most (dflash2.block) +/// Block rows a batch takes: every stream's block in one forward (launch_blocks). +pub const most_rows = block * st.max_streams; +/// Copy items a batch takes: every layer's keys and values of every stream, two pieces a head (or the head's rows). +const most_copies = @max(dl.layers * st.max_streams * 2 * dl.kv_heads * 2, 2 * most_rows); + +/// A stream's drafting context: each layer's keys and values ([kv heads, len, head dim], two buffers to swap). +pub const Context = struct { + arena: st.Arena, + keys: [dl.layers][2]u64, + values: [dl.layers][2]u64, + cur: u1 = 0, + len: usize = 0, // rows held (at most dl.window) + end: usize = 0, // the target position after the last absorbed row + + pub fn bytes() usize { + return dl.layers * 4 * std.mem.alignForward(usize, dl.kv_heads * dl.window * D * bf, 256); + } + + pub fn init(d: *const cuda.Driver) !Context { + var x: Context = .{ .arena = try st.Arena.init(d, bytes()), .keys = undefined, .values = undefined }; + for (0..dl.layers) |l| for (0..2) |j| { + x.keys[l][j] = try x.arena.take(dl.kv_heads * dl.window * D * bf); + x.values[l][j] = try x.arena.take(dl.kv_heads * dl.window * D * bf); + }; + return x; + } + + pub fn deinit(x: *Context) void { + x.arena.deinit(); + } + + /// dflash2.skip: `n` rows go untapped, so the context starts again after them. + pub fn skip(x: *Context, n: usize) void { + x.len = 0; + x.end += n; + } +}; + +/// One stream's share of a batch absorb: its context and how many of the gathered tap rows are its own (in turn). +pub const Absorb = struct { ctx: *Context, rows: usize }; +/// One stream's block: its context and the pending token heading it. +pub const Block = struct { ctx: *const Context, pending: u32 }; + +pub const DFlash2 = struct { + gpa: std.mem.Allocator, + w: dl.Weights, + target: *const wts.Weights, + ops: kern.Ops, + t: tri.Tri, + arena: st.Arena, + inv_freq: u64, // (D / 2) fp32 on the device, as dflash2 makes them + positions: []i32, // a call's row positions on the host + // a block's buffers (block rows) and absorbing's (up to the window's rows) + x: u64, + normed: u64, + conv: u64, + dyn: u64, + xs: u64, + qkv: u64, + q: u64, + k: u64, + v: u64, + attn: u64, + o: u64, + gate: u64, + up: u64, + act: u64, + act_xs: u64, + head_h: u64, + selected: u64, + logits: u64, + logits_tail: u64, + logits32: u64, + cand_values: u64, + cand_ids: u64, + topk_scratch: u64, + pos: u64, + cos: u64, + sin: u64, + table: u64, + copies: u64, + ids: u64, + // host copies of a round's candidates + host_ids: []i64, + host_values: []f32, + host_selected: []u16, + cands: []u32, + unary: []f64, + projected: []f64, + tree: policy.Tree = .{}, + launched: usize = 0, // the last batch's block rows a stream (0: none) + + pub fn init(gpa: std.mem.Allocator, io: std.Io, ops: kern.Ops, t: tri.Tri, target: *const wts.Weights, dir: []const u8, target_dir: []const u8) !*DFlash2 { + const x = try gpa.create(DFlash2); + errdefer gpa.destroy(x); + x.* = undefined; + x.gpa = gpa; + x.ops = ops; + x.t = t; + x.target = target; + x.tree = .{}; + x.w = try dl.load(gpa, io, ops, dir, target_dir); + errdefer x.w.deinit(); + const R = dl.window + 1; // rows absorbed at once at most (the prompt's tail) + const W = most_rows; + x.arena = .{ .buf = null }; + for (0..2) |pass| { + if (pass == 1) x.arena = try st.Arena.init(ops.k.d, x.arena.used); + x.arena.used = 0; + const A = &x.arena; + x.x = try A.take(R * H * bf); + x.normed = try A.take(R * H * bf); + x.conv = try A.take(W * H * bf); + x.dyn = try A.take(W * 4 * (H / dl.conv_group) * bf); + x.xs = try A.take(R * (H * 5) / 64 * 4); + x.qkv = try A.take(R * (dl.heads + 2 * dl.kv_heads) * D * bf); + x.q = try A.take(W * dl.heads * D * bf); + x.k = try A.take(R * dl.kv_heads * D * bf); + x.v = try A.take(R * dl.kv_heads * D * bf); + x.attn = try A.take(W * dl.heads * D * bf); + x.o = try A.take(W * H * bf); + x.gate = try A.take(W * 17408 * bf); + x.up = try A.take(W * 17408 * bf); + x.act = try A.take(W * 17408 * bf); + x.act_xs = try A.take(W * 17408 / 64 * 4); + x.head_h = try A.take(W * H * bf); + x.selected = try A.take(W * dl.selector_rank * bf); + x.logits = try A.take(W * dl.head_rows * bf); + x.logits_tail = try A.take(W * (dl.spans[1][1] - dl.spans[1][0]) * bf); + x.logits32 = try A.take(W * dl.head_rows * 4); + x.cand_values = try A.take(W * policy.top * 4); + x.cand_ids = try A.take(W * policy.top * 8); + x.topk_scratch = try A.take(kern.torch_ops.topkScratchBytes(W, dl.head_rows)); + x.inv_freq = try A.take(D / 2 * 4); + x.pos = try A.take(R * 4); + x.cos = try A.take(R * D / 2 * 4); + x.sin = try A.take(R * D / 2 * 4); + x.table = try A.take(dl.layers * st.max_streams * 24); + x.copies = try A.take(most_copies * @sizeOf(kern.Copy)); + x.ids = try A.take(W * 4); + } + errdefer x.arena.deinit(); + { + var a: cuda.Args = .{}; + a.add(x.inv_freq); + a.add(@as(f32, @floatCast(dl.theta))); + a.add(@as(c_int, D / 2)); + a.add(@as(c_int, D)); + try cuda.launch.launch(ops.k.inv_freq, .{ .grid = .{ .x = 1 }, .block = .{ .x = D / 2 } }, ops.s, &a); + } + x.positions = try gpa.alloc(i32, dl.window + 1); + errdefer gpa.free(x.positions); + x.host_ids = try gpa.alloc(i64, W * policy.top); + x.host_values = try gpa.alloc(f32, W * policy.top); + x.host_selected = try gpa.alloc(u16, W * dl.selector_rank); + x.cands = try gpa.alloc(u32, W * policy.top); + x.unary = try gpa.alloc(f64, W * policy.top); + x.projected = try gpa.alloc(f64, W * dl.selector_rank); + return x; + } + + pub fn deinit(x: *DFlash2) void { + x.gpa.free(x.positions); + x.gpa.free(x.host_ids); + x.gpa.free(x.host_values); + x.gpa.free(x.host_selected); + x.gpa.free(x.cands); + x.gpa.free(x.unary); + x.gpa.free(x.projected); + x.tree.tokens.deinit(x.gpa); + x.tree.parents.deinit(x.gpa); + x.tree.scores.deinit(x.gpa); + x.arena.deinit(); + x.w.deinit(); + const gpa = x.gpa; + gpa.destroy(x); + } + + /// _lin: a packed projection of `rows` rows (each row's bits its own at any count), group sums first. + fn lin(x: *DFlash2, in: u64, q: kern.QLinear, out: u64, rows: usize) !void { + try x.t.groupSums(in, x.xs, q.k, rows, q.k); + try x.ops.dense(in, x.xs, q, out, rows); + } + + /// F.rms_norm over `rows` contiguous rows of the hidden width (the layers' and the head's norms). + fn torchNorm(x: *DFlash2, in: u64, weight: u64, out: u64, rows: usize) !void { + var a: cuda.Args = .{}; + a.add(in); + a.add(weight); + a.add(out); + a.add(@as(u64, 0)); + a.add(@as(f32, 1e-6)); + try cuda.launch.launch(x.ops.k.torch_rms, .{ .grid = .{ .x = @intCast(rows) }, .block = .{ .x = 128 } }, x.ops.s, &a); + } + + fn rmsNorm(x: *DFlash2, in: u64, stride: usize, weight: u64, out: u64, rows: usize) !void { + var a: cuda.Args = .{}; + a.add(in); + a.add(@as(c_int, @intCast(stride))); + a.add(weight); + a.add(out); + a.add(@as(c_int, H)); + a.add(@as(f32, 1e-6)); + try cuda.launch.launch(x.ops.k.rms_norm, .{ .grid = .{ .x = @intCast(rows) }, .block = .{ .x = 256 } }, x.ops.s, &a); + } + + /// dflash2._rotary: cos and sin of each run's positions [start, start + rows) times the inverse frequencies, fp32. + fn rotary(x: *DFlash2, runs: []const [2]usize) !void { + var rows: usize = 0; + for (runs) |run| for (0..run[1]) |j| { + x.positions[rows] = @intCast(run[0] + j); + rows += 1; + }; + try x.ops.upload(x.pos, std.mem.sliceAsBytes(x.positions[0..rows])); + var a: cuda.Args = .{}; + for ([_]u64{ x.pos, x.inv_freq, x.cos, x.sin }) |v| a.add(v); + a.add(@as(c_int, D / 2)); + try cuda.launch.launch(x.ops.k.rotary, .{ .grid = .{ .x = @intCast(rows) }, .block = .{ .x = D / 2 } }, x.ops.s, &a); + } + + /// dflash2._prep: q (heads, rows, D), k and v (kv heads, rows, D) from [q | k | v] rows, normed and rotated. + fn prep(x: *DFlash2, qkv: u64, layer: usize, rows: usize, with_q: bool, q: u64, k: u64, v: u64) !void { + const l = x.w.layers[layer]; + const h: usize = if (with_q) dl.heads else 0; + const stride = (h + 2 * dl.kv_heads) * D; + const p = aot.ptr; + try x.t.set.run(x.t.s, "_prep_kernel", .{ @intCast(rows), @intCast(h + 2 * dl.kv_heads), 1 }, &.{ + p("QKV", "*bf16", qkv), p("QN", "*bf16", l.q_norm), p("KN", "*bf16", l.k_norm), p("COS", "*fp32", x.cos), p("SIN", "*fp32", x.sin), + p("QO", "*bf16", if (with_q) q else qkv), p("KO", "*bf16", k), p("VO", "*bf16", v), aot.int("L", @intCast(rows)), aot.int("stride", @intCast(stride)), + aot.float("eps", 1e-6), + }, &.{ aot.ci("H", @intCast(h)), aot.ci("HKV", dl.kv_heads), aot.ci("HALF", D / 2) }); + } + + /// dflash2._dconv: the two-tap dynamic conv of each `seg`-row block, plus a residual. + fn dconv(x: *DFlash2, in: u64, dyn: u64, base: u64, branch: usize, res: ?u64, out: u64, rows: usize, seg: usize) !void { + const p = aot.ptr; + try x.t.set.run(x.t.s, "_dconv_kernel", .{ @intCast(rows), (H + 1023) / 1024, 1 }, &.{ + p("X", "*bf16", in), p("DYN", "*bf16", dyn), p("BASE", "*bf16", base), p("RES", "*bf16", res orelse in), p("OUT", "*bf16", out), aot.int("SEG", @intCast(seg)), + }, &.{ aot.ci("D", H), aot.ci("G", H / dl.conv_group), aot.ci("GS", dl.conv_group), aot.ci("BRANCH", @intCast(branch)), aot.ci("HAS_RES", @intFromBool(res != null)), aot.ci("BLOCK", 1024) }); + } + + /// add_taps: the target's taps (rows of 5 x hidden bf16) projected into the context's layers. + pub fn absorb(x: *DFlash2, ctx: *Context, taps: u64, rows: usize) !void { + return x.absorbMany(&.{.{ .ctx = ctx, .rows = rows }}, taps); + } + + /// add_taps_streams: every stream's rows (in turn) through each projection once, each context its last window. + pub fn absorbMany(x: *DFlash2, items: []const Absorb, taps: u64) !void { + var total: usize = 0; + for (items) |it| total += it.rows; + if (total == 0) return; + if (total > dl.window + 1 or items.len > st.max_streams) return error.TooManyDraftTaps; + try x.lin(taps, x.w.fc, x.qkv, total); // fc's (rows, hidden) output, in a buffer of the window's rows + try x.rmsNorm(x.qkv, H, x.w.hidden_norm, x.normed, total); + var runs: [st.max_streams][2]usize = undefined; + for (items, 0..) |it, j| runs[j] = .{ it.ctx.end, it.rows }; + try x.rotary(runs[0..items.len]); + // every layer's copies: the last `keep` rows of [context | new rows], a head at a time, both halves once + var list: std.ArrayList(kern.Copy) = .empty; + defer list.deinit(x.gpa); + var starts: [dl.layers + 1]usize = undefined; + for (0..dl.layers) |l| { + starts[l] = list.items.len; + var first: usize = 0; + for (items) |it| { + const ctx = it.ctx; + const keep = @min(dl.window, ctx.len + it.rows); + const old_n = ctx.len; + const drop = old_n + it.rows - keep; // rows of [old | new] that fall out of the window + const from_old = old_n -| drop; + const new_from = drop -| old_n; + const nxt: u1 = ctx.cur ^ 1; + for ([_][2]u64{ x.ctxPair(ctx.keys[l], ctx.cur, nxt), x.ctxPair(ctx.values[l], ctx.cur, nxt) }, [_]u64{ x.k, x.v }) |pair, fresh| { + for (0..dl.kv_heads) |h| { + if (from_old > 0) try list.append(x.gpa, .{ .dst = pair[1] + h * keep * D * bf, .src = pair[0] + (h * old_n + drop) * D * bf, .bytes = from_old * D * bf }); + try list.append(x.gpa, .{ .dst = pair[1] + (h * keep + from_old) * D * bf, .src = fresh + (h * total + first + new_from) * D * bf, .bytes = (it.rows - new_from) * D * bf }); + } + } + first += it.rows; + } + } + starts[dl.layers] = list.items.len; + if (list.items.len > most_copies) return error.TooManyDraftTaps; + try x.ops.upload(x.copies, std.mem.sliceAsBytes(list.items)); + for (0..dl.layers) |l| { + try x.lin(x.normed, x.w.layers[l].kv, x.qkv, total); + try x.prep(x.qkv, l, total, false, 0, x.k, x.v); + try x.ops.copies(x.copies + starts[l] * @sizeOf(kern.Copy), starts[l + 1] - starts[l]); + } + try x.ops.s.synchronize(); + for (items) |it| { + it.ctx.cur ^= 1; + it.ctx.len = @min(dl.window, it.ctx.len + it.rows); + it.ctx.end += it.rows; + } + } + + fn ctxPair(_: *DFlash2, bufs: [2]u64, cur: u1, nxt: u1) [2]u64 { + return .{ bufs[cur], bufs[nxt] }; + } + + /// launch_blocks: each stream's pending token then masks through the layers in one forward; candidates on the host. + pub fn launchMany(x: *DFlash2, blocks: []const Block, L: usize) !void { + const S = blocks.len; + const rows = S * L; + if (S == 0 or S > st.max_streams or L < 2 or L > block) return error.NoDraftContext; + for (blocks) |b| if (b.ctx.len == 0) return error.NoDraftContext; + var ids: [most_rows]i32 = @splat(dl.mask_id); + var runs: [st.max_streams][2]usize = undefined; + // each layer's table: the streams' (keys, values) pointers, then their context lengths + const per = std.mem.alignForward(usize, S * 20, 16); // 16-byte aligned tables, as fresh torch tensors are + var table: [dl.layers * (st.max_streams * 20 + 16)]u8 align(8) = undefined; + for (blocks, 0..) |b, j| { + ids[j * L] = @intCast(b.pending); + runs[j] = .{ b.ctx.end, L }; + for (0..dl.layers) |l| { + const at = table[l * per ..]; + std.mem.writeInt(u64, at[j * 16 ..][0..8], b.ctx.keys[l][b.ctx.cur], .little); + std.mem.writeInt(u64, at[j * 16 + 8 ..][0..8], b.ctx.values[l][b.ctx.cur], .little); + std.mem.writeInt(i32, at[S * 16 + j * 4 ..][0..4], @intCast(b.ctx.len), .little); + } + } + try x.ops.upload(x.ids, std.mem.sliceAsBytes(ids[0..rows])); + try x.ops.upload(x.table, table[0 .. dl.layers * per]); + const tw = x.target; + try x.t.embed(x.ids, tw.embed.w, tw.embed.s, tw.embed.b, x.x, rows, H); + try x.rotary(runs[0..S]); + const scale: f32 = @floatCast(std.math.pow(f64, D, -0.5)); + for (x.w.layers, 0..) |l, i| { + try x.torchNorm(x.x, l.in_norm, x.normed, rows); + try x.lin(x.normed, l.attn_proj, x.dyn, rows); + try x.dconv(x.normed, x.dyn, l.attn_base, 0, null, x.conv, rows, L); + try x.lin(x.conv, l.qkv, x.qkv, rows); + try x.prep(x.qkv, i, rows, true, x.q, x.k, x.v); + const tb = x.table + i * per; + const p = aot.ptr; + try x.t.set.run(x.t.s, "_block_attention", .{ @intCast(S), dl.kv_heads, 1 }, &.{ + p("Q", "*bf16", x.q), p("KB", "*bf16", x.k), p("VB", "*bf16", x.v), p("TABLE", "*i64", tb), p("LENS", "*i32", tb + S * 16), p("O", "*bf16", x.attn), + aot.float("scale", scale), aot.int("window", dl.window), aot.int("R", @intCast(rows)), + }, &.{ aot.ci("G", dl.heads / dl.kv_heads), aot.ci("HKV", dl.kv_heads), aot.ci("L", @intCast(L)), aot.ci("LP", @intCast(@max(16, std.math.ceilPowerOfTwo(usize, L) catch unreachable))), aot.ci("D", D), aot.ci("BN", 64), aot.ci("CAUSAL", 0) }); + try x.lin(x.attn, l.o, x.o, rows); + try x.dconv(x.o, x.dyn, l.attn_base, 1, x.x, x.x, rows, L); + try x.torchNorm(x.x, l.post_norm, x.normed, rows); + try x.lin(x.normed, l.mlp_proj, x.dyn, rows); + try x.dconv(x.normed, x.dyn, l.mlp_base, 0, null, x.conv, rows, L); + try x.t.groupSums(x.conv, x.xs, H, rows, H); + try x.ops.dense(x.conv, x.xs, l.gate, x.gate, rows); + try x.ops.dense(x.conv, x.xs, l.up, x.up, rows); + try x.t.swiglu(x.gate, x.up, x.act, x.act_xs, rows, l.gate.n); + try x.ops.dense(x.act, x.act_xs, l.down, x.o, rows); + try x.dconv(x.o, x.dyn, l.mlp_base, 1, x.x, x.x, rows, L); + } + // every block but its pending row, normed into stacked depths + const depths = L - 1; + const total = S * depths; + for (0..S) |j| try x.torchNorm(x.x + (j * L + 1) * H * bf, x.w.norm, x.head_h + j * depths * H * bf, depths); + try x.lin(x.head_h, x.w.select, x.selected, total); + // matmul_rows: the head's spans at the stacked rows' K split (1), then joined column-wise + try x.t.groupSums(x.head_h, x.xs, H, total, H); + const head = tw.head; + const first: kern.QLinear = .{ .w = head.w, .s = head.s, .b = head.b, .n = dl.spans[0][1], .k = head.k, .npad = head.npad }; + try x.ops.denseSplit(x.head_h, x.xs, first, x.logits32, total, 1); // a scratch of the right size + try x.ops.denseSplit(x.head_h, x.xs, x.w.head_tail, x.logits_tail, total, 1); + var items: std.ArrayList(kern.Copy) = .empty; + defer items.deinit(x.gpa); + const na = dl.spans[0][1]; + const nb = dl.spans[1][1] - dl.spans[1][0]; + for (0..total) |r| { + try items.append(x.gpa, .{ .dst = x.logits + r * dl.head_rows * bf, .src = x.logits32 + r * na * bf, .bytes = na * bf }); + try items.append(x.gpa, .{ .dst = x.logits + (r * dl.head_rows + na) * bf, .src = x.logits_tail + r * nb * bf, .bytes = nb * bf }); + } + try x.ops.upload(x.copies, std.mem.sliceAsBytes(items.items)); + try x.ops.copies(x.copies, items.items.len); + const torch = x.ops.torch(); + try torch.toF32(x.logits, x.logits32, total * dl.head_rows); + try torch.topk(x.logits32, dl.head_rows, total, policy.top, x.cand_values, x.cand_ids, x.topk_scratch); + try x.ops.download(std.mem.sliceAsBytes(x.host_values[0 .. total * policy.top]), x.cand_values); + try x.ops.download(std.mem.sliceAsBytes(x.host_ids[0 .. total * policy.top]), x.cand_ids); + try x.ops.download(std.mem.sliceAsBytes(x.host_selected[0 .. total * dl.selector_rank]), x.selected); + try x.ops.s.synchronize(); + for (0..total * policy.top) |j| { + const local: usize = @intCast(x.host_ids[j]); + x.cands[j] = @intCast(if (local < na) local else dl.spans[1][0] + local - na); + x.unary[j] = x.host_values[j]; + } + for (0..total * dl.selector_rank) |j| x.projected[j] = @as(f32, @bitCast(@as(u32, x.host_selected[j]) << 16)); + x.launched = L; + } + + /// finish_tree for the batch's `j`th block: best_first into `out` (parents -1: under the pending row). + pub fn finish(x: *DFlash2, j: usize, pending: u32, context_length: usize, max_nodes: usize, sampling: ?lanes.Sampling, out: *policy.Tree) !void { + const depths = x.launched - 1; + const at = j * depths; + try policy.bestFirst(x.gpa, .{ .ids = x.cands[at * policy.top ..][0 .. depths * policy.top], .unary = x.unary[at * policy.top ..][0 .. depths * policy.top], .projected = x.projected[at * dl.selector_rank ..][0 .. depths * dl.selector_rank], .depths = depths }, x.w.pred, x.w.succ, pending, @min(127, max_nodes), sampling, context_length, out); + } + + /// finish_tree: best_first over the launched block's candidates; nodes and parents (-1: under the pending row). + pub fn propose(x: *DFlash2, ctx: *const Context, pending: u32, context_length: usize, max_nodes: usize, sampling: ?lanes.Sampling) !struct { tokens: []const u32, parents: []const i32 } { + if (ctx.len == 0 or max_nodes == 0) return .{ .tokens = &.{}, .parents = &.{} }; + const rows: usize = @min(block, max_nodes + 1); + try x.launchMany(&.{.{ .ctx = ctx, .pending = pending }}, rows); + try x.finish(0, pending, context_length, max_nodes, sampling, &x.tree); + return .{ .tokens = x.tree.tokens.items, .parents = x.tree.parents.items }; + } +}; diff --git a/zig/src/families/qwen27/cuda/draft_load.zig b/zig/src/families/qwen27/cuda/draft_load.zig new file mode 100644 index 0000000000..8f9b0a7404 --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_load.zig @@ -0,0 +1,274 @@ +//! DFlash2's weights as dflash2 holds them: projections quantize4'd and tiled, codebooks fp32 on the host. + +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const kern = @import("kernels.zig"); +const st = @import("state.zig"); + +pub const QLinear = kern.QLinear; +const Tensor = core.checkpoint.Tensor; + +pub const hidden = 5120; +pub const layers = 5; +pub const heads = 32; +pub const kv_heads = 8; +pub const head_dim = 128; +pub const conv_group = 16; +pub const mask_id = 248070; +pub const window = 2047; // sliding_window - 1: context rows each layer keeps +pub const taps = [_]usize{ 5, 19, 33, 47, 61 }; // dflash_config.target_layer_ids +pub const selector_rank = 256; +pub const theta: f64 = 10000000; +/// The head's rows the drafter reads: the first 98,304 tokens and the last 288 (dflash2.vocab_spans). +pub const spans = [2][2]usize{ .{ 0, 98304 }, .{ 248032, 248320 } }; +pub const head_rows = spans[0][1] - spans[0][0] + spans[1][1] - spans[1][0]; + +pub const Layer = struct { + in_norm: u64, + post_norm: u64, + attn_proj: QLinear, // attention_conv.kernel_projection: the conv's dynamic taps (2 x 2 x 320) + attn_base: u64, // attention_conv.base_kernel (2, 2, 5120) bf16 + mlp_proj: QLinear, + mlp_base: u64, + qkv: QLinear, // [q | k | v] stacked + kv: QLinear, // [k | v] stacked: the context's projection + o: QLinear, + q_norm: u64, + k_norm: u64, + gate: QLinear, + up: QLinear, + down: QLinear, +}; + +pub const Weights = struct { + gpa: std.mem.Allocator, + layers: [layers]Layer = undefined, + fc: QLinear = undefined, // five taps (25,600) to the hidden width + hidden_norm: u64 = 0, + norm: u64 = 0, + select: QLinear = undefined, // candidate_selector.hidden_projection + head_tail: QLinear = undefined, // the target head's last 288 rows, tiled on their own (rows() copies them) + pred: []f32 = &.{}, // candidate_selector.predecessor_codebook (vocab, 256) + succ: []f32 = &.{}, + buffers: std.ArrayList(cuda.DeviceBuffer) = .empty, + + pub fn deinit(w: *Weights) void { + for (w.buffers.items) |*b| b.free(); + w.buffers.deinit(w.gpa); + w.gpa.free(w.pred); + w.gpa.free(w.succ); + w.* = undefined; + } +}; + +fn bf16Value(word: u16) f32 { + return @bitCast(@as(u32, word) << 16); +} + +fn toBf16(x: f32) u16 { + const b: u32 = @bitCast(x); + return @intCast((b + 0x7FFF + ((b >> 16) & 1)) >> 16); +} + +/// torch.round: halves to even. +fn roundEven(x: f32) f32 { + const r = @round(x); + return if (@abs(x - @trunc(x)) == 0.5) 2.0 * @round(x / 2.0) else r; +} + +/// dflash2.quantize4 on the host, bit for bit: (n, k) bf16 to MLX affine 4-bit words, bf16 scales and biases. +pub fn quantize4(src: []const u8, n: usize, k: usize, words: []u32, scales: []u16, biases: []u16) void { + const kg = k / 64; + for (0..n) |r| for (0..kg) |gi| { + var vals: [64]f32 = undefined; + for (&vals, 0..) |*v, j| v.* = bf16Value(std.mem.readInt(u16, src[(r * k + gi * 64 + j) * 2 ..][0..2], .little)); + var lo = vals[0]; + var hi = vals[0]; + for (vals[1..]) |v| { + lo = @min(lo, v); + hi = @max(hi, v); + } + const s = toBf16(@max((hi - lo) / 15.0, 1e-8)); + const b = toBf16(lo); + scales[r * kg + gi] = s; + biases[r * kg + gi] = b; + const sf = bf16Value(s); + const bf = bf16Value(b); + for (0..8) |wi| { + var word: u32 = 0; + for (0..8) |j| { + const q = std.math.clamp(roundEven((vals[wi * 8 + j] - bf) / sf), 0.0, 15.0); + word |= @as(u32, @intFromFloat(q)) << @intCast(4 * j); + } + words[r * (k / 8) + gi * 8 + wi] = word; + } + }; +} + +const Loader = struct { + gpa: std.mem.Allocator, + ops: kern.Ops, + w: *Weights, + words: std.ArrayList(u32) = .empty, + scales: std.ArrayList(u16) = .empty, + biases: std.ArrayList(u16) = .empty, + staging: ?cuda.DeviceBuffer = null, + + fn alloc(L: *Loader, len: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(L.ops.k.d, len); + try L.w.buffers.append(L.gpa, b); + return b.ptr; + } + + fn raw(L: *Loader, t: Tensor) !u64 { + const ptr = try L.alloc(t.bytes.len); + try L.ops.upload(ptr, t.bytes); + try L.ops.s.synchronize(); + return ptr; + } + + /// qmm_fast.tile of host MLX words, scales and biases (n rows of k inputs): the lane matmul's layout. + fn tile(L: *Loader, words: []const u32, scales: []const u16, biases: []const u16, n: usize, k: usize) !QLinear { + const k8 = k / 8; + const kg = k / 64; + const npad = (n + 127) / 128 * 128; + const sizes = [3]usize{ n * k8 * 4, n * kg * 2, n * kg * 2 }; + const total_in = sizes[0] + sizes[1] + sizes[2]; + if (L.staging == null or L.staging.?.len < total_in) { + if (L.staging) |*b| b.free(); + L.staging = try cuda.DeviceBuffer.alloc(L.ops.k.d, total_in); + } + const base = L.staging.?.ptr; + try L.ops.upload(base, std.mem.sliceAsBytes(words)); + try L.ops.upload(base + sizes[0], std.mem.sliceAsBytes(scales)); + try L.ops.upload(base + sizes[0] + sizes[1], std.mem.sliceAsBytes(biases)); + const q: QLinear = .{ .w = try L.alloc(npad * k / 2), .s = try L.alloc(kg * npad * 2), .b = try L.alloc(kg * npad * 2), .n = n, .k = k, .npad = npad }; + const total: u64 = @as(u64, npad / 64) * kg * 512; + var a: cuda.Args = .{}; + a.add(base); + for ([_]usize{ n, k8, kg }) |v| a.add(@as(c_int, @intCast(v))); + a.add(q.w); + a.add(@as(c_longlong, @intCast(total))); + try cuda.launch.launch(L.ops.k.pack_dense, .{ .grid = .{ .x = @intCast((total + 255) / 256) }, .block = .{ .x = 256 } }, L.ops.s, &a); + for ([_]u64{ q.s, q.b }, [_]usize{ sizes[0], sizes[0] + sizes[1] }) |out, off| { + var t: cuda.Args = .{}; + t.add(base + off); + for ([_]usize{ n, kg, npad }) |v| t.add(@as(c_int, @intCast(v))); + t.add(out); + const cells = @as(u64, kg) * npad; + try cuda.launch.launch(L.ops.k.transpose16, .{ .grid = .{ .x = @intCast((cells + 255) / 256) }, .block = .{ .x = 256 } }, L.ops.s, &t); + } + try L.ops.s.synchronize(); // the staging buffer and host arrays are reused next + return q; + } + + /// One bf16 (n, k) checkpoint tensor (or several stacked by rows) quantized and tiled. + fn quantized(L: *Loader, parts: []const Tensor, k: usize) !QLinear { + var n: usize = 0; + for (parts) |t| { + if (t.dtype != .bf16 or t.rank != 2 or t.dim(1) != k) return error.UnexpectedDraftTensor; + n += t.dim(0); + } + try L.words.resize(L.gpa, n * k / 8); + try L.scales.resize(L.gpa, n * k / 64); + try L.biases.resize(L.gpa, n * k / 64); + var row: usize = 0; + for (parts) |t| { + const r = t.dim(0); + quantize4(t.bytes, r, k, L.words.items[row * k / 8 ..][0 .. r * k / 8], L.scales.items[row * k / 64 ..][0 .. r * k / 64], L.biases.items[row * k / 64 ..][0 .. r * k / 64]); + row += r; + } + return L.tile(L.words.items, L.scales.items, L.biases.items, n, k); + } + + fn get(ck: *core.Checkpoint, comptime fmt: []const u8, args: anytype) !Tensor { + var buf: [160]u8 = undefined; + return ck.get(try std.fmt.bufPrint(&buf, fmt, args)); + } + + /// The codebook as fp32 on the host (dflash2 keeps tensor.float().numpy()). + fn codebook(L: *Loader, t: Tensor) ![]f32 { + if (t.dtype != .bf16 or t.rank != 2 or t.dim(1) != selector_rank) return error.UnexpectedDraftTensor; + const out = try L.gpa.alloc(f32, t.dim(0) * selector_rank); + for (out, 0..) |*o, i| o.* = bf16Value(std.mem.readInt(u16, t.bytes[2 * i ..][0..2], .little)); + return out; + } +}; + +/// The drafter at `dir`, and the target head's last span tiled from the target checkpoint at `target_dir`. +pub fn load(gpa: std.mem.Allocator, io: std.Io, ops: kern.Ops, dir: []const u8, target_dir: []const u8) !Weights { + var w: Weights = .{ .gpa = gpa }; + errdefer w.deinit(); + var L: Loader = .{ .gpa = gpa, .ops = ops, .w = &w }; + defer { + L.words.deinit(gpa); + L.scales.deinit(gpa); + L.biases.deinit(gpa); + if (L.staging) |*b| b.free(); + } + var ck = try core.Checkpoint.openModel(gpa, io, dir); + defer ck.close(); + for (&w.layers, 0..) |*l, i| { + const q = try Loader.get(&ck, "layers.{d}.self_attn.q_proj.weight", .{i}); + const k = try Loader.get(&ck, "layers.{d}.self_attn.k_proj.weight", .{i}); + const v = try Loader.get(&ck, "layers.{d}.self_attn.v_proj.weight", .{i}); + l.qkv = try L.quantized(&.{ q, k, v }, hidden); + l.kv = try L.quantized(&.{ k, v }, hidden); + l.o = try L.quantized(&.{try Loader.get(&ck, "layers.{d}.self_attn.o_proj.weight", .{i})}, heads * head_dim); + l.q_norm = try L.raw(try Loader.get(&ck, "layers.{d}.self_attn.q_norm.weight", .{i})); + l.k_norm = try L.raw(try Loader.get(&ck, "layers.{d}.self_attn.k_norm.weight", .{i})); + l.in_norm = try L.raw(try Loader.get(&ck, "layers.{d}.input_layernorm.weight", .{i})); + l.post_norm = try L.raw(try Loader.get(&ck, "layers.{d}.post_attention_layernorm.weight", .{i})); + l.attn_proj = try L.quantized(&.{try Loader.get(&ck, "layers.{d}.attention_conv.kernel_projection.weight", .{i})}, hidden); + l.attn_base = try L.raw(try Loader.get(&ck, "layers.{d}.attention_conv.base_kernel", .{i})); + l.mlp_proj = try L.quantized(&.{try Loader.get(&ck, "layers.{d}.mlp_conv.kernel_projection.weight", .{i})}, hidden); + l.mlp_base = try L.raw(try Loader.get(&ck, "layers.{d}.mlp_conv.base_kernel", .{i})); + l.gate = try L.quantized(&.{try Loader.get(&ck, "layers.{d}.mlp.gate_proj.weight", .{i})}, hidden); + l.up = try L.quantized(&.{try Loader.get(&ck, "layers.{d}.mlp.up_proj.weight", .{i})}, hidden); + const down = try Loader.get(&ck, "layers.{d}.mlp.down_proj.weight", .{i}); + l.down = try L.quantized(&.{down}, down.dim(1)); + } + const fc = try ck.get("fc.weight"); + w.fc = try L.quantized(&.{fc}, fc.dim(1)); + w.hidden_norm = try L.raw(try ck.get("hidden_norm.weight")); + w.norm = try L.raw(try ck.get("norm.weight")); + w.select = try L.quantized(&.{try ck.get("candidate_selector.hidden_projection.weight")}, hidden); + w.pred = try L.codebook(try ck.get("candidate_selector.predecessor_codebook")); + w.succ = try L.codebook(try ck.get("candidate_selector.successor_codebook")); + if (ck.unused() != 0) return error.UnusedDraftTensors; + // dflash2's sub-head: rows() of the tiled head copies a span off the 64-row tile grid, packed on its own + var tk = try core.Checkpoint.openModelPrefix(gpa, io, target_dir, "language_model.lm_head"); + defer tk.close(); + const hw = try tk.get("language_model.lm_head.weight"); + const hs = try tk.get("language_model.lm_head.scales"); + const hb = try tk.get("language_model.lm_head.biases"); + const a = spans[1][0]; + const n = spans[1][1] - a; + const k8 = hidden / 8; + const kg = hidden / 64; + // mapped tensors may sit at odd offsets: the rows go through aligned host arrays + try L.words.resize(gpa, n * k8); + try L.scales.resize(gpa, n * kg); + try L.biases.resize(gpa, n * kg); + @memcpy(std.mem.sliceAsBytes(L.words.items), hw.bytes[a * k8 * 4 ..][0 .. n * k8 * 4]); + @memcpy(std.mem.sliceAsBytes(L.scales.items), hs.bytes[a * kg * 2 ..][0 .. n * kg * 2]); + @memcpy(std.mem.sliceAsBytes(L.biases.items), hb.bytes[a * kg * 2 ..][0 .. n * kg * 2]); + w.head_tail = try L.tile(L.words.items, L.scales.items, L.biases.items, n, hidden); + return w; +} + +test "quantize4 rounds as torch does" { + // one group: values 0..63 / 4 -> lo 0, scale bf16(63/4/15 = 1.05), each q round-half-even of v / 1.046875 + var src: [64 * 2]u8 = undefined; + for (0..64) |j| std.mem.writeInt(u16, src[2 * j ..][0..2], toBf16(@as(f32, @floatFromInt(j)) / 4.0), .little); + var words: [8]u32 = undefined; + var scales: [1]u16 = undefined; + var biases: [1]u16 = undefined; + quantize4(&src, 1, 64, &words, &scales, &biases); + try std.testing.expectEqual(toBf16(63.0 / 4.0 / 15.0), scales[0]); + try std.testing.expectEqual(@as(u16, 0), biases[0]); + try std.testing.expectEqual(@as(u32, 15), words[7] >> 28); + try std.testing.expectEqual(roundEven(2.5), 2.0); + try std.testing.expectEqual(roundEven(3.5), 4.0); +} diff --git a/zig/src/families/qwen27/cuda/draft_policy.zig b/zig/src/families/qwen27/cuda/draft_policy.zig new file mode 100644 index 0000000000..83888d435a --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_policy.zig @@ -0,0 +1,122 @@ +//! draft_tree.best_first on the host: which of DFlash2's candidates a round verifies, and where each hangs. + +const std = @import("std"); +const lanes = @import("lanes"); + +pub const branch = 4; +const edge_weight = 0.6; +const noise_weight = 0.7; +const scale = 1.5; +pub const rank = 256; // the selector's codebook width +pub const top = 16; // candidates a depth + +/// A drafted block's candidates: each depth's 16 token ids, their logits (unary) and the selector's projection. +pub const Candidates = struct { ids: []const u32, unary: []const f64, projected: []const f64, depths: usize }; + +/// The tree: nodes in pop order (parents first), each node's parent (-1: hangs from the anchor), path scores. +pub const Tree = struct { tokens: std.ArrayList(u32) = .empty, parents: std.ArrayList(i32) = .empty, scores: std.ArrayList(f64) = .empty }; + +const Item = struct { score: f64, parent: i32, token: u32, depth: usize }; + +fn before(_: void, a: Item, b: Item) std.math.Order { + // heapq's tuple order: score, then parent, then token, then depth + if (a.score != b.score) return std.math.order(a.score, b.score); + if (a.parent != b.parent) return std.math.order(a.parent, b.parent); + if (a.token != b.token) return std.math.order(a.token, b.token); + return std.math.order(a.depth, b.depth); +} + +/// best_first: expand the anchor's candidates, then pop the best path score until `max_nodes` are placed. +pub fn bestFirst(gpa: std.mem.Allocator, c: Candidates, pred: []const f32, succ: []const f32, anchor: u32, max_nodes: usize, sampling: ?lanes.Sampling, first_position: u64, out: *Tree) !void { + out.tokens.clearRetainingCapacity(); + out.parents.clearRetainingCapacity(); + out.scores.clearRetainingCapacity(); + const temp: f64 = if (sampling) |s| @max(s.temperature, 1e-6) else 1.0; + var heap: std.PriorityQueue(Item, void, before) = .empty; + defer heap.deinit(gpa); + const Ctx = struct { + gpa: std.mem.Allocator, + c: Candidates, + pred: []const f32, + succ: []const f32, + temp: f64, + sampling: ?lanes.Sampling, + first_position: u64, + heap: *std.PriorityQueue(Item, void, before), + + fn expand(x: @This(), token: u32, depth: usize, parent: i32, path: f64) !void { + var values: [top]f64 = undefined; + const p = x.pred[@as(usize, token) * rank ..][0..rank]; + const proj = x.c.projected[depth * rank ..][0..rank]; + var weighted: [rank]f64 = undefined; + for (&weighted, p, proj) |*wgt, a, b| wgt.* = @as(f64, a) * b; + for (&values, 0..) |*v, i| { + const cand = x.c.ids[depth * top + i]; + const s = x.succ[@as(usize, cand) * rank ..][0..rank]; + var edge: f64 = 0; + for (s, weighted) |a, b| edge += @as(f64, a) * b; + v.* = (x.c.unary[depth * top + i] + edge_weight * edge) / x.temp; + if (x.sampling) |smp| v.* += noise_weight * -@log(-@log(lanes.sampling.uniform(smp.seed, x.first_position + depth, cand))); + v.* /= scale; + } + var most = values[0]; + for (values[1..]) |v| most = @max(most, v); + var total: f64 = 0; + for (&values) |*v| { + v.* -= most; + total += @exp(v.*); + } + const log_total = @log(total); + var order: [top]usize = undefined; + for (&order, 0..) |*o, i| o.* = i; + std.mem.sort(usize, &order, &values, struct { + fn more(vals: *const [top]f64, a: usize, b: usize) bool { + return vals[a] > vals[b] or (vals[a] == vals[b] and a < b); + } + }.more); + for (order[0..branch]) |i| { + const logp = values[i] - log_total; + if (!std.math.isFinite(logp)) break; + try x.heap.push(x.gpa, .{ .score = path - logp, .parent = parent, .token = x.c.ids[depth * top + i], .depth = depth }); + } + } + }; + const ctx: Ctx = .{ .gpa = gpa, .c = c, .pred = pred, .succ = succ, .temp = temp, .sampling = sampling, .first_position = first_position, .heap = &heap }; + try ctx.expand(anchor, 0, -1, 0.0); + while (heap.count() > 0 and out.tokens.items.len < max_nodes) { + const it = heap.pop().?; + const me: i32 = @intCast(out.tokens.items.len); + try out.tokens.append(gpa, it.token); + try out.parents.append(gpa, it.parent); + try out.scores.append(gpa, it.score); + if (it.depth + 1 < c.depths) try ctx.expand(it.token, it.depth + 1, me, it.score); + } +} + +test "best_first pops by path score, parents before children, ties by parent" { + const gpa = std.testing.allocator; + const vocab = 32; + const pred = try gpa.alloc(f32, vocab * rank); + defer gpa.free(pred); + const succ = try gpa.alloc(f32, vocab * rank); + defer gpa.free(succ); + @memset(pred, 0); + @memset(succ, 0); + var ids: [2 * top]u32 = undefined; + var unary: [2 * top]f64 = undefined; + for (0..2) |d| for (0..top) |i| { + ids[d * top + i] = @intCast(i); + unary[d * top + i] = -@as(f64, @floatFromInt(i)); + }; + const projected: [2 * rank]f64 = @splat(0); + var tree: Tree = .{}; + defer { + tree.tokens.deinit(gpa); + tree.parents.deinit(gpa); + tree.scores.deinit(gpa); + } + try bestFirst(gpa, .{ .ids = &ids, .unary = &unary, .projected = &projected, .depths = 2 }, pred, succ, 3, 5, null, 100, &tree); + // logp_i = -i / 1.5 - ln(sum): the anchor's 0 and 1 (score 0.72, 1.39) come before 0's child 0 (1.44) + try std.testing.expectEqualSlices(u32, &.{ 0, 1, 0, 2, 1 }, tree.tokens.items); + try std.testing.expectEqualSlices(i32, &.{ -1, -1, 0, -1, 0 }, tree.parents.items); +} diff --git a/zig/src/families/qwen27/cuda/drafts.zig b/zig/src/families/qwen27/cuda/drafts.zig new file mode 100644 index 0000000000..02e3659524 --- /dev/null +++ b/zig/src/families/qwen27/cuda/drafts.zig @@ -0,0 +1,103 @@ +//! DFlash2 for a shared round's streams (multi.py's drafter halves): copies first, else every block in one batch. + +const std = @import("std"); +const lanes = @import("lanes"); +const st = @import("state.zig"); +const kern = @import("kernels.zig"); +const dr = @import("draft.zig"); +const dl = @import("draft_load.zig"); +const lone = @import("lone.zig"); + +const bf = 2; +pub const taps_width = dl.taps.len * dl.hidden; +/// A shared window's drafts at most (MultiDecoder's max_rows - 1). +pub const max_nodes = st.max_rows - 1; + +pub const Kind = enum { none, copy, tree }; + +/// One stream's drafting: its drafter context, its copy index, and the drafts it holds for its next window. +pub const Stream = struct { + gpa: std.mem.Allocator, + ctx: dr.Context, + copies: lone.CopyIndex, + kind: Kind = .none, + tokens: std.ArrayList(u32) = .empty, + parents: std.ArrayList(i32) = .empty, // each draft's parent draft (-1: under the pending row) + scores: std.ArrayList(f64) = .empty, // each draft's path score (best_first's; 0 for a copy) + + pub fn init(gpa: std.mem.Allocator, ctx: dr.Context) Stream { + return .{ .gpa = gpa, .ctx = ctx, .copies = .{ .gpa = gpa } }; + } + + pub fn deinit(x: *Stream) void { + x.ctx.deinit(); + x.copies.deinit(); + x.tokens.deinit(x.gpa); + x.parents.deinit(x.gpa); + x.scores.deinit(x.gpa); + } + + fn clear(x: *Stream) void { + x.kind = .none; + x.tokens.clearRetainingCapacity(); + x.parents.clearRetainingCapacity(); + x.scores.clearRetainingCapacity(); + } +}; + +/// A stream asking for its next drafts: its context through the pending token, and its sampling. +pub const Ask = struct { x: *Stream, context: []const u32, sampling: ?lanes.Sampling }; + +/// _mode, launch_blocks and finish_tree: a copied continuation where the context repeats, else a DFlash2 tree. +pub fn propose(d: *dr.DFlash2, asks: []const Ask) !void { + var blocks: [st.max_streams]dr.Block = undefined; + var which: [st.max_streams]usize = undefined; + var n: usize = 0; + for (asks, 0..) |a, k| { + const x = a.x; + x.clear(); + const copied = try x.copies.propose(a.context, max_nodes); + if (copied.len > 0) { + x.kind = .copy; + try x.tokens.appendSlice(x.gpa, copied); + for (0..copied.len) |r| { + try x.parents.append(x.gpa, @as(i32, @intCast(r)) - 1); + try x.scores.append(x.gpa, 0); + } + } else if (x.ctx.len > 0) { + blocks[n] = .{ .ctx = &x.ctx, .pending = a.context[a.context.len - 1] }; + which[n] = k; + n += 1; + } + } + if (n == 0) return; + try d.launchMany(blocks[0..n], @min(dr.block, max_nodes + 1)); + for (blocks[0..n], which[0..n], 0..) |b, k, j| { + const a = asks[k]; + try d.finish(j, b.pending, a.context.len, max_nodes, a.sampling, &d.tree); + a.x.kind = .tree; + try a.x.tokens.appendSlice(a.x.gpa, d.tree.tokens.items); + try a.x.parents.appendSlice(a.x.gpa, d.tree.parents.items); + try a.x.scores.appendSlice(a.x.gpa, d.tree.scores.items); + } +} + +/// A stream's kept rows of the last round (rows of the round's taps, in path order). +pub const Kept = struct { x: *Stream, rows: []const u32 }; + +/// add_taps_streams: every stream's kept rows' taps gathered after the round's rows, then absorbed in one batch. +pub fn absorb(d: *dr.DFlash2, ops: kern.Ops, taps: u64, copy_items: u64, kept: []const Kept) !void { + var items: std.ArrayList(kern.Copy) = .empty; + defer items.deinit(d.gpa); + var absorbs: [st.max_streams]dr.Absorb = undefined; + const row = taps_width * bf; + const gathered = taps + st.round_rows * row; + for (kept, 0..) |k, j| { + for (k.rows) |r| try items.append(d.gpa, .{ .dst = gathered + items.items.len * row, .src = taps + r * row, .bytes = row }); + absorbs[j] = .{ .ctx = &k.x.ctx, .rows = k.rows.len }; + } + if (items.items.len == 0) return; + try ops.upload(copy_items, std.mem.sliceAsBytes(items.items)); + try ops.copies(copy_items, items.items.len); + try d.absorbMany(absorbs[0..kept.len], gathered); +} diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig new file mode 100644 index 0000000000..52576d7cd3 --- /dev/null +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -0,0 +1,118 @@ +//! The 27B's CUDA engine for one stream: the weights, kernels and scratch, prefill and greedy serial decode. + +const std = @import("std"); +const cuda = @import("cuda"); +const c = @import("shape.zig"); +const kern = @import("kernels.zig"); +const tri = @import("triton.zig"); +const st = @import("state.zig"); +const wts = @import("weights.zig"); +const Forward = @import("forward.zig").Forward; +const Part = @import("forward.zig").Part; + +/// prefill.CHUNK: a prompt chunk's rows (prompt_rows on a GPU of 80 GB or more). +pub const prompt_rows = 4096; + +/// prefill.chunks: even chunks of at most `size` rows of [start, end); any bounds give the same bits. +pub fn chunkBounds(start: usize, end: usize, size: usize, j: usize) [2]usize { + const n = (end - start + size - 1) / size; + return .{ start + (end - start) * j / n, start + (end - start) * (j + 1) / n }; +} + +pub const Options = struct { context: usize }; + +pub const Engine = struct { + gpa: std.mem.Allocator, + ctx: *const cuda.Context, + s: cuda.Stream, + k: kern.Kernels, + w: wts.Weights, + scratch: st.Scratch, + context: usize, + + pub fn init(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: []const u8, o: Options) !*Engine { + const cfg = try c.Config.read(gpa, io, dir); + const g = cfg.g; + const e = try gpa.create(Engine); + errdefer gpa.destroy(e); + e.gpa = gpa; + e.ctx = ctx; + e.context = o.context; + e.s = try cuda.Stream.init(ctx.d, true); + errdefer e.s.deinit(); + e.k = try kern.Kernels.load(gpa, io, ctx, kernels, g.linear_v_heads); + errdefer e.k.deinit(); + e.w = try wts.load(gpa, io, e.ops(), dir, cfg); + errdefer e.w.deinit(); + e.scratch = try st.Scratch.init(gpa, ctx.d, g, prompt_rows, o.context); + errdefer e.scratch.deinit(); + try e.s.synchronize(); + return e; + } + + pub fn deinit(e: *Engine) void { + e.scratch.deinit(); + e.w.deinit(); + e.k.deinit(); + e.s.deinit(); + e.gpa.destroy(e); + } + + pub fn ops(e: *Engine) kern.Ops { + return .{ .k = &e.k, .s = e.s }; + } + + pub fn forward(e: *Engine) Forward { + return .{ .gpa = e.gpa, .w = &e.w, .ops = e.ops(), .t = .{ .set = &e.k.triton, .s = e.s }, .s = &e.scratch }; + } + + /// A stream's state for `capacity` positions (its caches, recurrences and conv windows), empty. + pub fn sequence(e: *Engine, capacity: usize) !st.Seq { + return st.Seq.init(e.gpa, e.ops(), e.w.g, capacity); + } + + /// prefill_state from an empty state: the prompt in even chunks of at most `prompt_rows`, its last row normed. + pub fn prefill(e: *Engine, seq: *st.Seq, prompt: []const u32) !void { + if (prompt.len == 0) return error.EmptyPrompt; + if (prompt.len >= seq.capacity) return error.PromptTooLong; + try seq.reset(e.ops(), e.w.g); + var f = e.forward(); + const n = (prompt.len + prompt_rows - 1) / prompt_rows; + for (0..n) |j| { + const b = chunkBounds(0, prompt.len, prompt_rows, j); + try f.chunk(prompt[b[0]..b[1]], seq, j == n - 1); + } + } + + /// decode.serial_decode, greedy: one-row windows from `pending` until `count` tokens or an end token. + pub fn decode(e: *Engine, seq: *st.Seq, pending: u32, count: usize, out: *std.ArrayList(u32)) !void { + var f = e.forward(); + try out.append(e.gpa, pending); + var pick: [1]u32 = undefined; + while (out.items.len < count and !e.w.isEos(out.items[out.items.len - 1])) { + const last = out.items[out.items.len - 1]; + const part = [_]Part{.{ .seq = seq, .tokens = &.{last} }}; + try f.round(&part); + try f.picks(1, &pick); + try f.commit(&part, &.{&.{0}}); + try out.append(e.gpa, pick[0]); + } + } + + /// The prompt, then up to `max_tokens` greedy tokens (the first included), as generate(draft=False) gives them. + pub fn generate(e: *Engine, seq: *st.Seq, prompt: []const u32, max_tokens: usize, out: *std.ArrayList(u32)) !void { + const room = @min(max_tokens, seq.capacity - prompt.len); + try e.prefill(seq, prompt); + var f = e.forward(); + try f.head(seq); + var pick: [1]u32 = undefined; + try f.picks(1, &pick); + try e.decode(seq, pick[0], @max(1, room), out); + } +}; + +test "prefill.chunks splits evenly" { + try std.testing.expectEqual([2]usize{ 0, 2539 }, chunkBounds(0, 5078, 4096, 0)); + try std.testing.expectEqual([2]usize{ 2539, 5078 }, chunkBounds(0, 5078, 4096, 1)); + try std.testing.expectEqual([2]usize{ 0, 40 }, chunkBounds(0, 40, 4096, 0)); +} diff --git a/zig/src/families/qwen27/cuda/forward.zig b/zig/src/families/qwen27/cuda/forward.zig new file mode 100644 index 0000000000..4e4964be4e --- /dev/null +++ b/zig/src/families/qwen27/cuda/forward.zig @@ -0,0 +1,436 @@ +//! The 27B's forward on CUDA: verify rounds (tree_forward, multi_tree_forward), their commit, and prompt chunks. + +const std = @import("std"); +const c = @import("shape.zig"); +const kern = @import("kernels.zig"); +const tri = @import("triton.zig"); +const st = @import("state.zig"); +const wts = @import("weights.zig"); +const rnd = @import("round.zig"); + +const bf = 2; +pub const Part = rnd.Part; +/// The target layers DFlash2 reads (dflash_config.target_layer_ids). +pub const tap_layers = [_]usize{ 5, 19, 33, 47, 61 }; + +pub const Forward = struct { + gpa: std.mem.Allocator, + w: *const wts.Weights, + ops: kern.Ops, + t: tri.Tri, + s: *st.Scratch, + dump: ?Dump = null, // a debugging copy of every norm's (h, y) rows, as capture.py's hook writes them + taps: ?u64 = null, // rows of the drafter's five layer taps (5 x hidden bf16), written by round and chunk + + pub const Dump = struct { io: std.Io, dir: []const u8, n: usize = 0, rows: usize = 0 }; + + fn g(f: *const Forward) c.Geometry { + return f.w.g; + } + + fn gdnShape(f: *const Forward) tri.Gdn { + const g_ = f.g(); + return .{ .c = g_.convDim(), .kh = g_.linear_k_heads, .vh = g_.linear_v_heads, .dk = c.linear_dim, .keep = c.conv_taps - 1 }; + } + + fn heads(f: *const Forward) kern.Heads { + return .{ .hk = f.g().linear_k_heads, .hv = f.g().linear_v_heads, .dv = c.linear_dim }; + } + + fn attnShape(f: *const Forward) tri.Attn { + return .{ .heads = f.g().query_heads, .kv_heads = f.g().kv_heads, .dim = c.head_dim, .half = c.rotary_dim / 2 }; + } + + fn upload32(f: *Forward, dst: u64, values: []const i32) !void { + try f.ops.upload(dst, std.mem.sliceAsBytes(values)); + try f.ops.s.synchronize(); // the host copy must outlive the async upload + } + + /// add_rmsnorm into the scratch's other residual buffer; returns the residual stream's new home. + fn norm(f: *Forward, x: u64, r: ?u64, weight: u64, rows: usize) !u64 { + const out = if (r == null) x else if (x == f.s.x) f.s.h else f.s.x; + try f.t.addRmsnorm(x, r, weight, out, f.s.y, f.s.xs, rows, f.g().hidden, c.eps); + if (f.dump) |*d| try f.saveRows(d, out, f.s.y, rows); + return out; + } + + /// Layer `i`'s tap, when the drafter reads it: (x + pending) of the call's rows into the taps buffer. + fn tapLayer(f: *Forward, i: usize, x: u64, rows: usize) !void { + const out = f.taps orelse return; + const j = std.mem.indexOfScalar(usize, &tap_layers, i) orelse return; + const h = f.g().hidden; + try f.ops.tap(x, f.s.pending, out, rows, h, tap_layers.len * h, j * h); + } + + fn saveRows(f: *Forward, d: *Dump, h: u64, y: u64, rows: usize) !void { + const bytes = try f.gpa.alloc(u8, rows * f.g().hidden * bf); + defer f.gpa.free(bytes); + for ([_]u64{ h, y }, [_][]const u8{ "h", "y" }) |src, name| { + try f.ops.download(bytes, src); + try f.ops.s.synchronize(); + var buf: [256]u8 = undefined; + const path = try std.fmt.bufPrint(&buf, "{s}/{d:0>3}_{s}.bin", .{ d.dir, d.n, name }); + try std.Io.Dir.cwd().writeFile(d.io, .{ .sub_path = path, .data = bytes }); + } + d.n += 1; + } + + /// The window of one stream at its position (tree_forward): logits (rows, vocab) bf16 in the scratch, records kept. + pub fn window(f: *Forward, tokens: []const u32, seq: *st.Seq) !void { + return f.round(&.{.{ .seq = seq, .tokens = tokens }}); + } + + /// Several streams' chain windows in one forward (multi_tree_forward; one stream: tree_forward), rows in order. + pub fn round(f: *Forward, parts: []const Part) !void { + const s = f.s; + const g_ = f.g(); + const w = f.w; + const r = try rnd.stage(f.gpa, f.ops, f.t, s, g_, parts); + const W = r.rows; + try f.t.embed(r.ids, w.embed.w, w.embed.s, w.embed.b, s.x, W, g_.hidden); + var x = s.x; + var pending: ?u64 = null; + var lin: usize = 0; + var att: usize = 0; + for (w.layers, 0..) |layer, i| { + x = try f.norm(x, pending, layer.input_norm, W); + switch (layer.mixer) { + .delta => |d| { + const rec = s.records[i]; + try f.ops.dense(s.y, s.xs, d.qkv, rec.qkv, W); + try f.ops.dense(s.y, s.xs, d.z, s.z, W); + try f.ops.dense(s.y, s.xs, d.b, s.b, W); + try f.ops.dense(s.y, s.xs, d.a, s.a, W); + const sid: ?u64 = if (r.multi) r.sids else null; + try f.t.gdnPre(rec.qkv, r.conv(parts, s, g_, i, lin), d.conv, r.windows, sid, s.a, s.b, d.a_log, d.dt_bias, rec.q, rec.k, rec.v, rec.g, rec.beta, W, f.gdnShape()); + const state = if (r.multi) 0 else parts[0].seq.rec[i]; + const table = if (r.multi) r.table(lin) else 0; + try f.ops.gdnTree(rec.q, rec.k, rec.v, rec.g, rec.beta, state, table, if (r.multi) r.starts else 0, r.plan, s.yr, W, r.streams, r.slots, r.most, f.heads()); + try f.t.gatedNorm(s.yr, s.z, d.norm, s.gated, s.gated_xs, W, g_.linear_v_heads, c.linear_dim, c.eps); + try f.ops.dense(s.gated, s.gated_xs, d.out, s.r, W); + lin += 1; + }, + .attention => |a| { + const rec = s.attn_records[i]; + try f.ops.dense(s.y, s.xs, a.q, s.qg, W); + try f.ops.dense(s.y, s.xs, a.k, s.key, W); + try f.ops.dense(s.y, s.xs, a.v, rec.v, W); + try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, r.pos, w.inv_freq, s.q_rot, rec.k, W, f.attnShape(), c.eps); + try f.t.attention(s.q_rot, rec.k, rec.v, s.origin, r.offsets(i, att), r.attn, .{ .o = s.part_o, .m = s.part_m, .l = s.part_l }, s.attn, f.attnShape()); + try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); + try f.ops.dense(s.gated, s.gated_xs, a.o, s.r, W); + att += 1; + }, + } + x = try f.norm(x, s.r, layer.post_norm, W); + try f.ops.dense(s.y, s.xs, layer.gate, s.gate, W); + try f.ops.dense(s.y, s.xs, layer.up, s.up, W); + try f.t.swiglu(s.gate, s.up, s.act, s.act_xs, W, g_.intermediate); + try f.ops.dense(s.act, s.act_xs, layer.down, s.pending, W); + pending = s.pending; + try f.tapLayer(i, x, W); + } + const spare = if (x == s.x) s.h else s.x; + try f.t.addRmsnorm(x, pending, w.norm, spare, s.last_h, s.last_xs, W, g_.hidden, c.eps); + if (f.dump) |*d| try f.saveRows(d, spare, s.last_h, W); + try f.ops.dense(s.last_h, s.last_xs, w.head, s.logits, W); + } + + /// commit_streams after a round over `parts`: each stream keeps the window rows of its accepted path. + pub fn commit(f: *Forward, parts: []const Part, paths: []const []const u32) !void { + const s = f.s; + const g_ = f.g(); + const S = parts.len; + var width: usize = 0; + for (parts, paths) |p, path| { + if (path.len < 1 or path.len > p.tokens.len) return error.InvalidQwenCommit; + for (path) |row| if (row >= p.tokens.len) return error.InvalidQwenCommit; + width += p.tokens.len; + } + // gdn.replay_table: each DeltaNet layer's k, v, g, beta, then each stream's state of each layer + var table: std.ArrayList(u64) = .empty; + defer table.deinit(f.gpa); + var layers: usize = 0; + for (0..g_.layers) |i| if (c.linear(i)) { + const rc = s.records[i]; + try table.appendSlice(f.gpa, &.{ rc.k, rc.v, rc.g, rc.beta }); + layers += 1; + }; + for (parts) |p| for (0..g_.layers) |i| if (c.linear(i)) try table.append(f.gpa, p.seq.rec[i]); + try f.ops.upload(s.replay_table, std.mem.sliceAsBytes(table.items)); + // path_indices: each stream's accepted rows (round rows) padded to the round's width, then its count + var rows: std.ArrayList(i32) = .empty; + defer rows.deinit(f.gpa); + var base: usize = 0; + for (parts, paths) |p, path| { + for (0..width) |j| try rows.append(f.gpa, if (j < path.len) @intCast(base + path[j]) else 0); + try rows.append(f.gpa, @intCast(path.len)); + base += p.tokens.len; + } + try f.ops.upload(s.replay_rows, std.mem.sliceAsBytes(rows.items)); + try f.ops.s.synchronize(); + try f.ops.gdnReplay(s.replay_table, layers, S, s.replay_rows, width + 1, s.replay_rows + width * 4, width + 1, f.heads()); + const row = g_.convDim() * bf; + const keep = c.conv_taps - 1; + const kv = g_.kvInner() * bf; + // conv rows: the last three of [committed | accepted rows]; old ones saved first, then written with the rest + var save: std.ArrayList(kern.Copy) = .empty; + defer save.deinit(f.gpa); + var write: std.ArrayList(kern.Copy) = .empty; + defer write.deinit(f.gpa); + base = 0; + for (parts, paths, 0..) |p, path, k| { + const n = path.len; + var lin: usize = 0; + for (0..g_.layers) |i| { + if (c.linear(i)) { + const old = s.conv_old + (k * g_.layers + lin) * keep * row; + const stay = keep -| n; // committed rows that stay: old[n..3) go to new[0..3-n) + if (stay > 0) { + try save.append(f.gpa, .{ .dst = old, .src = p.seq.conv[i] + n * row, .bytes = stay * row }); + try write.append(f.gpa, .{ .dst = p.seq.conv[i], .src = old, .bytes = stay * row }); + } + for (path[n - (keep - stay) ..], stay..) |acc, j| try write.append(f.gpa, .{ .dst = p.seq.conv[i] + j * row, .src = s.records[i].qkv + (base + acc) * row, .bytes = row }); + lin += 1; + } else { + for (path, 0..) |acc, j| { + const at = (p.seq.pos + j) * kv; + try write.append(f.gpa, .{ .dst = p.seq.keys[i] + at, .src = s.attn_records[i].k + (base + acc) * kv, .bytes = kv }); + try write.append(f.gpa, .{ .dst = p.seq.values[i] + at, .src = s.attn_records[i].v + (base + acc) * kv, .bytes = kv }); + } + } + } + base += p.tokens.len; + p.seq.pos += n; + } + const second = s.copy_items + st.copy_items * @sizeOf(kern.Copy); + try f.ops.upload(s.copy_items, std.mem.sliceAsBytes(save.items)); + try f.ops.upload(second, std.mem.sliceAsBytes(write.items)); + try f.ops.copies(s.copy_items, save.items.len); + try f.ops.copies(second, write.items.len); + try f.ops.s.synchronize(); // the tables' host copies go when this returns + } + + /// prefill_chunk: `ids` committed at the stream's position; `last` leaves the last row normed, with group_sums. + pub fn chunk(f: *Forward, ids: []const u32, seq: *st.Seq, last: bool) !void { + const W = ids.len; + const s = f.s; + const g_ = f.g(); + const w = f.w; + if (W < 1 or W > s.rows) return error.ChunkTooLong; + if (seq.pos + W > seq.capacity) return error.PromptTooLong; + const p0 = seq.pos; + { + const host = try f.gpa.alloc(i32, W * (2 + c.conv_taps)); + defer f.gpa.free(host); + for (ids, 0..) |tok, r| { + host[r] = @intCast(tok); + host[W + r] = @intCast(p0 + r); + for (0..c.conv_taps) |j| host[2 * W + r * c.conv_taps + j] = @intCast(r + j); + } + try f.upload32(s.ids, host[0..W]); + try f.upload32(s.pos, host[W .. 2 * W]); + try f.upload32(s.windows, host[2 * W ..]); + } + try f.t.embed(s.ids, w.embed.w, w.embed.s, w.embed.b, s.x, W, g_.hidden); + var x = s.x; + var pending: ?u64 = null; + const kv = g_.kvInner() * bf; + const row = g_.convDim() * bf; + const keep = c.conv_taps - 1; + for (w.layers, 0..) |layer, i| { + x = try f.norm(x, pending, layer.input_norm, W); + switch (layer.mixer) { + .delta => |d| { + try f.ops.prefillDense(s.y, d.qkv, s.qkv, W); + try f.ops.prefillDense(s.y, d.z, s.z, W); + try f.ops.prefillDense(s.y, d.b, s.b, W); + try f.ops.prefillDense(s.y, d.a, s.a, W); + try f.t.gdnPre(s.qkv, seq.conv[i], d.conv, s.windows, null, s.a, s.b, d.a_log, d.dt_bias, s.gq, s.gk, s.gv, s.gg, s.gbeta, W, f.gdnShape()); + // each thread reads its state rows before it writes them: the chain may end in place + try f.ops.gdnPrompt(s.gq, s.gk, s.gv, s.gg, s.gbeta, seq.rec[i], seq.rec[i], s.yr, W, f.heads()); + if (W >= keep) { + try f.ops.copy(seq.conv[i], s.qkv + (W - keep) * row, keep * row); + } else { + try f.ops.copy(s.conv_tmp, seq.conv[i], keep * row); + try f.ops.copy(s.conv_tmp + keep * row, s.qkv, W * row); + try f.ops.copy(seq.conv[i], s.conv_tmp + W * row, keep * row); + } + try f.t.gatedNorm(s.yr, s.z, d.norm, s.gated, s.gated_xs, W, g_.linear_v_heads, c.linear_dim, c.eps); + try f.ops.prefillDense(s.gated, d.out, s.r, W); + }, + .attention => |a| { + try f.ops.prefillDense(s.y, a.q, s.qg, W); + try f.ops.prefillDense(s.y, a.k, s.key, W); + try f.ops.prefillDense(s.y, a.v, seq.values[i] + p0 * kv, W); + try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, seq.keys[i] + p0 * kv, W, f.attnShape(), c.eps); + const scale: f32 = @floatCast(std.math.pow(f64, c.head_dim, -0.5)); + try f.ops.prefillAttention(s.q_rot, seq.keys[i], seq.values[i], s.attn, p0, W, g_.query_heads, g_.kv_heads, scale); + try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); + try f.ops.prefillDense(s.gated, a.o, s.r, W); + }, + } + x = try f.norm(x, s.r, layer.post_norm, W); + try f.ops.prefillDense(s.y, layer.gate, s.gate, W); + try f.ops.prefillDense(s.y, layer.up, s.up, W); + try f.t.swiglu(s.gate, s.up, s.act, s.act_xs, W, g_.intermediate); + try f.ops.prefillDense(s.act, layer.down, s.pending, W); + pending = s.pending; + try f.tapLayer(i, x, W); + } + seq.pos = p0 + W; + if (!last) return; + const at = (W - 1) * g_.hidden * bf; + const spare = if (x == s.x) s.h else s.x; + try f.t.addRmsnorm(x + at, pending.? + at, w.norm, spare, seq.head_in, s.last_xs, 1, g_.hidden, c.eps); + try f.t.groupSums(seq.head_in, seq.head_xs, g_.hidden, 1, g_.hidden); + } + + /// One stream's share of a batched prompt pass: its ids, committed at its position; `last` normalizes its last row. + pub const Piece = struct { ids: []const u32, seq: *st.Seq, last: bool }; + + /// prefill_rows: several streams' prompt chunks in one pass (each stream's conv, DeltaNet and attention its own). + pub fn chunks(f: *Forward, pieces: []const Piece) !void { + const s = f.s; + const g_ = f.g(); + const w = f.w; + const S = pieces.len; + if (S == 0 or S > st.max_streams) return error.InvalidQwenWindows; + var starts: [st.max_streams + 1]usize = undefined; + starts[0] = 0; + for (pieces, 0..) |p, k| { + if (p.ids.len == 0) return error.ChunkTooLong; + if (p.seq.pos + p.ids.len > p.seq.capacity) return error.PromptTooLong; + starts[k + 1] = starts[k] + p.ids.len; + } + const W = starts[S]; + if (W > s.rows) return error.ChunkTooLong; + const keep = c.conv_taps - 1; + const kv = g_.kvInner() * bf; + const row = g_.convDim() * bf; + { + // ids, positions, each row's stream and its conv window: [the streams' kept rows; every prompt row] + const host = try f.gpa.alloc(i32, W * (3 + c.conv_taps)); + defer f.gpa.free(host); + for (pieces, 0..) |p, k| for (p.ids, 0..) |tok, r| { + const at = starts[k] + r; + host[at] = @intCast(tok); + host[W + at] = @intCast(p.seq.pos + r); + host[2 * W + at] = @intCast(k); + for (0..c.conv_taps) |j| host[3 * W + at * c.conv_taps + j] = @intCast(if (r + j < keep) r + j else r + j + starts[k]); + }; + try f.upload32(s.ids, host[0..W]); + try f.upload32(s.pos, host[W .. 2 * W]); + try f.upload32(s.sids, host[2 * W .. 3 * W]); + try f.upload32(s.windows, host[3 * W ..]); + } + // copies the pass needs: every stream's conv rows stacked by DeltaNet layer, then each layer's writes + var items: std.ArrayList(kern.Copy) = .empty; + defer items.deinit(f.gpa); + var lin_n: usize = 0; + for (0..g_.layers) |i| if (c.linear(i)) { + const cat = s.conv_cat + lin_n * st.max_streams * keep * row; + for (pieces, 0..) |p, k| try items.append(f.gpa, .{ .dst = cat + k * keep * row, .src = p.seq.conv[i], .bytes = keep * row }); + lin_n += 1; + }; + const staged = items.items.len; + var at_layer: [256]usize = undefined; // each layer's first write, then the end + lin_n = 0; + for (0..g_.layers) |i| { + at_layer[i] = items.items.len; + if (c.linear(i)) { + // the last `keep` rows of [kept rows; the stream's prompt rows] + const cat = s.conv_cat + lin_n * st.max_streams * keep * row; + for (pieces, 0..) |p, k| { + const n = p.ids.len; + const o = starts[k]; + if (n >= keep) { + try items.append(f.gpa, .{ .dst = p.seq.conv[i], .src = s.qkv + (o + n - keep) * row, .bytes = keep * row }); + } else { + try items.append(f.gpa, .{ .dst = p.seq.conv[i], .src = cat + (k * keep + n) * row, .bytes = (keep - n) * row }); + try items.append(f.gpa, .{ .dst = p.seq.conv[i] + (keep - n) * row, .src = s.qkv + o * row, .bytes = n * row }); + } + } + lin_n += 1; + } else { + for (pieces, 0..) |p, k| { + const o = starts[k]; + const n = p.ids.len; + try items.append(f.gpa, .{ .dst = p.seq.keys[i] + p.seq.pos * kv, .src = s.prompt_keys + o * kv, .bytes = n * kv }); + try items.append(f.gpa, .{ .dst = p.seq.values[i] + p.seq.pos * kv, .src = s.prompt_values + o * kv, .bytes = n * kv }); + } + } + } + at_layer[g_.layers] = items.items.len; + if (items.items.len > 2 * st.copy_items) return error.InvalidQwenWindows; + try f.ops.upload(s.copy_items, std.mem.sliceAsBytes(items.items)); + try f.ops.copies(s.copy_items, staged); + const item = @sizeOf(kern.Copy); + try f.t.embed(s.ids, w.embed.w, w.embed.s, w.embed.b, s.x, W, g_.hidden); + var x = s.x; + var pending: ?u64 = null; + lin_n = 0; + for (w.layers, 0..) |layer, i| { + x = try f.norm(x, pending, layer.input_norm, W); + switch (layer.mixer) { + .delta => |d| { + try f.ops.prefillDense(s.y, d.qkv, s.qkv, W); + try f.ops.prefillDense(s.y, d.z, s.z, W); + try f.ops.prefillDense(s.y, d.b, s.b, W); + try f.ops.prefillDense(s.y, d.a, s.a, W); + const cat = s.conv_cat + lin_n * st.max_streams * keep * row; + try f.t.gdnPre(s.qkv, cat, d.conv, s.windows, s.sids, s.a, s.b, d.a_log, d.dt_bias, s.gq, s.gk, s.gv, s.gg, s.gbeta, W, f.gdnShape()); + for (pieces, 0..) |p, k| { + const o = starts[k]; + // each thread reads its state rows before it writes them: the chain may end in place + try f.ops.gdnPrompt(s.gq + o * g_.kInner() * bf, s.gk + o * g_.kInner() * bf, s.gv + o * g_.vInner() * bf, s.gg + o * g_.linear_v_heads * 4, s.gbeta + o * g_.linear_v_heads * 4, p.seq.rec[i], p.seq.rec[i], s.yr + o * g_.vInner() * bf, p.ids.len, f.heads()); + } + try f.ops.copies(s.copy_items + at_layer[i] * item, at_layer[i + 1] - at_layer[i]); + try f.t.gatedNorm(s.yr, s.z, d.norm, s.gated, s.gated_xs, W, g_.linear_v_heads, c.linear_dim, c.eps); + try f.ops.prefillDense(s.gated, d.out, s.r, W); + lin_n += 1; + }, + .attention => |a| { + try f.ops.prefillDense(s.y, a.q, s.qg, W); + try f.ops.prefillDense(s.y, a.k, s.key, W); + try f.ops.prefillDense(s.y, a.v, s.prompt_values, W); + try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, s.prompt_keys, W, f.attnShape(), c.eps); + try f.ops.copies(s.copy_items + at_layer[i] * item, at_layer[i + 1] - at_layer[i]); + const scale: f32 = @floatCast(std.math.pow(f64, c.head_dim, -0.5)); + const qrow = g_.query_heads * c.head_dim * bf; + for (pieces, 0..) |p, k| try f.ops.prefillAttention(s.q_rot + starts[k] * qrow, p.seq.keys[i], p.seq.values[i], s.attn + starts[k] * qrow, p.seq.pos, p.ids.len, g_.query_heads, g_.kv_heads, scale); + try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); + try f.ops.prefillDense(s.gated, a.o, s.r, W); + }, + } + x = try f.norm(x, s.r, layer.post_norm, W); + try f.ops.prefillDense(s.y, layer.gate, s.gate, W); + try f.ops.prefillDense(s.y, layer.up, s.up, W); + try f.t.swiglu(s.gate, s.up, s.act, s.act_xs, W, g_.intermediate); + try f.ops.prefillDense(s.act, layer.down, s.pending, W); + pending = s.pending; + try f.tapLayer(i, x, W); + } + const spare = if (x == s.x) s.h else s.x; + for (pieces, 0..) |p, k| { + p.seq.pos += p.ids.len; + if (!p.last) continue; + const at = (starts[k + 1] - 1) * g_.hidden * bf; + try f.t.addRmsnorm(x + at, pending.? + at, w.norm, spare, p.seq.head_in, s.last_xs, 1, g_.hidden, c.eps); + try f.t.groupSums(p.seq.head_in, p.seq.head_xs, g_.hidden, 1, g_.hidden); + } + try f.ops.s.synchronize(); // the copy table's host list goes when this returns + } + + /// The head over the stream's last prompt row: logits (1, vocab) in the scratch. + pub fn head(f: *Forward, seq: *const st.Seq) !void { + try f.ops.dense(seq.head_in, seq.head_xs, f.w.head, f.s.logits, 1); + } + + /// Greedy picks of the scratch's first `rows` logits rows (torch.argmax, the first maximum). + pub fn picks(f: *Forward, rows: usize, out: []u32) !void { + try f.ops.argmax(f.s.logits, c.vocab, c.vocab, f.s.picks, rows); + try f.ops.download(std.mem.sliceAsBytes(out[0..rows]), f.s.picks); + try f.ops.s.synchronize(); + } +}; diff --git a/zig/src/families/qwen27/cuda/kernels.zig b/zig/src/families/qwen27/cuda/kernels.zig new file mode 100644 index 0000000000..4b486922f8 --- /dev/null +++ b/zig/src/families/qwen27/cuda/kernels.zig @@ -0,0 +1,365 @@ +//! The 27B's CUDA kernels: our .cu fatbins with the Python wrappers' launch logic, and the captured Triton set. + +const std = @import("std"); +const cuda = @import("cuda"); +pub const torch_ops = @import("nemotron").torch_ops; + +/// Mangled names of the instantiations the copies in zig/kernels/cuda export (cuobjdump -symbols of each fatbin). +const sym = struct { + const group = "_ZN12tf_qmm_group12group_kernelILi64ELi16ELi64ELi1ELi4ELi8ELb0ELb0ELb0ELb0EEEvPK13__nv_bfloat16PKfNS_5PartsEiiiii"; + const wide = [_][:0]const u8{ + "_ZN12tf_qmm_group12group_kernelILi64ELi32ELi64ELi1ELi4ELi4ELb0ELb0ELb0ELb0EEEvPK13__nv_bfloat16PKfNS_5PartsEiiiii", + "_ZN12tf_qmm_group12group_kernelILi64ELi64ELi64ELi1ELi4ELi4ELb0ELb0ELb0ELb0EEEvPK13__nv_bfloat16PKfNS_5PartsEiiiii", + "_ZN12tf_qmm_group12group_kernelILi64ELi64ELi128ELi2ELi4ELi3ELb0ELb0ELb0ELb0EEEvPK13__nv_bfloat16PKfNS_5PartsEiiiii", + }; + const gemv = "_ZN12tf_lane_gemv11gemv_kernelILi64ELi64ELi8EEEvPK13__nv_bfloat16PKfNS_4PartEiii"; + const prefill_mm = "_ZN14tf_qmm_prefill14prefill_kernelILi64ELi128ELi128ELi2ELi2ELi2ELb0EEEvPK13__nv_bfloat16PKjS3_S3_Pviiiiii"; + const pattn = "_ZN20tf_prefill_attention12pattn_kernelILi256ELi8ELi2ELi2EEEvPK13__nv_bfloat16S3_S3_PS1_iiiiif"; + const chain128 = "_ZN14tf_gdn_prefill12chain_kernelI13__nv_bfloat16Li128ELi32EEEvPKT_S4_PKS1_PKfS8_S8_PfPS1_iii"; + const chain64 = "_ZN14tf_gdn_prefill12chain_kernelI13__nv_bfloat16Li64ELi32EEEvPKT_S4_PKS1_PKfS8_S8_PfPS1_iii"; + const argmax = "tf_argmax_rows_i32_kernel"; +}; + +/// qmm_group.cu's Part and Parts, passed by value: four projections at most, one used here. +pub const Part = extern struct { w: u64, scales: u64, biases: u64, out: u64, n: c_int, npad: c_int, sk: c_int, tiles: c_int, first: c_int }; +pub const Parts = extern struct { p: [4]Part, count: c_int }; + +/// lane_gemv.cu's Part: one projection whose column tiles the CTAs share out. +pub const GemvPart = extern struct { w: u64, scales: u64, biases: u64, out: u64, n: c_int, npad: c_int, sk: c_int, tiles: c_int }; + +/// gdn.cu's Pending (the tree kernel's rows still to fold): all null here, commits replay instead. +pub const Pending = extern struct { k: u64 = 0, v: u64 = 0, g: u64 = 0, beta: u64 = 0, rows: u64 = 0, row_stride: c_int = 0, counts: u64 = 0, count_stride: c_int = 0 }; + +comptime { + std.debug.assert(@sizeOf(Part) == 56 and @sizeOf(Parts) == 232 and @sizeOf(GemvPart) == 48 and @sizeOf(Pending) == 64); +} + +pub const group_smem: u32 = 35328; // LaneTile<64, 16, 64, 1, 4, 8>::SMEM +/// qmm_group's GB10 tiles 3-5 for wider rounds: rows a tile, columns a tile, threads, LaneTile::SMEM. +pub const Wide = struct { bm: usize, bn: usize, threads: u32, smem: u32 }; +pub const wide = [_]Wide{ .{ .bm = 32, .bn = 64, .threads = 128, .smem = 26112 }, .{ .bm = 64, .bn = 64, .threads = 128, .smem = 43008 }, .{ .bm = 64, .bn = 128, .threads = 256, .smem = 39168 } }; +pub const prefill_mm_smem: u32 = 41984; // Tile<64, 128, 128, 2, 2, 2>::SMEM: prompt_tile 9, two stages +pub const pattn_smem: u32 = 98304; // two 32-key slots of 256 dims beside eight warps' staged queries + +/// One item of a batched copy: 16-byte aligned pointers, a multiple of 16 bytes. +pub const Copy = extern struct { dst: u64, src: u64, bytes: u64 }; + +/// A tiled 4-bit projection: packed words, (kg, npad) scales and biases, n outputs from k inputs. +pub const QLinear = struct { w: u64, s: u64, b: u64, n: usize, k: usize, npad: usize }; + +pub const Kernels = struct { + d: *const cuda.Driver, + mods: [15]cuda.Module, + triton: cuda.aot.Set, + group: cuda.Function, + wide: [wide.len]cuda.Function, // tiles 3-5: rows <= 32, <= 64, more + gemv: cuda.Function, + prefill_mm: cuda.Function, + pattn: cuda.Function, + trees: [cuda.kernels.gdn_variants.len]cuda.Function, // tree_kernel by gdn_variants: [0] the chain one + chip: [3]c_int, // major, minor, SMs: what dispatch_tree reads + replay: cuda.Function, + prompt_chain: cuda.Function, + prompt_rows: usize, // the prompt chain's value rows a block: 64 when heads < SMs + pack_dense: cuda.Function, + transpose16: cuda.Function, + argmax: cuda.Function, + torch: torch_ops.Functions, + copy_batch: cuda.Function, + tap: cuda.Function, + rms_norm: cuda.Function, + inv_freq: cuda.Function, + torch_rms: cuda.Function, // F.rms_norm's bits on bf16 rows of 5120 (torch_ops/norm_rotary.cu) + rotary: cuda.Function, + gemv_blocks: usize, // resident lane_gemv CTAs: per SM times SMs + gemv_rows: usize, // windows of at most this many rows take lane_gemv (none; TF_QWEN27_GEMV_ROWS) + gb10: bool, + discrete: bool, // the card has its own memory: checkpoint bytes reach it through page-locked slots + + /// Loads every module; `triton_dir` holds the captured aot.json and cubins for this GPU. + pub fn load(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, triton_dir: []const u8, heads: usize) !Kernels { + const d = ctx.d; + if (!cuda.kernels.available) return error.BuiltWithoutKernels; + var k: Kernels = undefined; + k.d = d; + const kk = cuda.kernels; + const images = [_][]const u8{ kk.qmm_group, kk.qmm_prefill, kk.lane_gemv, kk.prefill_attention, kk.gdn, kk.gdn_prefill, kk.affine4_pack, kk.torch_argmax, kk.torch_topk, kk.torch_pointwise, kk.torch_indexing, kk.torch_movement, kk.torch_nemotron_constants, kk.qwen_ops, kk.torch_norm_rotary }; + var loaded: usize = 0; + errdefer for (k.mods[0..loaded]) |*m| m.unload(); + for (images, 0..) |img, i| { + k.mods[i] = try cuda.Module.load(d, img); + loaded += 1; + } + k.group = try k.mods[0].function(sym.group); + for (&k.wide, sym.wide) |*f, name| f.* = try k.mods[0].function(name); + k.prefill_mm = try k.mods[1].function(sym.prefill_mm); + k.gemv = try k.mods[2].function(sym.gemv); + k.pattn = try k.mods[3].function(sym.pattn); + for (&k.trees, cuda.kernels.gdn_variants) |*f, v| f.* = try k.mods[4].function(v.symbol); + k.replay = try k.mods[4].function(cuda.kernels.gdn_symbols.replay_bf16); + k.pack_dense = try k.mods[6].function(cuda.qlinear.symbols.pack_words); + k.transpose16 = try k.mods[6].function(cuda.qlinear.symbols.pack_scales); + k.argmax = try k.mods[7].function(sym.argmax); + k.torch = try torch_ops.Functions.resolve(k.mods[7..13]); + k.copy_batch = try k.mods[13].function("tf_copy_batch"); + k.tap = try k.mods[13].function("tf_tap"); + k.rms_norm = try k.mods[13].function("tf_rms_norm"); + k.inv_freq = try k.mods[13].function("tf_inv_freq"); + k.rotary = try k.mods[13].function("tf_rotary"); + k.torch_rms = try k.mods[14].function("tf_dflash_rms5120_bf16_fma_kernel"); + const sms: usize = @intCast(try ctx.attribute(.multiprocessor_count)); + // gdn_prefill_cuda: a block takes 128 value rows, or 64 when there are fewer heads than SMs + k.prompt_rows = if (heads >= sms) 128 else 64; + k.prompt_chain = try k.mods[5].function(if (k.prompt_rows == 128) sym.chain128 else sym.chain64); + k.triton = try cuda.aot.Set.load(gpa, io, d, ctx.device, triton_dir); + errdefer k.triton.deinit(); + try k.group.allowDynamicShared(group_smem); + for (k.wide, wide) |f, t| try f.allowDynamicShared(t.smem); + try k.gemv.allowDynamicShared(group_smem); + try k.prefill_mm.allowDynamicShared(prefill_mm_smem); + try k.pattn.allowDynamicShared(pattn_smem); + k.gemv_blocks = @max(1, try k.gemv.occupancy(128, group_smem)) * sms; + const major = try ctx.attribute(.compute_capability_major); + const minor = try ctx.attribute(.compute_capability_minor); + k.gb10 = major == 12 and minor == 1; + k.discrete = try ctx.attribute(.integrated) == 0; + k.gemv_rows = 0; // a GB10 runs the 27B's projections faster in Python's clusters, 1 to 16 rows alike + if (std.c.getenv("TF_QWEN27_GEMV_ROWS")) |v| k.gemv_rows = std.fmt.parseInt(usize, std.mem.span(v), 10) catch k.gemv_rows; + k.chip = .{ major, minor, @intCast(sms) }; + return k; + } + + pub fn deinit(k: *Kernels) void { + k.triton.deinit(); + for (&k.mods) |*m| m.unload(); + } +}; + +/// qmm.split_k: K slices fixed by the weight's shape, never by the row count. +pub fn splitK(n: usize, k: usize) usize { + const tiles = (n + 63) / 64; + const groups = k / 64; + var sk: usize = 1; + while (sk < 8 and tiles * sk < 192 and groups % (sk * 2) == 0 and groups / (sk * 2) >= 8) sk *= 2; + return sk; +} + +fn int(x: usize) c_int { + return @intCast(x); +} + +fn u(x: usize) u32 { + return @intCast(x); +} + +/// The GDN shapes every launch takes: key heads, value heads, value rows a head. +pub const Heads = struct { hk: usize, hv: usize, dv: usize }; + +/// Launch helpers on one stream; each mirrors the Python wrapper it replaces. +pub const Ops = struct { + k: *const Kernels, + s: cuda.Stream, + + fn go(o: Ops, f: cuda.Function, grid: [3]usize, block: u32, shared: u32, args: *cuda.Args) !void { + try cuda.launch.launch(f, .{ .grid = .{ .x = u(grid[0]), .y = u(grid[1]), .z = u(grid[2]) }, .block = .{ .x = block }, .shared = shared }, o.s, args); + } + + /// qmm.matmul on sm_12x: x (rows, k) bf16 with group sums xs -> out (rows, n) bf16, qmm_group's bits. + pub fn dense(o: Ops, x: u64, xs: u64, q: QLinear, out: u64, rows: usize) !void { + const sk = splitK(q.n, q.k); + if (rows > 16) return o.wideTile(x, xs, q, out, rows, sk); + // Python's clusters, a CTA a K slice; lane_gemv (byte-equal) when TF_QWEN27_GEMV_ROWS asks for it + return if (sk > 1 and rows <= o.k.gemv_rows) o.gemv(x, xs, q, out, rows, sk) else o.cluster(x, xs, q, out, rows, sk); + } + + /// qmm.matmul at a K split the caller names (matmul_rows takes the stacked weight's for each part). + pub fn denseSplit(o: Ops, x: u64, xs: u64, q: QLinear, out: u64, rows: usize, sk: usize) !void { + if (rows > 16) return o.wideTile(x, xs, q, out, rows, sk); + return if (sk > 1 and rows <= o.k.gemv_rows) o.gemv(x, xs, q, out, rows, sk) else o.cluster(x, xs, q, out, rows, sk); + } + + /// tree_forward's taps: bf16(x + pending) of `rows` rows into columns [col, col + d) of rows `stride` wide. + pub fn tap(o: Ops, x: u64, pending: u64, out: u64, rows: usize, d: usize, stride: usize, col: usize) !void { + var a: cuda.Args = .{}; + for ([_]u64{ x, pending, out }) |v| a.add(v); + for ([_]usize{ d, stride, col }) |v| a.add(int(v)); + try o.go(o.k.tap, .{ rows, 4, 1 }, 256, 0, &a); + } + + /// lane_gemv: every K slice of a column tile in one CTA, summed in slice order, CTAs looping over the tiles. + fn gemv(o: Ops, x: u64, xs: u64, q: QLinear, out: u64, rows: usize, sk: usize) !void { + const tiles = (q.n + 63) / 64; + var a: cuda.Args = .{}; + a.add(x); + a.add(xs); + a.add(GemvPart{ .w = q.w, .scales = q.s, .biases = q.b, .out = out, .n = int(q.n), .npad = int(q.npad), .sk = int(sk), .tiles = int(tiles) }); + for ([_]usize{ rows, q.k, q.k }) |v| a.add(int(v)); + const cfg: cuda.Config = .{ .grid = .{ .x = u(@min(tiles, o.k.gemv_blocks)) }, .block = .{ .x = 128 }, .shared = group_smem, .pdl = o.k.gb10 }; + try cuda.launch.launch(o.k.gemv, cfg, o.s, &a); + } + + /// qmm_group's tile 2 with one K slice: a CTA a column tile. + fn cluster(o: Ops, x: u64, xs: u64, q: QLinear, out: u64, rows: usize, sk: usize) !void { + const tiles = (q.n + 63) / 64; + var parts: Parts = std.mem.zeroes(Parts); + parts.count = 1; + parts.p[0] = .{ .w = q.w, .scales = q.s, .biases = q.b, .out = out, .n = int(q.n), .npad = int(q.npad), .sk = int(sk), .tiles = int(tiles), .first = 0 }; + const rows_t = (rows + 15) / 16; + var a: cuda.Args = .{}; + a.add(x); + a.add(xs); + a.add(parts); + for ([_]usize{ rows, q.k, q.k, rows_t, sk }) |v| a.add(int(v)); + const cfg: cuda.Config = .{ + .grid = .{ .x = u(rows_t * tiles * sk) }, + .block = .{ .x = 128 }, + .shared = group_smem, + .cluster = if (sk > 1) .{ .x = u(sk) } else null, + .pdl = o.k.gb10, + }; + try cuda.launch.launch(o.k.group, cfg, o.s, &a); + } + + /// qmm_group's dispatch on a GB10 past 16 rows: tile 3, 4 or 5, a cluster of `sk` CTAs a column tile. + fn wideTile(o: Ops, x: u64, xs: u64, q: QLinear, out: u64, rows: usize, sk: usize) !void { + const i: usize = if (rows <= 32) 0 else if (rows <= 64) 1 else 2; + const t = wide[i]; + const tiles = (q.n + t.bn - 1) / t.bn; + const rows_t = (rows + t.bm - 1) / t.bm; + var parts: Parts = std.mem.zeroes(Parts); + parts.count = 1; + parts.p[0] = .{ .w = q.w, .scales = q.s, .biases = q.b, .out = out, .n = int(q.n), .npad = int(q.npad), .sk = int(sk), .tiles = int(tiles), .first = 0 }; + var a: cuda.Args = .{}; + a.add(x); + a.add(xs); + a.add(parts); + for ([_]usize{ rows, q.k, q.k, rows_t, sk }) |v| a.add(int(v)); + const cfg: cuda.Config = .{ + .grid = .{ .x = u(rows_t * tiles * sk) }, + .block = .{ .x = t.threads }, + .shared = t.smem, + .cluster = if (sk > 1) .{ .x = u(sk) } else null, + .pdl = o.k.gb10, + }; + try cuda.launch.launch(o.k.wide[i], cfg, o.s, &a); + } + + /// qmm.prefill_matmul at prompt_tile 9: weights rounded once to bf16, one fp32 chain over K; bf16 out. + pub fn prefillDense(o: Ops, x: u64, q: QLinear, out: u64, rows: usize) !void { + const rows_t = (rows + 127) / 128; + const band = (12 << 20) / (128 * q.k * 2); + const group = @max(1, @min(rows_t, band)); + var a: cuda.Args = .{}; + for ([_]u64{ x, q.w, q.s, q.b, out }) |v| a.add(v); + for ([_]usize{ rows, q.n, q.k, q.npad, q.k, group }) |v| a.add(int(v)); + try o.go(o.k.prefill_mm, .{ rows_t * ((q.n + 127) / 128), 1, 1 }, 128, prefill_mm_smem, &a); + } + + /// gdn.tree (dispatch_tree): y from one `state`, or each stream's own by `table` and `starts`; `slots` 0: chains. + pub fn gdnTree(o: Ops, q: u64, k: u64, v: u64, g: u64, beta: u64, state: u64, table: u64, starts: u64, plan: u64, y: u64, rows: usize, streams: usize, slots: usize, most: usize, h: Heads) !void { + const variant = cuda.kernels.treeVariant(@intCast(slots), @intCast(streams), o.k.chip[0], o.k.chip[1], o.k.chip[2]); + const index = for (cuda.kernels.gdn_variants, 0..) |x, n| { + if (std.mem.eql(u8, x.symbol, variant.symbol)) break n; + } else unreachable; + var a: cuda.Args = .{}; + for ([_]u64{ q, k, v, g, beta, state, table, starts, plan }) |x| a.add(x); + a.add(int(rows)); + a.add(y); + for ([_]usize{ h.hk, h.hv, h.dv }) |x| a.add(int(x)); + a.add(Pending{}); + a.add(@as(u64, 0)); + a.add(@as(u64, 0)); + a.add(h.dv % variant.r == 0 and v % (2 * @as(u64, @min(variant.r, 8))) == 0); + const shared = 16 * @as(usize, variant.warps) * variant.slots * variant.r * 32 + (if (variant.chain) 0 else 4 * 3 * most); + try o.go(o.k.trees[index], .{ (h.dv + variant.r * variant.warps - 1) / (variant.r * variant.warps), h.hv, streams }, 32 * variant.warps, @intCast(shared), &a); + } + + /// gdn.replay in place: each stream's accepted rows stepped into the states the table names. + pub fn gdnReplay(o: Ops, table: u64, layers: usize, streams: usize, rows: u64, row_stride: usize, counts: u64, count_stride: usize, h: Heads) !void { + var a: cuda.Args = .{}; + a.add(table); + a.add(int(layers)); + a.add(rows); + a.add(int(row_stride)); + a.add(counts); + a.add(int(count_stride)); + a.add(@as(u64, 0)); + for ([_]usize{ h.hk, h.hv, h.dv }) |x| a.add(int(x)); + try o.go(o.k.replay, .{ (h.dv + 31) / 32, h.hv, layers * streams }, 128, 0, &a); + } + + /// gdn.chain: a prompt chunk's rows from `state` (read only) to `last`, chunk-invariant bits. + pub fn gdnPrompt(o: Ops, q: u64, k: u64, v: u64, g: u64, beta: u64, state: u64, last: u64, y: u64, rows: usize, h: Heads) !void { + var a: cuda.Args = .{}; + for ([_]u64{ q, k, v, g, beta, state, last, y }) |x| a.add(x); + for ([_]usize{ rows, h.hk, h.hv }) |x| a.add(int(x)); + const r = o.k.prompt_rows; + try o.go(o.k.prompt_chain, .{ h.hv, 128 / r, 1 }, u(2 * r), 0, &a); + } + + /// prefill_attention (head dim 256, two query heads a block): q (rows, heads, 256) at positions p0.. + pub fn prefillAttention(o: Ops, q: u64, kc: u64, vc: u64, out: u64, p0: usize, rows: usize, heads: usize, kv_heads: usize, scale: f32) !void { + const g = heads / kv_heads; + const hpc = 2; + var a: cuda.Args = .{}; + for ([_]u64{ q, kc, vc, out }) |v| a.add(v); + for ([_]usize{ p0, rows, heads, kv_heads, g }) |v| a.add(int(v)); + a.add(scale); + const block_rows = 16 * (8 / hpc); + try o.go(o.k.pattn, .{ (rows + block_rows - 1) / block_rows, kv_heads * (g / hpc), 1 }, 256, pattn_smem, &a); + } + + /// torch.argmax(rows of bf16 logits) as int32: the first maximum, a NaN first. + pub fn argmax(o: Ops, logits: u64, vocab: usize, ld: usize, out: u64, rows: usize) !void { + var a: cuda.Args = .{}; + a.add(logits); + a.add(out); + for ([_]usize{ rows, vocab, ld }) |v| a.add(@as(i64, @intCast(v))); + a.add(@as(i32, 0)); // dtype 0: bf16 + try o.go(o.k.argmax, .{ rows, 1, 1 }, 1024, 0, &a); + } + + /// The torch-op replacements on this stream (topk and casts for sampled draws). + pub fn torch(o: Ops) torch_ops.Torch { + return .{ .f = &o.k.torch, .s = o.s }; + } + + /// qwen_ops.cu's tf_copy_batch: `count` (dst, src, bytes) items from the device table `items`, one launch. + pub fn copies(o: Ops, items: u64, count: usize) !void { + if (count == 0) return; + var a: cuda.Args = .{}; + a.add(items); + try o.go(o.k.copy_batch, .{ count, 4, 1 }, 256, 0, &a); + } + + pub fn copy(o: Ops, dst: u64, src: u64, bytes: usize) !void { + if (bytes == 0) return; + try o.k.d.check(o.k.d.api.cuMemcpyDtoDAsync_v2(dst, src, bytes, o.s.handle), "cuMemcpyDtoDAsync"); + } + + pub fn fill32(o: Ops, dst: u64, value: u32, words: usize) !void { + try o.k.d.check(o.k.d.api.cuMemsetD32Async(dst, value, words, o.s.handle), "cuMemsetD32Async"); + } + + pub fn upload(o: Ops, dst: u64, bytes: []const u8) !void { + if (bytes.len == 0) return; + try o.k.d.check(o.k.d.api.cuMemcpyHtoDAsync_v2(dst, bytes.ptr, bytes.len, o.s.handle), "cuMemcpyHtoDAsync"); + } + + pub fn download(o: Ops, dst: []u8, src: u64) !void { + if (dst.len == 0) return; + try o.k.d.check(o.k.d.api.cuMemcpyDtoHAsync_v2(dst.ptr, src, dst.len, o.s.handle), "cuMemcpyDtoHAsync"); + } +}; + +test "split_k follows the 27B's shapes" { + try std.testing.expectEqual(@as(usize, 2), splitK(10240, 5120)); // qkv + try std.testing.expectEqual(@as(usize, 2), splitK(6144, 5120)); // z + try std.testing.expectEqual(@as(usize, 8), splitK(48, 5120)); // b, a + try std.testing.expectEqual(@as(usize, 1), splitK(17408, 5120)); // gate, up + try std.testing.expectEqual(@as(usize, 4), splitK(5120, 17408)); // down + try std.testing.expectEqual(@as(usize, 4), splitK(5120, 6144)); // out_proj, o_proj + try std.testing.expectEqual(@as(usize, 1), splitK(12288, 5120)); // q with its gate + try std.testing.expectEqual(@as(usize, 8), splitK(1024, 5120)); // k, v + try std.testing.expectEqual(@as(usize, 1), splitK(248320, 5120)); // head +} diff --git a/zig/src/families/qwen27/cuda/lanes.zig b/zig/src/families/qwen27/cuda/lanes.zig new file mode 100644 index 0000000000..8a5f86fad8 --- /dev/null +++ b/zig/src/families/qwen27/cuda/lanes.zig @@ -0,0 +1,549 @@ +//! The 27B's CUDA engine behind the lane core: every stream's window in one verify, DFlash2 drafts in one batch. + +const std = @import("std"); +const lanes = @import("lanes"); +const c = @import("shape.zig"); +const st = @import("state.zig"); +const Engine = @import("engine.zig").Engine; +const Part = @import("forward.zig").Part; +const Forward = @import("forward.zig").Forward; +const prompt_rows = @import("engine.zig").prompt_rows; +const dl = @import("draft_load.zig"); +const dr = @import("draft.zig"); +const drafts = @import("drafts.zig"); +const lone = @import("lone.zig"); +const trees = @import("tree.zig"); +const be = lanes.backend; + +/// The row counts calibrate times (calibration_rows without the tile steps a GB10 skips), to 16 a stream. +const calibration = [_]usize{ 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256 }; + +/// The last verified round, committed when the next call needs the streams' states (keep may cut it first). +const Pending = struct { + streams: [st.max_streams]*lanes.Stream = undefined, + parts: [st.max_streams]Part = undefined, + paths: [st.max_streams][]const u32 = undefined, + path_rows: [st.round_rows]u32 = undefined, // the round's kept rows, each stream's path in its own slice + tokens: [st.round_rows]u32 = undefined, + parents: [st.round_rows]i32 = undefined, + starts: [st.max_streams]usize = undefined, // each window's first row in the round + n: usize = 0, +}; + +pub const Cuda = struct { + gpa: std.mem.Allocator, + e: *Engine, + streams: u32, + seqs: std.AutoHashMapUnmanaged(*lanes.Stream, *st.Seq) = .empty, + logits: []u16 = &.{}, // a whole row on the host, for a draw with top_k off + cand_values: []f32 = &.{}, // the rows' top candidates on the host + cand_ids: []i64 = &.{}, + values: []f64 = &.{}, + pending: Pending = .{}, + drafter: ?*dr.DFlash2 = null, // DFlash2, when its checkpoint is pulled and drafts are on + drafting: std.AutoHashMapUnmanaged(*lanes.Stream, *drafts.Stream) = .empty, + costs: [calibration.len + 1]lanes.config.Cost = undefined, // the forward's ms by a round's rows (calibrate) + cost_count: usize = 0, + + pub fn init(gpa: std.mem.Allocator, e: *Engine, streams: u32) !Cuda { + return .{ + .gpa = gpa, + .e = e, + .streams = streams, + .logits = try gpa.alloc(u16, c.vocab), + .cand_values = try gpa.alloc(f32, st.round_rows * st.max_candidates), + .cand_ids = try gpa.alloc(i64, st.round_rows * st.max_candidates), + .values = try gpa.alloc(f64, st.max_candidates), + }; + } + + pub fn deinit(x: *Cuda) void { + var it = x.seqs.valueIterator(); + while (it.next()) |seq| { + seq.*.deinit(); + x.gpa.destroy(seq.*); + } + x.seqs.deinit(x.gpa); + var dit = x.drafting.valueIterator(); + while (dit.next()) |ds| { + ds.*.deinit(); + x.gpa.destroy(ds.*); + } + x.drafting.deinit(x.gpa); + x.gpa.free(x.logits); + x.gpa.free(x.cand_values); + x.gpa.free(x.cand_ids); + x.gpa.free(x.values); + } + + /// The facts the round loop reads: 16-row windows in one forward; with DFlash2, host trees with node chances. + pub fn facts(x: *const Cuda) lanes.Model { + var m: lanes.Model = .{ .exact_width = st.max_rows, .hidden_rows = true, .batch_rows = st.round_rows, .max_streams = @min(x.streams, st.max_streams), .streams_exact = true }; + if (x.drafter != null) { + m.mtp = true; + m.speculate = true; + m.speculate_early = false; + m.drafts = drafts.max_nodes; + m.draft_probabilities = true; + m.draft_streams = true; + } + const timed = x.costs[0..x.cost_count]; + var narrow: usize = 0; + while (narrow < timed.len and timed[narrow].width <= st.max_rows) narrow += 1; + m.window_costs = timed[0..narrow]; + m.shared_costs = timed; + return m; + } + + /// multi.calibrate: the forward's ms (median of three) by the rows `streams` windows bring, on a fresh state. + pub fn calibrate(x: *Cuda, io: std.Io, streams: usize) !void { + var seq = x.e.sequence(st.max_rows + 2) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + defer seq.deinit(); + const zeros: [st.max_rows]u32 = @splat(0); + const most: usize = st.max_rows * @as(usize, @min(streams, st.max_streams)); + var rows: [calibration.len + 1]usize = undefined; + var n: usize = 0; + for (calibration) |r| if (r < most) { + rows[n] = r; + n += 1; + }; + rows[n] = most; + n += 1; + var f = x.e.forward(); + const clock = std.Io.Clock.awake; + for (rows[0..n], x.costs[0..n]) |r, *cost| { + const parts_n = (r + st.max_rows - 1) / st.max_rows; + var parts: [st.max_streams]Part = undefined; + for (parts[0..parts_n], 0..) |*part, i| part.* = .{ .seq = &seq, .tokens = zeros[0 .. r / parts_n + @intFromBool(i < r % parts_n)] }; + var times: [3]f64 = undefined; + for (0..4) |rep| { + try f.ops.s.synchronize(); + const t0 = clock.now(io).toNanoseconds(); + try f.round(parts[0..parts_n]); + try f.ops.s.synchronize(); + if (rep > 0) times[rep - 1] = @as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t0)) / 1e6; + } + std.mem.sort(f64, ×, {}, std.sort.asc(f64)); + cost.* = .{ .width = @intCast(r), .ms = times[1] }; + } + x.cost_count = n; + } + + pub fn backend(x: *Cuda) be.Backend { + return .{ .ptr = x, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany } }; + } + + /// The stream's drafting (its DFlash2 context and copy index), made on first use. + pub fn draftingOf(x: *Cuda, s: *lanes.Stream) !*drafts.Stream { + if (x.drafting.get(s)) |ds| return ds; + const ds = try x.gpa.create(drafts.Stream); + errdefer x.gpa.destroy(ds); + const ctx = dr.Context.init(x.e.ctx.d) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + ds.* = drafts.Stream.init(x.gpa, ctx); + errdefer ds.deinit(); + try x.drafting.put(x.gpa, s, ds); + return ds; + } + + fn cast(p: *anyopaque) *Cuda { + return @ptrCast(@alignCast(p)); + } + + fn seqOf(x: *Cuda, s: *lanes.Stream) !*st.Seq { + return x.seqs.get(s) orelse error.UnknownQwenStream; + } + + /// Commits the last verified round: every stream its kept rows (its whole window unless keep cut it). + fn settle(x: *Cuda) !void { + const p = &x.pending; + if (p.n == 0) return; + const n = p.n; + p.n = 0; + var f = x.e.forward(); + try f.commit(p.parts[0..n], p.paths[0..n]); + } + + /// The stream's state, the last round committed first. + fn settled(x: *Cuda, s: *lanes.Stream) !*st.Seq { + try x.settle(); + return x.seqOf(s); + } + + /// The round's logits rows drawn: greedy by one argmax, sampled from top (top_k + 8) GPU candidates or whole rows. + pub fn drawRows(x: *Cuda, rules: []const ?lanes.Sampling, positions: []const u64, out: []u32) !void { + const rows = positions.len; + var f = x.e.forward(); + const s = f.s; + try f.picks(rows, out[0..rows]); + var want: usize = 0; + for (rules) |r| if (r) |rule| if (rule.temperature > 0 and rule.top_k != 0) { + want = @max(want, @min(c.vocab, @as(usize, rule.top_k) + 8)); + }; + if (want > st.max_candidates) want = 0; // past the GPU's candidates: whole rows below + if (want > 0) { + const t = f.ops.torch(); + try t.toF32(s.logits, s.logits32, rows * c.vocab); + try t.topk(s.logits32, c.vocab, rows, want, s.cand_values, s.cand_ids, s.topk_scratch); + try f.ops.download(std.mem.sliceAsBytes(x.cand_values[0 .. rows * want]), s.cand_values); + try f.ops.download(std.mem.sliceAsBytes(x.cand_ids[0 .. rows * want]), s.cand_ids); + try f.ops.s.synchronize(); + } + for (rules, positions, out[0..rows], 0..) |rule_, pos, *o, r| { + const rule = rule_ orelse continue; + if (rule.temperature <= 0) continue; + if (rule.top_k != 0 and want > 0) { + const vals = x.values[0..want]; + for (vals, x.cand_values[r * want ..][0..want]) |*v, w| v.* = w; + const ids = x.cand_ids[r * want ..][0..want]; + o.* = @intCast(try lanes.sampling.choose(x.gpa, vals, @ptrCast(ids), pos, rule)); + continue; + } + try f.ops.download(std.mem.sliceAsBytes(x.logits), s.logits + r * c.vocab * 2); + try f.ops.s.synchronize(); + o.* = try draw(x.gpa, x.logits, rule, pos); + } + } + + /// A new stream's state, sized for its prompt and reply. + pub fn open(x: *Cuda, s: *lanes.Stream) !*st.Seq { + try x.settle(); + const ids = s.prompt(); + if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; + const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); + if (ids.len >= capacity) return error.PromptTooLong; + const seq = try x.gpa.create(st.Seq); + errdefer x.gpa.destroy(seq); + seq.* = x.e.sequence(capacity) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + errdefer seq.deinit(); + try x.seqs.put(x.gpa, s, seq); + return seq; + } + + fn prefill(p: *anyopaque, s: *lanes.Stream) !void { + const x = cast(p); + const seq = try x.open(s); + errdefer x.dropSeq(s); + try x.prefillOpened(s, seq); + } + + /// prefill_state into an opened stream: the prompt in chunks, a drafting stream's last window of taps absorbed. + fn prefillOpened(x: *Cuda, s: *lanes.Stream, seq: *st.Seq) !void { + if (x.drafter) |d| if (s.drafts) { + const ds = try x.draftingOf(s); + return lone.prefill(x.e, seq, &ds.ctx, d, s.prompt()); + }; + try x.e.prefill(seq, s.prompt()); + } + + /// prefill_batch: prompts in shared passes of at most a chunk's rows (a longer one alone, in its chunks). + fn prefillMany(p: *anyopaque, streams: []const *lanes.Stream, errs: []?anyerror) !void { + const x = cast(p); + var pieces: [st.max_streams]Forward.Piece = undefined; + var which: [st.max_streams]usize = undefined; + var n: usize = 0; + var rows: usize = 0; + for (streams, errs, 0..) |s, *err, i| { + const seq = x.open(s) catch |e| { + err.* = e; + continue; + }; + const ids = s.prompt(); + if (ids.len > prompt_rows) { + x.prefillOpened(s, seq) catch |e| { + x.dropSeq(s); + err.* = e; + }; + continue; + } + if (n == st.max_streams or rows + ids.len > prompt_rows) { + x.pass(streams, errs, pieces[0..n], which[0..n]); + n = 0; + rows = 0; + } + pieces[n] = .{ .ids = ids, .seq = seq, .last = true }; + which[n] = i; + n += 1; + rows += ids.len; + } + if (n > 0) x.pass(streams, errs, pieces[0..n], which[0..n]); + } + + /// One shared prompt pass; on a failure every stream in it fails. + fn pass(x: *Cuda, streams: []const *lanes.Stream, errs: []?anyerror, pieces: []const Forward.Piece, which: []const usize) void { + x.passPieces(streams, pieces, which) catch |e| for (which) |i| { + x.dropSeq(streams[i]); + errs[i] = e; + }; + } + + fn passPieces(x: *Cuda, streams: []const *lanes.Stream, pieces: []const Forward.Piece, which: []const usize) !void { + var f = x.e.forward(); + const d = x.drafter orelse return f.chunks(pieces); + f.taps = x.e.scratch.taps; + try f.chunks(pieces); + // each drafting stream's last window of taps, consecutive whole prompts absorbed together + const row = drafts.taps_width * 2; + var group: [st.max_streams]dr.Absorb = undefined; + var g: usize = 0; + var g_rows: usize = 0; + var g_from: usize = 0; // the group's first tap row + var at: usize = 0; // this piece's first row in the pass + for (pieces, which) |pc, i| { + defer at += pc.ids.len; + const s = streams[i]; + if (!s.drafts) continue; + const ds = try x.draftingOf(s); + ds.ctx.len = 0; + ds.ctx.end = 0; + const tap_from = pc.ids.len -| dl.window; + ds.ctx.skip(tap_from); + const taken = pc.ids.len - tap_from; + if (g > 0 and (tap_from > 0 or g_from + g_rows != at or g_rows + taken > dl.window + 1)) { + try d.absorbMany(group[0..g], x.e.scratch.taps + g_from * row); + g = 0; + } + if (g == 0) { + g_from = at + tap_from; + g_rows = 0; + } + group[g] = .{ .ctx = &ds.ctx, .rows = taken }; + g += 1; + g_rows += taken; + } + if (g > 0) try d.absorbMany(group[0..g], x.e.scratch.taps + g_from * row); + } + + fn dropSeq(x: *Cuda, s: *lanes.Stream) void { + if (x.seqs.fetchRemove(s)) |entry| { + entry.value.deinit(); + x.gpa.destroy(entry.value); + } + } + + fn first(p: *anyopaque, s: *lanes.Stream, position: u64) !u64 { + const x = cast(p); + const seq = try x.settled(s); + if (position != seq.pos) return error.InvalidQwenPosition; + var f = x.e.forward(); + try f.head(seq); + var out: [1]u32 = undefined; + try x.drawRows(&.{s.sampling}, &.{position}, &out); + return out[0]; + } + + fn queue(p: *anyopaque, s: *lanes.Stream, feed: be.Feed, position: u64) !u64 { + const x = cast(p); + const seq = try x.settled(s); + if (position != seq.pos + 1) return error.InvalidQwenPosition; + const token: u32 = switch (feed) { + .value => |v| v, + .handle => |h| @intCast(h), + }; + var f = x.e.forward(); + const part = [_]Part{.{ .seq = seq, .tokens = &.{token} }}; + try f.round(&part); + var out: [1]u32 = undefined; + try x.drawRows(&.{s.sampling}, &.{position}, &out); + try f.commit(&part, &.{&.{0}}); + return out[0]; + } + + fn read(_: *anyopaque, handle: u64) !u32 { + if (handle >= c.vocab) return error.InvalidQwenToken; + return @intCast(handle); + } + + /// One forward over every window (one stream: tree_forward's, several: multi_tree_forward's), each row drawn. + fn verify(p: *anyopaque, windows: []const be.Window, out: []be.Verified) !void { + const x = cast(p); + if (windows.len != out.len or windows.len == 0 or windows.len > st.max_streams) return error.InvalidQwenWindows; + try x.settle(); + const pend = &x.pending; + var total: usize = 0; + for (windows, out, 0..) |w, o, k| { + const rows = w.rows(); + if (w.held != 0 or w.positions.len != rows or rows > st.max_rows or total + rows > st.round_rows) return error.UnsupportedQwenWindow; + if (o.sampled.len != rows or o.drafts.len != w.tokens.len) return error.InvalidQwenOutputs; + const seq = try x.seqOf(w.stream); + pend.tokens[total] = w.pending; + @memcpy(pend.tokens[total + 1 ..][0..w.tokens.len], w.tokens); + const parents: ?[]const i32 = if (w.parents) |given| blk: { + if (given.len != rows) return error.InvalidQwenWindows; + @memcpy(pend.parents[total..][0..rows], given); + break :blk pend.parents[total..][0..rows]; + } else null; + // each row's draw is keyed at its depth past the committed position + var depth: [st.max_rows]usize = undefined; + if (parents) |ps| { + try trees.depths(ps, depth[0..rows]); + } else for (depth[0..rows], 0..) |*d, r| { + d.* = r; + } + for (w.positions, depth[0..rows]) |pos, d| if (pos != seq.pos + d + 1) return error.InvalidQwenPosition; + pend.parts[k] = .{ .seq = seq, .tokens = pend.tokens[total..][0..rows], .parents = parents }; + pend.starts[k] = total; + pend.streams[k] = w.stream; + // a window kept whole: its rows in order (keep replaces them with its path) + for (pend.path_rows[total..][0..rows], 0..) |*row, r| row.* = @intCast(r); + pend.paths[k] = pend.path_rows[total..][0..rows]; + total += rows; + } + var f = x.e.forward(); + if (x.drafter != null) f.taps = x.e.scratch.taps; + try f.round(pend.parts[0..windows.len]); + pend.n = windows.len; + var rules: [st.round_rows]?lanes.Sampling = undefined; + var positions: [st.round_rows]u64 = undefined; + var drawn: [st.round_rows]u32 = undefined; + var row: usize = 0; + for (windows) |w| for (w.positions) |pos| { + rules[row] = w.stream.sampling; + positions[row] = pos; + row += 1; + }; + try x.drawRows(rules[0..total], positions[0..total], &drawn); + row = 0; + for (windows, out) |w, o| { + @memcpy(o.sampled, drawn[row..][0..o.sampled.len]); + row += o.sampled.len; + @memcpy(o.drafts, w.tokens); + } + } + + /// A window keeps its accepted path (a chain's prefix, or a tree's root-to-node rows); verify keeps all of it. + fn keep(p: *anyopaque, windows: []const be.Window, paths: []const []const u32) !void { + const x = cast(p); + const pend = &x.pending; + for (windows, paths) |w, path| { + const k = for (pend.streams[0..pend.n], 0..) |s, k| { + if (s == w.stream) break k; + } else return error.UnknownQwenStream; + const rows = pend.parts[k].tokens.len; + if (path.len == 0 or path.len > rows) return error.InvalidQwenCommit; + const slot: [*]u32 = @constCast(pend.paths[k].ptr); + for (path, 0..) |r, i| { + if (r >= rows) return error.InvalidQwenCommit; + slot[i] = r; + } + pend.paths[k] = slot[0..path.len]; + } + } + + /// The kept rows' taps into each stream's drafter context, then each stream's next drafts. + fn draft(p: *anyopaque, requests: []const be.DraftRequest) !void { + const x = cast(p); + const d = x.drafter orelse { + for (requests) |r| if (r.depth != 0) return error.QwenCheckpointHasNoDraftHead; + return; + }; + const pend = &x.pending; + var kept: [st.max_streams]drafts.Kept = undefined; + var rows: [st.round_rows]u32 = undefined; + var asks: [st.max_streams]drafts.Ask = undefined; + var firsts: [st.max_streams]std.ArrayList(u32) = @splat(.empty); + defer for (&firsts) |*l| l.deinit(x.gpa); + var nk: usize = 0; + var nr: usize = 0; + var na: usize = 0; + for (requests) |r| { + if (na == st.max_streams) return error.InvalidQwenWindows; + const ds = x.drafting.get(r.stream) orelse continue; + var context = r.stream.context.items; + if (r.rows) |path| { + const k = for (pend.streams[0..pend.n], 0..) |s, k| { + if (s == r.stream) break k; + } else return error.UnknownQwenStream; + for (path, rows[nr..][0..path.len]) |row, *g| g.* = @intCast(pend.starts[k] + row); + kept[nk] = .{ .x = ds, .rows = rows[nr..][0..path.len] }; + nk += 1; + nr += path.len; + } else if (r.first) |feed| { + // the prompt's taps went in with its prefill; the first token ends the context the copies read + const token: u32 = switch (feed) { + .value => |v| v, + .handle => |h| try read(p, h), + }; + try firsts[na].appendSlice(x.gpa, context); + try firsts[na].append(x.gpa, token); + context = firsts[na].items; + } + asks[na] = .{ .x = ds, .context = context, .sampling = r.stream.sampling }; + na += 1; + } + const f = x.e.forward(); + try drafts.absorb(d, f.ops, x.e.scratch.taps, x.e.scratch.copy_items, kept[0..nk]); + try drafts.propose(d, asks[0..na]); + } + + /// The drafts a stream holds as host tokens and parents (a copy's as a chain). + fn tree(p: *anyopaque, s: *lanes.Stream, gpa: std.mem.Allocator) !?lanes.stream.Held { + const x = cast(p); + const ds = x.drafting.get(s) orelse return null; + const tokens = try gpa.dupe(u32, ds.tokens.items); + errdefer gpa.free(tokens); + const parents: ?[]i32 = if (ds.kind == .tree) try gpa.dupe(i32, ds.parents.items) else null; + return .{ .count = @intCast(tokens.len), .tokens = tokens, .parents = parents }; + } + + /// Each held draft's chance of landing: a tree node's exp(-path score) (allocate's), a copied token's 1. + fn probabilities(p: *anyopaque, s: *lanes.Stream, out: []f64) !bool { + const x = cast(p); + const ds = x.drafting.get(s) orelse return false; + if (out.len > ds.scores.items.len) return false; + for (out, ds.scores.items[0..out.len]) |*o, score| o.* = @exp(-score); + return true; + } + + fn release(p: *anyopaque, s: *lanes.Stream) void { + const x = cast(p); + x.settle() catch |err| std.log.err("committing the last round before a release: {s}", .{@errorName(err)}); + if (x.seqs.fetchRemove(s)) |entry| { + entry.value.deinit(); + x.gpa.destroy(entry.value); + } + if (x.drafting.fetchRemove(s)) |entry| { + entry.value.deinit(); + x.gpa.destroy(entry.value); + } + } +}; + +fn value(word: u16) f64 { + return @as(f32, @bitCast(@as(u32, word) << 16)); +} + +/// The Metal backend's draw: the top-k candidates by value then id, then the shared fp64 keyed sampler. +pub fn draw(gpa: std.mem.Allocator, logits: []const u16, settings: ?lanes.Sampling, position: u64) !u32 { + var best: usize = 0; + for (logits, 0..) |word, id| { + const x = value(word); + if (!std.math.isFinite(x)) return error.NonfiniteQwenLogits; + if (x > value(logits[best])) best = id; + } + const sampling = settings orelse return @intCast(best); + if (sampling.temperature <= 0) return @intCast(best); + const k = if (sampling.top_k == 0) logits.len else @min(logits.len, sampling.top_k); + const values = try gpa.alloc(f64, k); + defer gpa.free(values); + const ids = try gpa.alloc(u64, k); + defer gpa.free(ids); + var filled: usize = 0; + for (logits, 0..) |word, id| { + const x = value(word); + if (k == logits.len) { + values[id] = x; + ids[id] = id; + continue; + } + if (filled == k and x <= values[k - 1]) continue; + var at = @min(filled, k - 1); + while (at > 0 and x > values[at - 1]) : (at -= 1) { + values[at] = values[at - 1]; + ids[at] = ids[at - 1]; + } + values[at] = x; + ids[at] = id; + filled = @min(filled + 1, k); + } + return @intCast(try lanes.sampling.choose(gpa, values, ids, position, sampling)); +} diff --git a/zig/src/families/qwen27/cuda/lone.zig b/zig/src/families/qwen27/cuda/lone.zig new file mode 100644 index 0000000000..0397ea706f --- /dev/null +++ b/zig/src/families/qwen27/cuda/lone.zig @@ -0,0 +1,181 @@ +//! decode.draft_decode for one stream: copies from the context, else a DFlash2 tree; verify, walk, commit, absorb. + +const std = @import("std"); +const lanes = @import("lanes"); +const c = @import("shape.zig"); +const st = @import("state.zig"); +const tree = @import("tree.zig"); +const kern = @import("kernels.zig"); +const dr = @import("draft.zig"); +const dl = @import("draft_load.zig"); +const Engine = @import("engine.zig").Engine; +const Forward = @import("forward.zig").Forward; +const Part = @import("forward.zig").Part; +const Lanes = @import("lanes.zig").Cuda; + +const bf = 2; +const taps_width = dl.taps.len * dl.hidden; + +/// decode.CopyIndex: earlier 8-token matches of the context's suffix, the longest continuation proposed. +pub const CopyIndex = struct { + pub const n = 8; + const Key = [n]u32; + gpa: std.mem.Allocator, + positions: std.AutoHashMapUnmanaged(Key, std.ArrayList(usize)) = .empty, + indexed: usize = 0, + + pub fn deinit(x: *CopyIndex) void { + var it = x.positions.valueIterator(); + while (it.next()) |l| l.deinit(x.gpa); + x.positions.deinit(x.gpa); + } + + fn update(x: *CopyIndex, context: []const u32) !void { + if (context.len < n) return; + const last = context.len - n; // the trailing query itself stays out of the index + var start = x.indexed; + while (start < last) : (start += 1) { + const gop = try x.positions.getOrPut(x.gpa, context[start..][0..n].*); + if (!gop.found_existing) gop.value_ptr.* = .empty; + try gop.value_ptr.append(x.gpa, start); + } + x.indexed = @max(x.indexed, last); + } + + /// The longest continuation after an earlier occurrence of the last 8 tokens (latest first), at least 8 long. + pub fn propose(x: *CopyIndex, context: []const u32, max_nodes: usize) ![]const u32 { + if (context.len < 2 * n) return &.{}; + try x.update(context); + const starts = x.positions.get(context[context.len - n ..][0..n].*) orelse return &.{}; + var best: []const u32 = &.{}; + var i = starts.items.len; + while (i > 0) { + i -= 1; + const from = starts.items[i] + n; + const cont = context[from..@min(context.len, from + max_nodes)]; + if (cont.len > best.len) { + best = cont; + if (best.len == max_nodes) break; + } + } + return if (best.len >= n) best else &.{}; + } +}; + +/// decode.next_copy_rows: a copy's next window, twice as wide after a whole copy, half after a broken one. +pub fn nextCopyRows(rows: usize, whole: bool, tree_rows: usize, max_rows: usize) usize { + const first = @min(max_rows, @max(tree_rows, 16)); + return if (whole) @min(max_rows, @max(rows, first) * 2) else @max(first, @min(rows, max_rows) / 2); +} + +pub const Stats = struct { rounds: usize = 0, drafted: usize = 0, accepted: usize = 0 }; + +/// prefill_state with a drafter: the prompt in even chunks, its last `window` rows' taps into the context. +pub fn prefill(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, prompt: []const u32) !void { + try seq.reset(e.ops(), e.w.g); + ctx.len = 0; + ctx.end = 0; + const tap_from = prompt.len -| dl.window; + ctx.skip(tap_from); + var f = e.forward(); + f.taps = e.scratch.taps; + const rows = @import("engine.zig").prompt_rows; + const chunks = (prompt.len + rows - 1) / rows; + for (0..chunks) |j| { + const b = @import("engine.zig").chunkBounds(0, prompt.len, rows, j); + try f.chunk(prompt[b[0]..b[1]], seq, j == chunks - 1); + if (b[1] <= tap_from) continue; + const skip = tap_from -| b[0]; + try d.absorb(ctx, e.scratch.taps + skip * taps_width * bf, b[1] - b[0] - skip); + } +} + +/// draft_decode, greedy: rounds until `count` tokens or an end token; the first token is `pending`. +pub fn decode(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, prompt: []const u32, pending: u32, count: usize, max_rows: usize, out: *std.ArrayList(u32), stats: *Stats) !void { + const gpa = e.gpa; + var context: std.ArrayList(u32) = .empty; + defer context.deinit(gpa); + try context.appendSlice(gpa, prompt); + try out.append(gpa, pending); + try context.append(gpa, pending); + var copies: CopyIndex = .{ .gpa = gpa }; + defer copies.deinit(); + const tree_rows = max_rows; + var copy_rows = nextCopyRows(tree_rows, false, tree_rows, max_rows); + var tokens: [st.max_rows]u32 = undefined; + var parents: [st.max_rows]i32 = undefined; + var sampled: [st.max_rows]u32 = undefined; + var path: [st.max_rows]u32 = undefined; + var f = e.forward(); + f.taps = e.scratch.taps; + while (out.items.len < count and !e.w.isEos(out.items[out.items.len - 1])) { + const last = out.items[out.items.len - 1]; + tokens[0] = last; + parents[0] = -1; + var rows: usize = 1; + const copied = try copies.propose(context.items, copy_rows - 1); + if (copied.len > 0) { + for (copied, 1..) |t, r| { + tokens[r] = t; + parents[r] = @as(i32, @intCast(r)) - 1; + } + rows += copied.len; + } else if (ctx.len > 0) { + const p = try d.propose(ctx, last, context.items.len, tree_rows - 1, null); + for (p.tokens, p.parents, 1..) |t, pr, r| { + tokens[r] = t; + parents[r] = if (pr < 0) 0 else pr + 1; + } + rows += p.tokens.len; + } + const part = [_]Part{.{ .seq = seq, .tokens = tokens[0..rows], .parents = parents[0..rows] }}; + try f.round(&part); + try f.picks(rows, &sampled); + // the path: from the root, each next token the child whose token the target drew + var n: usize = 1; + path[0] = 0; + var terminal = sampled[0]; + while (out.items.len + n < count and !e.w.isEos(terminal)) { + const child = for (1..rows) |r| { + if (parents[r] == @as(i32, @intCast(path[n - 1])) and tokens[r] == terminal) break r; + } else break; + path[n] = @intCast(child); + n += 1; + terminal = sampled[child]; + } + try f.commit(&part, &.{path[0..n]}); + // the drafter reads the kept rows' taps, gathered in path order + var items: [st.max_rows]kern.Copy = undefined; + const gathered = e.scratch.taps + st.max_rows * taps_width * bf; + for (path[0..n], 0..) |row, j| items[j] = .{ .dst = gathered + j * taps_width * bf, .src = e.scratch.taps + row * taps_width * bf, .bytes = taps_width * bf }; + try f.ops.upload(e.scratch.copy_items, std.mem.sliceAsBytes(items[0..n])); + try f.ops.copies(e.scratch.copy_items, n); + try d.absorb(ctx, gathered, n); + for (path[1..n]) |row| { + try out.append(gpa, tokens[row]); + try context.append(gpa, tokens[row]); + } + try out.append(gpa, terminal); + try context.append(gpa, terminal); + if (copied.len > 0) copy_rows = nextCopyRows(copy_rows, n == rows, tree_rows, max_rows); + stats.rounds += 1; + stats.drafted += rows - 1; + stats.accepted += n - 1; + } +} + +test "copy windows grow after whole copies and shrink after broken ones" { + try std.testing.expectEqual(@as(usize, 12), nextCopyRows(12, false, 12, 12)); + try std.testing.expectEqual(@as(usize, 12), nextCopyRows(12, true, 12, 12)); + try std.testing.expectEqual(@as(usize, 16), nextCopyRows(16, false, 12, 64)); + try std.testing.expectEqual(@as(usize, 32), nextCopyRows(16, true, 12, 64)); +} + +test "copies propose the latest earlier continuation of the last 8 tokens" { + const gpa = std.testing.allocator; + var x: CopyIndex = .{ .gpa = gpa }; + defer x.deinit(); + const ctx = [_]u32{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 1, 2, 3, 4, 5, 6, 7, 8 }; + try std.testing.expectEqualSlices(u32, &.{ 9, 10, 11, 12, 13, 14, 15, 16 }, try x.propose(&ctx, 8)); + try std.testing.expectEqual(@as(usize, 0), (try x.propose(ctx[0..15], 8)).len); +} diff --git a/zig/src/families/qwen27/cuda/native.zig b/zig/src/families/qwen27/cuda/native.zig new file mode 100644 index 0000000000..6e3f362fdf --- /dev/null +++ b/zig/src/families/qwen27/cuda/native.zig @@ -0,0 +1,111 @@ +//! Qwen3.5-family dense models on CUDA behind the native family interface (native/cuda.zig's registry). + +const std = @import("std"); +const cuda = @import("cuda"); +const lanes = @import("lanes"); +const c = @import("shape.zig"); +const st = @import("state.zig"); +const engine = @import("engine.zig"); +const Lanes = @import("lanes.zig").Cuda; +const dr = @import("draft.zig"); + +pub const model_type = "qwen3_5"; +pub const formats: []const []const u8 = &.{"mlx-q4g64"}; +pub const default_context: i64 = 32768; +pub const max_segments: u32 = 1; +pub const prompt_rows: u32 = engine.prompt_rows; + +pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, drafter: ?[]const u8 = null }; + +/// A lone drafted stream's own driver (none here: one stream decodes in the lane core too, as under --parallel). +pub const LoneRun = *const fn (ctx: *anyopaque, s: *lanes.Stream, hooks: *anyopaque, committed: *const fn (*anyopaque) void, yield: *const fn (*anyopaque) bool) anyerror!bool; + +/// What the native server drives: the lane backend, the facts its round loop reads, and how to free it. +pub const Loaded = struct { + backend: lanes.backend.Backend, + facts: lanes.Model, + rows: u32, + /// Device bytes each admitted stream allocates for its own state and caches. + stream_bytes: usize, + /// Streams that allocate nothing (none here: every stream's state is its own). + free_streams: u32 = 0, + /// What a --slide trainer keeps out of the streams' room (none: the 27B does not learn on CUDA yet). + learn_bytes: usize = 0, + ctx: *anyopaque, + deinit: *const fn (*anyopaque) void, + lone: ?LoneRun = null, +}; + +const Owned = struct { gpa: std.mem.Allocator, e: *engine.Engine, lanes: Lanes, draft: ?*dr.DFlash2 = null }; + +/// The drafter the Python engine pairs with the 27B once pulled (z-lab/Qwen3.8-27B-DFlash2). +const drafter_repo = "models--z-lab--Qwen3.8-27B-DFlash2"; + +fn getenv(name: [:0]const u8) ?[]const u8 { + return std.mem.span(std.c.getenv(name) orelse return null); +} + +/// --drafter, else the pulled drafter's snapshot in the Hugging Face cache; null when neither holds one. +fn drafterDir(a: std.mem.Allocator, io: std.Io, given: ?[]const u8) !?[]const u8 { + if (given) |d| return try a.dupe(u8, d); + const hub = if (getenv("HF_HUB_CACHE")) |h| try a.dupe(u8, h) else if (getenv("HF_HOME")) |h| try std.fs.path.join(a, &.{ h, "hub" }) else blk: { + const home = getenv("HOME") orelse return null; + break :blk try std.fs.path.join(a, &.{ home, ".cache", "huggingface", "hub" }); + }; + defer a.free(hub); + const snaps = try std.fs.path.join(a, &.{ hub, drafter_repo, "snapshots" }); + defer a.free(snaps); + var dir = std.Io.Dir.cwd().openDir(io, snaps, .{ .iterate = true }) catch return null; + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |entry| if (entry.kind == .directory) return try std.fs.path.join(a, &.{ snaps, entry.name }); + return null; +} + +/// The streams the lane core may admit at most; the server's memory admission picks how many fit. +const most_streams = 32; + +pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: ?[]const u8, o: Options) !Loaded { + // the 27B's glue runs the Python engine's Triton kernels, captured on this chip (no own glue kernels yet) + const e = try engine.Engine.init(gpa, io, ctx, dir, kernels orelse return error.QwenNeedsCapturedKernels, .{ .context = o.context }); + errdefer e.deinit(); + const own = try gpa.create(Owned); + errdefer gpa.destroy(own); + own.* = .{ .gpa = gpa, .e = e, .lanes = try Lanes.init(gpa, e, most_streams) }; + errdefer own.lanes.deinit(); + if (o.drafts) if (try drafterDir(gpa, io, o.drafter)) |draft_dir| { + defer gpa.free(draft_dir); + const f = e.forward(); + own.draft = try dr.DFlash2.init(gpa, io, f.ops, f.t, &e.w, draft_dir, dir); + own.lanes.drafter = own.draft; + std.log.info("DFlash2 drafts every stream: {s}", .{draft_dir}); + // the forward's ms by rows: a shared round's tree widths follow the curve + try own.lanes.calibrate(io, most_streams); + }; + return .{ + .backend = own.lanes.backend(), + .facts = own.lanes.facts(), + .rows = st.max_rows, + .stream_bytes = st.Seq.bytes(e.w.g, o.context) + if (own.draft != null) dr.Context.bytes() else 0, + .ctx = own, + .deinit = release, + }; +} + +/// A request this engine refuses, in words; null: none of its own. +pub fn explain(_: ?*anyopaque, err: anyerror) ?[]const u8 { + return switch (err) { + error.PromptTooLong => "the prompt and its reply exceed this server's context window: shorten it or lower max_tokens", + error.OutOfDeviceMemory => "the GPU had no memory left for this request's caches: retry once another request ends", + error.QwenCheckpointHasNoDraftHead => "this checkpoint has no draft head on CUDA yet: serve with --no-drafts", + else => null, + }; +} + +fn release(p: *anyopaque) void { + const own: *Owned = @ptrCast(@alignCast(p)); + own.lanes.deinit(); + if (own.draft) |d| d.deinit(); + own.e.deinit(); + own.gpa.destroy(own); +} diff --git a/zig/src/families/qwen27/cuda/round.zig b/zig/src/families/qwen27/cuda/round.zig new file mode 100644 index 0000000000..2b2d4e2697 --- /dev/null +++ b/zig/src/families/qwen27/cuda/round.zig @@ -0,0 +1,230 @@ +//! A verify round's inputs laid out as tree_forward or multi_tree_forward lays them (Triton variants follow alignment). + +const std = @import("std"); +const c = @import("shape.zig"); +const kern = @import("kernels.zig"); +const tri = @import("triton.zig"); +const st = @import("state.zig"); +const tree = @import("tree.zig"); + +const bf = 2; + +/// One stream's window: its tokens from its committed position, and each row's parent (null: a chain). +pub const Part = struct { seq: *st.Seq, tokens: []const u32, parents: ?[]const i32 = null }; + +/// Where a round's inputs live on the device, and what the layer loop launches with. +pub const Round = struct { + rows: usize, + streams: usize, + multi: bool, + ids: u64, + pos: u64, + windows: u64, + sids: u64, // multi: each row's stream; else the windows again (gdn_pre passes WIN as SID) + plan: u64, // GDN schedule entries (rows, 3) + starts: u64, // stream row starts (streams + 1) + slots: usize, // the GDN schedule's live states (0: every window a chain) + most: usize, // the most rows a stream's window holds + tables: u64, // multi: (DeltaNet layers, streams) state pointers, a layer's row at its DeltaNet index + attn: tri.Tri.Plan, + offs: u64, // (attention layers, streams, 2) int64 cache offsets; single: indexed by layer + starts_host: [st.max_streams + 1]usize, + + /// The conv rows gdn_pre reads for layer `i` (its DeltaNet index `j`): the stream's own, or the stacked copy. + pub fn conv(r: *const Round, parts: []const Part, s: *const st.Scratch, g: c.Geometry, i: usize, j: usize) u64 { + if (!r.multi) return parts[0].seq.conv[i]; + return s.conv_cat + j * st.max_streams * (c.conv_taps - 1) * g.convDim() * bf; + } + + /// Layer `i`'s attention cache offsets: tree_forward's (one stream, by layer) or the packed (layer, streams) rows. + pub fn offsets(r: *const Round, i: usize, a: usize) u64 { + return if (r.multi) r.offs + a * r.streams * 16 else r.offs + i * 16; + } + + /// The DeltaNet state table of DeltaNet layer `j` (multi only). + pub fn table(r: *const Round, j: usize) u64 { + return r.tables + j * r.streams * 8; + } +}; + +/// A part's parents: its own tree's, or a chain's. +pub fn parentsOf(p: Part, buf: []i32) []const i32 { + if (p.parents) |x| return x; + for (buf[0..p.tokens.len], 0..) |*x, r| x.* = @as(i32, @intCast(r)) - 1; + return buf[0..p.tokens.len]; +} + +/// attention.plan_host: rows, (first, rows, keys, slots) per stream, items, then window parents (global rows). +fn attnPlan(gpa: std.mem.Allocator, out: *std.ArrayList(i32), parts: []const Part, group: usize) !usize { + for (parts, 0..) |p, s| for (p.tokens) |_| try out.append(gpa, @intCast(s)); + var start: usize = 0; + for (parts) |p| { + const w = p.tokens.len; + try out.appendSlice(gpa, &.{ @intCast(start), @intCast(w), @intCast(p.seq.pos), @intCast(tri.slots(p.seq.pos, w)) }); + start += w; + } + var n_items: usize = 0; + for (parts, 0..) |p, s| { + const w = p.tokens.len; + const folded = tri.groups(p.seq.pos, w); + const loose = p.seq.pos / tri.chunk - folded * tri.group_chunks; + for (0..folded + loose) |j| { + const code: i32 = if (j < folded) @intCast(j) else -1 - @as(i32, @intCast(j - folded)); + var first: usize = 0; + while (first < w * group) : (first += 16) { + try out.appendSlice(gpa, &.{ @intCast(s), @intCast(first), code }); + n_items += 1; + } + } + } + start = 0; + var buf: [tree.max_nodes]i32 = undefined; + for (parts) |p| { + for (parentsOf(p, &buf)) |x| try out.append(gpa, if (x < 0) -1 else @intCast(start + @as(usize, @intCast(x)))); + start += p.tokens.len; + } + return n_items; +} + +/// Each stream's key and value caches as bf16 element offsets from the scratch's origin. +fn offsetOf(s: *const st.Scratch, ptr: u64) i64 { + return @divExact(@as(i64, @bitCast(ptr -% s.origin)), bf); +} + +/// Every stream's host plans: positions (pos + depth), the GDN schedule (nodes offset by its start), conv windows. +const Host = struct { pos: std.ArrayList(i32) = .empty, entries: std.ArrayList(i32) = .empty, windows: std.ArrayList(i32) = .empty, slots: usize = 0, most: usize = 0 }; + +fn plans(gpa: std.mem.Allocator, parts: []const Part, starts: []const usize, multi: bool, h: *Host) !void { + const keep = c.conv_taps - 1; + var buf: [tree.max_nodes]i32 = undefined; + var dep: [tree.max_nodes]usize = undefined; + var win: [tree.max_nodes * c.conv_taps]i32 = undefined; + for (parts, 0..) |p, k| { + const parents = parentsOf(p, &buf); + const w = parents.len; + try tree.depths(parents, dep[0..w]); + for (dep[0..w]) |d| try h.pos.append(gpa, @intCast(p.seq.pos + d)); + const s = try tree.schedule(gpa, parents); + defer gpa.free(s.entries); + for (s.entries, 0..) |x, j| try h.entries.append(gpa, if (j % 3 == 0) x + @as(i32, @intCast(starts[k])) else x); + h.slots = @max(h.slots, s.slots); + h.most = @max(h.most, w); + // one stream's windows index [its state; its rows]; several streams' index [their states; all rows] + tree.convWindows(parents, keep, if (multi) starts[k] else 0, win[0 .. w * (keep + 1)]); + try h.windows.appendSlice(gpa, win[0 .. w * (keep + 1)]); + } +} + +/// Uploads the round's inputs and runs attention's _paths; the caller's layer loop reads the returned Round. +pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, g: c.Geometry, parts: []const Part) !Round { + if (parts.len == 0 or parts.len > st.max_streams) return error.InvalidQwenWindows; + var W: usize = 0; + var r: Round = undefined; + r.starts_host[0] = 0; + for (parts, 0..) |p, k| { + if (p.tokens.len == 0 or p.tokens.len > tree.max_nodes) return error.InvalidQwenWindows; + if (p.parents) |x| if (x.len != p.tokens.len) return error.InvalidQwenWindows; + if (p.seq.pos + p.tokens.len > p.seq.capacity) return error.PromptTooLong; + W += p.tokens.len; + r.starts_host[k + 1] = W; + } + if (W > st.round_rows) return error.WindowTooWide; + r.rows = W; + r.streams = parts.len; + r.multi = parts.len > 1; + const S = parts.len; + const group = g.query_heads / g.kv_heads; + var h: Host = .{}; + defer { + h.pos.deinit(gpa); + h.entries.deinit(gpa); + h.windows.deinit(gpa); + } + try plans(gpa, parts, r.starts_host[0 .. S + 1], r.multi, &h); + r.slots = h.slots; + r.most = h.most; + var host: std.ArrayList(i32) = .empty; + defer host.deinit(gpa); + var offs: std.ArrayList(i64) = .empty; + defer offs.deinit(gpa); + if (!r.multi) { + // tree_forward: tokens, positions, plan (entries then starts), windows and attention's plan apart + const p = parts[0]; + for (p.tokens) |tok| try host.append(gpa, @intCast(tok)); + try ops.upload(s.ids, std.mem.sliceAsBytes(host.items)); + try ops.upload(s.pos, std.mem.sliceAsBytes(h.pos.items)); + try ops.upload(s.windows, std.mem.sliceAsBytes(h.windows.items)); + try ops.s.synchronize(); + host.clearRetainingCapacity(); + try host.appendSlice(gpa, h.entries.items); + try host.appendSlice(gpa, &.{ 0, @intCast(W) }); + try ops.upload(s.plan, std.mem.sliceAsBytes(host.items)); + try ops.s.synchronize(); + host.clearRetainingCapacity(); + const n_items = try attnPlan(gpa, &host, parts, group); + try ops.upload(s.attn_plan, std.mem.sliceAsBytes(host.items)); + try ops.s.synchronize(); + for (0..g.layers) |i| try offs.appendSlice(gpa, if (c.linear(i)) &.{ 0, 0 } else &.{ offsetOf(s, p.seq.keys[i]), offsetOf(s, p.seq.values[i]) }); + try ops.upload(s.offs, std.mem.sliceAsBytes(offs.items)); + try ops.s.synchronize(); + r.ids = s.ids; + r.pos = s.pos; + r.windows = s.windows; + r.sids = s.windows; + r.plan = s.plan; + r.starts = s.plan + 3 * W * 4; + r.tables = 0; + r.offs = s.offs; + r.attn = .{ .rows = s.attn_plan, .streams = s.attn_plan + W * 4, .items = s.attn_plan + (W + 4) * 4, .n_items = n_items, .paths = s.paths, .depths = s.depths, .width = W }; + try t.paths(s.attn_plan + (W + 4 + 3 * n_items) * 4, s.paths, s.depths, W); + return r; + } + // multi_tree_forward: positions, stream ids, plan entries, starts, ids, windows, attention's plan in one copy + try host.appendSlice(gpa, h.pos.items); + for (parts, 0..) |p, k| for (p.tokens) |_| try host.append(gpa, @intCast(k)); + try host.appendSlice(gpa, h.entries.items); + for (r.starts_host[0 .. S + 1]) |x| try host.append(gpa, @intCast(x)); + for (parts) |p| for (p.tokens) |tok| try host.append(gpa, @intCast(tok)); + try host.appendSlice(gpa, h.windows.items); + const at_attn = host.items.len; + const n_items = try attnPlan(gpa, &host, parts, group); + try ops.upload(s.round, std.mem.sliceAsBytes(host.items)); + try ops.s.synchronize(); + const base = s.round; + r.pos = base; + r.sids = base + W * 4; + r.plan = base + 2 * W * 4; + r.starts = base + 5 * W * 4; + r.ids = base + (5 * W + S + 1) * 4; + r.windows = base + (6 * W + S + 1) * 4; + const ab = base + at_attn * 4; + r.attn = .{ .rows = ab, .streams = ab + W * 4, .items = ab + (W + 4 * S) * 4, .n_items = n_items, .paths = s.paths, .depths = s.depths, .width = W }; + // the state pointer table and cache offsets, one copy each; every stream's conv rows stacked by layer + var ptrs: std.ArrayList(u64) = .empty; + defer ptrs.deinit(gpa); + var stack: std.ArrayList(kern.Copy) = .empty; + defer stack.deinit(gpa); + const keep = c.conv_taps - 1; + var j: usize = 0; + for (0..g.layers) |i| { + if (!c.linear(i)) { + for (parts) |p| try offs.appendSlice(gpa, &.{ offsetOf(s, p.seq.keys[i]), offsetOf(s, p.seq.values[i]) }); + continue; + } + const cat = s.conv_cat + j * st.max_streams * keep * g.convDim() * bf; + for (parts, 0..) |p, k| { + try ptrs.append(gpa, p.seq.rec[i]); + try stack.append(gpa, .{ .dst = cat + k * keep * g.convDim() * bf, .src = p.seq.conv[i], .bytes = keep * g.convDim() * bf }); + } + j += 1; + } + try ops.upload(s.tables, std.mem.sliceAsBytes(ptrs.items)); + try ops.upload(s.offs, std.mem.sliceAsBytes(offs.items)); + try ops.upload(s.copy_items, std.mem.sliceAsBytes(stack.items)); + try ops.copies(s.copy_items, stack.items.len); + try ops.s.synchronize(); + r.tables = s.tables; + r.offs = s.offs; + try t.paths(ab + (W + 4 * S + 3 * n_items) * 4, s.paths, s.depths, W); + return r; +} diff --git a/zig/src/families/qwen27/cuda/shape.zig b/zig/src/families/qwen27/cuda/shape.zig new file mode 100644 index 0000000000..b018dc22a1 --- /dev/null +++ b/zig/src/families/qwen27/cuda/shape.zig @@ -0,0 +1,102 @@ +//! The shapes the CUDA engine is built for: the family's parsed config (config.zig), held to the 27B's. +const std = @import("std"); +const config = @import("../config.zig"); + +pub const vocab = 248320; +pub const linear_dim = 128; // a DeltaNet key or value head's width +pub const conv_taps = 4; +pub const head_dim = 256; +pub const rotary_dim = 64; +pub const eps: f32 = 1e-6; +pub const theta: f32 = 10000000; + +/// Whether layer `index` is a DeltaNet layer: every fourth is attention. +pub fn linear(index: usize) bool { + return index % 4 != 3; +} + +/// The widths a forward sizes its buffers by. +pub const Geometry = struct { + hidden: usize, + intermediate: usize, + layers: usize, + linear_k_heads: usize, // DeltaNet key (and query) heads + linear_v_heads: usize, // DeltaNet value heads, each key head serving a run of them + query_heads: usize, + kv_heads: usize, + + /// The DeltaNet input projection's row: queries, keys, then values. + pub fn convDim(g: Geometry) usize { + return (2 * g.linear_k_heads + g.linear_v_heads) * linear_dim; + } + pub fn kInner(g: Geometry) usize { + return g.linear_k_heads * linear_dim; + } + pub fn vInner(g: Geometry) usize { + return g.linear_v_heads * linear_dim; + } + pub fn qInner(g: Geometry) usize { + return g.query_heads * head_dim; + } + pub fn kvInner(g: Geometry) usize { + return g.kv_heads * head_dim; + } + /// The widest row an activation buffer holds: the recurrence's values or the attention's heads. + pub fn inner(g: Geometry) usize { + return @max(g.vInner(), g.qInner()); + } + /// One layer's recurrent state (fp32) and its conv window (bf16). + pub fn deltaBytes(g: Geometry) usize { + return g.linear_v_heads * linear_dim * linear_dim * 4; + } + pub fn convBytes(g: Geometry) usize { + return (conv_taps - 1) * g.convDim() * 2; + } +}; + +/// Qwen3.8-27B, the one checkpoint shape the CUDA engine serves. +pub const the_27b: Geometry = .{ .hidden = 5120, .intermediate = 17408, .layers = 64, .linear_k_heads = 16, .linear_v_heads = 48, .query_heads = 24, .kv_heads = 4 }; + +/// A checkpoint's config.json (and generation_config.json's end tokens), refused unless it is the 27B's. +pub const Config = struct { + g: Geometry, + eos: [config.max_eos]u32, + eos_count: usize, + + pub fn read(gpa: std.mem.Allocator, io: std.Io, dir: []const u8) !Config { + const text = try file(gpa, io, dir, "config.json") orelse return error.MissingConfig; + defer gpa.free(text); + const generation = try file(gpa, io, dir, "generation_config.json"); + defer if (generation) |x| gpa.free(x); + return check(try config.parse(gpa, text, generation)); + } + + fn file(gpa: std.mem.Allocator, io: std.Io, dir: []const u8, name: []const u8) !?[]u8 { + const path = try std.fs.path.join(gpa, &.{ dir, name }); + defer gpa.free(path); + return std.Io.Dir.cwd().readFileAlloc(io, path, gpa, .limited(1 << 22)) catch |err| switch (err) { + error.FileNotFound => null, + else => err, + }; + } +}; + +/// The parsed config as the CUDA engine's Config, when every shape the kernels assume is the 27B's. +pub fn check(c: config.Config) !Config { + const g: Geometry = .{ .hidden = c.hidden, .intermediate = c.intermediate, .layers = c.layers, .linear_k_heads = c.k_heads, .linear_v_heads = c.v_heads, .query_heads = c.heads, .kv_heads = c.kv_heads }; + if (!std.meta.eql(g, the_27b) or c.vocab != vocab or c.head_dim != head_dim or c.dk != linear_dim or c.dv != linear_dim) return error.UnsupportedQwenGeometry; + if (c.conv_kernel != conv_taps or c.rope_dims != rotary_dim or c.eps != eps or c.rope_theta != theta) return error.UnsupportedQwenGeometry; + for (c.kinds[0..c.layers], 0..) |kind, i| if ((kind == .linear) != linear(i)) return error.UnsupportedQwenGeometry; + return .{ .g = g, .eos = c.eos, .eos_count = c.eos_count }; +} + +test "the 27B's shapes pass and another hidden size is refused" { + var c: config.Config = .{ .hidden = 5120, .intermediate = 17408, .layers = 64, .vocab = vocab, .heads = 24, .kv_heads = 4, .head_dim = head_dim, .k_heads = 16, .v_heads = 48, .dk = linear_dim, .dv = linear_dim, .conv_kernel = conv_taps, .eps = eps, .rope_dims = rotary_dim, .rope_theta = theta }; + for (c.kinds[0..64], 0..) |*k, i| k.* = if (linear(i)) .linear else .attention; + c.eos[0] = 248044; + c.eos_count = 1; + const ok = try check(c); + try std.testing.expectEqual(@as(usize, 1), ok.eos_count); + c.hidden = 2048; + try std.testing.expectError(error.UnsupportedQwenGeometry, check(c)); +} diff --git a/zig/src/families/qwen27/cuda/state.zig b/zig/src/families/qwen27/cuda/state.zig new file mode 100644 index 0000000000..4cbe06c07e --- /dev/null +++ b/zig/src/families/qwen27/cuda/state.zig @@ -0,0 +1,304 @@ +//! A stream's committed state (DeltaNet recurrences, conv windows, attention caches) and the forward's scratch. + +const std = @import("std"); +const cuda = @import("cuda"); +const c = @import("shape.zig"); +const kern = @import("kernels.zig"); +const tri = @import("triton.zig"); +const torch_ops = kern.torch_ops; + +/// The most rows a stream's verify window takes (the multi-stream GDN tree kernel's limit). +pub const max_rows = 16; +/// The most streams a shared round takes. +pub const max_streams = 16; +/// The most rows a verify round takes: every stream's whole window. +pub const round_rows = max_rows * max_streams; +/// The most items a batched copy takes: every stream's layers, three rows or a key and a value each. +pub const copy_items = max_streams * 64 * 3 + round_rows * 64 * 2; +/// The most candidates a sampled row reads from the GPU: top_k up to 1,024, and exact_sampling.MARGIN past it. +pub const max_candidates = 1024 + 8; + +/// A bump allocator over one device buffer, every slice 256-byte aligned. +pub const Arena = struct { + buf: ?cuda.DeviceBuffer, // null: a measuring pass, every take counted and none backed + used: usize = 0, + + pub fn init(d: *const cuda.Driver, len: usize) !Arena { + return .{ .buf = try cuda.DeviceBuffer.alloc(d, len) }; + } + + pub fn deinit(a: *Arena) void { + if (a.buf) |*b| b.free(); + } + + pub fn take(a: *Arena, len: usize) !u64 { + const at = std.mem.alignForward(usize, a.used, 256); + a.used = at + len; + const b = a.buf orelse return 0; + if (a.used > b.len) return error.ArenaFull; + return b.ptr + at; + } +}; + +/// One stream's committed state: DeltaNet recurrences and conv windows, attention keys and values for `capacity`. +pub const Seq = struct { + arena: Arena, + rec: []u64, // by layer: (hv, dv, dk) fp32, 0 for attention layers + conv: []u64, // by layer: (3, conv dim) bf16 + keys: []u64, // by layer: (capacity, kv heads, head dim) bf16 + values: []u64, + head_in: u64, // the last prompt row normed, (hidden) bf16, and its group sums: the first draw's input + head_xs: u64, + capacity: usize, + pos: usize = 0, + gpa: std.mem.Allocator, + + /// Device bytes a stream of `capacity` positions takes. + pub fn bytes(g: c.Geometry, capacity: usize) usize { + var n: usize = std.mem.alignForward(usize, g.hidden * 2, 256) + std.mem.alignForward(usize, g.hidden / 64 * 4, 256); + const a = std.mem.alignForward; + for (0..g.layers) |i| n += if (c.linear(i)) a(usize, g.deltaBytes(), 256) + a(usize, g.convBytes(), 256) else 2 * a(usize, capacity * g.kvInner() * 2, 256); + return n; + } + + /// A fresh stream: zero recurrences and conv windows, caches for `capacity` positions. + pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize) !Seq { + var s: Seq = .{ .gpa = gpa, .arena = try Arena.init(ops.k.d, bytes(g, capacity)), .rec = &.{}, .conv = &.{}, .keys = &.{}, .values = &.{}, .head_in = 0, .head_xs = 0, .capacity = capacity }; + errdefer s.arena.deinit(); + s.rec = try gpa.alloc(u64, g.layers); + errdefer gpa.free(s.rec); + s.conv = try gpa.alloc(u64, g.layers); + errdefer gpa.free(s.conv); + s.keys = try gpa.alloc(u64, g.layers); + errdefer gpa.free(s.keys); + s.values = try gpa.alloc(u64, g.layers); + errdefer gpa.free(s.values); + const bf = 2; + s.head_in = try s.arena.take(g.hidden * 2); + s.head_xs = try s.arena.take(g.hidden / 64 * 4); + for (0..g.layers) |i| { + s.rec[i] = 0; + s.conv[i] = 0; + s.keys[i] = 0; + s.values[i] = 0; + const A = &s.arena; + if (c.linear(i)) { + s.rec[i] = try A.take(g.deltaBytes()); + s.conv[i] = try A.take(g.convBytes()); + } else { + s.keys[i] = try A.take(capacity * g.kvInner() * bf); + s.values[i] = try A.take(capacity * g.kvInner() * bf); + } + } + try s.reset(ops, g); + return s; + } + + pub fn deinit(s: *Seq) void { + s.arena.deinit(); + s.gpa.free(s.rec); + s.gpa.free(s.conv); + s.gpa.free(s.keys); + s.gpa.free(s.values); + s.* = undefined; + } + + /// Back to an empty prompt: zero recurrences and conv windows (State.__init__'s zeros). + pub fn reset(s: *Seq, ops: kern.Ops, g: c.Geometry) !void { + for (0..g.layers) |i| if (c.linear(i)) { + try ops.fill32(s.rec[i], 0, g.deltaBytes() / 4); + try ops.fill32(s.conv[i], 0, g.convBytes() / 4); + }; + s.pos = 0; + } +}; + +/// One DeltaNet layer's window record, kept until commit: what the replay steps and the conv window takes. +pub const GdnRecord = struct { q: u64, k: u64, v: u64, g: u64, beta: u64, qkv: u64 }; +/// One attention layer's window record: the rotated keys and the values the window's rows would add. +pub const AttnRecord = struct { k: u64, v: u64 }; + +/// The forward's device buffers, sized for `rows` rows a call (a decode window, or a prompt chunk). +pub const Scratch = struct { + arena: Arena, + rows: usize, + // the residual stream and the normed rows with their group sums + x: u64, + h: u64, + y: u64, + xs: u64, + r: u64, + pending: u64, + // DeltaNet + qkv: u64, + z: u64, + b: u64, + a: u64, + yr: u64, + gq: u64, // a prompt chunk's DeltaNet inputs (a window's live in its records) + gk: u64, + gv: u64, + gg: u64, + gbeta: u64, + gated: u64, + gated_xs: u64, + // attention + qg: u64, + key: u64, + value: u64, + q_rot: u64, + attn: u64, + // MLP + gate: u64, + up: u64, + act: u64, + act_xs: u64, + // the head's input, logits and picks + last_h: u64, + last_xs: u64, + logits: u64, + picks: u64, + logits32: u64, // the round's logits as fp32, the top-k input + cand_values: u64, // (rows, max_candidates) fp32 and int64: each row's top candidates + cand_ids: u64, + topk_scratch: u64, + // inputs a call uploads: ids, positions, conv windows, the GDN plan, the attention plan + ids: u64, + pos: u64, + sids: u64, // a batched prompt's rows' streams + prompt_keys: u64, // a batched prompt's rotated keys and values, before each stream's cache takes its rows + prompt_values: u64, + windows: u64, + plan: u64, + attn_plan: u64, + paths: u64, + depths: u64, + offs: u64, + part_o: u64, + part_m: u64, + part_l: u64, + part_chunks: usize, + origin: u64, // the bf16 origin cache offsets count from + conv_tmp: u64, + conv_old: u64, // a round's streams' conv rows that survive its commit, by stream and DeltaNet layer + copy_items: u64, // two batched-copy tables (kern.Copy), one for each phase of a commit + taps: u64, // a call's rows of the drafter's taps: (rows, 5 x hidden) bf16 + replay_table: u64, + replay_rows: u64, + round: u64, // a shared round's packed int32 inputs (multi_tree_forward's one host copy) + tables: u64, // (DeltaNet layers, streams) int64: each stream's state of each layer + conv_cat: u64, // by DeltaNet layer: every stream's conv rows, stacked + records: []GdnRecord, // by layer: the round's DeltaNet inputs, kept until commit + attn_records: []AttnRecord, // by layer: the round's rotated keys and values + gpa: std.mem.Allocator, + + /// Scratch for prompt chunks of `prompt_rows` and windows of `max_rows`, over caches of `capacity` positions. + pub fn init(gpa: std.mem.Allocator, d: *const cuda.Driver, g: c.Geometry, prompt_rows: usize, capacity: usize) !Scratch { + var s: Scratch = undefined; + s.gpa = gpa; + s.records = try gpa.alloc(GdnRecord, g.layers); + errdefer gpa.free(s.records); + s.attn_records = try gpa.alloc(AttnRecord, g.layers); + errdefer gpa.free(s.attn_records); + s.arena = .{ .buf = null }; + try s.layout(g, prompt_rows, capacity); + s.arena = try Arena.init(d, s.arena.used); + errdefer s.arena.deinit(); + try s.layout(g, prompt_rows, capacity); + return s; + } + + fn layout(s: *Scratch, g: c.Geometry, prompt_rows: usize, capacity: usize) !void { + const R = @max(prompt_rows, round_rows); + const W = round_rows; + const bf = 2; + const chunks = (capacity + W + tri.chunk - 1) / tri.chunk; + // attention.plan_host: a query tile's items per committed chunk and folded group + const codes = capacity / tri.chunk + capacity / (tri.chunk * tri.group_chunks); + const items = codes * ((W * (g.query_heads / g.kv_heads) + 15) / 16); + s.rows = R; + s.part_chunks = chunks; + s.arena.used = 0; + const A = &s.arena; + s.origin = try A.take(128); + s.x = try A.take(R * g.hidden * bf); + s.h = try A.take(R * g.hidden * bf); + s.y = try A.take(R * g.hidden * bf); + s.r = try A.take(R * g.hidden * bf); + s.pending = try A.take(R * g.hidden * bf); + s.xs = try A.take(R * g.hidden / 64 * 4); + s.qkv = try A.take(R * g.convDim() * bf); + s.z = try A.take(R * g.vInner() * bf); + s.b = try A.take(R * g.linear_v_heads * bf); + s.a = try A.take(R * g.linear_v_heads * bf); + s.yr = try A.take(R * g.vInner() * bf); + s.gq = try A.take(R * g.kInner() * bf); + s.gk = try A.take(R * g.kInner() * bf); + s.gv = try A.take(R * g.vInner() * bf); + s.gg = try A.take(R * g.linear_v_heads * 4); + s.gbeta = try A.take(R * g.linear_v_heads * 4); + s.gated = try A.take(R * g.inner() * bf); + s.gated_xs = try A.take(R * g.inner() / 64 * 4); + s.qg = try A.take(R * 2 * g.qInner() * bf); + s.key = try A.take(R * g.kvInner() * bf); + s.value = try A.take(R * g.kvInner() * bf); + s.q_rot = try A.take(R * g.qInner() * bf); + s.attn = try A.take(R * g.qInner() * bf); + s.gate = try A.take(R * g.intermediate * bf); + s.up = try A.take(R * g.intermediate * bf); + s.act = try A.take(R * g.intermediate * bf); + s.act_xs = try A.take(R * g.intermediate / 64 * 4); + s.last_h = try A.take(W * g.hidden * bf); + s.last_xs = try A.take(W * g.hidden / 64 * 4); + s.logits = try A.take(W * c.vocab * bf); + s.picks = try A.take(W * 4); + s.logits32 = try A.take(W * c.vocab * 4); + s.cand_values = try A.take(W * max_candidates * 4); + s.cand_ids = try A.take(W * max_candidates * 8); + s.topk_scratch = try A.take(torch_ops.topkScratchBytes(W, c.vocab)); + s.ids = try A.take(R * 4); + s.pos = try A.take(R * 4); + s.sids = try A.take(R * 4); + s.prompt_keys = try A.take(R * g.kvInner() * bf); + s.prompt_values = try A.take(R * g.kvInner() * bf); + s.windows = try A.take(R * c.conv_taps * 4); + s.plan = try A.take(R * 3 * 4); + s.attn_plan = try A.take((W + 4 + 3 * items + W) * 4); + s.paths = try A.take(W * tri.max_nodes * 4); + s.depths = try A.take(W * 4); + s.offs = try A.take(g.layers * max_streams * 2 * 8); + s.part_o = try A.take(chunks * W * g.query_heads * c.head_dim * 4); + s.part_m = try A.take(chunks * W * g.query_heads * 4); + s.part_l = try A.take(chunks * W * g.query_heads * 4); + s.conv_tmp = try A.take((c.conv_taps - 1 + R) * g.convDim() * bf); + s.conv_old = try A.take(max_streams * g.layers * (c.conv_taps - 1) * g.convDim() * bf); + s.copy_items = try A.take(2 * copy_items * @sizeOf(kern.Copy)); + s.taps = try A.take(R * 5 * g.hidden * bf); + s.replay_table = try A.take(g.layers * (4 + max_streams) * 8); + s.replay_rows = try A.take(max_streams * (W + 1) * 4); + s.round = try A.take((6 * W + max_streams + 1 + W * c.conv_taps + W + 4 * max_streams + 3 * items * max_streams + W) * 4); + s.tables = try A.take(g.layers * max_streams * 8); + s.conv_cat = try A.take(g.layers * max_streams * (c.conv_taps - 1) * g.convDim() * bf); + for (0..g.layers) |i| { + if (c.linear(i)) { + s.records[i] = .{ + .q = try A.take(W * g.kInner() * bf), + .k = try A.take(W * g.kInner() * bf), + .v = try A.take(W * g.vInner() * bf), + .g = try A.take(W * g.linear_v_heads * 4), + .beta = try A.take(W * g.linear_v_heads * 4), + .qkv = try A.take(W * g.convDim() * bf), + }; + } else { + s.attn_records[i] = .{ .k = try A.take(W * g.kvInner() * bf), .v = try A.take(W * g.kvInner() * bf) }; + } + } + } + + pub fn deinit(s: *Scratch) void { + s.arena.deinit(); + s.gpa.free(s.records); + s.gpa.free(s.attn_records); + s.* = undefined; + } +}; diff --git a/zig/src/families/qwen27/cuda/tree.zig b/zig/src/families/qwen27/cuda/tree.zig new file mode 100644 index 0000000000..f8cdf7187d --- /dev/null +++ b/zig/src/families/qwen27/cuda/tree.zig @@ -0,0 +1,181 @@ +//! A verify tree's host plans: gdn.schedule's slots and visiting order, and each node's conv window and depth. + +const std = @import("std"); + +/// The most nodes a stream's tree takes (forward._paths), and the most live states a schedule may hold. +pub const max_nodes = 128; +pub const max_slots = 32; + +/// gdn.schedule's result: (node, source, dest) entries in visiting order and the slots they use. +pub const Schedule = struct { entries: []i32, slots: usize }; + +/// forward._paths: each row's depth from the root (parents topologically sorted, -1 at row 0 only). +pub fn depths(parents: []const i32, out: []usize) !void { + if (parents.len == 0 or parents.len > max_nodes or parents[0] != -1) return error.InvalidQwenTree; + for (parents, 0..) |p, row| { + if (row == 0) { + out[0] = 0; + continue; + } + if (p < 0 or p >= row) return error.InvalidQwenTree; + out[row] = out[@intCast(p)] + 1; + } +} + +/// Whether every row's parent is the row before it: a chain. +pub fn chain(parents: []const i32) bool { + for (parents, 0..) |p, row| if (p != @as(i32, @intCast(row)) - 1) return false; + return true; +} + +/// gdn._order_slots: entries for one visiting order, and the slots it needs. +fn orderSlots(gpa: std.mem.Allocator, parents: []const i32, order: []const usize) !Schedule { + const n = parents.len; + const pos = try gpa.alloc(usize, n); + defer gpa.free(pos); + for (order, 0..) |node, i| pos[node] = i; + // each node's last child in this order, and whether all its children follow it directly + const last = try gpa.alloc(?usize, n); + defer gpa.free(last); + @memset(last, null); + const direct = try gpa.alloc(bool, n); + defer gpa.free(direct); + @memset(direct, true); + for (parents, 0..) |p, node| { + if (p < 0) continue; + const pu: usize = @intCast(p); + if (last[pu] == null or pos[node] > pos[last[pu].?]) last[pu] = node; + if (pos[node] != pos[pu] + 1) direct[pu] = false; + } + const slot_of = try gpa.alloc(?i32, n); + defer gpa.free(slot_of); + @memset(slot_of, null); + var free: std.ArrayList(i32) = .empty; + defer free.deinit(gpa); + var entries = try gpa.alloc(i32, 3 * n); + errdefer gpa.free(entries); + var used: i32 = 0; + for (order, 0..) |node, i| { + const parent = parents[node]; + var source: i32 = -1; + if (parent >= 0) { + const pu: usize = @intCast(parent); + source = if (i > 0 and order[i - 1] == pu) -2 else slot_of[pu] orelse return error.InvalidQwenTree; + // its last child reads it now: the slot may be reused + if (slot_of[pu] != null and pos[last[pu].?] == i) { + try free.append(gpa, slot_of[pu].?); + slot_of[pu] = null; + } + } + var dest: i32 = -1; + if (last[node] != null and !direct[node]) { + if (free.items.len > 0) { + std.mem.sort(i32, free.items, {}, std.sort.asc(i32)); + dest = free.orderedRemove(0); + } else { + dest = used; + used += 1; + } + slot_of[node] = dest; + } + entries[3 * i ..][0..3].* = .{ @intCast(node), source, dest }; + } + return .{ .entries = entries, .slots = @intCast(used) }; +} + +/// gdn.schedule: depth-first or level order, whichever needs fewer slots (depth-first on a tie). +pub fn schedule(gpa: std.mem.Allocator, parents: []const i32) !Schedule { + const n = parents.len; + const d = try gpa.alloc(usize, n); + defer gpa.free(d); + try depths(parents, d); + // depth-first: children in row order, as the Python stack pushes them reversed + const dfs = try gpa.alloc(usize, n); + defer gpa.free(dfs); + var stack: std.ArrayList(usize) = .empty; + defer stack.deinit(gpa); + try stack.append(gpa, 0); + var at: usize = 0; + while (stack.pop()) |node| { + dfs[at] = node; + at += 1; + var child = n; + while (child > node + 1) { + child -= 1; + if (parents[child] == @as(i32, @intCast(node))) try stack.append(gpa, child); + } + } + const level = try gpa.alloc(usize, n); + defer gpa.free(level); + for (level, 0..) |*l, i| l.* = i; + std.mem.sort(usize, level, d, struct { + fn less(ds: []const usize, a: usize, b: usize) bool { + return ds[a] < ds[b] or (ds[a] == ds[b] and a < b); + } + }.less); + const first = try orderSlots(gpa, parents, dfs); + const second = orderSlots(gpa, parents, level) catch |err| { + gpa.free(first.entries); + return err; + }; + const best, const other = if (second.slots < first.slots) .{ second, first } else .{ first, second }; + gpa.free(other.entries); + if (best.slots > max_slots) { + gpa.free(best.entries); + return error.QwenTreeTooWide; + } + return best; +} + +/// forward._conv_windows: each node's last `keep` inputs along its path, then its own row (`keep + base + row`). +pub fn convWindows(parents: []const i32, keep: usize, base: usize, out: []i32) void { + const w = keep + 1; + for (parents, 0..) |p, row| { + const dst = out[row * w ..][0..w]; + if (p < 0) { + for (0..keep) |j| dst[j] = @intCast(j); + } else { + const src = out[@as(usize, @intCast(p)) * w ..][0..w]; + @memcpy(dst[0..keep], src[1..w]); + } + dst[keep] = @intCast(keep + base + row); + } +} + +test "a chain's schedule keeps no slot and reads each predecessor" { + const gpa = std.testing.allocator; + const s = try schedule(gpa, &.{ -1, 0, 1, 2 }); + defer gpa.free(s.entries); + try std.testing.expectEqual(@as(usize, 0), s.slots); + try std.testing.expectEqualSlices(i32, &.{ 0, -1, -1, 1, -2, -1, 2, -2, -1, 3, -2, -1 }, s.entries); +} + +test "a fork saves the shared parent in a slot until its last child" { + const gpa = std.testing.allocator; + // root 0 with children 1 and 2, 1 with child 3: depth-first 0 1 3 2 needs the root kept for 2 + const s = try schedule(gpa, &.{ -1, 0, 0, 1 }); + defer gpa.free(s.entries); + try std.testing.expectEqual(@as(usize, 1), s.slots); + try std.testing.expectEqualSlices(i32, &.{ 0, -1, 0, 1, -2, -1, 3, -2, -1, 2, 0, -1 }, s.entries); +} + +test "schedules match gdn.schedule on random trees" { + const gpa = std.testing.allocator; + const cases = .{ + .{ &[_]i32{ -1, 0, 1, 2, 0, 3 }, &[_]i32{ 0, -1, 0, 1, -2, -1, 2, -2, -1, 3, -2, -1, 5, -2, -1, 4, 0, -1 }, 1 }, + .{ &[_]i32{ -1, 0, 0, 0, 0, 2, 3, 6, 3 }, &[_]i32{ 0, -1, 0, 1, -2, -1, 2, 0, -1, 5, -2, -1, 3, 0, 1, 6, -2, -1, 7, -2, -1, 8, 1, -1, 4, 0, -1 }, 2 }, + .{ &[_]i32{ -1, 0, 0, 2, 1, 0, 5, 1, 6, 4, 2, 6 }, &[_]i32{ 0, -1, 0, 1, -2, 1, 4, -2, -1, 9, -2, -1, 7, 1, -1, 2, 0, 1, 3, -2, -1, 10, 1, -1, 5, 0, -1, 6, -2, 0, 8, -2, -1, 11, 0, -1 }, 2 }, + }; + inline for (cases) |case| { + const s = try schedule(gpa, case[0]); + defer gpa.free(s.entries); + try std.testing.expectEqual(@as(usize, case[2]), s.slots); + try std.testing.expectEqualSlices(i32, case[1], s.entries); + } +} + +test "conv windows follow each node's path" { + var out: [4 * 4]i32 = undefined; + convWindows(&.{ -1, 0, 0, 1 }, 3, 0, &out); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 1, 2, 3, 4, 1, 2, 3, 5, 2, 3, 4, 6 }, &out); +} diff --git a/zig/src/families/qwen27/cuda/triton.zig b/zig/src/families/qwen27/cuda/triton.zig new file mode 100644 index 0000000000..5206ff382d --- /dev/null +++ b/zig/src/families/qwen27/cuda/triton.zig @@ -0,0 +1,141 @@ +//! The 27B's Triton kernels from the captured cubins: each launch has the Python wrapper's grid and constexprs. + +const std = @import("std"); +const cuda = @import("cuda"); +const aot = cuda.aot; + +const p = aot.ptr; + +fn int(name: []const u8, v: usize) aot.Arg { + return aot.int(name, @intCast(v)); +} + +fn ci(name: []const u8, v: usize) aot.Const { + return aot.ci(name, @intCast(v)); +} + +fn u(x: usize) u32 { + return @intCast(x); +} + +fn cdiv(a: usize, b: usize) u32 { + return u((a + b - 1) / b); +} + +fn pow2(n: usize) usize { + return std.math.ceilPowerOfTwo(usize, n) catch unreachable; +} + +/// attention.py's layout constants: 512-key chunks, folded four to a group, up to 128 window rows. +pub const chunk = 512; +pub const group_chunks = 4; +pub const max_nodes = 128; +const merge_columns = 64; + +/// The DeltaNet shapes glue.gdn_pre takes: projection width, key and value heads, head dim, kept conv rows. +pub const Gdn = struct { c: usize, kh: usize, vh: usize, dk: usize, keep: usize }; + +/// The attention shapes: query and key heads, head dim, rotary half. +pub const Attn = struct { heads: usize, kv_heads: usize, dim: usize, half: usize }; + +pub const Tri = struct { + set: *const aot.Set, + s: cuda.Stream, + + fn run(t: Tri, name: []const u8, grid: [3]u32, args: []const aot.Arg, consts: []const aot.Const) !void { + try t.set.run(t.s, name, grid, args, consts); + } + + /// glue.embed: a 4-bit table row dequantized, one program a row and 64-group. + pub fn embed(t: Tri, ids: u64, w: u64, s: u64, b: u64, out: u64, rows: usize, d: usize) !void { + try t.run("_embed", .{ u(rows), u(d / 64), 1 }, &.{ p("IDS", "*i32", ids), p("Wt", "*i32", w), p("S", "*bf16", s), p("B", "*bf16", b), p("OUT", "*bf16", out) }, &.{ci("D", d)}); + } + + /// glue.add_rmsnorm: h = x + r (or x itself without r), y = rmsnorm(h) * w, xs = y's 64-group sums. + pub fn addRmsnorm(t: Tri, x: u64, r: ?u64, w: u64, h: u64, y: u64, xs: u64, rows: usize, d: usize, eps: f32) !void { + try t.run("_add_rmsnorm", .{ u(rows), 1, 1 }, &.{ p("X", "*bf16", x), p("R", "*bf16", r orelse x), p("W", "*bf16", w), p("H", "*bf16", if (r != null) h else x), p("Y", "*bf16", y), p("XS", "*fp32", xs), aot.float("eps", eps) }, &.{ ci("D", d), ci("BLOCK", pow2(d)), ci("HAS_R", @intFromBool(r != null)) }); + } + + /// glue.gdn_pre: conv over each row's window of [state; qkv], q/k norms, g and beta; `sid`: several streams. + pub fn gdnPre(t: Tri, qkv: u64, cs: u64, cw: u64, win: u64, sid: ?u64, a: u64, b: u64, alog: u64, dtb: u64, q: u64, k: u64, v: u64, g: u64, beta: u64, rows: usize, m: Gdn) !void { + try t.run("_gdn_pre", .{ u(rows), u(2 * m.kh + m.vh), 1 }, &.{ + p("QKV", "*bf16", qkv), p("CS", "*bf16", cs), p("CW", "*bf16", cw), p("WIN", "*i32", win), p("A", "*bf16", a), p("B", "*bf16", b), p("ALOG", "*fp32", alog), + p("DTB", "*fp32", dtb), p("Q", "*bf16", q), p("K", "*bf16", k), p("V", "*bf16", v), p("G", "*fp32", g), p("BETA", "*fp32", beta), p("SID", "*i32", sid orelse win), + }, &.{ ci("C", m.c), ci("KH", m.kh), ci("VH", m.vh), ci("DK", m.dk), ci("NKEEP", m.keep), ci("MULTI", @intFromBool(sid != null)) }); + } + + /// glue.gated_norm: silu(z) * rmsnorm(y) * w per value head, and its group sums. + pub fn gatedNorm(t: Tri, y: u64, z: u64, w: u64, out: u64, xs: u64, rows: usize, vh: usize, dv: usize, eps: f32) !void { + try t.run("_gated_norm", .{ u(rows), u(vh), 1 }, &.{ p("Yr", "*bf16", y), p("Z", "*bf16", z), p("W", "*bf16", w), p("OUT", "*bf16", out), p("XS", "*fp32", xs), aot.float("eps", eps) }, &.{ ci("VH", vh), ci("DV", dv) }); + } + + /// glue.swiglu: silu(gate) * up in 1024-column blocks, and its group sums. + pub fn swiglu(t: Tri, gate: u64, up: u64, out: u64, xs: u64, rows: usize, n: usize) !void { + try t.run("_swiglu", .{ u(rows), cdiv(n, 1024), 1 }, &.{ p("GATE", "*bf16", gate), p("UP", "*bf16", up), p("OUT", "*bf16", out), p("XS", "*fp32", xs) }, &.{ ci("N", n), ci("BLOCK", 1024) }); + } + + /// glue.attn_prep: q (from the [q | gate] rows) and k normed, then rotated at each row's position. + pub fn attnPrep(t: Tri, qg: u64, kv: u64, qn: u64, kn: u64, pos: u64, inv: u64, qout: u64, kout: u64, rows: usize, a: Attn, eps: f32) !void { + try t.run("_attn_prep", .{ u(rows), u(a.heads + a.kv_heads), 1 }, &.{ p("QG", "*bf16", qg), p("KV", "*bf16", kv), p("QN", "*bf16", qn), p("KN", "*bf16", kn), p("POS", "*i32", pos), p("INV", "*fp32", inv), p("QOUT", "*bf16", qout), p("KOUT", "*bf16", kout), aot.float("eps", eps) }, &.{ + ci("H", a.heads), ci("HKV", a.kv_heads), ci("D", a.dim), ci("HALF", a.half), ci("MROPE", 0), ci("ROWS", 0), ci("HSEC", 11), ci("WSEC", 10), + }); + } + + /// glue.gate_mul: the attention output times sigmoid(gate) from the [q | gate] rows, and its group sums. + pub fn gateMul(t: Tri, o: u64, qg: u64, out: u64, xs: u64, rows: usize, heads: usize, dim: usize) !void { + try t.run("_gate_mul", .{ u(rows), u(heads), 1 }, &.{ p("O", "*bf16", o), p("QG", "*bf16", qg), p("OUT", "*bf16", out), p("XS", "*fp32", xs) }, &.{ ci("H", heads), ci("D", dim) }); + } + + /// qmm.group_sums: each row's 64-group input sums in fp32 (rows may be strided by `ld`). + pub fn groupSums(t: Tri, x: u64, xs: u64, ld: usize, rows: usize, k: usize) !void { + try t.run("_group_sums", .{ u(rows), cdiv(k / 64, 16), 1 }, &.{ p("X", "*bf16", x), p("XS", "*fp32", xs), int("ldx", ld) }, &.{ ci("KG", k / 64), ci("GS", 64), ci("GB", 16) }); + } + + /// attention.from_packed's _paths: each row's root-to-row window rows and its depth. + pub fn paths(t: Tri, parents: u64, out: u64, depths: u64, rows: usize) !void { + try t.run("_paths", .{ u(rows), 1, 1 }, &.{ p("PARENTS", "*i32", parents), p("PATHS", "*i32", out), p("DEPTHS", "*i32", depths) }, &.{ci("MAXD", max_nodes)}); + } + + /// The layout attention.attention reads: the plan's tables and the partial buffers it writes. + pub const Plan = struct { streams: u64, items: u64, n_items: usize, rows: u64, paths: u64, depths: u64, width: usize }; + pub const Partials = struct { o: u64, m: u64, l: u64 }; + + /// attention.attention: committed keys in 512-key chunks, the row's own path in its tail, merged in order. + pub fn attention(t: Tri, q: u64, kn: u64, vn: u64, base: u64, offs: u64, plan: Plan, part: Partials, out: u64, a: Attn) !void { + const g = a.heads / a.kv_heads; + const scale: f32 = @floatCast(std.math.pow(f64, @floatFromInt(a.dim), -0.5)); + const w = plan.width; + if (plan.n_items > 0) try t.run("_shared", .{ u(plan.n_items * a.kv_heads), 1, 1 }, &.{ + p("Q", "*bf16", q), p("KC", "*bf16", base), p("VC", "*bf16", base), p("OFF", "*i64", offs), p("STREAM", "*i32", plan.streams), + p("ITEMS", "*i32", plan.items), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), + }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), aot.cf("SCALE", scale), ci("GR", group_chunks) }); + const tails = 1 + (max_nodes + chunk - 1) / chunk; + try t.run("_tail", .{ u(w), u(a.kv_heads), u(tails) }, &.{ + p("Q", "*bf16", q), p("KN", "*bf16", kn), p("VN", "*bf16", vn), p("KC", "*bf16", base), p("VC", "*bf16", base), p("OFF", "*i64", offs), p("STREAM", "*i32", plan.streams), + p("ROWS", "*i32", plan.rows), p("PATHS", "*i32", plan.paths), p("DEPTHS", "*i32", plan.depths), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), + }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), ci("MAXD", max_nodes), aot.cf("SCALE", scale), ci("GR", group_chunks) }); + try t.run("_merge", .{ u(w), u(a.kv_heads), u(a.dim / merge_columns) }, &.{ + p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), p("OUT", "*bf16", out), p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), int("W", w), + }, &.{ ci("H", a.heads), ci("D", a.dim), ci("G", g), ci("DS", merge_columns), ci("CH", chunk), ci("GR", group_chunks) }); + } +}; + +/// attention.groups: whole committed 2048-key groups fold in their own programs only when they fill the GPU. +pub fn groups(p_: usize, w: usize) usize { + const g = p_ / (chunk * group_chunks); + return if (g * w >= 64) g else 0; +} + +/// attention.slots: one partial slot per folded group, then one per chunk through the window's last key. +pub fn slots(p_: usize, w: usize) usize { + const g = groups(p_, w); + return g + (p_ + w + chunk - 1) / chunk - g * group_chunks; +} + +test "attention slots follow the Python plan" { + try std.testing.expectEqual(@as(usize, 1), slots(0, 1)); + try std.testing.expectEqual(@as(usize, 2), slots(512, 1)); + try std.testing.expectEqual(@as(usize, 9), slots(4096, 1)); // 8 chunks + the window's own + try std.testing.expectEqual(@as(usize, 13), slots(6144, 16)); // 3 groups x 16 rows < 64: every chunk its own + try std.testing.expectEqual(@as(usize, 4), slots(6144, 32)); // 3 folded groups, then the window's chunk +} diff --git a/zig/src/families/qwen27/cuda/weights.zig b/zig/src/families/qwen27/cuda/weights.zig new file mode 100644 index 0000000000..5c796bd43a --- /dev/null +++ b/zig/src/families/qwen27/cuda/weights.zig @@ -0,0 +1,252 @@ +//! The 27B's weights on the GPU in the Python engine's layouts: tiled projections, the stored embedding, fp32 gates. + +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const source = @import("nemotron").source; +const c = @import("shape.zig"); +const config_eos = @import("../config.zig").max_eos; +const kern = @import("kernels.zig"); + +pub const QLinear = kern.QLinear; +const Tensor = core.checkpoint.Tensor; +const prefix = "language_model."; + +/// The token table as stored (MLX words, scales, biases): a row lookup, not a matmul. +pub const Embed = struct { w: u64, s: u64, b: u64 }; +pub const Delta = struct { qkv: QLinear, z: QLinear, b: QLinear, a: QLinear, out: QLinear, conv: u64, a_log: u64, dt_bias: u64, norm: u64 }; +pub const Attention = struct { q: QLinear, k: QLinear, v: QLinear, o: QLinear, q_norm: u64, k_norm: u64 }; +pub const Mixer = union(enum) { delta: Delta, attention: Attention }; +pub const Layer = struct { input_norm: u64, post_norm: u64, mixer: Mixer, gate: QLinear, up: QLinear, down: QLinear }; + +pub const Weights = struct { + gpa: std.mem.Allocator, + g: c.Geometry, + embed: Embed = undefined, + layers: []Layer = &.{}, + norm: u64 = 0, + head: QLinear = undefined, + inv_freq: u64 = 0, // (rotary_dim / 2,) fp32 + eos: [config_eos]u32 = undefined, // config.json's end tokens, then generation_config.json's + eos_count: usize = 0, + buffers: std.ArrayList(cuda.DeviceBuffer) = .empty, + + pub fn deinit(w: *Weights) void { + for (w.buffers.items) |*b| b.free(); + w.buffers.deinit(w.gpa); + w.gpa.free(w.layers); + w.* = undefined; + } + + pub fn isEos(w: *const Weights, token: u32) bool { + for (w.eos[0..w.eos_count]) |e| if (e == token) return true; + return false; + } + + /// Device bytes the weights hold. + pub fn bytes(w: *const Weights) usize { + var n: usize = 0; + for (w.buffers.items) |b| n += b.len; + return n; + } +}; + +const Loader = struct { + gpa: std.mem.Allocator, + ops: kern.Ops, + w: *Weights, + src: *source.Source, + scratch: cuda.DeviceBuffer, + host: std.ArrayList(u8) = .empty, + + fn alloc(L: *Loader, len: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(L.ops.k.d, len); + try L.w.buffers.append(L.gpa, b); + return b.ptr; + } + + /// Device scratch of at least `len` bytes; growing waits for the stream so pending packs keep their inputs. + fn tmp(L: *Loader, len: usize) !u64 { + if (L.scratch.len < len) { + try L.src.flush(); + try L.ops.s.synchronize(); + L.scratch.free(); + L.scratch = try cuda.DeviceBuffer.alloc(L.ops.k.d, len); + } + return L.scratch.ptr; + } + + /// The host staging buffer, once the stream has read its last contents. + fn staging(L: *Loader, len: usize) ![]u8 { + try L.src.flush(); + try L.ops.s.synchronize(); + try L.host.resize(L.gpa, len); + return L.host.items; + } + + fn raw(L: *Loader, t: Tensor) !u64 { + const ptr = try L.alloc(t.bytes.len); + try L.src.upload(ptr, t.bytes); + return ptr; + } + + /// `.float()` of a bf16 tensor (or an f32 one as stored): exact. + fn widen(L: *Loader, t: Tensor) !u64 { + if (t.dtype == .f32) return L.raw(t); + if (t.dtype != .bf16) return error.UnexpectedTensor; + const n = t.bytes.len / 2; + const out = std.mem.bytesAsSlice(u32, try L.staging(n * 4)); + const in = try L.src.view(t.bytes); + for (out, 0..) |*o, i| o.* = @as(u32, std.mem.readInt(u16, in[2 * i ..][0..2], .little)) << 16; + const ptr = try L.alloc(n * 4); + try L.ops.upload(ptr, L.host.items); + return ptr; + } + + /// qmm_fast.tile: MLX words, scales and biases packed into the lane matmul's tiled layout. + fn tile(L: *Loader, ck: *core.Checkpoint, name: []const u8, n: usize, k: usize) !QLinear { + var parts: [3]Tensor = undefined; + var buf: [192]u8 = undefined; + for ([_][]const u8{ "weight", "scales", "biases" }, 0..) |s, i| parts[i] = try ck.get(try std.fmt.bufPrint(&buf, "{s}{s}.{s}", .{ prefix, name, s })); + const k8 = k / 8; + const kg = k / 64; + if (parts[0].dtype != .u32 or parts[1].dtype != .bf16 or parts[2].dtype != .bf16) return error.UnsupportedQuantization; + if (!parts[0].is(.u32, &.{ n, k8 }) or !parts[1].is(.bf16, &.{ n, kg }) or !parts[2].is(.bf16, &.{ n, kg })) { + std.log.err("{s}: not a 4-bit group-64 ({d}, {d}) projection", .{ name, n, k }); + return error.UnexpectedTensor; + } + const npad = (n + 127) / 128 * 128; + const sizes = [3]usize{ n * k8 * 4, n * kg * 2, n * kg * 2 }; + const base = try L.tmp(sizes[0] + sizes[1] + sizes[2]); + const at = [3]usize{ 0, sizes[0], sizes[0] + sizes[1] }; + for (0..3) |j| try L.src.upload(base + at[j], parts[j].bytes); + const q: QLinear = .{ .w = try L.alloc(npad * k / 2), .s = try L.alloc(kg * npad * 2), .b = try L.alloc(kg * npad * 2), .n = n, .k = k, .npad = npad }; + const total: u64 = @as(u64, npad / 64) * kg * 512; + try L.src.flush(); + var a: cuda.Args = .{}; + a.add(base); + for ([_]usize{ n, k8, kg }) |v| a.add(@as(c_int, @intCast(v))); + a.add(q.w); + a.add(@as(c_longlong, @intCast(total))); + try cuda.launch.launch(L.ops.k.pack_dense, .{ .grid = .{ .x = @intCast((total + 255) / 256) }, .block = .{ .x = 256 } }, L.ops.s, &a); + for ([_]u64{ q.s, q.b }, [_]usize{ at[1], at[2] }) |out, off| { + var t: cuda.Args = .{}; + t.add(base + off); + for ([_]usize{ n, kg, npad }) |v| t.add(@as(c_int, @intCast(v))); + t.add(out); + const cells = @as(u64, kg) * npad; + try cuda.launch.launch(L.ops.k.transpose16, .{ .grid = .{ .x = @intCast((cells + 255) / 256) }, .block = .{ .x = 256 } }, L.ops.s, &t); + } + return q; + } + + fn vector(L: *Loader, ck: *core.Checkpoint, name: []const u8, len: usize) !u64 { + var buf: [192]u8 = undefined; + return L.raw(try ck.expect(try std.fmt.bufPrint(&buf, "{s}{s}", .{ prefix, name }), .bf16, &.{len})); + } + + fn delta(L: *Loader, ck: *core.Checkpoint, p: []const u8, g: c.Geometry) !Delta { + var b1: [160]u8 = undefined; + var b2: [192]u8 = undefined; + const at = struct { + fn f(buf: []u8, pre: []const u8, name: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "{s}linear_attn.{s}", .{ pre, name }); + } + }.f; + var d: Delta = undefined; + d.qkv = try L.tile(ck, try at(&b1, p, "in_proj_qkv"), g.convDim(), g.hidden); + d.z = try L.tile(ck, try at(&b1, p, "in_proj_z"), g.vInner(), g.hidden); + d.b = try L.tile(ck, try at(&b1, p, "in_proj_b"), g.linear_v_heads, g.hidden); + d.a = try L.tile(ck, try at(&b1, p, "in_proj_a"), g.linear_v_heads, g.hidden); + d.out = try L.tile(ck, try at(&b1, p, "out_proj"), g.hidden, g.vInner()); + // conv1d.weight (C, 4, 1) is (C, 4) as stored: the reshape moves no bytes + d.conv = try L.raw(try ck.expect(try std.fmt.bufPrint(&b2, "{s}{s}", .{ prefix, try at(&b1, p, "conv1d.weight") }), .bf16, &.{ g.convDim(), c.conv_taps, 1 })); + d.a_log = try L.widen(try ck.get(try std.fmt.bufPrint(&b2, "{s}{s}", .{ prefix, try at(&b1, p, "A_log") }))); + d.dt_bias = try L.widen(try ck.get(try std.fmt.bufPrint(&b2, "{s}{s}", .{ prefix, try at(&b1, p, "dt_bias") }))); + d.norm = try L.vector(ck, try at(&b1, p, "norm.weight"), c.linear_dim); + return d; + } + + fn attention(L: *Loader, ck: *core.Checkpoint, p: []const u8, g: c.Geometry) !Attention { + var b1: [160]u8 = undefined; + const at = struct { + fn f(buf: []u8, pre: []const u8, name: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "{s}self_attn.{s}", .{ pre, name }); + } + }.f; + return .{ + .q = try L.tile(ck, try at(&b1, p, "q_proj"), 2 * g.qInner(), g.hidden), + .k = try L.tile(ck, try at(&b1, p, "k_proj"), g.kvInner(), g.hidden), + .v = try L.tile(ck, try at(&b1, p, "v_proj"), g.kvInner(), g.hidden), + .o = try L.tile(ck, try at(&b1, p, "o_proj"), g.hidden, g.qInner()), + .q_norm = try L.vector(ck, try at(&b1, p, "q_norm.weight"), c.head_dim), + .k_norm = try L.vector(ck, try at(&b1, p, "k_norm.weight"), c.head_dim), + }; + } +}; + +/// weights.inv_freq: theta ** (-i / half) in float64, rounded to fp32. +pub fn invFreq(out: []f32) void { + const half: f64 = @floatFromInt(out.len); + for (out, 0..) |*f, i| f.* = @floatCast(std.math.pow(f64, c.theta, -@as(f64, @floatFromInt(i)) / half)); +} + +/// weights.load (tiled, one GPU): the model folder's text tensors on the GPU, leftovers refused. +pub fn load(gpa: std.mem.Allocator, io: std.Io, ops: kern.Ops, dir: []const u8, cfg: c.Config) !Weights { + const g = cfg.g; + var w: Weights = .{ .gpa = gpa, .g = g, .eos = cfg.eos, .eos_count = cfg.eos_count }; + errdefer w.deinit(); + var src = try source.Source.init(gpa, .{ .d = ops.k.d, .s = ops.s, .discrete = ops.k.discrete }); + defer src.deinit(); + var L: Loader = .{ .gpa = gpa, .ops = ops, .w = &w, .src = &src, .scratch = try cuda.DeviceBuffer.alloc(ops.k.d, 1 << 20) }; + defer { + src.flush() catch {}; + ops.s.synchronize() catch {}; + L.scratch.free(); + L.host.deinit(gpa); + } + // the vision tower and the MTP head stay on disk, as weights._Tensors skips them + var ck = try core.Checkpoint.openModelPrefix(gpa, io, dir, prefix); + defer ck.close(); + try src.add(&ck); + w.layers = try gpa.alloc(Layer, g.layers); + for (w.layers, 0..) |*layer, i| { + var pb: [64]u8 = undefined; + var nb: [160]u8 = undefined; + const p = try std.fmt.bufPrint(&pb, "model.layers.{d}.", .{i}); + const join = struct { + fn f(buf: []u8, a: []const u8, b: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "{s}{s}", .{ a, b }); + } + }.f; + layer.input_norm = try L.vector(&ck, try join(&nb, p, "input_layernorm.weight"), g.hidden); + layer.mixer = if (c.linear(i)) .{ .delta = try L.delta(&ck, p, g) } else .{ .attention = try L.attention(&ck, p, g) }; + layer.gate = try L.tile(&ck, try join(&nb, p, "mlp.gate_proj"), g.intermediate, g.hidden); + layer.up = try L.tile(&ck, try join(&nb, p, "mlp.up_proj"), g.intermediate, g.hidden); + layer.down = try L.tile(&ck, try join(&nb, p, "mlp.down_proj"), g.hidden, g.intermediate); + layer.post_norm = try L.vector(&ck, try join(&nb, p, "post_attention_layernorm.weight"), g.hidden); + } + w.embed = .{ + .w = try L.raw(try ck.expect(prefix ++ "model.embed_tokens.weight", .u32, &.{ c.vocab, g.hidden / 8 })), + .s = try L.raw(try ck.expect(prefix ++ "model.embed_tokens.scales", .bf16, &.{ c.vocab, g.hidden / 64 })), + .b = try L.raw(try ck.expect(prefix ++ "model.embed_tokens.biases", .bf16, &.{ c.vocab, g.hidden / 64 })), + }; + w.norm = try L.vector(&ck, "model.norm.weight", g.hidden); + w.head = try L.tile(&ck, "lm_head", c.vocab, g.hidden); + if (ck.unused() != 0) return error.UnusedCheckpointTensors; + var inv: [c.rotary_dim / 2]f32 = undefined; + invFreq(&inv); + w.inv_freq = try L.alloc(@sizeOf(@TypeOf(inv))); + try ops.upload(w.inv_freq, std.mem.asBytes(&inv)); + try src.flush(); + try ops.s.synchronize(); + return w; +} + +test "inv_freq: theta ** (-i / 32) rounded to fp32" { + var inv: [32]f32 = undefined; + invFreq(&inv); + try std.testing.expectEqual(@as(f32, 1.0), inv[0]); + try std.testing.expectEqual(@as(f32, 0.6042963862419128), inv[1]); + try std.testing.expectEqual(@as(f32, 0.22067341208457947), inv[3]); +} diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 296aa21409..032f64c603 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -4,13 +4,14 @@ const cuda = @import("cuda"); const api = @import("engine_api"); const lanes = @import("lanes"); const nemotron = @import("nemotron"); +const qwen27 = @import("qwen27"); const Allocator = std.mem.Allocator; const budget = @import("cuda_memory.zig"); const nemotron_slide = @import("nemotron_slide.zig"); const Pool = budget.Pool; /// CUDA families provide metadata, open their lane backend and explain their refusals. -const registry = .{nemotron.native}; +const registry = .{ nemotron.native, qwen27.native }; /// Nemotron's prompt-cache copies. A file-level value so the store's pointer stays valid for the process. const nemotron_snaps: api.prompt_cache.Snapshots.VTable = .{ @@ -410,7 +411,10 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O problem.* = try std.fmt.allocPrint(a, "the checkpoint's {d:.1} GiB of weights do not fit the {d:.1} GiB the CUDA memory budget grants ({d:.1} GiB free less a {d:.1} GiB reserve{s}); free device memory or adjust TENSORFOLD_MEMORY_RESERVE_GIB / TENSORFOLD_CUDA_MEMORY_LIMIT_GB", .{ toGib(weights), toGib(before.room(held0)), toGib(before.free), toGib(before.reserve), if (before.limit != null) ", under TENSORFOLD_CUDA_MEMORY_LIMIT_GB" else "" }); return null; } - const loaded = F.open(gpa, io, &g.ctx, o.dir, kernels, .{ .context = @intCast(window), .drafts = o.drafts, .segments = segments, .slide = o.slide }) catch |e| { + var family_options: F.Options = .{ .context = @intCast(window), .drafts = o.drafts, .segments = segments }; + if (@hasField(F.Options, "slide")) family_options.slide = o.slide; + if (@hasField(F.Options, "drafter")) family_options.drafter = o.drafter; + const loaded = F.open(gpa, io, &g.ctx, o.dir, kernels, family_options) catch |e| { problem.* = try std.fmt.allocPrint(a, "the native CUDA engine cannot load {s} with {s} glue kernels ({s})", .{ o.dir, kernels orelse "its own", @errorName(e) }); return null; }; @@ -429,13 +433,14 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O const h = try gpa.create(Host); errdefer gpa.destroy(h); h.* = .{ .gpa = gpa, .gpu = g, .family = loaded.ctx, .release = loaded.deinit, .inner = loaded.backend, .vtable = undefined, .cfg = undefined, .clock = undefined, .core = undefined, .host = undefined, .lone = loaded.lone }; - const cache = budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)); + // prompt-cache snapshots are Nemotron's (cuda_reuse.zig); a family without them serves without a cache + const cache = if (@hasDecl(F, "snap_bytes")) budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)) else 0; h.startup = try std.fmt.allocPrint(gpa, "CUDA sm_{d} device {d} ({s}{s}): model {d:.2} GiB{s}; {d} stream{s} at once, {d:.2} GiB each at a {d}-token window, of {d:.1} GiB left after a {d:.1} GiB reserve; prompts in {d}-row chunks{s}{s}; {s}", .{ capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache", .{toGib(cache)}) else ", no prompt cache", if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", }); errdefer gpa.free(h.startup); - if (cache > 0) { + if (@hasDecl(F, "snap_bytes") and cache > 0) { const pc = api.prompt_cache; const store = try gpa.create(pc.Store); // states kept and resumed on the family's grid, where a cache-off pass cuts; the head reads one token on @@ -446,7 +451,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O store.deinit(); gpa.destroy(store); }; - if (loaded.learns) |e| { + if (@hasDecl(F, "slide")) if (loaded.learns) |e| { const Slide = nemotron_slide.Adapter(F.slide); const s = try gpa.create(Slide); s.* = Slide.init(gpa, io, e); @@ -459,7 +464,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O } }.f; h.learn = s.hook(); - } + }; errdefer if (h.slide) |s| h.slide_free.?(gpa, s); // the family cuts its own prompt grid from position 0, as `tensorfold run` does: prefill_step 0 try h.serve(io, loaded.facts, loaded.rows, .{ .lanes = streams, .context_window = @intCast(window), .startup = h.startup, .prompt_cache = h.store != null, .logprobs = loaded.backend.vtable.first_row != null }, .{ .ctx = loaded.ctx, .text = F.explain }); diff --git a/zig/tests/cuda/qwen27/capture.py b/zig/tests/cuda/qwen27/capture.py new file mode 100644 index 0000000000..a21dd828eb --- /dev/null +++ b/zig/tests/cuda/qwen27/capture.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""The 27B's Python CUDA engine as the Zig engine's oracle: Triton launches, per-layer dumps, tokens.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path + +TEXTS = { + "story": "Write a short story about a lighthouse keeper who finds a message in a bottle.", + "code": "Write a Python function that merges two sorted lists, with a docstring and three tests.", + "facts": "Explain how a refrigerator moves heat out of its cabinet, step by step.", +} +LONG_BODY = 5000 # tokens of module source in the long prompt: two 4096-row prompt chunks +TEACHER = 32 # teacher-forced decode steps (one row each, from an empty state) +DUMP_STEPS = (0, 1, TEACHER - 1) + + +def chat(user: str) -> str: + return f"<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + + +def prompts(model: Path) -> dict[str, list[int]]: + from tokenizers import Tokenizer + import textwrap + + tok = Tokenizer.from_file(str(model / "tokenizer.json")) + out = {name: tok.encode(chat(text), add_special_tokens=False).ids for name, text in TEXTS.items()} + words = Path(textwrap.__file__).read_text() + body = tok.encode(words * (1 + LONG_BODY // max(1, len(words) // 3)), add_special_tokens=False).ids[:LONG_BODY] + head = tok.encode("<|im_start|>user\nReview this module and list its public functions:\n\n", + add_special_tokens=False).ids + tail = tok.encode("<|im_end|>\n<|im_start|>assistant\n\n\n\n\n", add_special_tokens=False).ids + out["long"] = head + body + tail + return out + + +def sha12(tokens: list[int]) -> str: + return hashlib.sha256(json.dumps([int(t) for t in tokens]).encode()).hexdigest()[:12] + + +def digest(t) -> str: + import torch + + raw = t.detach().contiguous() + return hashlib.sha256(raw.view(torch.uint8).cpu().numpy().tobytes()).hexdigest() + + +class Dumps: + """Raw little-endian tensors by name under the current directory; None turns dumping off.""" + + def __init__(self) -> None: + self.dir: Path | None = None + self.i = 0 + + def at(self, d: Path | None) -> None: + self.dir, self.i = d, 0 + if d is not None: + d.mkdir(parents=True, exist_ok=True) + + def save(self, name: str, t) -> None: + import torch + + if self.dir is None: + return + torch.cuda.synchronize() + t.detach().contiguous().view(torch.uint8).cpu().numpy().tofile(self.dir / f"{name}.bin") + + +def hook(dumps: Dumps) -> None: + """Every add_rmsnorm call's (h, y, xs), in call order: two a layer, then the final norm.""" + + from tensorfold.families.qwen3_5.cuda import glue + + norm = glue.add_rmsnorm + + def traced(x, r, w, eps): + h, y, xs = norm(x, r, w, eps) + if dumps.dir is not None: + i = dumps.i + dumps.i += 1 + dumps.save(f"{i:03d}_h", h) + dumps.save(f"{i:03d}_y", y) + dumps.save(f"{i:03d}_xs", xs) + return h, y, xs + + glue.add_rmsnorm = traced + + +def specialization() -> dict: + """Each launched JIT function's parameter names and which ones Triton never specializes.""" + + from triton.runtime.jit import JITFunction + import gc + + out = {} + for fn in gc.get_objects(): + if isinstance(fn, JITFunction): + name = f"{fn.fn.__module__}.{fn.fn.__qualname__}" + out[name] = {"params": [p.name for p in fn.params], + "do_not_specialize": [p.name for p in fn.params if p.do_not_specialize], + "no_align": [p.name for p in fn.params if p.do_not_specialize_on_alignment]} + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--tools", default="", help="folder of triton_aot_manifest.py: record every Triton launch") + ap.add_argument("--max-tokens", type=int, default=128) + ap.add_argument("--repeats", type=int, default=2) + ap.add_argument("--context", type=int, default=16384) + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + rec = None + if a.tools: + sys.path.insert(0, a.tools) + import triton_aot_manifest as aot + + rec = aot.Recorder().install() + + import torch + from contextlib import nullcontext + + def phase(name: str, detail: bool = False): + return rec.scope(name, detail) if rec is not None else nullcontext() + + model = Path(a.model) + ids = prompts(model) + (out / "prompts.json").write_text(json.dumps(ids) + "\n") + dumps = Dumps() + hook(dumps) + from tensorfold.families.qwen3_5.cuda.engine import Qwen27Engine + + t0 = time.perf_counter() + with phase("startup"): + eng = Qwen27Engine(model, None, streams=1, context=a.context, context_explicit=True) + if rec is not None: + from tensorfold.cuda.kernels import gdn, prefill_attention, qmm + + rec.wrap(qmm._ext(), ("qmm", "qmm_group", "qmm_prefill"), "qmm") + rec.wrap(gdn._ext(), ("tree", "replay", "prefill"), "gdn") + rec.wrap(prefill_attention._ext(), ("prefill_attention",), "prefill_attention") + info = {"startup_s": round(time.perf_counter() - t0, 2), "context": eng.context_window, "eos": list(eng.eos), + "prompt_rows": eng.w.prompt_rows, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0)} + print(json.dumps(info), flush=True) + + results: dict[str, dict] = {} + for name, prompt in ids.items(): + runs = [] + for _ in range(a.repeats): + toks: list[int] = [] + with phase(f"serial-{name}"): + torch.cuda.synchronize() + start = time.perf_counter() + stats = eng.generate(prompt, a.max_tokens, None, lambda new: toks.extend(map(int, new)) and False, + draft=False, stop_eos=True) + torch.cuda.synchronize() + wall = time.perf_counter() - start + runs.append({"tokens": toks, "sha": sha12(toks), "wall_s": round(wall, 4), **stats}) + last = runs[-1] + same = all(r["tokens"] == last["tokens"] for r in runs) + step = last.get("decode_s", 0.0) / max(1, len(last["tokens"]) - 1) * 1e3 + results[name] = {**last, "repeats_same": same, "ms_per_token": round(step, 4)} + print(f"serial {name}: {len(last['tokens'])} tokens sha {last['sha']} prefill {last.get('prefill_s'):.3f}s " + f"{step:.3f} ms/token same {same}", flush=True) + + from tensorfold.families.qwen3_5.cuda.decode import prefill + from tensorfold.families.qwen3_5.cuda.forward import State, commit, tree_forward + + w = eng.w + tf = ids["code"][:TEACHER] + sampled, logit_sha = [], [] + st = State(w) + with phase("teacher", detail=True): + for i, t in enumerate(tf): + dumps.at(out / "teacher" / f"step{i:03d}" if i in DUMP_STEPS else None) + logits, record = tree_forward(w, torch.tensor([t], dtype=torch.int32, device="cuda"), [-1], st) + dumps.save("logits", logits) + sampled.append(int(logits.argmax(dim=-1)[0])) + logit_sha.append(digest(logits)) + commit(st, record, [0]) + dumps.at(None) + pre_out = {} + for name in ("code", "long"): + dumps.at(out / "prefill" / name if name == "code" else None) + with phase(f"prefill-{name}", detail=name == "code"): + _, pending = prefill(w, ids[name], None, limit=a.context) + dumps.at(None) + pre_out[name] = {"pending": pending} + (out / "teacher.json").write_text(json.dumps({"tokens": tf, "sampled": sampled, "logits_sha256": logit_sha, + "dump_steps": list(DUMP_STEPS), "prefill": pre_out}) + "\n") + (out / "results.json").write_text(json.dumps({"info": info, "results": results}, indent=1) + "\n") + if rec is not None: + rec.dump(out / "launches.json") + (out / "jit.json").write_text(json.dumps(specialization(), indent=1, sort_keys=True) + "\n") + print("done", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/zig/tests/cuda/qwen27/capture_draft.py b/zig/tests/cuda/qwen27/capture_draft.py new file mode 100644 index 0000000000..b5aa457ebc --- /dev/null +++ b/zig/tests/cuda/qwen27/capture_draft.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""The 27B's DFlash2 drafting as the Zig engine's oracle: drafted replies (equal to serial), rounds, Triton launches.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path + + +def sha12(tokens: list[int]) -> str: + return hashlib.sha256(json.dumps([int(t) for t in tokens]).encode()).hexdigest()[:12] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", required=True) + ap.add_argument("--draft", required=True) + ap.add_argument("--prompts", required=True, help="capture.py's prompts.json") + ap.add_argument("--out", required=True) + ap.add_argument("--tools", default="") + ap.add_argument("--max-tokens", type=int, default=128) + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + rec = None + if a.tools: + sys.path.insert(0, a.tools) + import triton_aot_manifest as aot + + rec = aot.Recorder().install() + import torch + + from tensorfold.families.qwen3_5.cuda.decode import draft_decode, prefill + from tensorfold.families.qwen3_5.cuda.engine import Qwen27Engine + + eng = Qwen27Engine(Path(a.model), Path(a.draft), streams=1, context=16384, context_explicit=True) + if rec is not None: + from tensorfold.cuda.kernels import gdn, prefill_attention, qmm + + rec.wrap(qmm._ext(), ("qmm", "qmm_group", "qmm_prefill"), "qmm") + rec.wrap(gdn._ext(), ("tree", "replay", "prefill"), "gdn") + rec.wrap(prefill_attention._ext(), ("prefill_attention",), "prefill_attention") + prompts = json.loads(Path(a.prompts).read_text()) + results = {} + for name, ids in prompts.items(): + for _ in range(2): # the second run is the timed one + drafter = eng.draft + drafter.restore(([None] * drafter.layers, [None] * drafter.layers, 0, 0)) + torch.cuda.synchronize() + t0 = time.perf_counter() + st, pending = prefill(eng.w, ids, None, drafter, limit=16384) + trace: list = [] + r = draft_decode(eng.w, st, ids, pending, a.max_tokens, None, drafter, max_rows=eng.max_rows, + tree_rows=eng.tree_rows, allow_copy=True, stop_eos=True, trace=trace, inplace=True) + torch.cuda.synchronize() + wall = time.perf_counter() - t0 + results[name] = {"tokens": r.tokens, "sha": sha12(r.tokens), "rounds": r.rounds, "drafted": r.drafted_rows, + "accepted": r.accepted_drafts, "decode_s": round(r.seconds, 4), "wall_s": round(wall, 4), + "widths": r.widths, "paths": [len(t.get("path", [])) for t in trace]} + print(f"{name}: {len(r.tokens)} tokens sha {sha12(r.tokens)} rounds {r.rounds} accepted {r.accepted_drafts} " + f"{len(r.tokens) / r.seconds:.1f} tok/s", flush=True) + (out / "draft.json").write_text(json.dumps({"max_rows": eng.max_rows, "results": results}) + "\n") + if rec is not None: + rec.dump(out / "launches.json") + print("done", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/zig/tests/cuda/qwen27/capture_multi.py b/zig/tests/cuda/qwen27/capture_multi.py new file mode 100644 index 0000000000..6d18232073 --- /dev/null +++ b/zig/tests/cuda/qwen27/capture_multi.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""The 27B's drafted shared rounds as the Zig engine's oracle: MultiDecoder's windows each round, and every reply.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +TOPICS = ["lighthouses", "volcanoes", "bread", "rivers", "chess", "glaciers", "owls", "trains"] +CODE = '''def fibonacci(n): + """Return the nth Fibonacci number.""" + if n < 2: + return n + a, b = 0, 1 + for _ in range(n - 1): + a, b = b, a + b + return b +''' + + +def chat(user: str) -> str: + return f"<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + + +def sha12(tokens: list[int]) -> str: + return hashlib.sha256(json.dumps([int(t) for t in tokens]).encode()).hexdigest()[:12] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", required=True) + ap.add_argument("--draft", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--tools", default="") + ap.add_argument("--streams", type=int, default=6) + ap.add_argument("--max-tokens", type=int, default=96) + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + rec = None + if a.tools: + sys.path.insert(0, a.tools) + import triton_aot_manifest as aot + + rec = aot.Recorder().install() + from tokenizers import Tokenizer + + from tensorfold.cuda.streams import Stream + from tensorfold.families.qwen3_5.cuda.engine import Qwen27Engine + from tensorfold.families.qwen3_5.cuda.multi import MultiDecoder + + model = Path(a.model) + tok = Tokenizer.from_file(str(model / "tokenizer.json")) + users = [f"Rewrite this function with a docstring that explains each step, keeping the code the same:\n\n{CODE}"] + users += [f"Write four sentences about {t}." for t in TOPICS] + prompts = [tok.encode(chat(u), add_special_tokens=False).ids for u in users[:a.streams]] + eng = Qwen27Engine(model, Path(a.draft), streams=1, context=8192, context_explicit=True) + if rec is not None: + from tensorfold.cuda.kernels import gdn, prefill_attention, qmm + + rec.wrap(qmm._ext(), ("qmm", "qmm_group", "qmm_prefill"), "qmm") + rec.wrap(gdn._ext(), ("tree", "replay", "prefill"), "gdn") + rec.wrap(prefill_attention._ext(), ("prefill_attention",), "prefill_attention") + # GB10's planning: no measured costs (every tree node verified), the drafter's block 16 rows a stream + md = MultiDecoder(eng.w, eng.draft, allow_copy=True, context=8192, keep=1) + md.depth = False + rounds: list[list[dict]] = [] + windows = md._windows + + def logged(plan, copied, blocks): + wins = windows(plan, copied, blocks) + rounds.append([{"stream": sid, "mode": ["copy", "tree", "one"][mode], "tokens": t, "parents": p} + for (sid, mode, _, _), (t, p) in zip(plan, wins)]) + return wins + + md._windows = logged + scores: list[list[float]] = [] # each tree's path scores, in finish order + finish = md.draft.finish_tree + + def scored(*args, **kw): + t = finish(*args, **kw) + scores.append([float(x) for x in t[2]]) + return t + + md.draft.finish_tree = scored + streams = [Stream(list(p), a.max_tokens, None, draft=True, stop_eos=True) for p in prompts] + for s in streams: + md.admit(s) + while any(not s.done for s in streams): + md.finish(md.round()) + results = [{"tokens": s.out, "sha": sha12(s.out), "rounds": s.rounds, "drafted": s.drafted, + "accepted": s.accepted} for s in streams] + for k, r in enumerate(results): + print(f"stream {k}: {len(r['tokens'])} tokens sha {r['sha']} rounds {r['rounds']} accepted {r['accepted']}", + flush=True) + (out / "multi.json").write_text(json.dumps({"prompts": prompts, "max_tokens": a.max_tokens, "results": results, + "rounds": rounds, "scores": scores}) + "\n") + if rec is not None: + rec.dump(out / "launches.json") + print(f"{len(rounds)} rounds", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/zig/tests/cuda/qwen27/capture_windows.py b/zig/tests/cuda/qwen27/capture_windows.py new file mode 100644 index 0000000000..78b7558951 --- /dev/null +++ b/zig/tests/cuda/qwen27/capture_windows.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""The 27B's verify windows as the Zig engine's oracle: chains of 1-16 rows and shared forwards, each row hashed.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +TOPICS = ["lighthouses", "volcanoes", "bread", "rivers", "chess", "glaciers", "owls", "trains", + "deserts", "bees", "clocks", "ferns", "kites", "salt", "comets", "violins"] +REPLY = 24 # serial tokens each stream decodes first: the chain windows' tokens + + +def chat(user: str) -> str: + return f"<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + + +def sha(t) -> str: + import torch + + return hashlib.sha256(t.detach().contiguous().view(torch.uint8).cpu().numpy().tobytes()).hexdigest() + + +def rows(logits) -> list[dict]: + return [{"sha": sha(logits[r]), "argmax": int(logits[r].argmax())} for r in range(logits.shape[0])] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--tools", default="") + ap.add_argument("--wide", action="store_true", help="also rounds past 16 rows") + ap.add_argument("--sweep", action="store_true", help="also every alignment of a shared round's packed inputs") + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + rec = None + if a.tools: + sys.path.insert(0, a.tools) + import triton_aot_manifest as aot + + rec = aot.Recorder().install() + import torch + from tokenizers import Tokenizer + + from tensorfold.families.qwen3_5.cuda.decode import clone_state, prefill, serial_decode + from tensorfold.families.qwen3_5.cuda.engine import Qwen27Engine + from tensorfold.families.qwen3_5.cuda.forward import commit_streams, multi_tree_forward, tree_forward + + model = Path(a.model) + tok = Tokenizer.from_file(str(model / "tokenizer.json")) + prompts = [tok.encode(chat(f"Write four sentences about {t}."), add_special_tokens=False).ids for t in TOPICS] + eng = Qwen27Engine(model, None, streams=1, context=4096, context_explicit=True) + if rec is not None: + from tensorfold.cuda.kernels import gdn, prefill_attention, qmm + + rec.wrap(qmm._ext(), ("qmm", "qmm_group", "qmm_prefill"), "qmm") + rec.wrap(gdn._ext(), ("tree", "replay", "prefill"), "gdn") + rec.wrap(prefill_attention._ext(), ("prefill_attention",), "prefill_attention") + w = eng.w + states, replies = [], [] + for p in prompts: + st, pending = prefill(w, p, None, limit=4096) + states.append(st) + replies.append(serial_decode(w, st, pending, REPLY, None, stop_eos=False).tokens) + dev = w.norm.device + cases = [] + + def tokens(ids): + return torch.tensor(ids, dtype=torch.int32, device=dev) + + # chains of one stream: the serial reply's first W tokens, each row's draw the next serial token + for width in range(1, 17): + st = clone_state(states[0]) + logits, _ = tree_forward(w, tokens(replies[0][:width]), list(range(-1, width - 1)), st) + cases.append({"kind": "chain", "streams": [{"stream": 0, "tokens": replies[0][:width]}], "rows": rows(logits)}) + # shared forwards: each stream its own chain from its own state, rows in stream order + layouts = [[1, 1], [1] * 4, [1] * 8, [1] * 16, [4, 4, 4, 4], [2, 5, 1, 8], [3] * 5, [16]] + if a.wide: # rounds past 16 rows: the wider matmul tiles (to 32, 64, more) + layouts += [[16, 16], [5, 16, 3], [16] * 4, [9, 16, 2, 16, 7, 16, 1, 5], [12] * 8, [16] * 16] + if a.sweep: # the packed copy's pointer alignments: rows and streams mod 4, 16-row multiples + for n in range(2, 6): + for total in (16 * n, 16 * n - 12, 16 * n - 11, 16 * n - 10, 16 * n - 9): + layouts.append([total // n + (i < total % n) for i in range(n)]) + for layout in layouts: + sts = [clone_state(states[s]) for s in range(len(layout))] + wins = [(replies[s][:n], list(range(-1, n - 1)), sts[s]) for s, n in enumerate(layout)] + logits, record, _, starts = multi_tree_forward(w, wins) + case = {"kind": "shared", "streams": [{"stream": s, "tokens": replies[s][:n]} for s, n in enumerate(layout)], + "rows": rows(logits)} + # each stream keeps a prefix of its rows (as multi._commit names them: window rows), then one more row each + keep = [list(range(max(1, n - 1))) for n in layout] + commit_streams(sts, record, [[starts[s] + r for r in k] for s, k in enumerate(keep)]) + nxt = [([replies[s][len(k)]], [-1], sts[s]) for s, k in enumerate(keep)] + after, _, _, _ = multi_tree_forward(w, nxt) + case["keep"] = [len(k) for k in keep] + case["after"] = rows(after) + cases.append(case) + # trees: each stream's serial reply as the main chain, with other tokens hung beside it + def tree(stream: int, shape: str): + r = replies[stream] + alt = lambda t: (t + 7919) % w.config.vocab # noqa: E731 + if shape == "fork": # 0 r0; 1 r1 (0); 2 alt (0); 3 r2 (1); 4 alt (1); 5 r3 (3); 6 alt (2) + return [r[0], r[1], alt(r[1]), r[2], alt(r[2]), r[3], alt(r[3])], [-1, 0, 0, 1, 1, 3, 2] + if shape == "fan": # the root's six children (r1 first), then r2 under r1 + return [r[0], r[1]] + [alt(r[1] + j) for j in range(5)] + [r[2]], [-1, 0, 0, 0, 0, 0, 0, 1] + raise ValueError(shape) + + tree_cases = [([(0, "fork")], [[0, 1, 3, 5]]), ([(1, "fan")], [[0, 1, 7]]), + ([(2, "fork"), (3, "fork")], [[0, 1, 3, 5], [0, 2, 6]])] + if a.wide: # every stream a tree in one round, 120 rows + tree_cases.append(([(s, "fork" if s % 2 else "fan") for s in range(16)], + [[0, 1, 3, 5] if s % 2 else [0, 1, 7] for s in range(16)])) + for spec, paths in tree_cases: + sts = [clone_state(states[s]) for s, _ in spec] + wins = [(*tree(s, shape), st) for (s, shape), st in zip(spec, sts)] + if len(spec) == 1: + logits, record = tree_forward(w, tokens(wins[0][0]), wins[0][1], sts[0]) + starts = [0, len(wins[0][0])] + else: + logits, record, _, starts = multi_tree_forward(w, [(t, p, st) for t, p, st in wins]) + case = {"kind": "tree", "streams": [{"stream": s, "tokens": t, "parents": p} for (s, _), (t, p, _) in + zip(spec, wins)], "rows": rows(logits), "paths": paths} + nxt = [int(logits[starts[k] + path[-1]].argmax()) for k, path in enumerate(paths)] + commit_streams(sts, record, [[starts[k] + r for r in path] for k, path in enumerate(paths)]) + after, _, _, _ = multi_tree_forward(w, [([t], [-1], st) for t, st in zip(nxt, sts)]) + case["next"] = nxt + case["after"] = rows(after) + cases.append(case) + torch.cuda.synchronize() + (out / "windows.json").write_text(json.dumps({"prompts": prompts, "replies": replies, "cases": cases}) + "\n") + if rec is not None: + rec.dump(out / "launches.json") + print(f"{len(cases)} cases", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 907e013d15d2e5e4c8b9cc946acdcf97e9cb574d Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:57:08 -0500 Subject: [PATCH 04/19] Native image input: images from a request's image parts, prepared before admission; GLM-5.3-Flash reads them on Metal Server (engine-independent): - `Info.vision` is an engine's offer: a `prepare` hook (an image's bytes to its tower's patches, grid, tokens and a content hash, decoded by the platform's own decoder), the image token, the wrapper's tokens, and a request's visual-token budget and image limit. `--vision`, `--vision-max-images` and `--vision-image-tokens` reach native engines. - User messages keep their image_url parts (data URLs) beside their text for the template. Images are counted from those parts, never from text: a marker that a user's words or a tool's output spells is rendered as ordinary text tokens (each quoted marker stands in as a private sentinel while the template renders, then is read without added-token matching), so every image token in the prompt is the template's, one wrapper a part (#511). A prompt that quotes no marker gets exactly the ids it got before. - Each image's placeholder is expanded to its tokens before the window, the chunks and the cache see the prompt; `Request.images` carries the prepared patches and positions, and `Request.cache_key` replaces each image's rows by ids derived from its content, so a kept state of one image never resumes another under the same placeholders. - 0.6.6's limits and refusals: 4 images, 10 MB each and 20 MB a request, 8192 px a side and 16 MP, a single frame, a shared visual-token budget, detail low at 256 tokens; PNG and JPEG (images.py's formats), WebP refused by name until a decoder exists; remote URLs refused. GLM-5.3-Flash on Metal: the image tower (24 pre-norm blocks with 2-D rotary attention, the 2x2 merge and the merger, its projections on gemm_nax), its rows written over the placeholders' embeddings in the prompt pass and one position on for the MTP head; preprocessing as 0.6.6's vision/images.py and mlx-vlm's Glm5NextImageProcessor (EXIF orientation, alpha over white, PIL's 8-bit bicubic resampler). tf-glm-vision compares a reference dump's stages. tools/zig/glm_vision_fixtures.py draws the fixed image set and dumps the Python frontend's tensors; tools/zig/vision_served.py checks a running server. --- build.zig | 3 + docs/api.md | 3 +- docs/vision.md | 20 + tools/zig/glm_vision_fixtures.py | 165 ++++++++ tools/zig/vision_served.py | 173 ++++++++ zig/kernels/metal/glm_vision.metal | 229 +++++++++++ zig/kernels/metal/sources.zig | 2 + zig/src/core/engine_api.zig | 42 ++ zig/src/core/lane_host.zig | 16 +- zig/src/core/lane_host_reuse_test.zig | 88 +++++ zig/src/core/lanes/stream.zig | 7 + zig/src/core/tokenizer/tokenizer.zig | 24 +- zig/src/families/glm/backend.zig | 6 + zig/src/families/glm/checks.zig | 2 +- zig/src/families/glm/engine.zig | 16 +- zig/src/families/glm/forward.zig | 28 +- zig/src/families/glm/glm.zig | 4 + zig/src/families/glm/image.zig | 453 +++++++++++++++++++++ zig/src/families/glm/mtp.zig | 1 + zig/src/families/glm/prompt.zig | 3 +- zig/src/families/glm/slot_images.zig | 37 ++ zig/src/families/glm/slots.zig | 1 + zig/src/families/glm/vision.zig | 541 ++++++++++++++++++++++++++ zig/src/native/glm_host.zig | 16 +- zig/src/native/metal.zig | 10 +- zig/src/server/anthropic.zig | 2 +- zig/src/server/chat.zig | 35 +- zig/src/server/cli.zig | 20 +- zig/src/server/hf_text.zig | 5 + zig/src/server/images.zig | 350 +++++++++++++++++ zig/src/server/messages.zig | 51 ++- zig/src/server/model_text.zig | 7 + zig/src/server/openai.zig | 4 +- zig/src/server/prompt.zig | 21 +- zig/src/server/reply_text.zig | 29 ++ zig/src/server/root.zig | 1 + zig/tests/glm_vision.zig | 148 +++++++ 37 files changed, 2495 insertions(+), 68 deletions(-) create mode 100644 tools/zig/glm_vision_fixtures.py create mode 100644 tools/zig/vision_served.py create mode 100644 zig/kernels/metal/glm_vision.metal create mode 100644 zig/src/families/glm/image.zig create mode 100644 zig/src/families/glm/slot_images.zig create mode 100644 zig/src/families/glm/vision.zig create mode 100644 zig/src/server/images.zig create mode 100644 zig/tests/glm_vision.zig diff --git a/build.zig b/build.zig index 054e688280..b2c9e0d8e4 100644 --- a/build.zig +++ b/build.zig @@ -410,6 +410,7 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b .{ .name = "tf-grid-sync-bench", .path = "zig/tests/grid_sync_bench.zig", .about = "A GPU-wide barrier in one persistent dispatch against dependent relaunches" }, .{ .name = "tf-weight-read-check", .path = "zig/tests/weight_read_check.zig", .about = "Check native file reads and failed-read cleanup", .c_source = "zig/tests/pread_fault.c" }, .{ .name = "tf-glm-run", .path = "zig/tests/glm_run.zig", .about = "GLM-5.3-Flash greedy replies at each draft depth against depth 0 and reference tokens" }, + .{ .name = "tf-glm-vision", .path = "zig/tests/glm_vision.zig", .about = "GLM-5.3-Flash's image tower on an image or a reference dump's patches, compared with the dump" }, .{ .name = "tf-glm-attn-probe", .path = "zig/tests/glm_attn_probe.zig", .about = "GLM-5.3-Flash's dense latent attention kernels on given inputs (scores, probabilities, outputs)" }, .{ .name = "tf-moe-bench", .path = "zig/tests/moe_bench.zig", .about = "Routed-expert prompt kernels from .metal files on a skewed routing: chained speed, bits against the first" }, .{ .name = "tf-mm-bench", .path = "zig/tests/mm_bench.zig", .about = "Prompt matmul kernels from .metal files: chained speed at one shape, bits against the first, error against fp64" }, @@ -540,6 +541,8 @@ fn metalEngineModules(b: *std.Build, target: std.Build.ResolvedTarget, optimize: metal.linkFramework("Metal", .{}); metal.linkFramework("Foundation", .{}); metal.linkFramework("IOSurface", .{}); // the Neural Engine's prompt share passes IOSurfaces + metal.linkFramework("ImageIO", .{}); // GLM-5.3-Flash's image input: decoding (families/glm/image.zig) + metal.linkFramework("CoreGraphics", .{}); metal.linkSystemLibrary("objc", .{}); const lanes = b.createModule(.{ .root_source_file = b.path("zig/src/core/lanes/lanes.zig"), .target = target, .optimize = optimize, .link_libc = true }); const fabric = b.createModule(.{ .root_source_file = b.path("zig/src/fabric/fabric.zig"), .target = target, .optimize = optimize, .link_libc = true }); diff --git a/docs/api.md b/docs/api.md index dc5ff8bb1a..76a950230b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -67,7 +67,8 @@ chat fields that shape the prompt (`tools`, `reasoning_effort`, `chat_template_k `tokens`, and `token_strs` with `return_token_strs: true`. The IDs are the ones the chat route runs for the same request, images expanded. `/detokenize` takes `tokens` and returns `prompt`. With `--vision`, supported Qwen3.5/3.8 dense checkpoints accept `image_url` content parts alongside text in user -messages and in tool results (`role: "tool"`), such as an agent's screenshots. +messages and in tool results (`role: "tool"`), such as an agent's screenshots. The native server does the same for +GLM-5.3-Flash on Metal, with data URLs only ([image input](vision.md#the-native-server)). See [image input](vision.md) for data URLs, public image URLs, limits and cache behavior. Unsupported image input, audio, video and non-text output requests receive HTTP 400. diff --git a/docs/vision.md b/docs/vision.md index 5e6bfa608b..75a37c1c58 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -129,6 +129,26 @@ This prevents identical image-placeholder token IDs from reusing another image's Multi-turn image conversations work when the request includes the original image content parts, but image-prefix reuse and persisted image KV are not implemented. For Qwen, each image request carries its own multimodal rotary positions and continuation offset, including during concurrent lane rounds. GLM uses its native KDA/NoPE attention state. +## The native server + +`tensorfold-native serve` takes `--vision`, `--vision-max-images` (default 4) and `--vision-image-tokens` (default +4,096) for engines that read images; GLM-5.3-Flash on one Mac reads them so far, and other native engines refuse the +flag at startup. + +- Images arrive as `image_url` data URLs in user messages and tool results. Remote URLs are refused, since + `--vision-urls` isn't on the native server yet. +- PNG and JPEG are decoded by the platform's decoder (ImageIO on macOS). WebP is refused by name until a decoder + exists; other formats are refused. +- The limits and the processing are 0.6.6's: EXIF orientation, alpha composited over white, the GLM5-Next processor's + sizing and PIL's bicubic resampler. Six drawn PNG fixtures give patches bit-equal to 0.6.6 with mlx-vlm + (`tools/zig/glm_vision_fixtures.py`). JPEG patches differ slightly, because ImageIO and libjpeg-turbo decode + differently. +- Images are counted from the request's image parts. A user or tool message whose text spells the model's image + markers (`<|image|>`, `<|begin_of_image|>`, `<|end_of_image|>`) is tokenized as ordinary text, so only the chat + template's own markers, one wrapper for each image part, become image tokens. +- The prompt cache keys each image's placeholder rows by the image's content, so a kept state resumes the same image + and never another one under the same placeholders. Image conversations reuse their history like text ones. + ## Verification Compare image requests with `draft: true` and `draft: false` at identical sampling settings and seed, then compare concurrent requests with their solo results. diff --git a/tools/zig/glm_vision_fixtures.py b/tools/zig/glm_vision_fixtures.py new file mode 100644 index 0000000000..ee17c15eb5 --- /dev/null +++ b/tools/zig/glm_vision_fixtures.py @@ -0,0 +1,165 @@ +"""GLM-5.3-Flash image input against the Python frontend, on a fixed image set (#565's gate). + + python -I tools/zig/glm_vision_fixtures.py --model DIR --out DIR [--tower zig-out/bin/tf-glm-vision] + +Draws the image set (each case a preprocessing path: padding only, an antialiased downscale, an upscale, alpha over +white, a palette with a transparent entry, gray with alpha, EXIF orientation, baseline JPEG), then for each image +writes what TensorFold 0.6.6's --vision gives (vision/images.py's decode: EXIF orientation, alpha over white; then +mlx-vlm's Glm5NextImageProcessor and tower, the code 0.6.6 serves): + patches.f32 (grid_h*grid_w, 1176) normalized patches, temporal frames duplicated + embed.f32 / block0.f32 / blocks.f32 / features.f32 the tower's stages, features the rows that replace <|image|> +and meta.json. With --tower it then runs tf-glm-vision on each image: the native patches compared value by value, +the native tower's rows by cosine (bf16 against bf16). + +Needs TensorFold 0.6.6 with its vision extra (mlx-vlm 0.7.x) and Pillow. Loads only model.visual.* (about 1 GB). +""" + +from __future__ import annotations + +import argparse +import io +import json +import re +import subprocess +import sys +from pathlib import Path + +import mlx.core as mx +import numpy as np +from PIL import Image, ImageDraw + + +def draw_shapes(size: tuple[int, int]) -> Image.Image: + w, h = size + im = Image.new("RGB", size, (245, 240, 230)) + d = ImageDraw.Draw(im) + d.ellipse((w * 0.08, h * 0.25, w * 0.42, h * 0.7), fill=(210, 40, 40)) + d.rectangle((w * 0.55, h * 0.25, w * 0.9, h * 0.7), fill=(40, 70, 200)) + for x in range(int(w * 0.1), int(w * 0.9), 6): + d.line((x, h * 0.8, x, h * 0.92), fill=(20, 20, 20), width=2) + d.text((w * 0.4, h * 0.06), "GLM 7", fill=(0, 0, 0)) + return im + + +def fixtures(folder: Path) -> list[Path]: + folder.mkdir(parents=True, exist_ok=True) + out = [] + + def save(name: str, im: Image.Image, **kw) -> None: + p = folder / name + im.save(p, **kw) + out.append(p) + + save("shapes.png", draw_shapes((517, 389))) # padding only + big = draw_shapes((2000, 1500)) + px = big.load() + for y in range(0, 1500, 3): # fine detail for the antialias + for x in range(0, 2000, 7): + px[x, y] = ((x * 3) % 256, (y * 5) % 256, (x + y) % 256) + save("large.png", big) # antialiased downscale + save("tiny.png", draw_shapes((20, 15))) # upscale to the token floor + logo = Image.new("RGBA", (96, 96), (0, 0, 0, 0)) + d = ImageDraw.Draw(logo) + d.rounded_rectangle((4, 4, 92, 92), 16, fill=(30, 30, 40, 255)) + d.polygon([(26, 28), (70, 28), (30, 68), (70, 68), (70, 74), (24, 74), (64, 34), (26, 34)], fill=(255, 255, 255, 230)) + save("logo.png", logo) # RGBA upscale, alpha over white + pal = draw_shapes((517, 389)).quantize(32) + pal.info["transparency"] = 0 + save("palette.png", pal, transparency=0) # palette, a transparent entry + la = Image.new("LA", (240, 180), (128, 0)) + d = ImageDraw.Draw(la) + for r in range(80, 0, -8): + d.ellipse((120 - r, 90 - r, 120 + r, 90 + r), fill=(255 - 3 * r, 255 - 2 * r)) + save("gray-alpha.png", la) # gray with alpha + rot = draw_shapes((300, 200)) + exif = Image.Exif() + exif[0x0112] = 6 # rotate 90 degrees clockwise + save("rotated.jpg", rot, quality=90, exif=exif.tobytes()) # EXIF orientation + save("shapes.jpg", draw_shapes((517, 389)), quality=85) # baseline JPEG + return out + + +def reference(model: Path, image_path: Path, out: Path, tower, processor, max_tokens: int) -> dict: + from tensorfold.vision.images import DEFAULT_LIMITS, _decode + + out.mkdir(parents=True, exist_ok=True) + image = _decode(image_path.read_bytes(), "auto", DEFAULT_LIMITS, DEFAULT_LIMITS.max_total_pixels).to_pil() + done = processor([image], return_tensors="np", min_image_tokens=min(16, max_tokens), max_image_tokens=max_tokens) + patches = np.asarray(done["pixel_values"], dtype=np.float32) + grid = np.asarray(done["image_grid_thw"], dtype=np.int64) + dtype = tower.patch_embed.proj.weight.dtype + x = mx.array(patches).astype(dtype) + g = mx.array(grid, dtype=mx.int32) + h = tower.patch_embed(x) + embed = h + pos = tower._rotary_embeddings(g) + cu = mx.array([0, int(h.shape[0])]) + block0 = None + for i, block in enumerate(tower.blocks): + h = block(h, cu, pos) + if i == 0: + block0 = h + blocks = h + features = tower(x, g) + mx.eval(embed, block0, blocks, features) + for name, a in (("patches", patches), ("embed", embed), ("block0", block0), ("blocks", blocks), ("features", features)): + np.asarray(a.astype(mx.float32) if isinstance(a, mx.array) else a, dtype=np.float32).tofile(out / f"{name}.f32") + meta = {"image": str(image_path), "size": list(image.size), "mode": Image.open(image_path).mode, "grid_thw": grid[0].tolist(), + "max_tokens": max_tokens, "tokens": int(features.shape[0]), "patch_width": int(patches.shape[1]), "dtype": str(dtype)} + (out / "meta.json").write_text(json.dumps(meta, indent=1) + "\n") + return meta + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--model", required=True, help="the GLM-5.3-Flash checkpoint folder") + ap.add_argument("--out", required=True, help="where the images and their references go") + ap.add_argument("--max-tokens", type=int, default=4096, help="the visual-token budget an image is sized to") + ap.add_argument("--tower", help="tf-glm-vision: also compare the native patches and tower with each reference") + args = ap.parse_args() + + from mlx_vlm.models.glm5_next.config import VisionConfig + from mlx_vlm.models.glm5_next.processing import Glm5NextImageProcessor + from mlx_vlm.models.glm5_next.vision import VisionModel + + model, out = Path(args.model), Path(args.out) + config = json.loads((model / "config.json").read_text()) + proc_cfg = json.loads((model / "processor_config.json").read_text())["image_processor"] + proc_cfg.pop("image_processor_type", None) + processor = Glm5NextImageProcessor(**proc_cfg) + tower = VisionModel(VisionConfig.from_dict(config["vision_config"])) + index = json.loads((model / "model.safetensors.index.json").read_text())["weight_map"] + weights = {} + for shard in sorted({f for k, f in index.items() if k.startswith("model.visual.")}): + for k, v in mx.load(str(model / shard)).items(): + if k.startswith("model.visual."): + weights[k[len("model.visual."):]] = v + tower.load_weights(list(tower.sanitize(weights).items()), strict=True) + mx.eval(tower.parameters()) + + failures = 0 + for image in fixtures(out / "images"): + ref = out / image.name.replace(".", "-") + meta = reference(model, image, ref, tower, processor, args.max_tokens) + line = f"{image.name:15} {meta['size'][0]}x{meta['size'][1]} {meta['mode']:5} grid {meta['grid_thw']} {meta['tokens']} tokens" + if args.tower: + run = subprocess.run([args.tower, str(model), str(ref), "features", str(image)], capture_output=True, text=True) + text = run.stdout + run.stderr + p = re.search(r"patches: (\d+) of (\d+) values differ", text) + c = re.search(r"cosine mean ([0-9.]+) worst ([0-9.]+)", text) + if run.returncode != 0 or not p or not c: + failures += 1 + line += f" FAIL: {text.strip().splitlines()[-1] if text.strip() else run.returncode}" + else: + differ = int(p.group(1)) + exact = differ == 0 + # PNG patches must be bit-exact; JPEG patches differ by decoder (ImageIO against libjpeg-turbo) + failures += (not exact) and image.suffix == ".png" + line += f" patches {'equal' if exact else f'{differ} of {p.group(2)} differ'}, rows' cosine {c.group(1)} (worst {c.group(2)})" + print(line) + print("PASS" if failures == 0 else f"FAIL: {failures}") + return 0 if failures == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/zig/vision_served.py b/tools/zig/vision_served.py new file mode 100644 index 0000000000..49bd56bca6 --- /dev/null +++ b/tools/zig/vision_served.py @@ -0,0 +1,173 @@ +"""Image input served end to end on a running ``tensorfold-native --vision`` (#565's served checks). + + python tools/zig/vision_served.py --url http://127.0.0.1:8000 --images DIR + +DIR holds PNG and JPEG test images. Checks: an image request answers, and text or a tool result that quotes the +model's image marker beside a real image is ordinary text (images come from the request's image parts, #511); drafted +equals ``"draft": false`` and concurrent equals alone with images; an image conversation's next turn resumes past +its image, and another image under the same placeholders resumes none of the first one's rows; and the refusals (WebP by name, other formats, an image outside a +user or tool message, too many images, remote URLs, bad data URLs, a bad detail). +""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures as cf +import json +import sys +import urllib.error +import urllib.request +from pathlib import Path + + +def post(url: str, body: dict) -> tuple[int, dict]: + req = urllib.request.Request(url + "/v1/chat/completions", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=1800) as r: + return r.status, json.loads(r.read()) + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b"{}") + + +def data_url(raw: bytes, mime: str) -> str: + return f"data:{mime};base64,{base64.b64encode(raw).decode()}" + + +def image_part(url: str, detail: str | None = None) -> dict: + inner = {"url": url} + if detail: + inner["detail"] = detail + return {"type": "image_url", "image_url": inner} + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--url", default="http://127.0.0.1:8000") + ap.add_argument("--images", required=True) + ap.add_argument("--tokens", type=int, default=48) + ap.add_argument("--fresh-url", help="the same model served with --prompt-cache-gib 0: the cache check's reference reply") + args = ap.parse_args() + with urllib.request.urlopen(args.url + "/v1/models") as r: + model = json.loads(r.read())["data"][0]["id"] + folder = Path(args.images) + pngs = sorted(folder.glob("*.png")) + jpgs = sorted(folder.glob("*.jpg")) + first, second = pngs[0], pngs[1] + url_a = data_url(first.read_bytes(), "image/png") + url_b = data_url(second.read_bytes(), "image/png") + failures = 0 + + def ask(messages: list, draft: bool = True, tokens: int | None = None) -> tuple[int, dict]: + b = {"model": model, "messages": messages, "max_tokens": tokens or args.tokens, "temperature": 0, + "chat_template_kwargs": {"enable_thinking": False}} + if not draft: + b["draft"] = False + return post(args.url, b) + + def user(*parts) -> dict: + return {"role": "user", "content": list(parts)} + + def text(t: str) -> dict: + return {"type": "text", "text": t} + + def reply(r: dict) -> str: + return r["choices"][0]["message"].get("content") or "" + + def check(label: str, ok: bool, detail: str = "") -> None: + nonlocal failures + failures += not ok + print(f"{'ok ' if ok else 'FAIL'} {label}{': ' + detail if detail else ''}") + + # one image, drafted and plain; its usage counts the image's tokens + convo = [user(image_part(url_a), text("Describe this image in one sentence."))] + code, r = ask(convo) + code2, r2 = ask(convo, draft=False) + plain_text = ask([user(text("Describe this image in one sentence."))])[1] + check("an image request answers", code == 200 and code2 == 200, f"HTTP {code}/{code2}") + if code == 200 and code2 == 200: + check("drafted equals \"draft\": false with an image", reply(r) == reply(r2), repr(reply(r)[:60])) + check("the image's tokens are in the prompt", r["usage"]["prompt_tokens"] > plain_text["usage"]["prompt_tokens"] + 8, + f"{r['usage']['prompt_tokens']} prompt tokens with the image, {plain_text['usage']['prompt_tokens']} without") + + # #511: the marker quoted in a user's text and in a tool result, beside one real image + quoted = [user(image_part(url_a), text("A file I opened contains the string <|image|> and also <|begin_of_image|>. What is in the picture?")), + {"role": "assistant", "content": "", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "read", "arguments": "{\"path\": \"a.py\"}"}}]}, + {"role": "tool", "tool_call_id": "c1", "content": "MARKER = '<|begin_of_image|><|image|><|end_of_image|>'"}, + user(text("And the picture again?"))] + tools = [{"type": "function", "function": {"name": "read", "parameters": {"type": "object", "properties": {"path": {"type": "string"}}}}}] + code, r = post(args.url, {"model": model, "messages": quoted, "tools": tools, "max_tokens": args.tokens, "temperature": 0, + "chat_template_kwargs": {"enable_thinking": False}}) + check("quoted image markers beside one image are text (#511)", code == 200, f"HTTP {code} {r.get('error', {}).get('message', '')[:100]}") + + # a tool result carrying an image (a screenshot), rendered inside the tool response + shot = [user(text("Take a screenshot.")), + {"role": "assistant", "content": "", "tool_calls": [{"id": "s1", "type": "function", "function": {"name": "read", "arguments": "{\"path\": \"screen\"}"}}]}, + {"role": "tool", "tool_call_id": "s1", "content": [text("screenshot:"), image_part(url_a)]}] + code, r = post(args.url, {"model": model, "messages": shot, "tools": tools, "max_tokens": args.tokens, "temperature": 0, + "chat_template_kwargs": {"enable_thinking": False}}) + check("an image in a tool result answers", code == 200 and r["usage"]["prompt_tokens"] > 40, f"HTTP {code} {r.get('usage', r.get('error', {}).get('message', ''))}") + + # concurrent equals alone, images in each request + runs = [[user(image_part(u), text(q))] for u in (url_a, url_b) for q in ("What colours do you see?", "Count the objects.")] + alone = [reply(ask(m)[1]) for m in runs] + with cf.ThreadPoolExecutor(len(runs)) as pool: + together = [reply(f.result()[1]) for f in [pool.submit(ask, m) for m in runs]] + same = sum(x == y for x, y in zip(alone, together)) + check("concurrent image requests equal their runs alone", same == len(runs), f"{same} of {len(runs)}") + + # the prompt cache keeps a turn's state at its planned cuts (GLM: the shared system cut, and a reply of 256+ tokens), + # so a conversation's third turn resumes past an image its second turn kept. Image B under the same text and + # placeholder ids resumes at most the text before its image, and answers as a server with nothing kept + system = {"role": "system", "content": " ".join(f"Rule {i}: describe what you see plainly and briefly." for i in range(120))} + q, q2, q3 = "What is shown?", "Say that again in three words.", "And once more." + head = ask([system, user(text(q))], tokens=1)[1]["usage"]["prompt_tokens"] # the text alone: past where an image starts + first = " ".join(f"Detail {i}: the picture shows shapes, colours and lines." for i in range(40)) # a reply of 256+ tokens + + def turns(url: str) -> tuple[list, list]: + two = [system, user(image_part(url), text(q)), {"role": "assistant", "content": first}, user(text(q2))] + return two, two + [{"role": "assistant", "content": "Three words here."}, user(text(q3))] + + two_a, three_a = turns(url_a) + two_b, three_b = turns(url_b) + if args.fresh_url: # B's reply with nothing kept + rf = post(args.fresh_url, {"model": model, "messages": three_b, "max_tokens": 24, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}})[1] + ask(two_a, tokens=8) # keeps A's state at the long reply's start, past its image + code, ra = ask(three_a, tokens=24) + cached_a = ra.get("usage", {}).get("prompt_tokens_details", {}).get("cached_tokens", 0) if code == 200 else -1 + check("an image conversation's later turn resumes past its image", code == 200 and cached_a > head, f"cached {cached_a} of {ra.get('usage', {}).get('prompt_tokens')}; the text alone is {head}") + code, rb = ask(three_b, tokens=24) + cached_b = rb.get("usage", {}).get("prompt_tokens_details", {}).get("cached_tokens", 0) if code == 200 else -1 + check("another image under the same placeholders resumes only the text before it", code == 200 and cached_b < head, + f"cached {cached_b} of {rb.get('usage', {}).get('prompt_tokens')}") + if args.fresh_url: + check("and answers as a server with nothing kept", reply(rf) == reply(rb), repr(reply(rb)[:60])) + + # refusals, before any token + webp = b"RIFF\x24\x00\x00\x00WEBPVP8 \x18\x00\x00\x00" + bytes(24) + gif = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\xff\xff\xff\x00\x00\x00!\xf9\x04\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + refusals = [ + ("WebP, by name", [user(image_part(data_url(webp, "image/webp")), text("hi"))], "WebP"), + ("GIF", [user(image_part(data_url(gif, "image/gif")), text("hi"))], "not PNG or JPEG"), + ("an image in an assistant message", [user(text("hi")), {"role": "assistant", "content": [image_part(url_a)]}, user(text("ok"))], "user and tool messages only"), + ("too many images", [user(*[image_part(url_a)] * 5, text("hi"))], "at most"), + ("a remote URL", [user(image_part("https://example.com/cat.png"), text("hi"))], "data: URL"), + ("bad base64", [user(image_part("data:image/png;base64,@@@@"), text("hi"))], "encoding"), + ("a bad detail", [user(image_part(url_a, "medium"), text("hi"))], "detail"), + ("bytes that decode as nothing", [user(image_part(data_url(b"\x89PNG\r\n\x1a\n" + bytes(32), "image/png")), text("hi"))], "could not be decoded"), + ] + for label, messages, words in refusals: + code, r = ask(messages) + message = r.get("error", {}).get("message", json.dumps(r)) + check(f"refused {label}", code == 400 and words in message, f"HTTP {code} {message[:100]!r}") + if jpgs: + code, r = ask([user(image_part(data_url(jpgs[0].read_bytes(), "image/jpeg")), text("Describe it."))]) + check("a JPEG answers", code == 200, f"HTTP {code}") + code, r = ask([user(image_part(url_a, "low"), text("Describe it."))]) + check("detail low answers", code == 200, f"HTTP {code}") + print("PASS" if failures == 0 else f"FAIL: {failures}") + return 0 if failures == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/zig/kernels/metal/glm_vision.metal b/zig/kernels/metal/glm_vision.metal new file mode 100644 index 0000000000..c0fcc513e4 --- /dev/null +++ b/zig/kernels/metal/glm_vision.metal @@ -0,0 +1,229 @@ +// GLM-5.3-Flash's image tower (GLM5-Next vision: 24 pre-norm blocks of 1024, 16 heads of 64, 2-D rotary, a 2x2 merge +// and a SwiGLU merger to the language model's 4096). Compiled after prefill/gemm_nax.metal, whose tfp::gemm_nax runs +// every projection; the rest is MLX's per-op arithmetic as mlx-vlm's glm5_next/vision.py runs it in bf16. + +// ---- the projections and attention products: tfp::gemm_nax (bm64 bn128 bk256 wm2 wn4) by operand layout ---- +#define GLMV_GEMM(NAME, T, TA, TB, AM, AN) \ + [[max_total_threads_per_threadgroup(256)]] [[kernel]] void NAME( \ + const device T* A [[buffer(0)]], const device T* B [[buffer(1)]], const device int32_t* P [[buffer(2)]], \ + device T* D [[buffer(3)]], uint sg [[simdgroup_index_in_threadgroup]], \ + uint lane [[thread_index_in_simdgroup]], uint3 tg [[threadgroup_position_in_grid]]) { \ + tfp::gemm_nax(A, B, D, P, tg, sg, lane); \ + } +// x [M, K] w^T: every linear layer (bf16 in and out, fp32 sums) +GLMV_GEMM(glmv_gemm_bf16_nt_00, bfloat16_t, false, true, false, false) +GLMV_GEMM(glmv_gemm_bf16_nt_01, bfloat16_t, false, true, false, true) +GLMV_GEMM(glmv_gemm_bf16_nt_10, bfloat16_t, false, true, true, false) +GLMV_GEMM(glmv_gemm_bf16_nt_11, bfloat16_t, false, true, true, true) +// q k^T per head in fp32 (the scores MLX's attention keeps in fp32) +GLMV_GEMM(glmv_gemm_f32_nt_00, float, false, true, false, false) +GLMV_GEMM(glmv_gemm_f32_nt_01, float, false, true, false, true) +GLMV_GEMM(glmv_gemm_f32_nt_10, float, false, true, true, false) +GLMV_GEMM(glmv_gemm_f32_nt_11, float, false, true, true, true) +// p v per head in fp32 +GLMV_GEMM(glmv_gemm_f32_nn_00, float, false, false, false, false) +GLMV_GEMM(glmv_gemm_f32_nn_01, float, false, false, false, true) +GLMV_GEMM(glmv_gemm_f32_nn_10, float, false, false, true, false) +GLMV_GEMM(glmv_gemm_f32_nn_11, float, false, false, true, true) + +struct GlmvRows { + uint rows; + uint width; + float eps; +}; + +inline float glmv_bf(float x) { return float(bfloat(x)); } + +// One threadgroup's sum of every thread's `v` (at most 1024 threads), returned to all of them. +inline float glmv_group_sum(float v, threadgroup float* part, uint t, uint n, uint lane, uint sg) { + v = simd_sum(v); + if (lane == 0) part[sg] = v; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sg == 0) { + float s = lane < (n + 31) / 32 ? part[lane] : 0.0f; + s = simd_sum(s); + if (lane == 0) part[32] = s; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + const float out = part[32]; + threadgroup_barrier(mem_flags::mem_threadgroup); + return out; +} + +// y[r, c] = y + b[c] (a linear layer's bias, added after its product as nn.Linear's bf16 add rounds it) +[[kernel]] void glmv_bias(device bfloat* y [[buffer(0)]], const device bfloat* b [[buffer(1)]], + constant uint& width [[buffer(2)]], uint2 at [[thread_position_in_grid]]) { + const ulong i = ulong(at.y) * width + at.x; + y[i] = bfloat(float(y[i]) + float(b[at.x])); +} + +// x += y (a block's residual) +[[kernel]] void glmv_add(device bfloat* x [[buffer(0)]], const device bfloat* y [[buffer(1)]], + uint i [[thread_position_in_grid]]) { + x[i] = bfloat(float(x[i]) + float(y[i])); +} + +// mx.fast.rms_norm, one threadgroup a row: w * bf16(x * rsqrt(mean(x^2) + eps)) +[[kernel]] void glmv_rms(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]], + device bfloat* y [[buffer(2)]], constant GlmvRows& p [[buffer(3)]], + uint row [[threadgroup_position_in_grid]], uint t [[thread_position_in_threadgroup]], + uint n [[threads_per_threadgroup]], uint lane [[thread_index_in_simdgroup]], + uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float part[33]; + const device bfloat* xr = x + ulong(row) * p.width; + device bfloat* yr = y + ulong(row) * p.width; + float s = 0.0f; + for (uint i = t; i < p.width; i += n) { + const float v = float(xr[i]); + s += v * v; + } + s = glmv_group_sum(s, part, t, n, lane, sg); + const float inv = metal::precise::rsqrt(s / float(p.width) + p.eps); + for (uint i = t; i < p.width; i += n) yr[i] = bfloat(float(w[i]) * glmv_bf(float(xr[i]) * inv)); +} + +// mx.fast.layer_norm with weight and bias, one threadgroup a row: w * bf16((x - mean) * rsqrt(var + eps)) + b +[[kernel]] void glmv_layer_norm(const device bfloat* x [[buffer(0)]], const device bfloat* w [[buffer(1)]], + const device bfloat* b [[buffer(2)]], device bfloat* y [[buffer(3)]], + constant GlmvRows& p [[buffer(4)]], uint row [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], uint n [[threads_per_threadgroup]], + uint lane [[thread_index_in_simdgroup]], uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float part[33]; + const device bfloat* xr = x + ulong(row) * p.width; + device bfloat* yr = y + ulong(row) * p.width; + float s = 0.0f; + for (uint i = t; i < p.width; i += n) s += float(xr[i]); + const float mean = glmv_group_sum(s, part, t, n, lane, sg) / float(p.width); + float v = 0.0f; + for (uint i = t; i < p.width; i += n) { + const float d = float(xr[i]) - mean; + v += d * d; + } + const float inv = metal::precise::rsqrt(glmv_group_sum(v, part, t, n, lane, sg) / float(p.width) + p.eps); + for (uint i = t; i < p.width; i += n) { + yr[i] = bfloat(glmv_bf(float(w[i]) * glmv_bf((float(xr[i]) - mean) * inv)) + float(b[i])); + } +} + +struct GlmvQkv { + uint L; // rows (patches) of this image + float eps; // the q and k norms' +}; + +// One simdgroup per (row, head, q|k|v) of qkv [L, 3, 16, 64]: q and k RMS-normed, rotated by the row's 2-D angles +// (cos and sin [L, 64]: the height's 16 frequencies, then the width's, twice), and written head-major in fp32 +// ([16, L, 64], each value a bf16 as MLX hands its attention); v as stored. +[[kernel]] void glmv_qkv(const device bfloat* qkv [[buffer(0)]], const device bfloat* qn [[buffer(1)]], + const device bfloat* kn [[buffer(2)]], const device float* cs [[buffer(3)]], + const device float* sn [[buffer(4)]], device float* Q [[buffer(5)]], + device float* K [[buffer(6)]], device float* V [[buffer(7)]], + constant GlmvQkv& p [[buffer(8)]], uint g [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_simdgroup]]) { + const uint which = g % 3, h = (g / 3) % 16, r = g / 48; + const device bfloat* src = qkv + ulong(r) * 3072 + which * 1024 + h * 64; + const ulong out = (ulong(h) * p.L + r) * 64; + const float a = float(src[lane]), b = float(src[lane + 32]); + if (which == 2) { + V[out + lane] = a; + V[out + lane + 32] = b; + return; + } + const device bfloat* w = which == 0 ? qn : kn; + const float inv = metal::precise::rsqrt(simd_sum(a * a + b * b) / 64.0f + p.eps); + const float na = glmv_bf(float(w[lane]) * glmv_bf(a * inv)); + const float nb = glmv_bf(float(w[lane + 32]) * glmv_bf(b * inv)); + const device float* c = cs + ulong(r) * 64; + const device float* s = sn + ulong(r) * 64; + device float* dst = which == 0 ? Q : K; + dst[out + lane] = glmv_bf(na * c[lane] - nb * s[lane]); + dst[out + lane + 32] = glmv_bf(nb * c[lane + 32] + na * s[lane + 32]); +} + +// Each row of S [rows, n] (fp32 scores) as softmax(scale * s), in place; one threadgroup a row. +[[kernel]] void glmv_softmax(device float* S [[buffer(0)]], constant uint& n [[buffer(1)]], + constant float& scale [[buffer(2)]], uint row [[threadgroup_position_in_grid]], + uint t [[thread_position_in_threadgroup]], uint tn [[threads_per_threadgroup]], + uint lane [[thread_index_in_simdgroup]], uint sg [[simdgroup_index_in_threadgroup]]) { + threadgroup float part[33]; + device float* s = S + ulong(row) * n; + float m = -INFINITY; + for (uint i = t; i < n; i += tn) m = max(m, s[i] * scale); + m = simd_max(m); + if (lane == 0) part[sg] = m; + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sg == 0) { + float v = lane < (tn + 31) / 32 ? part[lane] : -INFINITY; + v = simd_max(v); + if (lane == 0) part[32] = v; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + m = part[32]; + threadgroup_barrier(mem_flags::mem_threadgroup); + float z = 0.0f; + for (uint i = t; i < n; i += tn) { + const float e = metal::precise::exp(s[i] * scale - m); + s[i] = e; + z += e; + } + const float inv = 1.0f / glmv_group_sum(z, part, t, tn, lane, sg); + for (uint i = t; i < n; i += tn) s[i] *= inv; +} + +// O [16, rows, 64] (fp32, one chunk's attention) into rows [r0, r0 + rows) of out [L, 1024] (bf16, heads side by side) +[[kernel]] void glmv_heads(const device float* O [[buffer(0)]], device bfloat* out [[buffer(1)]], + constant uint2& p [[buffer(2)]], uint2 at [[thread_position_in_grid]]) { + const uint rows = p.x, r0 = p.y, h = at.x / 64, d = at.x % 64, r = at.y; + out[ulong(r0 + r) * 1024 + at.x] = bfloat(O[(ulong(h) * rows + r) * 64 + d]); +} + +// The vision MLP's and merger's gate: silu(min(g, limit)) * clip(u, -limit, limit), each op rounded as MLX does. +// gu: g then u, `width` each, a row at a time (stride 2 * width); act [rows, width]. +[[kernel]] void glmv_swiglu(const device bfloat* g [[buffer(0)]], const device bfloat* u [[buffer(1)]], + device bfloat* act [[buffer(2)]], constant float& limit [[buffer(3)]], + uint i [[thread_position_in_grid]]) { + const float gate = glmv_bf(min(float(g[i]), limit)); + const float up = glmv_bf(clamp(float(u[i]), -limit, limit)); + const float sig = glmv_bf(1.0f / (1.0f + metal::precise::exp(-gate))); + act[i] = bfloat(glmv_bf(gate * sig) * up); +} + +// erf to fp32 precision (W. J. Cody's rational approximations, as libm's erff). +inline float glmv_erf(float x) { + const float ax = fabs(x); + if (ax < 0.84375f) { + const float z = x * x; + const float r = 1.28379167e-01f + z * (-3.25042108e-01f + z * (-2.84817495e-02f + z * (-5.77027029e-03f + z * -2.37630166e-05f))); + const float s = 1.0f + z * (3.97917233e-01f + z * (6.50222499e-02f + z * (5.08130628e-03f + z * (1.32494738e-04f + z * -3.96022827e-06f)))); + return x + x * (r / s); + } + if (ax < 1.25f) { + const float s = ax - 1.0f; + const float P = -2.36211857e-03f + s * (4.14856106e-01f + s * (-3.72207481e-01f + s * (3.18346620e-01f + s * (-1.10894694e-01f + s * (3.54783031e-02f + s * -2.16637559e-03f))))); + const float Q = 1.0f + s * (1.06420882e-01f + s * (5.40397942e-01f + s * (7.18286547e-02f + s * (1.26171216e-01f + s * (1.36370836e-02f + s * 1.19844998e-02f))))); + const float e = 8.45062911e-01f + P / Q; + return x >= 0 ? e : -e; + } + if (ax >= 6.0f) return x >= 0 ? 1.0f : -1.0f; + const float s = 1.0f / (ax * ax); + float R, S; + if (ax < 1.0f / 0.35f) { + R = -9.86494403e-03f + s * (-6.93858570e-01f + s * (-1.05586262e+01f + s * (-6.23753357e+01f + s * (-1.62396660e+02f + s * (-1.84605087e+02f + s * (-8.12874374e+01f + s * -9.81432934e+00f)))))); + S = 1.0f + s * (1.96512714e+01f + s * (1.37657754e+02f + s * (4.34565887e+02f + s * (6.45387268e+02f + s * (4.29008148e+02f + s * (1.08635002e+02f + s * (6.57024977e+00f + s * -6.04244141e-02f))))))); + } else { + R = -9.86494310e-03f + s * (-7.99283242e-01f + s * (-1.77579556e+01f + s * (-1.60636383e+02f + s * (-6.37566467e+02f + s * (-1.02509509e+03f + s * -4.83519196e+02f))))); + S = 1.0f + s * (3.03380604e+01f + s * (3.25792511e+02f + s * (1.53672961e+03f + s * (3.19985820e+03f + s * (2.55305029e+03f + s * (4.74528534e+02f + s * -2.24409515e+01f)))))); + } + const float zh = as_type(as_type(ax) & 0xfffff000u); + const float r = metal::precise::exp(-zh * zh - 0.5625f) * metal::precise::exp((zh - ax) * (zh + ax) + R / S); + const float e = 1.0f - r / ax; + return x >= 0 ? e : -e; +} + +// nn.gelu (exact): x * (1 + erf(x / sqrt(2))) / 2, each op rounded to bf16 as the compiled MLX graph does +[[kernel]] void glmv_gelu(device bfloat* x [[buffer(0)]], uint i [[thread_position_in_grid]]) { + const float v = float(x[i]); + const float t = glmv_bf(v / 1.41421356237309504880f); + const float e = glmv_bf(glmv_erf(t)); + const float w = glmv_bf(v * glmv_bf(1.0f + e)); + x[i] = bfloat(w / 2.0f); +} diff --git a/zig/kernels/metal/sources.zig b/zig/kernels/metal/sources.zig index d5fdc24b18..87da9a439a 100644 --- a/zig/kernels/metal/sources.zig +++ b/zig/kernels/metal/sources.zig @@ -71,6 +71,8 @@ pub const glm = @import("glm/kernels.zig"); pub const glm_glue = @embedFile("glm_glue.metal"); /// GLM-5.3-Flash's latent attention for rows that read every key: 64 heads as one matrix on the tensor units. pub const glm_attn = @embedFile("glm_attn.metal"); +/// GLM-5.3-Flash's image tower: the prefill GEMM template's projections and attention products, then its own steps. +pub const glm_vision = @embedFile("prefill/gemm_nax.metal") ++ @embedFile("glm_vision.metal"); /// A prompt chunk's KDA layer in three passes, appended to the generated kda_rows source (its helpers). pub const glm_kda_prompt = @embedFile("glm_kda_prompt.metal"); /// GLM-5.3-Flash speed-up mode's exchange kernels (families/glm/ep.zig). diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index f72c13b227..cbacca9162 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -9,6 +9,9 @@ pub const Id = u64; /// One request's sampling (Python's exact_sampling.Sampling); a request without one decodes greedily. pub const Sampling = lanes.Sampling; +/// One image of a request (``Request.images``): the tower's input and the placeholder rows its output replaces. +pub const Image = lanes.stream.Image; + /// A prompt's reproducible seed (``seed_for``), for requests that name none. pub const seedFor = lanes.sampling.seedFor; @@ -82,8 +85,43 @@ pub const Request = struct { decide: ?u32 = null, /// Logprob rows for each reply token with this many best tokens (0..20), when ``Info.logprobs``; null: none. logprobs: ?u8 = null, + /// Images whose rows replace placeholder tokens of ``prompt``, in order; only an engine whose ``Info.vision`` is set. + images: []const Image = &.{}, + /// The prompt as the prompt cache matches it (each image's placeholders by its content); empty: ``prompt``. + cache_key: []const u32 = &.{}, + + /// The prompt as the prompt cache matches it: an image request's placeholders stand for their images. + pub fn cacheKey(r: *const Request) []const u32 { + return if (r.cache_key.len > 0) r.cache_key else r.prompt; + } + + /// The decoded prompt rows, for a backend that keeps their bits (``decoded_rows``); otherwise none. + pub fn cacheSpans(r: *const Request, decoded_rows: bool) []const [2]u32 { + return if (decoded_rows) r.decode_spans else &.{}; + } }; +/// An image prepared for the engine's tower (``Vision.prepare``): patches, patch grid, tokens and a content hash. +pub const PreparedImage = struct { pixels: []f32, gh: u32, gw: u32, tokens: u32, hash: u64 }; + +/// Image input an engine serves (``Info.vision``): how the server turns an image's bytes into tower input and what +/// a request may hold. +pub const Vision = struct { + ctx: *anyopaque, + /// The image's bytes (any format the platform decodes) at most ``max_tokens`` tokens; allocated in ``a``. + prepare: *const fn (ctx: *anyopaque, a: Allocator, bytes: []const u8, max_tokens: u32) anyerror!PreparedImage, + /// The token the chat template writes once per image, expanded to the image's tokens. + image_token: u32, + /// The tokens the template wraps an image in (the image token among them): text that spells one stays text. + markers: []const u32 = &.{}, + /// Image tokens one request may hold in all, and images. + image_tokens: u32, + max_images: u32, +}; + +/// --vision: image input's limits (``Open.vision``). +pub const VisionOpen = struct { image_tokens: u32 = 4096, max_images: u32 = 4 }; + pub const Reason = enum { stop, length, cancelled, failed }; /// What a finished reply's rounds did, for its runtime fields and /metrics. @@ -148,6 +186,8 @@ pub const Info = struct { plain_only: bool = false, /// The immutable retained-prefix plan applied at startup, not a process memory limit or current cache occupancy. prompt_cache_plan: ?PromptCachePlan = null, + /// Image input (--vision); null: text only. + vision: ?Vision = null, /// The engine decodes greedily only: the server refuses a request with a temperature before admitting it. greedy_only: bool = false, /// Whether a request's caches fit beside the running streams' now (null: a free lane does); else it waits in order. @@ -213,6 +253,8 @@ pub const Open = struct { master_port: u16 = 29551, /// --policy: the engine's policy as `key=value,...`; empty: its defaults. policy: []const u8 = "", + /// --vision: load the checkpoint's image tower (null: text only; an engine without one refuses). + vision: ?VisionOpen = null, }; /// An opened engine; ``close`` stops its thread and frees its backend. diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index 09ff0fa2de..e8d5c5f8a0 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -71,7 +71,7 @@ pub const LaneHost = struct { fn kept(ptr: *anyopaque, s: *lanes.Stream, at: u32) void { const job: *Job = @ptrCast(@alignCast(ptr)); job.reported(); // before a keep can evict the entry the pass restored - if (job.host.cache) |store| _ = store.keep(job.request.prompt, at, s, job.request.chunks, job.host.spans(job.request)); + if (job.host.cache) |store| _ = store.keep(job.request.cacheKey(), at, s, job.request.chunks, job.request.cacheSpans(job.host.decoded_rows)); } /// Report a restored prefix or its failed copy; an untouched prefix remains kept. @@ -81,7 +81,7 @@ pub const LaneHost = struct { const store = job.host.cache orelse return; if (!job.started) return; job.stream.reuse.saved = null; // the backend restores before its first chunk; a later keep may free it - if (job.stream.reuse_failed) store.resumed(e, job.request.prompt, false) else if (job.stream.cached == e.at) store.resumed(e, job.request.prompt, true); + if (job.stream.reuse_failed) store.resumed(e, job.request.cacheKey(), false) else if (job.stream.cached == e.at) store.resumed(e, job.request.cacheKey(), true); } }; @@ -291,7 +291,7 @@ pub const LaneHost = struct { const r = job.request; var reuse: lanes.stream.Reuse = .{}; // the entry stays alive until the backend restores it: nothing keeps between here and this stream's own pass - if (h.cache) |store| if (store.lookupRewind(h.gpa, r.prompt, r.history_len, r.rewind_len, r.shared_prefixes, r.chunks, h.spans(r))) |l| { + if (h.cache) |store| if (store.lookupRewind(h.gpa, r.cacheKey(), r.history_len, r.rewind_len, r.shared_prefixes, r.chunks, r.cacheSpans(h.decoded_rows))) |l| { job.entry = l.entry; job.kept0 = store.counts.kept; job.marks = l.marks; @@ -322,8 +322,9 @@ pub const LaneHost = struct { job.proposer.deinit(); return h.drop(job, if (e == error.OutOfMemory) "out of memory" else "the reply's grammar could not start (xgrammar)"); } else null, - .decode_spans = h.spans(r), + .decode_spans = r.cacheSpans(h.decoded_rows), .decide = r.decide, + .images = r.images, }) catch { job.proposer.deinit(); return h.drop(job, "out of memory"); @@ -395,11 +396,6 @@ pub const LaneHost = struct { emit(job, .{ .prefilled = job.stream.cached }); } - /// A request's decoded prompt rows, for a backend that keeps their bits (`decoded_rows`); otherwise none. - fn spans(h: *const LaneHost, r: *const Request) []const [2]u32 { - return if (h.decoded_rows) r.decode_spans else &.{}; - } - /// A prompt-only pass on a `warm_turns` engine: the backend prefills and the cache keeps where it ends. fn warmPass(h: *LaneHost, job: *Job, began: i96) bool { const be = h.core.backend; @@ -409,7 +405,7 @@ pub const LaneHost = struct { return h.cancel(job); }; h.prefilled(job, began); - if (h.cache) |store| _ = store.keep(job.request.prompt, @intCast(job.request.prompt.len), &job.stream, job.request.chunks, h.spans(job.request)); + if (h.cache) |store| _ = store.keep(job.request.prompt, @intCast(job.request.prompt.len), &job.stream, job.request.chunks, job.request.cacheSpans(h.decoded_rows)); be.release(&job.stream); h.remove(job); h.finish(job, .length, ""); diff --git a/zig/src/core/lane_host_reuse_test.zig b/zig/src/core/lane_host_reuse_test.zig index 549121dd83..c3a1286c21 100644 --- a/zig/src/core/lane_host_reuse_test.zig +++ b/zig/src/core/lane_host_reuse_test.zig @@ -97,6 +97,94 @@ test "a lane host resumes a conversation from its kept prompt state and reports try std.testing.expectEqual(@as(usize, 2), store.entries.items.len); } +test "an image's placeholders are cached by its content: another image under the same ids resumes none of it" { + const gpa = std.testing.allocator; + var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7); + defer cfg.deinit(gpa); + var target: lanes.fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: lanes.fake.FixedClock = .{}; + var core = lanes.Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer core.deinit(); + const Snaps = struct { + fn bytes(_: *anyopaque, at: u32) u64 { + return 64 + 4 * @as(u64, at); + } + fn save(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!pc.Saved { + const f: *lanes.fake.Fake = @ptrCast(@alignCast(ptr)); + return f.save(@ptrCast(@alignCast(owner.?)), at); + } + fn restore(_: *anyopaque, _: ?*anyopaque, _: pc.Saved) anyerror!void { + return error.BackendRestores; // lane-core backends restore inside their own prompt pass + } + fn drop(ptr: *anyopaque, saved: pc.Saved) void { + const f: *lanes.fake.Fake = @ptrCast(@alignCast(ptr)); + f.drop(saved); + } + }; + var store = pc.Store.init(gpa, .{ .ptr = &target, .vtable = &.{ .bytes = Snaps.bytes, .save = Snaps.save, .restore = Snaps.restore, .drop = Snaps.drop } }, .{ .min_gap = 4, .min_prompt = 0 }, 1 << 20); + defer store.deinit(); + var host = LaneHost.init(gpa, std.testing.io, &core, .{ .lanes = 2 }); + host.cache = &store; + try host.start(); + defer host.stop(); + const Box = struct { + mutex: std.Io.Mutex = .init, + tokens: std.ArrayList(u32) = .empty, + cached: ?u32 = null, + done: ?Reason = null, + fn event(ctx: *anyopaque, _: Id, e: *const Event) void { + const b: *@This() = @ptrCast(@alignCast(ctx)); + b.mutex.lockUncancelable(std.testing.io); + defer b.mutex.unlock(std.testing.io); + switch (e.*) { + .prefilled => |c| b.cached = c, + .logprobs => {}, + .tokens => |t| b.tokens.appendSlice(gpa, t) catch {}, + .finished => |f| b.done = f.reason, + } + } + fn wait(b: *@This()) Reason { + while (true) { + b.mutex.lockUncancelable(std.testing.io); + const d = b.done; + b.mutex.unlock(std.testing.io); + if (d) |r| return r; + std.Io.sleep(std.testing.io, .fromMilliseconds(1), .awake) catch {}; + } + } + }; + const e = host.engine(); + // prompt ids 3..5 are one image's placeholders (token 9); cache_key holds ids derived from each image's content + const t1 = [_]u32{ 3, 1, 9, 9, 9, 9, 2, 6, 5, 3 }; + var key_a = t1; + var key_b = t1; + for (2..6) |k| { + key_a[k] = 0x8000_0000 + @as(u32, @intCast(k)); + key_b[k] = 0x9000_0000 + @as(u32, @intCast(k)); + } + var b1: Box = .{}; + defer b1.tokens.deinit(gpa); + const r1: Request = .{ .prompt = &t1, .max_tokens = 6, .history_len = 8, .cache_key = &key_a }; + try e.submit(1, &r1, .{ .ctx = &b1, .event = Box.event }); + try std.testing.expectEqual(Reason.length, b1.wait()); + try std.testing.expectEqual(@as(usize, 1), store.entries.items.len); // image A's state at 8, past its image + // the same text and placeholder ids with image B: the kept state is A's, so B resumes none of it + var b2: Box = .{}; + defer b2.tokens.deinit(gpa); + const r2: Request = .{ .prompt = &t1, .max_tokens = 6, .history_len = 8, .cache_key = &key_b }; + try e.submit(2, &r2, .{ .ctx = &b2, .event = Box.event }); + try std.testing.expectEqual(Reason.length, b2.wait()); + try std.testing.expectEqual(@as(?u32, 0), b2.cached); + // image A again resumes its own state + var b3: Box = .{}; + defer b3.tokens.deinit(gpa); + const r3: Request = .{ .prompt = &t1, .max_tokens = 6, .history_len = 8, .cache_key = &key_a }; + try e.submit(3, &r3, .{ .ctx = &b3, .event = Box.event }); + try std.testing.expectEqual(Reason.length, b3.wait()); + try std.testing.expectEqual(@as(?u32, 8), b3.cached); +} + test "a turn's mark evicting the state it resumed from: the hit is reported first and the entry freed once" { const gpa = std.testing.allocator; var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7); diff --git a/zig/src/core/lanes/stream.zig b/zig/src/core/lanes/stream.zig index d43d01b103..7096e1d517 100644 --- a/zig/src/core/lanes/stream.zig +++ b/zig/src/core/lanes/stream.zig @@ -97,6 +97,10 @@ pub const Held = struct { parents: ?[]i32 = null, }; +/// One image of a prompt: the tower's input (patches [gh * gw, width], the family's layout) and the `tokens` +/// placeholder rows from `at` its output replaces. +pub const Image = struct { pixels: []const f32, gh: u32, gw: u32, at: u32, tokens: u32 }; + pub const Spec = struct { id: []const u8, prompt: []const u32, @@ -121,6 +125,7 @@ pub const Spec = struct { logprobs: ?u8 = null, // the target's log probabilities for each committed token, with this many best tokens grammar: ?*grammar.Constraint = null, // structured output: the reply stays in it (the creator frees it) decode_spans: []const [2]u32 = &.{}, // prompt rows an earlier reply decoded (Request.decode_spans), for backends that keep their bits + images: []const Image = &.{}, // in prompt order; the backend's tower fills their rows }; pub const Stream = struct { @@ -150,6 +155,7 @@ pub const Stream = struct { reuse: Reuse = .{}, decode_spans: []const [2]u32 = &.{}, decide: ?u32 = null, + images: []const Image = &.{}, cached: u32 = 0, // prompt tokens the backend restored from `reuse` (its prompt pass started there) reuse_failed: bool = false, // the backend's restore of `reuse` failed: it prefilled from 0 context: std.ArrayList(u32) = .empty, @@ -209,6 +215,7 @@ pub const Stream = struct { .logprobs = spec.logprobs, .grammar = spec.grammar, .decode_spans = spec.decode_spans, + .images = spec.images, }; // under a grammar a forced close ends at the think end: the grammar takes the reply from the next token if (spec.grammar) |g| if (g.after) |end| if (std.mem.indexOfScalar(u32, s.think_close, end)) |at| { diff --git a/zig/src/core/tokenizer/tokenizer.zig b/zig/src/core/tokenizer/tokenizer.zig index bcb2f5ee8f..69456001ad 100644 --- a/zig/src/core/tokenizer/tokenizer.zig +++ b/zig/src/core/tokenizer/tokenizer.zig @@ -130,10 +130,32 @@ pub const Tokenizer = struct { return a.dupe(u32, ids.items); } + /// `encode` with no added token matched: text that spells one (a quoted marker) is ordinary text. + pub fn encodePlain(t: *const Tokenizer, a: Allocator, text: []const u8) Error![]u32 { + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + var ctx = steps.Context{ .a = arena.allocator(), .matcher = .{ .a = arena.allocator() } }; + var valid = text; + if (!std.unicode.utf8ValidateSlice(text)) { + var fixed: std.ArrayList(u8) = .empty; + try unicode.appendLossy(&fixed, ctx.a, text); + valid = fixed.items; + } + var ids: std.ArrayList(u32) = .empty; + if (valid.len > 0) try t.encodeTextWith(&ctx, valid, true, &ids, false); + return a.dupe(u32, ids.items); + } + fn encodeText(t: *const Tokenizer, ctx: *steps.Context, text: []const u8, first: bool, ids: *std.ArrayList(u32)) Error!void { + return t.encodeTextWith(ctx, text, first, ids, true); + } + + fn encodeTextWith(t: *const Tokenizer, ctx: *steps.Context, text: []const u8, first: bool, ids: *std.ArrayList(u32), added: bool) Error!void { var zero: usize = if (first) unicode.decodeAt(text, 0).len else 0; const normalized = if (t.normalizer) |*n| try n.apply(ctx, text, &zero) else text; - for (try t.splitAdded(ctx, &t.normalized, normalized)) |seg| { + const whole = [_]Segment{.{ .start = 0, .end = normalized.len, .id = null }}; + const segs: []const Segment = if (added) try t.splitAdded(ctx, &t.normalized, normalized) else &whole; + for (segs) |seg| { if (seg.id) |id| { try ids.append(ctx.a, id); continue; diff --git a/zig/src/families/glm/backend.zig b/zig/src/families/glm/backend.zig index d98f41932d..205c367bef 100644 --- a/zig/src/families/glm/backend.zig +++ b/zig/src/families/glm/backend.zig @@ -3,6 +3,7 @@ const std = @import("std"); const lanes = @import("lanes"); const st = @import("state.zig"); const slots_mod = @import("slots.zig"); +const slot_images = @import("slot_images.zig"); const pool = @import("pool.zig"); const mirror = @import("mirror.zig"); const snapshot = @import("snapshot.zig"); @@ -101,6 +102,11 @@ pub const Backend = struct { try mirror.send(e, .begin, b.words.items); try b.sl.begin(i, prompt, s.drafts); s.cached = 0; + if (s.images.len > 0) { // the images through the tower first: the chunks inject their rows + if (e.ep != null) return error.VisionOnPair; + try slot_images.encode(b.sl, i, s.images); + } + defer if (s.images.len > 0) slot_images.end(b.sl); var at: u32 = 0; if (s.reuse.saved) |saved| { // a kept state of this prompt's prefix: the pass starts there const snap: *snapshot.Snap = @ptrCast(@alignCast(saved)); diff --git a/zig/src/families/glm/checks.zig b/zig/src/families/glm/checks.zig index 4b521379c1..78ec7a274b 100644 --- a/zig/src/families/glm/checks.zig +++ b/zig/src/families/glm/checks.zig @@ -167,7 +167,7 @@ pub fn checkMatmul(e: *Engine, prompt: []const u32) !void { const out = try e.arena.buffer(@as(usize, rows) * N * 2); var x = e.ctx(); const b = e.begin(); - fwd.embed(&x, b.enc, e.prompt_ids, rows); + fwd.embed(&x, b.enc, e.prompt_ids, rows, 0); fwd.boundary(&x, b.enc, rows, false, L.hc.?[0], L.in_norm); fwd.qmv(&x, b.enc, e.k.qmv_kda_in, e.sc.normed, a.in_proj, out, rows); affine_mm.rowSums(b.enc, e.k.mm_bf16, 64, e.sc.normed, pr.sums, rows, a.in_proj.k); diff --git a/zig/src/families/glm/engine.zig b/zig/src/families/glm/engine.zig index 5affbae1f3..c37c166c98 100644 --- a/zig/src/families/glm/engine.zig +++ b/zig/src/families/glm/engine.zig @@ -4,6 +4,7 @@ const mtl = @import("metal"); const cfg = @import("config.zig"); const wts = @import("weights.zig"); const page_cache = @import("page_cache.zig"); +const vision_mod = @import("vision.zig"); const st = @import("state.zig"); const fwd = @import("forward.zig"); const mtp = @import("mtp.zig"); @@ -83,6 +84,9 @@ pub const Engine = struct { rank_log: bool = false, // GLM_RANKS=1: each MTP depth's logits kept, and the target's rank in them where drafts miss ep_arena: std.heap.ArenaAllocator, // the link settings, alive as long as the link residency: ?mtl.ResidencySet = null, + vision: ?*vision_mod.Vision = null, // --vision: the image tower + vision_rows: Ref = undefined, // bf16 [image tokens, hidden]: a prompt's image rows (vision only) + vision_limits: vision_mod.Limits = .{}, keepalive_sets: [1]mtl.ResidencySet = undefined, // the residency set, for the idle keepalive's commit keepalive_target: mtl.keepalive.Target = undefined, // the engine's queue, set at load for the server's ticker load_seconds: f64 = 0, @@ -94,11 +98,12 @@ pub const Engine = struct { /// The checkpoint in `dir`, caches for `cap` tokens; GLM_LAYERS=N: the first N layers only; GLM_EP=settings: half the experts. pub fn load(gpa: std.mem.Allocator, dir: []const u8, cap: u32) !*Engine { - return loadWith(gpa, dir, cap, if (std.c.getenv("GLM_EP")) |v| std.mem.span(v) else null, false); + return loadWith(gpa, dir, cap, if (std.c.getenv("GLM_EP")) |v| std.mem.span(v) else null, false, null); } /// `load` with expert parallel over the link in `ep_path` (this Mac's settings), or on one Mac when null. - pub fn loadWith(gpa: std.mem.Allocator, dir: []const u8, cap: u32, ep_path: ?[]const u8, learn: bool) !*Engine { + /// `vision`: the image tower too, with scratch and rows for that many image tokens a prompt. + pub fn loadWith(gpa: std.mem.Allocator, dir: []const u8, cap: u32, ep_path: ?[]const u8, learn: bool, vision: ?vision_mod.Limits) !*Engine { const e = try gpa.create(Engine); // undefined memory: every field is set below errdefer gpa.destroy(e); const pool = mtl.objc.Pool.push(); @@ -107,6 +112,7 @@ pub const Engine = struct { e.gpa = gpa; e.ev = 0; e.residency = null; + e.vision = null; e.gpu = .{ 0, 0 }; e.fused_route = if (std.c.getenv("GLM_ROUTE")) |v| v[0] != '0' else true; e.draft_vocab = if (std.c.getenv("GLM_DRAFT_VOCAB")) |v| std.fmt.parseInt(u32, std.mem.span(v), 10) catch 0 else 0; @@ -212,10 +218,13 @@ pub const Engine = struct { me.rest[2] = @intFromBool(learn); // --learn on both Macs or neither: each holds its half of a learned state e.ep = try ep_mod.Ep.init(gpa, e.device, s, me); } + if (vision) |lim| try vision_mod.attach(e, gpa, dir, f.bytes[0..f.size], lim); // --vision: the tower and its rows + errdefer if (e.vision) |v| v.deinit(); // opt-in: wiring 181 GB leaves macOS nothing to reclaim if another model shares the Mac (Flash Next runs without) - if (std.c.getenv("GLM_RESIDENCY") == null) {} else if (e.device.residencySet(e.w.buffers.items.len + e.arena.buffers.items.len)) |set| { + if (std.c.getenv("GLM_RESIDENCY") == null) {} else if (e.device.residencySet(e.w.buffers.items.len + e.arena.buffers.items.len + if (e.vision) |v| v.buffers().len else 0)) |set| { for (e.w.buffers.items) |b| set.add(b); for (e.arena.buffers.items) |b| set.add(b); + if (e.vision) |v| for (v.buffers()) |b| set.add(b); set.commit(); set.requestResidency(); e.queue.addResidencySet(set); @@ -265,6 +274,7 @@ pub const Engine = struct { set.deinit(); } if (e.ep) |ep| ep.deinit(gpa); + if (e.vision) |v| v.deinit(); e.ep_arena.deinit(); if (e.pr) |*p| p.deinit(); if (e.round_fence) |f| f.deinit(); diff --git a/zig/src/families/glm/forward.zig b/zig/src/families/glm/forward.zig index 018b99d3d3..84748ba88b 100644 --- a/zig/src/families/glm/forward.zig +++ b/zig/src/families/glm/forward.zig @@ -30,8 +30,29 @@ pub const Ctx = struct { draft_vocab: u32 = 154880, // the MTP head's tokens: the vocabulary's first this many m_row: u32 = 0, // the MTP head's logits row in m_logits (a rank log keeps each depth's in its own row) segs: []const Seg = &.{}, // a shared window's streams in row order (empty: every row is `s`'s) + inject: ?*const Inject = null, // a prompt pass's image rows over its placeholder embeddings }; +/// An image prompt's tower output: prompt rows [at, at + n) of each span embed as rows [row, row + n) of `rows`. +pub const Inject = struct { rows: Ref, spans: []const Span }; +pub const Span = struct { at: u32, n: u32, row: u32 }; + +/// The image rows among `rows` embedding rows (prompt positions pos..) written over `out`; nothing without an injection. +pub fn inject(x: *const Ctx, e: mtl.ComputeEncoder, out: Ref, rows: u32, pos: u32) void { + const inj = x.inject orelse return; + const D: usize = x.c.hidden; + for (inj.spans) |sp| { + const lo = @max(sp.at, pos); + const hi = @min(sp.at + sp.n, pos + rows); + if (lo >= hi) continue; + const n: usize = hi - lo; + e.setPipeline(x.k.copy_u32); + bind(e, 0, .{ inj.rows.at((sp.row + lo - sp.at) * D * 2), out.at((lo - pos) * D * 2) }); + e.setValue(@as(u32, @intCast(n * D / 2)), 2); + e.dispatchThreads(size(n * D / 2, 1, 1), size(256, 1, 1)); + } +} + /// One stream's rows in a shared window: its caches, its first row, its rows and the position of the first. pub const Seg = struct { s: *st.State, row0: u32, rows: u32, pos: u32 }; @@ -135,11 +156,12 @@ pub fn scale(x: *const Ctx, e: mtl.ComputeEncoder, in: Ref, out: Ref, rows: u32, e.dispatchThreads(size(width, rows, 1), size(@min(width, 256), 1, 1)); } -/// The window's tokens (`ids`, u32) as embedding rows in all four streams. -pub fn embed(x: *Ctx, e: mtl.ComputeEncoder, ids: Ref, rows: u32) void { +/// The window's tokens (`ids`, u32, at positions pos..) as embedding rows in all four streams, image rows injected. +pub fn embed(x: *Ctx, e: mtl.ComputeEncoder, ids: Ref, rows: u32, pos: u32) void { const sc = x.sc; const D = x.c.hidden; embedRows(x, e, ids, sc.h, rows); + inject(x, e, sc.h, rows, pos); e.setPipeline(x.k.streams); bind(e, 0, .{ sc.h, sc.x[0] }); e.setValue([2]u32{ D, rows }, 2); @@ -415,7 +437,7 @@ pub fn backbone(x: *Ctx, e: mtl.ComputeEncoder, ids: Ref, rows: u32, pos: u32) v const c = x.c; const sc = x.sc; const k = x.skip; - if (k & Class.ends == 0) embed(x, e, ids, rows); + if (k & Class.ends == 0) embed(x, e, ids, rows, pos); const plane = @as(usize, rows) * c.hidden * 2; snap(x, e, sc.h, plane); var pending = false; diff --git a/zig/src/families/glm/glm.zig b/zig/src/families/glm/glm.zig index 34bf628cc9..5b0ffe7425 100644 --- a/zig/src/families/glm/glm.zig +++ b/zig/src/families/glm/glm.zig @@ -13,6 +13,8 @@ pub const pool = @import("pool.zig"); pub const backend = @import("backend.zig"); pub const mirror = @import("mirror.zig"); pub const timing = @import("timing.zig"); +pub const vision = @import("vision.zig"); +pub const image = @import("image.zig"); test { _ = config; @@ -30,4 +32,6 @@ test { _ = pool; _ = mirror; _ = timing; + _ = vision; + _ = image; } diff --git a/zig/src/families/glm/image.zig b/zig/src/families/glm/image.zig new file mode 100644 index 0000000000..4da98c4f4e --- /dev/null +++ b/zig/src/families/glm/image.zig @@ -0,0 +1,453 @@ +//! An image's bytes to GLM5-Next vision patches as TensorFold 0.6.6's --vision makes them: decoded as its +//! vision/images.py does with PIL (ImageIO here: the stored values, no colour management; EXIF orientation applied; +//! alpha composited over white with PIL's integer blend), then mlx-vlm's Glm5NextImageProcessor: sized under the +//! token budget, resized with PIL's 8-bit bicubic resampler, padded with black, normalized, and cut into 2x2 windows +//! of 14x14 patches, each patch's frame given twice (temporal_patch_size 2). +const std = @import("std"); +const vision = @import("vision.zig"); + +pub const Rgb = struct { + w: u32, + h: u32, + px: []u8, // row-major RGB, 3 bytes a pixel + + pub fn deinit(r: Rgb, gpa: std.mem.Allocator) void { + gpa.free(r.px); + } +}; + +/// One image ready for the tower: its patches, its patch grid and its tokens (gh * gw / 4). +pub const Prepared = struct { + pixels: []f32, // [gh * gw, 1176] + gh: u32, + gw: u32, + tokens: u32, + hash: u64, // of the image's bytes: two prompts with the same placeholders but other images are other prompts +}; + +pub const Error = error{ BadImage, ImageTooLarge, Animated, ImageTooSmall, OutOfMemory }; + +/// images.py's ImageLimits: an image's encoded bytes, sides and pixels. +pub const max_bytes = 10 << 20; +pub const max_side = 8192; +pub const max_pixels = 16 << 20; + +// ---- ImageIO and CoreGraphics (C APIs; the server binary links both frameworks) ---- +const CF = *anyopaque; +const CGRect = extern struct { x: f64, y: f64, w: f64, h: f64 }; +extern "c" fn CFDataCreate(alloc: ?CF, bytes: [*]const u8, len: isize) ?CF; +extern "c" fn CFDataGetBytePtr(data: CF) [*]const u8; +extern "c" fn CFDataGetLength(data: CF) isize; +extern "c" fn CFRelease(cf: CF) void; +extern "c" fn CGImageSourceCreateWithData(data: CF, options: ?CF) ?CF; +extern "c" fn CGImageSourceGetCount(src: CF) usize; +extern "c" fn CGImageSourceCreateImageAtIndex(src: CF, index: usize, options: ?CF) ?CF; +extern "c" fn CGImageGetWidth(img: CF) usize; +extern "c" fn CGImageGetHeight(img: CF) usize; +extern "c" fn CGImageGetBitsPerComponent(img: CF) usize; +extern "c" fn CGImageGetBitsPerPixel(img: CF) usize; +extern "c" fn CGImageGetBytesPerRow(img: CF) usize; +extern "c" fn CGImageGetBitmapInfo(img: CF) u32; +extern "c" fn CGImageGetColorSpace(img: CF) ?CF; +extern "c" fn CGColorSpaceGetModel(cs: CF) i32; +extern "c" fn CGImageGetDataProvider(img: CF) ?CF; +extern "c" fn CGDataProviderCopyData(p: CF) ?CF; +extern "c" fn CGColorSpaceCreateDeviceRGB() ?CF; +extern "c" fn CGColorSpaceRelease(cs: CF) void; +extern "c" fn CGBitmapContextCreate(data: ?*anyopaque, w: usize, h: usize, bpc: usize, bpr: usize, cs: CF, info: u32) ?CF; +extern "c" fn CGContextDrawImage(ctx: CF, rect: CGRect, img: CF) void; +extern "c" fn CGContextRelease(ctx: CF) void; +extern "c" fn CGContextSetRGBFillColor(ctx: CF, r: f64, g: f64, b: f64, a: f64) void; +extern "c" fn CGContextFillRect(ctx: CF, rect: CGRect) void; +extern "c" fn CGImageSourceCopyPropertiesAtIndex(src: CF, index: usize, options: ?CF) ?CF; +extern "c" fn CFDictionaryGetValue(dict: CF, key: ?*const anyopaque) ?CF; +extern "c" fn CFNumberGetValue(num: CF, kind: isize, out: *anyopaque) bool; +extern var kCGImagePropertyOrientation: ?*const anyopaque; + +const alpha_mask: u32 = 0x1f; +const order_mask: u32 = 0x7000; +const order_32_little: u32 = 0x2000; +const order_16_little: u32 = 0x1000; +const float_components: u32 = 0x100; + +/// A PNG, JPEG, GIF, WebP, HEIC or any other single-frame image ImageIO reads, as 8-bit RGB: upright by its EXIF +/// orientation (PIL's exif_transpose) and over white where it has alpha (images.py's paste onto a white canvas). +pub fn decode(gpa: std.mem.Allocator, bytes: []const u8) Error!Rgb { + if (bytes.len == 0) return error.BadImage; + if (bytes.len > max_bytes) return error.ImageTooLarge; + const data = CFDataCreate(null, bytes.ptr, @intCast(bytes.len)) orelse return error.OutOfMemory; + defer CFRelease(data); + const src = CGImageSourceCreateWithData(data, null) orelse return error.BadImage; + defer CFRelease(src); + const frames = CGImageSourceGetCount(src); + if (frames == 0) return error.BadImage; + if (frames > 1) return error.Animated; // images.py: animated and multipage images are refused + const img = CGImageSourceCreateImageAtIndex(src, 0, null) orelse return error.BadImage; + defer CFRelease(img); + const w = CGImageGetWidth(img); + const h = CGImageGetHeight(img); + if (w == 0 or h == 0) return error.BadImage; + if (w > max_side or h > max_side or w * h > max_pixels) return error.ImageTooLarge; + const out = try gpa.alloc(u8, w * h * 3); + errdefer gpa.free(out); + if (!(try raw(img, w, h, out))) try drawn(img, w, h, out); + const rgb: Rgb = .{ .w = @intCast(w), .h = @intCast(h), .px = out }; + const o = orientation(src); + if (o <= 1 or o > 8) return rgb; + defer rgb.deinit(gpa); + return orient(gpa, rgb, o); +} + +/// The first image's EXIF orientation (1-8; 1 or 0: as stored). +fn orientation(src: CF) u32 { + const props = CGImageSourceCopyPropertiesAtIndex(src, 0, null) orelse return 1; + defer CFRelease(props); + const num = CFDictionaryGetValue(props, kCGImagePropertyOrientation) orelse return 1; + var v: i32 = 1; + if (!CFNumberGetValue(num, 9, &v)) return 1; // kCFNumberIntType + return if (v < 1) 1 else @intCast(v); +} + +/// PIL's exif_transpose: orientation 2-8 as FLIP_LEFT_RIGHT, ROTATE_180, FLIP_TOP_BOTTOM, TRANSPOSE, ROTATE_270, +/// TRANSVERSE, ROTATE_90; each output pixel read from its source pixel. +pub fn orient(gpa: std.mem.Allocator, src: Rgb, o: u32) Error!Rgb { + const w: usize = src.w; + const h: usize = src.h; + const swap = o >= 5; + const ow = if (swap) h else w; + const oh = if (swap) w else h; + const out = try gpa.alloc(u8, ow * oh * 3); + for (0..oh) |y| for (0..ow) |x| { + const s: [2]usize = switch (o) { + 2 => .{ w - 1 - x, y }, + 3 => .{ w - 1 - x, h - 1 - y }, + 4 => .{ x, h - 1 - y }, + 5 => .{ y, x }, + 6 => .{ y, h - 1 - x }, + 7 => .{ w - 1 - y, h - 1 - x }, + else => .{ w - 1 - y, x }, + }; + @memcpy(out[(y * ow + x) * 3 ..][0..3], src.px[(s[1] * w + s[0]) * 3 ..][0..3]); + }; + return .{ .w = @intCast(ow), .h = @intCast(oh), .px = out }; +} + +/// PIL's paste of a pixel with alpha `a` onto white: DIV255(255 * (255 - a) + c * a). +fn overWhite(c: u8, a: u8) u8 { + const t: u32 = 255 * (255 - @as(u32, a)) + @as(u32, c) * a + 128; + return @intCast(((t >> 8) + t) >> 8); +} + +/// The decoder's own values for 8- and 16-bit gray and RGB layouts (what PIL reads); false for any other layout. +fn raw(img: CF, w: usize, h: usize, out: []u8) Error!bool { + const info = CGImageGetBitmapInfo(img); + if (info & float_components != 0) return false; + const cs = CGImageGetColorSpace(img) orelse return false; + const model = CGColorSpaceGetModel(cs); // 0 monochrome, 1 RGB + if (model != 0 and model != 1) return false; + const bpc = CGImageGetBitsPerComponent(img); + const bpp = CGImageGetBitsPerPixel(img); + if (bpc != 8 and bpc != 16) return false; + const alpha = info & alpha_mask; // 0 none, 1/2 premultiplied last/first, 3/4 last/first, 5/6 skip last/first + if (alpha == 1 or alpha == 2 or alpha > 6) return false; + const colours: usize = if (model == 1) 3 else 1; + const comps: usize = colours + @intFromBool(alpha != 0); + if (bpp != comps * bpc) return false; + const first = alpha == 2 or alpha == 4 or alpha == 6; // the alpha (or skipped) component leads + const order = info & order_mask; + const little = (bpc == 8 and order == order_32_little) or (bpc == 16 and order == order_16_little); + if (bpc == 8 and order != 0 and order != order_32_little) return false; + if (bpc == 8 and little and comps != 4) return false; + const provider = CGImageGetDataProvider(img) orelse return false; + const data = CGDataProviderCopyData(provider) orelse return false; + defer CFRelease(data); + const base = CFDataGetBytePtr(data); + const len: usize = @intCast(CFDataGetLength(data)); + const bpr = CGImageGetBytesPerRow(img); + if (bpr * (h - 1) + w * bpp / 8 > len) return false; + const step = bpc / 8; + for (0..h) |y| { + const row = base + y * bpr; + for (0..w) |x| { + const px = row + x * comps * step; + var c: [4]u8 = undefined; + for (0..comps) |i| { + // a 32-bit little-endian pixel stores its components reversed; a 16-bit one each component's bytes + const slot = if (bpc == 8 and little) comps - 1 - i else i; + const at = px + slot * step; + c[i] = if (step == 1) at[0] else if (little) at[1] else at[0]; // 16 bits: the high byte, as PIL's 8-bit view + } + const lead: usize = @intFromBool(first and alpha != 0); + const o = out[(y * w + x) * 3 ..][0..3]; + if (colours == 3) { + o.* = .{ c[lead], c[lead + 1], c[lead + 2] }; + } else { + o.* = .{ c[lead], c[lead], c[lead] }; + } + if (alpha == 3 or alpha == 4) { // a real alpha band: over white, as images.py pastes it + const a = if (first) c[0] else c[comps - 1]; + for (o) |*v| v.* = overWhite(v.*, a); + } + } + } + return true; +} + +/// Any other layout (palette, CMYK, float, premultiplied): drawn into an RGB context over white. +fn drawn(img: CF, w: usize, h: usize, out: []u8) Error!void { + const cs = CGColorSpaceCreateDeviceRGB() orelse return error.BadImage; + defer CGColorSpaceRelease(cs); + const tmp = std.heap.page_allocator.alloc(u8, w * h * 4) catch return error.OutOfMemory; + defer std.heap.page_allocator.free(tmp); + @memset(tmp, 0); + const ctx = CGBitmapContextCreate(tmp.ptr, w, h, 8, w * 4, cs, 5) orelse return error.BadImage; // RGBX + defer CGContextRelease(ctx); + CGContextSetRGBFillColor(ctx, 1, 1, 1, 1); + CGContextFillRect(ctx, .{ .x = 0, .y = 0, .w = @floatFromInt(w), .h = @floatFromInt(h) }); + CGContextDrawImage(ctx, .{ .x = 0, .y = 0, .w = @floatFromInt(w), .h = @floatFromInt(h) }, img); + for (0..w * h) |i| @memcpy(out[i * 3 ..][0..3], tmp[i * 4 ..][0..3]); +} + +// ---- the processor's geometry ---- + +pub const Geometry = struct { target_h: u32, target_w: u32, content_h: u32, content_w: u32 }; + +fn alignUp(v: u64, f: u64) u64 { + return (v + f - 1) / f * f; +} + +/// smart_resize and _resize_geometry (num_frames = temporal_patch_size, as _process_one calls it): the padded canvas +/// under `max_tokens` (at least `min_tokens`), and the content size the image is resized to inside it. +pub fn geometry(h: u32, w: u32, min_tokens: u32, max_tokens: u32) Error!Geometry { + const factor: u64 = vision.patch * vision.merge; + const frames: u64 = vision.temporal; + const per_token = frames * factor * factor; + const min_px = @as(u64, min_tokens) * per_token; + const max_px = @as(u64, max_tokens) * per_token; + if (min_tokens == 0 or max_tokens < min_tokens) return error.ImageTooSmall; + const H: u64 = h; + const W: u64 = w; + var ah = alignUp(H, factor); + var aw = alignUp(W, factor); + var pixels = frames * ah * aw; + if (pixels < min_px) { + const scale = @sqrt(@as(f64, @floatFromInt(min_px)) / @as(f64, @floatFromInt(frames * H * W))); + ah = alignUp(@max(1, @as(u64, @intFromFloat(@ceil(@as(f64, @floatFromInt(H)) * scale)))), factor); + aw = alignUp(@max(1, @as(u64, @intFromFloat(@ceil(@as(f64, @floatFromInt(W)) * scale)))), factor); + pixels = frames * ah * aw; + } + if (pixels > max_px) { // the tallest content that fits, by bisection on its height + if (max_px < frames * factor * factor) return error.ImageTooSmall; + var best: [2]u64 = .{ factor, factor }; + var low: i64 = 1; + var high: i64 = @intCast(H); + while (low <= high) { + const ch: u64 = @intCast(@divFloor(low + high, 2)); + const cw: u64 = @max(1, W * ch / H); // math.floor(width * content_h / height) + const cand: [2]u64 = .{ alignUp(ch, factor), alignUp(cw, factor) }; + if (frames * cand[0] * cand[1] <= max_px) { + best = cand; + low = @as(i64, @intCast(ch)) + 1; + } else high = @as(i64, @intCast(ch)) - 1; + } + ah = best[0]; + aw = best[1]; + } + var scale = @min(@as(f64, @floatFromInt(ah)) / @as(f64, @floatFromInt(H)), @as(f64, @floatFromInt(aw)) / @as(f64, @floatFromInt(W))); + if (frames * H * W >= min_px) scale = @min(1.0, scale); + const chf: u64 = @intFromFloat(@floor(@as(f64, @floatFromInt(H)) * scale)); + const cwf: u64 = @intFromFloat(@floor(@as(f64, @floatFromInt(W)) * scale)); + return .{ .target_h = @intCast(ah), .target_w = @intCast(aw), .content_h = @intCast(@max(1, @min(ah, chf))), .content_w = @intCast(@max(1, @min(aw, cwf))) }; +} + +// ---- PIL's ImagingResample for 8-bit images (Resample.c), bicubic (a = -0.5) ---- + +const precision_bits = 32 - 8 - 2; + +fn bicubic(x0: f64) f64 { + const a = -0.5; + const x = @abs(x0); + if (x < 1.0) return ((a + 2.0) * x - (a + 3.0)) * x * x + 1; + if (x < 2.0) return (((x - 5) * x + 8) * x - 4) * a; + return 0; +} + +const Coeffs = struct { bounds: []u32, kk: []i32, ksize: usize }; + +/// precompute_coeffs and normalize_coeffs_8bpc: each output pixel's first input pixel, count and fixed-point weights. +fn coeffs(gpa: std.mem.Allocator, in_size: u32, out_size: u32) Error!Coeffs { + const scale = @as(f64, @floatFromInt(in_size)) / @as(f64, @floatFromInt(out_size)); + const filterscale = @max(scale, 1.0); + const support = 2.0 * filterscale; + const ksize: usize = @as(usize, @intFromFloat(@ceil(support))) * 2 + 1; + const bounds = try gpa.alloc(u32, @as(usize, out_size) * 2); + errdefer gpa.free(bounds); + const kk = try gpa.alloc(i32, @as(usize, out_size) * ksize); + errdefer gpa.free(kk); + var pre = try gpa.alloc(f64, ksize); + defer gpa.free(pre); + const ss = 1.0 / filterscale; + for (0..out_size) |xx| { + const center = (@as(f64, @floatFromInt(xx)) + 0.5) * scale; + var xmin_i: i64 = @intFromFloat(center - support + 0.5); // (int) truncates toward zero, as C does + if (xmin_i < 0) xmin_i = 0; + var xmax_i: i64 = @intFromFloat(center + support + 0.5); + if (xmax_i > in_size) xmax_i = in_size; + xmax_i -= xmin_i; + const xmin: usize = @intCast(xmin_i); + const xmax: usize = @intCast(xmax_i); + var ww: f64 = 0; + for (0..xmax) |x| { + const w = bicubic((@as(f64, @floatFromInt(x + xmin)) - center + 0.5) * ss); + pre[x] = w; + ww += w; + } + for (0..ksize) |x| { + const k = if (x < xmax) (if (ww != 0) pre[x] / ww else pre[x]) else 0; + const v = k * @as(f64, @floatFromInt(@as(i64, 1) << precision_bits)); + kk[xx * ksize + x] = @intFromFloat(if (v < 0) v - 0.5 else v + 0.5); // C's (int) of the rounded value + } + bounds[xx * 2] = @intCast(xmin); + bounds[xx * 2 + 1] = @intCast(xmax); + } + return .{ .bounds = bounds, .kk = kk, .ksize = ksize }; +} + +fn clip8(v: i64) u8 { + const s = v >> precision_bits; + return if (s < 0) 0 else if (s > 255) 255 else @intCast(s); +} + +/// Image.resize((w, h), BICUBIC) on an 8-bit RGB image: horizontal pass, then vertical, each rounded to 8 bits. +pub fn resize(gpa: std.mem.Allocator, src: Rgb, w: u32, h: u32) Error!Rgb { + var cur = src; + var owned = false; + errdefer if (owned) cur.deinit(gpa); + if (w != src.w) { // horizontal (PIL limits it to the rows the vertical pass reads: all of them here) + const c = try coeffs(gpa, src.w, w); + defer gpa.free(c.bounds); + defer gpa.free(c.kk); + const out = try gpa.alloc(u8, @as(usize, w) * cur.h * 3); + for (0..cur.h) |y| for (0..w) |xx| { + const xmin = c.bounds[xx * 2]; + const xmax = c.bounds[xx * 2 + 1]; + const k = c.kk[xx * c.ksize ..]; + for (0..3) |ch| { + var ss: i64 = 1 << (precision_bits - 1); + for (0..xmax) |x| ss += @as(i64, cur.px[(y * cur.w + x + xmin) * 3 + ch]) * k[x]; + out[(y * w + xx) * 3 + ch] = clip8(ss); + } + }; + if (owned) cur.deinit(gpa); + cur = .{ .w = w, .h = cur.h, .px = out }; + owned = true; + } + if (h != src.h) { + const c = try coeffs(gpa, src.h, h); + defer gpa.free(c.bounds); + defer gpa.free(c.kk); + const out = try gpa.alloc(u8, @as(usize, cur.w) * h * 3); + for (0..h) |yy| { + const ymin = c.bounds[yy * 2]; + const ymax = c.bounds[yy * 2 + 1]; + const k = c.kk[yy * c.ksize ..]; + for (0..cur.w) |x| for (0..3) |ch| { + var ss: i64 = 1 << (precision_bits - 1); + for (0..ymax) |y| ss += @as(i64, cur.px[((y + ymin) * cur.w + x) * 3 + ch]) * k[y]; + out[(yy * cur.w + x) * 3 + ch] = clip8(ss); + }; + } + if (owned) cur.deinit(gpa); + cur = .{ .w = cur.w, .h = h, .px = out }; + owned = true; + } + if (!owned) cur = .{ .w = src.w, .h = src.h, .px = try gpa.dupe(u8, src.px) }; + return cur; +} + +/// The content (already resized) padded with black to the canvas, scaled by 1/255, normalized by channel, cut into +/// 2x2 windows of 14x14 patches in row-major window order, each patch [channel, frame (twice), row, column]. +pub fn patchify(gpa: std.mem.Allocator, content: Rgb, g: Geometry, mean: [3]f32, sd: [3]f32) Error![]f32 { + const P = vision.patch; + const M = vision.merge; + const gh = g.target_h / P; + const gw = g.target_w / P; + const out = try gpa.alloc(f32, @as(usize, gh) * gw * vision.patch_width); + const rescale: f32 = @floatCast(@as(f64, 1.0) / 255.0); + var row: usize = 0; + for (0..gh / M) |wy| for (0..gw / M) |wx| for (0..M) |my| for (0..M) |mx| { + const py0 = (wy * M + my) * P; + const px0 = (wx * M + mx) * P; + const dst = out[row * vision.patch_width ..][0..vision.patch_width]; + for (0..3) |ch| for (0..vision.temporal) |t| for (0..P) |dy| for (0..P) |dx| { + const y = py0 + dy; + const x = px0 + dx; + const v: u8 = if (y < content.h and x < content.w) content.px[(y * content.w + x) * 3 + ch] else 0; + const f = @as(f32, @floatFromInt(v)) * rescale; + dst[((ch * vision.temporal + t) * P + dy) * P + dx] = (f - mean[ch]) / sd[ch]; + }; + row += 1; + }; + return out; +} + +/// Bytes to patches: decoded, sized for `max_tokens` (at least min(16, max_tokens)), resized, padded and cut. +pub fn prepare(gpa: std.mem.Allocator, bytes: []const u8, c: vision.Config, max_tokens: u32) Error!Prepared { + const img = try decode(gpa, bytes); + defer img.deinit(gpa); + return prepareRgb(gpa, img, std.hash.Wyhash.hash(0x1a6e, bytes), c, max_tokens); +} + +pub fn prepareRgb(gpa: std.mem.Allocator, img: Rgb, hash: u64, c: vision.Config, max_tokens: u32) Error!Prepared { + const g = try geometry(img.h, img.w, @min(c.min_tokens, max_tokens), max_tokens); + const content = try resize(gpa, img, g.content_w, g.content_h); + defer content.deinit(gpa); + const pixels = try patchify(gpa, content, g, c.mean, c.std); + const gh = g.target_h / vision.patch; + const gw = g.target_w / vision.patch; + return .{ .pixels = pixels, .gh = gh, .gw = gw, .tokens = gh * gw / vision.patches_per_token, .hash = hash }; +} + +test "geometry follows smart_resize" { + // the reference dumps' grids: 517x389 pads to 532x392 unscaled; 96x96 grows to 112; 20x15 to the 16-token floor + try std.testing.expectEqual(Geometry{ .target_h = 392, .target_w = 532, .content_h = 389, .content_w = 517 }, try geometry(389, 517, 16, 4096)); + try std.testing.expectEqual(Geometry{ .target_h = 112, .target_w = 112, .content_h = 112, .content_w = 112 }, try geometry(96, 96, 16, 4096)); + const tiny = try geometry(15, 20, 16, 4096); + try std.testing.expectEqual(@as(u32, 8 * 14), tiny.target_h); + try std.testing.expectEqual(@as(u32, 10 * 14), tiny.target_w); + // a large image under a small budget: the canvas fits it + const big = try geometry(1500, 2000, 16, 256); + try std.testing.expect(@as(u64, big.target_h) * big.target_w * 2 <= 256 * 2 * 28 * 28); + try std.testing.expect(big.content_h <= big.target_h and big.content_w <= big.target_w); +} + +test "orientations move pixels as PIL's transposes do" { + const gpa = std.testing.allocator; + // a 3x2 image of pixel indices: 0 1 2 / 3 4 5 + var px: [18]u8 = undefined; + for (0..6) |i| @memset(px[i * 3 ..][0..3], @intCast(i)); + const src: Rgb = .{ .w = 3, .h = 2, .px = &px }; + const want = [_][]const u8{ &.{ 2, 1, 0, 5, 4, 3 }, &.{ 5, 4, 3, 2, 1, 0 }, &.{ 3, 4, 5, 0, 1, 2 }, &.{ 0, 3, 1, 4, 2, 5 }, &.{ 3, 0, 4, 1, 5, 2 }, &.{ 5, 2, 4, 1, 3, 0 }, &.{ 2, 5, 1, 4, 0, 3 } }; + for (want, 2..) |w, o| { + const out = try orient(gpa, src, @intCast(o)); + defer out.deinit(gpa); + for (w, 0..) |v, i| try std.testing.expectEqual(v, out.px[i * 3]); + } +} + +test "alpha over white follows PIL's DIV255" { + try std.testing.expectEqual(@as(u8, 255), overWhite(0, 0)); + try std.testing.expectEqual(@as(u8, 10), overWhite(10, 255)); + try std.testing.expectEqual(@as(u8, 132), overWhite(10, 128)); // DIV255(255 * 127 + 10 * 128) +} + +test "resize keeps a flat image flat and its size" { + const gpa = std.testing.allocator; + const px = try gpa.alloc(u8, 7 * 5 * 3); + defer gpa.free(px); + @memset(px, 200); + const out = try resize(gpa, .{ .w = 7, .h = 5, .px = px }, 13, 3); + defer out.deinit(gpa); + try std.testing.expectEqual(@as(u32, 13), out.w); + for (out.px) |v| try std.testing.expectEqual(@as(u8, 200), v); +} diff --git a/zig/src/families/glm/mtp.zig b/zig/src/families/glm/mtp.zig index d683c2389f..109ebc91c3 100644 --- a/zig/src/families/glm/mtp.zig +++ b/zig/src/families/glm/mtp.zig @@ -35,6 +35,7 @@ pub fn absorb(x: *fwd.Ctx, e: mtl.ComputeEncoder, h: Ref, next: Ref, rows: u32, const L = &x.w.layers[c.layers]; const m = x.w.mtp.?; fwd.embedRows(x, e, next, sc.m_emb, rows); + fwd.inject(x, e, sc.m_emb, rows, pos + 1); // the head reads each row's next token: an image's rows one place on fwd.rms(x, e, sc.m_emb, m.enorm, sc.m_eh, rows, D, D, 2 * D, c.eps); fwd.rms(x, e, h, m.hnorm, sc.m_eh.at(@as(usize, D) * 2), rows, D, D, 2 * D, c.eps); fwd.qmv(x, e, x.k.qmv_kda_out, sc.m_eh, m.eh_proj, sc.m_x, rows); diff --git a/zig/src/families/glm/prompt.zig b/zig/src/families/glm/prompt.zig index e4f94f0600..007c4f779b 100644 --- a/zig/src/families/glm/prompt.zig +++ b/zig/src/families/glm/prompt.zig @@ -435,7 +435,7 @@ pub fn backbone(p: *const Prompt, x: *fwd.Ctx, e: mtl.ComputeEncoder, ids: Ref, std.debug.assert(x.sc == ss and M <= p.rows); const s = x.skip; // classes a profile leaves out const Class = fwd.Class; - if (s & Class.ends == 0) fwd.embed(x, e, ids, M); + if (s & Class.ends == 0) fwd.embed(x, e, ids, M, pos); const plane = @as(usize, M) * c.hidden * 2; // a trace's capture points: the decode backbone's fwd.snap(x, e, ss.h, plane); var pending = false; @@ -497,6 +497,7 @@ pub fn mtp(p: *const Prompt, x: *const fwd.Ctx, e: mtl.ComputeEncoder, h: Ref, n const L = &x.w.layers[c.layers]; const m = x.w.mtp.?; fwd.embedRows(x, e, next, p.m_emb, M); + fwd.inject(x, e, p.m_emb, M, pos + 1); // the head reads each row's next token: an image's rows one place on fwd.rms(x, e, p.m_emb, m.enorm, p.m_eh, M, D, D, 2 * D, c.eps); fwd.rms(x, e, h, m.hnorm, p.m_eh.at(@as(usize, D) * 2), M, D, D, 2 * D, c.eps); qmm(p, e, p.m_eh, m.eh_proj, p.m_x, M); diff --git a/zig/src/families/glm/slot_images.zig b/zig/src/families/glm/slot_images.zig new file mode 100644 index 0000000000..f3a0697e48 --- /dev/null +++ b/zig/src/families/glm/slot_images.zig @@ -0,0 +1,37 @@ +//! A stream's images through the tower into the engine's image rows, which its prompt chunks inject over their +//! placeholders (forward.zig's `inject`); the tower remembers whose rows it holds until the pass ends. +const std = @import("std"); +const mtl = @import("metal"); +const vision_mod = @import("vision.zig"); +const Slots = @import("slots.zig").Slots; +const Image = @import("lanes").stream.Image; + +/// Slot `i`'s images through the tower into the engine's image rows, injected over their placeholders by the chunks +/// that follow until `end`. One image a command buffer: the tower's host-written scratch is shared. +pub fn encode(sl: *Slots, i: u32, imgs: []const Image) !void { + const e = sl.e; + const v = e.vision orelse return error.NoVision; + if (imgs.len > e.vision_limits.max_images) return error.TooManyImages; + if (i >= sl.slots.len or !sl.slots[i].used) return error.SlotOutOfStep; + v.spans.clearRetainingCapacity(); + try sl.flush(); + e.sync(); + var row: u32 = 0; + for (imgs) |img| { + if (row + img.tokens > e.vision_limits.image_tokens or img.gh * img.gw != img.tokens * vision_mod.patches_per_token) return error.ImageTooLarge; + if (img.at + img.tokens > sl.slots[i].prompt_len) return error.ImageOutOfPrompt; + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const b = e.begin(); + try v.encode(b.enc, img.pixels, img.gh, img.gw, e.vision_rows.at(@as(usize, row) * e.c.hidden * 2), null); + try e.finish(b.cb, b.enc); + try v.spans.append(v.gpa, .{ .at = img.at, .n = img.tokens, .row = row }); + row += img.tokens; + } + v.inject = .{ .slot = i, .v = .{ .rows = e.vision_rows, .spans = v.spans.items } }; +} + +/// The stream's prompt pass is over: the chunks after it inject nothing. +pub fn end(sl: *Slots) void { + if (sl.e.vision) |v| v.inject = null; +} diff --git a/zig/src/families/glm/slots.zig b/zig/src/families/glm/slots.zig index c17d741a88..8693fbcdb3 100644 --- a/zig/src/families/glm/slots.zig +++ b/zig/src/families/glm/slots.zig @@ -330,6 +330,7 @@ pub const Slots = struct { const pool = mtl.objc.Pool.push(); defer pool.pop(); var x = sl.ctx(slot); + x.inject = if (e.vision) |v| v.injected(i) else null; // an image prompt's rows over its placeholders (slot_images.zig) const b = e.begin(); const ids = e.prompt_ids.at(@as(usize, at) * 4); const next = e.prompt_ids.at(@as(usize, at + 1) * 4); diff --git a/zig/src/families/glm/vision.zig b/zig/src/families/glm/vision.zig new file mode 100644 index 0000000000..8a5a410de3 --- /dev/null +++ b/zig/src/families/glm/vision.zig @@ -0,0 +1,541 @@ +//! GLM-5.3-Flash's image tower on the GPU: an image's patches to the rows that replace its `<|image|>` placeholders. +//! mlx-vlm's glm5_next/vision.py (what TensorFold 0.6.6's --vision serves) in bf16: a patch projection, 24 pre-norm +//! blocks (2-D rotary attention over the image's own patches, a clipped SwiGLU), a final norm, the 2x2 merge as a +//! projection of four neighbouring patches, and the merger's projection, layer norm, GELU and clipped SwiGLU. +const std = @import("std"); +const mtl = @import("metal"); +const sources = @import("kernel_sources"); +const frags = @import("../../core/frags.zig"); +const st = @import("../../core/safetensors.zig"); +const wts = @import("weights.zig"); +const fwd = @import("forward.zig"); +const readAll = @import("weights_fix.zig").readAll; +const Engine = @import("engine.zig").Engine; +const Ref = wts.Ref; + +// The tower this file reads; `Config.parse` refuses a checkpoint whose vision_config differs. +pub const hidden = 1024; +pub const heads = 16; +pub const head_dim = 64; +pub const depth = 24; +pub const mlp_inter = 4096; +pub const out_hidden = 4096; // the language model's width +pub const merger_inter = 10240; +pub const patch = 14; +pub const temporal = 2; +pub const merge = 2; +pub const patch_width = 3 * temporal * patch * patch; // 1176: a patch's values, channel by frame by row by column +pub const patches_per_token = merge * merge; + +/// --vision's limits: image tokens a prompt may hold in all (the tower's scratch and rows are sized for them), and images. +pub const Limits = struct { image_tokens: u32 = 4096, max_images: u32 = 4 }; + +pub const Config = struct { + eps: f32 = 1e-5, + limit: f32 = 10, // swiglu_limit + image_token: u32, + image_start: u32, + image_end: u32, + mean: [3]f32, + std: [3]f32, + min_tokens: u32, + max_tokens: u32, + + /// config.json's vision_config and image token ids, processor_config.json's image processor; null: no tower. + pub fn parse(arena: std.mem.Allocator, config_json: []const u8, processor_json: ?[]const u8) !?Config { + const doc = try std.json.parseFromSliceLeaky(std.json.Value, arena, config_json, .{}); + const vc = doc.object.get("vision_config") orelse return null; + if (vc != .object) return null; + const want = .{ .{ "depth", depth }, .{ "hidden_size", hidden }, .{ "num_heads", heads }, .{ "intermediate_size", mlp_inter }, .{ "out_hidden_size", out_hidden }, .{ "projection_intermediate_size", merger_inter }, .{ "patch_size", patch }, .{ "temporal_patch_size", temporal }, .{ "spatial_merge_size", merge }, .{ "in_channels", 3 } }; + inline for (want) |kv| { + const v = vc.object.get(kv[0]) orelse return error.VisionConfig; + if (v != .integer or v.integer != kv[1]) { + std.log.err("glm vision: vision_config.{s} is {f}; this tower reads {d}", .{ kv[0], std.json.fmt(v, .{}), kv[1] }); + return error.VisionConfig; + } + } + const act = vc.object.get("hidden_act") orelse return error.VisionConfig; + if (act != .string or !std.mem.eql(u8, act.string, "silu")) return error.VisionConfig; + var c: Config = .{ .image_token = 0, .image_start = 0, .image_end = 0, .mean = .{ 0.48145466, 0.4578275, 0.40821073 }, .std = .{ 0.26862954, 0.26130258, 0.27577711 }, .min_tokens = 16, .max_tokens = 8000 }; + if (vc.object.get("rms_norm_eps")) |v| c.eps = @floatCast(number(v) orelse return error.VisionConfig); + if (vc.object.get("swiglu_limit")) |v| c.limit = @floatCast(number(v) orelse return error.VisionConfig); + inline for (.{ .{ "image_token_id", "image_token" }, .{ "image_start_token_id", "image_start" }, .{ "image_end_token_id", "image_end" } }) |kv| { + const v = doc.object.get(kv[0]) orelse return error.VisionConfig; + if (v != .integer) return error.VisionConfig; + @field(c, kv[1]) = @intCast(v.integer); + } + const pj = processor_json orelse return c; + const pdoc = try std.json.parseFromSliceLeaky(std.json.Value, arena, pj, .{}); + const ip = pdoc.object.get("image_processor") orelse return c; + if (ip != .object) return error.VisionConfig; + inline for (.{ .{ "patch_size", patch }, .{ "temporal_patch_size", temporal }, .{ "merge_size", merge } }) |kv| { + if (ip.object.get(kv[0])) |v| if (v != .integer or v.integer != kv[1]) return error.VisionConfig; + } + if (ip.object.get("patch_expand_factor")) |v| if (v != .integer or v.integer != 1) return error.VisionConfig; + inline for (.{ "image_mean", "image_std" }, .{ &c.mean, &c.std }) |key, dst| if (ip.object.get(key)) |v| { + if (v != .array or v.array.items.len != 3) return error.VisionConfig; + for (v.array.items, 0..) |x, i| dst[i] = @floatCast(number(x) orelse return error.VisionConfig); + }; + if (ip.object.get("min_image_tokens")) |v| c.min_tokens = @intCast(v.integer); + if (ip.object.get("max_image_tokens")) |v| c.max_tokens = @intCast(v.integer); + return c; + } + + fn number(v: std.json.Value) ?f64 { + return switch (v) { + .float => |f| f, + .integer => |i| @floatFromInt(i), + else => null, + }; + } +}; + +const Kernels = struct { + lib: mtl.Library, + gemm: [3][2][2]mtl.Pipeline, // [bf16 nt, f32 nt, f32 nn][M % 64 == 0][N % 128 == 0] + bias: mtl.Pipeline, + add: mtl.Pipeline, + rms: mtl.Pipeline, + layer_norm: mtl.Pipeline, + qkv: mtl.Pipeline, + softmax: mtl.Pipeline, + heads_: mtl.Pipeline, + swiglu: mtl.Pipeline, + gelu: mtl.Pipeline, + + fn load(gpa: std.mem.Allocator, device: mtl.Device) !Kernels { + const src = try frags.source(device, gpa, sources.glm_vision); + defer gpa.free(src); + const lib = try mtl.Library.fromSource(device, src, mtl.CompileOptions.mlx()); + var k: Kernels = undefined; + k.lib = lib; + const kinds = [_][]const u8{ "bf16_nt", "f32_nt", "f32_nn" }; + for (kinds, 0..) |kind, ki| for (0..2) |m| for (0..2) |n| { + var name: [64]u8 = undefined; + const full = try std.fmt.bufPrint(&name, "glmv_gemm_{s}_{d}{d}", .{ kind, m, n }); + k.gemm[ki][m][n] = try mtl.Pipeline.init(device, lib, full, false); + }; + inline for (.{ .{ "bias", "glmv_bias" }, .{ "add", "glmv_add" }, .{ "rms", "glmv_rms" }, .{ "layer_norm", "glmv_layer_norm" }, .{ "qkv", "glmv_qkv" }, .{ "softmax", "glmv_softmax" }, .{ "heads_", "glmv_heads" }, .{ "swiglu", "glmv_swiglu" }, .{ "gelu", "glmv_gelu" } }) |kv| { + @field(k, kv[0]) = try mtl.Pipeline.init(device, lib, kv[1], false); + } + return k; + } + + fn deinit(k: *Kernels) void { + for (&k.gemm) |*a| for (a) |*b| for (b) |p| p.deinit(); + inline for (.{ "bias", "add", "rms", "layer_norm", "qkv", "softmax", "heads_", "swiglu", "gelu" }) |f| @field(k, f).deinit(); + k.lib.deinit(); + } +}; + +const Block = struct { norm1: Ref, norm2: Ref, qkv_w: Ref, qkv_b: Ref, proj_w: Ref, proj_b: Ref, q_norm: Ref, k_norm: Ref, gate_w: Ref, gate_b: Ref, up_w: Ref, up_b: Ref, down_w: Ref, down_b: Ref }; + +const Weights = struct { + buf: mtl.Buffer, + patch_w: Ref, + patch_b: Ref, + blocks: [depth]Block, + post_norm: Ref, + down_w: Ref, // [4096, 2, 2, 1024]: the merge's kernel with its taps outermost, as four neighbouring patch rows read it + down_b: Ref, + m_proj: Ref, + m_norm_w: Ref, + m_norm_b: Ref, + m_gate: Ref, + m_up: Ref, + m_down: Ref, +}; + +/// The tower: kernels, weights (bf16, about 0.9 GiB) and scratch for `max_tokens` image tokens a request. +pub const Vision = struct { + gpa: std.mem.Allocator, + inject: ?struct { slot: u32, v: fwd.Inject } = null, // an image prompt's rows while its pass runs (slot_images.zig) + spans: std.ArrayList(fwd.Span) = .empty, + c: Config, + k: Kernels, + w: Weights, + scratch: mtl.Buffer, + max_patches: u32, + chunk: u32, // query rows an attention pass scores at once (bounds the fp32 score buffer) + // scratch views + patches: Ref, // [P, 1176] bf16 + cos: Ref, // [P, 64] f32 + sin: Ref, + x: Ref, // [P, 1024] bf16: the residual stream + xn: Ref, // [P, 1024]: a norm's output; the merger's first [T, 4096] + att: Ref, // [P, 1024]: heads side by side; the merger's second [T, 4096] + branch: Ref, // [P, 1024] + qkv: Ref, // [P, 3072] + q: Ref, // [16, P, 64] f32 + k_: Ref, + v: Ref, + s: Ref, // [16, chunk, P] f32 + o: Ref, // [16, chunk, 64] f32 + g: Ref, // [P, 4096] bf16 (the merger's [T, 10240] fits: T * 10240 < P * 4096) + u: Ref, + act: Ref, + bytes: usize, + + pub fn load(gpa: std.mem.Allocator, device: mtl.Device, dir: []const u8, c: Config, max_tokens: u32) !*Vision { + const v = try gpa.create(Vision); + errdefer gpa.destroy(v); + v.gpa = gpa; + v.inject = null; + v.spans = .empty; + v.c = c; + v.k = try Kernels.load(gpa, device); + errdefer v.k.deinit(); + v.w = try loadWeights(gpa, device, dir); + errdefer v.w.buf.deinit(); + const P: usize = @as(usize, max_tokens) * patches_per_token; + v.max_patches = @intCast(P); + // the scores take at most 256 MiB: 16 heads of `chunk` rows by P keys in fp32 + v.chunk = @intCast(std.math.clamp((256 << 20) / (heads * P * 4) / 64 * 64, 64, 4096)); + const parts = [_]usize{ P * patch_width * 2, P * 64 * 4, P * 64 * 4, P * hidden * 2, P * hidden * 2, P * hidden * 2, P * hidden * 2, P * 3 * hidden * 2, P * hidden * 4, P * hidden * 4, P * hidden * 4, @as(usize, heads) * v.chunk * P * 4, @as(usize, heads) * v.chunk * head_dim * 4, P * mlp_inter * 2, P * mlp_inter * 2, P * mlp_inter * 2 }; + var total: usize = 0; + for (parts) |n| total += std.mem.alignForward(usize, n, 256); + v.scratch = try device.buffer(total, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + errdefer v.scratch.deinit(); + var at: usize = 0; + const views = [_]*Ref{ &v.patches, &v.cos, &v.sin, &v.x, &v.xn, &v.att, &v.branch, &v.qkv, &v.q, &v.k_, &v.v, &v.s, &v.o, &v.g, &v.u, &v.act }; + for (views, parts) |view, n| { + view.* = .{ .buf = v.scratch, .off = at }; + at += std.mem.alignForward(usize, n, 256); + } + v.bytes = total + v.w.buf.length(); + return v; + } + + pub fn deinit(v: *Vision) void { + v.spans.deinit(v.gpa); + v.scratch.deinit(); + v.w.buf.deinit(); + v.k.deinit(); + v.gpa.destroy(v); + } + + /// Slot `i`'s image rows for its prompt pass, while its images' pass runs; null for any other slot. + pub fn injected(v: *const Vision, i: u32) ?*const fwd.Inject { + if (v.inject) |*inj| if (inj.slot == i) return &inj.v; + return null; + } + + /// Buffers to keep resident with the engine's. + pub fn buffers(v: *const Vision) [2]mtl.Buffer { + return .{ v.w.buf, v.scratch }; + } + + /// One image's patches ([gh * gw, 1176] fp32, the processor's order) into its tokens' rows `out` ([gh * gw / 4, + /// 4096] bf16). The host writes the scratch: the caller has no work of this tower in flight. `stop`: after the + /// patch projection (0) or block `stop` (1-24), the residual stream is left in `x` and nothing more runs. + pub fn encode(v: *Vision, enc: mtl.ComputeEncoder, pixels: []const f32, gh: u32, gw: u32, out: Ref, stop: ?u32) !void { + const P: u32 = gh * gw; + if (P == 0 or gh % merge != 0 or gw % merge != 0) return error.ImageGrid; + if (P > v.max_patches) return error.ImageTooLarge; + if (pixels.len != @as(usize, P) * patch_width) return error.ImageGrid; + const T: u32 = P / patches_per_token; + // host: the patches in bf16 (MLX's astype), and each patch's rotary angles + const dst: [*]u16 = @ptrCast(@alignCast(v.patches.addr())); + for (pixels, 0..) |p, i| dst[i] = bf16(p); + rotary(v, gh, gw); + + v.gemmLinear(enc, v.patches, v.w.patch_w, v.x, P, hidden, patch_width); + v.biasAdd(enc, v.x, v.w.patch_b, P, hidden); + if (stop) |s| if (s == 0) return; + for (v.w.blocks, 0..) |b, i| { + v.block(enc, b, P); + if (stop) |s| if (s == i + 1) return; + } + v.rms(enc, v.x, v.w.post_norm, v.xn, P, hidden); + // the merge: four neighbouring patches' rows are one row of xn read as [T, 4096] + const m0 = v.xn; + const m1 = v.att; + v.gemmLinear(enc, m0, v.w.down_w, m1, T, out_hidden, patches_per_token * hidden); + v.biasAdd(enc, m1, v.w.down_b, T, out_hidden); + v.gemmLinear(enc, m1, v.w.m_proj, m0, T, out_hidden, out_hidden); + v.layerNorm(enc, m0, v.w.m_norm_w, v.w.m_norm_b, m1, T, out_hidden); + enc.setPipeline(v.k.gelu); + enc.setBuffer(m1.buf, m1.off, 0); + enc.dispatchThreads(mtl.Size.of(@as(usize, T) * out_hidden, 1, 1), mtl.Size.of(256, 1, 1)); + v.gemmLinear(enc, m1, v.w.m_gate, v.g, T, merger_inter, out_hidden); + v.gemmLinear(enc, m1, v.w.m_up, v.u, T, merger_inter, out_hidden); + v.swiglu(enc, @as(usize, T) * merger_inter); + v.gemmLinear(enc, v.act, v.w.m_down, out, T, out_hidden, merger_inter); + } + + fn block(v: *Vision, enc: mtl.ComputeEncoder, b: Block, P: u32) void { + v.rms(enc, v.x, b.norm1, v.xn, P, hidden); + v.gemmLinear(enc, v.xn, b.qkv_w, v.qkv, P, 3 * hidden, hidden); + v.biasAdd(enc, v.qkv, b.qkv_b, P, 3 * hidden); + enc.setPipeline(v.k.qkv); + const bufs = [_]Ref{ v.qkv, b.q_norm, b.k_norm, v.cos, v.sin, v.q, v.k_, v.v }; + for (bufs, 0..) |r, i| enc.setBuffer(r.buf, r.off, i); + enc.setValue(extern struct { L: u32, eps: f32 }{ .L = P, .eps = v.c.eps }, 8); + enc.dispatchGroups(mtl.Size.of(@as(usize, P) * heads * 3, 1, 1), mtl.Size.of(32, 1, 1)); + var q0: u32 = 0; + while (q0 < P) : (q0 += v.chunk) { + const lc = @min(v.chunk, P - q0); + // scores of this chunk's queries against every key, each head a batch + v.gemm(enc, 1, v.q.at(@as(usize, q0) * head_dim * 4), v.k_, v.s, lc, P, head_dim, head_dim, head_dim, P, heads, @as(usize, P) * head_dim, @as(usize, P) * head_dim, @as(usize, lc) * P); + enc.setPipeline(v.k.softmax); + enc.setBuffer(v.s.buf, v.s.off, 0); + enc.setValue(P, 1); + enc.setValue(@as(f32, 0.125), 2); // head_dim ** -0.5 + enc.dispatchGroups(mtl.Size.of(@as(usize, heads) * lc, 1, 1), mtl.Size.of(256, 1, 1)); + v.gemm(enc, 2, v.s, v.v, v.o, lc, head_dim, P, P, head_dim, head_dim, heads, @as(usize, lc) * P, @as(usize, P) * head_dim, @as(usize, lc) * head_dim); + enc.setPipeline(v.k.heads_); + enc.setBuffer(v.o.buf, v.o.off, 0); + enc.setBuffer(v.att.buf, v.att.off, 1); + enc.setValue([2]u32{ lc, q0 }, 2); + enc.dispatchThreads(mtl.Size.of(hidden, lc, 1), mtl.Size.of(256, 1, 1)); + } + v.gemmLinear(enc, v.att, b.proj_w, v.branch, P, hidden, hidden); + v.biasAdd(enc, v.branch, b.proj_b, P, hidden); + v.add(enc, v.x, v.branch, @as(usize, P) * hidden); + v.rms(enc, v.x, b.norm2, v.xn, P, hidden); + v.gemmLinear(enc, v.xn, b.gate_w, v.g, P, mlp_inter, hidden); + v.biasAdd(enc, v.g, b.gate_b, P, mlp_inter); + v.gemmLinear(enc, v.xn, b.up_w, v.u, P, mlp_inter, hidden); + v.biasAdd(enc, v.u, b.up_b, P, mlp_inter); + v.swiglu(enc, @as(usize, P) * mlp_inter); + v.gemmLinear(enc, v.act, b.down_w, v.branch, P, hidden, mlp_inter); + v.biasAdd(enc, v.branch, b.down_b, P, hidden); + v.add(enc, v.x, v.branch, @as(usize, P) * hidden); + } + + /// y [M, N] = x [M, K] w^T in bf16 (w as stored, [N, K]). + fn gemmLinear(v: *Vision, enc: mtl.ComputeEncoder, x: Ref, w: Ref, y: Ref, M: u32, N: u32, K: u32) void { + v.gemm(enc, 0, x, w, y, M, N, K, K, K, N, 1, 0, 0, 0); + } + + /// tfp::gemm_nax: d [batch, M, N] = a x b (kind 0: bf16 a b^T, 1: fp32 a b^T, 2: fp32 a b), batches `s*` elements apart. + fn gemm(v: *Vision, enc: mtl.ComputeEncoder, kind: usize, a: Ref, b: Ref, d: Ref, M: u32, N: u32, K: u32, lda: u32, ldb: u32, ldd: u32, batch: u32, sa: usize, sb: usize, sd: usize) void { + const tn = (N + 127) / 128; + const tm = (M + 63) / 64; + enc.setPipeline(v.k.gemm[kind][@intFromBool(M % 64 == 0)][@intFromBool(N % 128 == 0)]); + enc.setBuffer(a.buf, a.off, 0); + enc.setBuffer(b.buf, b.off, 1); + var p: [16]i32 = @splat(0); + const vals = [_]usize{ M, N, K, lda, ldb, ldd, tn, tm, 2, K / 256, sa, sb, sd, 0, 0 }; + for (vals, 0..) |x, i| p[i] = @intCast(x); + enc.setBytes(std.mem.asBytes(&p), 2); + enc.setBuffer(d.buf, d.off, 3); + enc.dispatchThreads(mtl.Size.of(@as(usize, tn << 2) * 32, (tm + 3) / 4 * 4, @as(usize, batch) * 2), mtl.Size.of(32, 4, 2)); + } + + fn biasAdd(v: *Vision, enc: mtl.ComputeEncoder, y: Ref, b: Ref, rows: u32, width: u32) void { + enc.setPipeline(v.k.bias); + enc.setBuffer(y.buf, y.off, 0); + enc.setBuffer(b.buf, b.off, 1); + enc.setValue(width, 2); + enc.dispatchThreads(mtl.Size.of(width, rows, 1), mtl.Size.of(256, 1, 1)); + } + + fn add(v: *Vision, enc: mtl.ComputeEncoder, x: Ref, y: Ref, n: usize) void { + enc.setPipeline(v.k.add); + enc.setBuffer(x.buf, x.off, 0); + enc.setBuffer(y.buf, y.off, 1); + enc.dispatchThreads(mtl.Size.of(n, 1, 1), mtl.Size.of(256, 1, 1)); + } + + fn rms(v: *Vision, enc: mtl.ComputeEncoder, x: Ref, w: Ref, y: Ref, rows: u32, width: u32) void { + enc.setPipeline(v.k.rms); + enc.setBuffer(x.buf, x.off, 0); + enc.setBuffer(w.buf, w.off, 1); + enc.setBuffer(y.buf, y.off, 2); + enc.setValue(extern struct { rows: u32, width: u32, eps: f32 }{ .rows = rows, .width = width, .eps = v.c.eps }, 3); + enc.dispatchGroups(mtl.Size.of(rows, 1, 1), mtl.Size.of(256, 1, 1)); + } + + fn layerNorm(v: *Vision, enc: mtl.ComputeEncoder, x: Ref, w: Ref, b: Ref, y: Ref, rows: u32, width: u32) void { + enc.setPipeline(v.k.layer_norm); + const bufs = [_]Ref{ x, w, b, y }; + for (bufs, 0..) |r, i| enc.setBuffer(r.buf, r.off, i); + enc.setValue(extern struct { rows: u32, width: u32, eps: f32 }{ .rows = rows, .width = width, .eps = 1e-5 }, 4); + enc.dispatchGroups(mtl.Size.of(rows, 1, 1), mtl.Size.of(256, 1, 1)); + } + + fn swiglu(v: *Vision, enc: mtl.ComputeEncoder, n: usize) void { + enc.setPipeline(v.k.swiglu); + enc.setBuffer(v.g.buf, v.g.off, 0); + enc.setBuffer(v.u.buf, v.u.off, 1); + enc.setBuffer(v.act.buf, v.act.off, 2); + enc.setValue(v.c.limit, 3); + enc.dispatchThreads(mtl.Size.of(n, 1, 1), mtl.Size.of(256, 1, 1)); + } +}; + +/// fp32 to bf16, round to nearest even (MLX's astype). +pub fn bf16(x: f32) u16 { + const b: u32 = @bitCast(x); + if (b & 0x7fffffff > 0x7f800000) return @truncate((b >> 16) | 0x40); // NaN stays NaN + return @truncate((b + 0x7fff + ((b >> 16) & 1)) >> 16); +} + +/// Each patch's 2-D rotary angles as vision.py builds them: the patch's row then column in the image's patch grid +/// (patches come in 2x2 merge windows, row-major windows), times the 16 frequencies 10000^(-k/16), cos and sin of +/// [h * f, w * f] twice over a head's 64 values. +fn rotary(v: *Vision, gh: u32, gw: u32) void { + var inv: [16]f32 = undefined; + for (&inv, 0..) |*f, k| f.* = 1.0 / @as(f32, @floatCast(std.math.pow(f64, 10000.0, @as(f64, @floatFromInt(2 * k)) / 32.0))); + const cs: [*]f32 = @ptrCast(@alignCast(v.cos.addr())); + const sn: [*]f32 = @ptrCast(@alignCast(v.sin.addr())); + const wx_n = gw / merge; + var i: usize = 0; + var wy: u32 = 0; + while (wy < gh / merge) : (wy += 1) { + var wx: u32 = 0; + while (wx < wx_n) : (wx += 1) { + for (0..merge) |my| for (0..merge) |mx| { + const h: f32 = @floatFromInt(wy * merge + my); + const w: f32 = @floatFromInt(wx * merge + mx); + for (0..32) |j| { + const angle: f32 = if (j < 16) h * inv[j] else w * inv[j - 16]; + const c: f32 = @floatCast(@cos(@as(f64, angle))); + const s: f32 = @floatCast(@sin(@as(f64, angle))); + cs[i * 64 + j] = c; + cs[i * 64 + j + 32] = c; + sn[i * 64 + j] = s; + sn[i * 64 + j + 32] = s; + } + i += 1; + }; + } + } +} + +/// model.visual.* from the checkpoint's shards into one buffer, the merge kernel's taps moved outermost. +fn loadWeights(gpa: std.mem.Allocator, device: mtl.Device, dir: []const u8) !Weights { + var arena_state = std.heap.ArenaAllocator.init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const Src = struct { path: [:0]const u8, off: u64, len: u64, dtype: st.DType, shape: []const usize }; + var names: std.StringHashMapUnmanaged(Src) = .empty; + { + const path = try std.fmt.allocPrintSentinel(arena, "{s}/model.safetensors.index.json", .{dir}, 0); + const f = try mtl.MappedFile.open(path); + defer f.deinit(); + const doc = try std.json.parseFromSliceLeaky(std.json.Value, arena, f.bytes[0..f.size], .{ .allocate = .alloc_always }); + var files: std.StringArrayHashMapUnmanaged(void) = .empty; + var it = doc.object.get("weight_map").?.object.iterator(); + while (it.next()) |kv| if (std.mem.startsWith(u8, kv.key_ptr.*, "model.visual.")) try files.put(arena, kv.value_ptr.string, {}); + if (files.count() == 0) return error.NoVisionTower; + for (files.keys()) |name| { + const full = try std.fmt.allocPrintSentinel(arena, "{s}/{s}", .{ dir, name }, 0); + const fd = std.c.open(full, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = std.c.close(fd); + var head: [8]u8 = undefined; + try readAll(fd, &head, 0); + const hlen = std.mem.readInt(u64, &head, .little); + const header = try arena.alloc(u8, hlen); + try readAll(fd, header, 8); + const end: u64 = @intCast(std.c.lseek(fd, 0, std.c.SEEK.END)); + // read here, not by core/safetensors.zig: the patch kernel is 5-D, past its max_rank + const doc2 = try std.json.parseFromSliceLeaky(std.json.Value, arena, header, .{}); + var eit = doc2.object.iterator(); + while (eit.next()) |kv| { + if (!std.mem.startsWith(u8, kv.key_ptr.*, "model.visual.")) continue; + const o = kv.value_ptr.object; + const dtype = st.DType.parse(o.get("dtype").?.string) orelse return error.UnsupportedDType; + const dims = o.get("shape").?.array.items; + const shape = try arena.alloc(usize, dims.len); + var n: u64 = dtype.size(); + for (dims, shape) |d, *s| { + s.* = @intCast(d.integer); + n *= s.*; + } + const offs = o.get("data_offsets").?.array.items; + const b: u64 = @intCast(offs[0].integer); + const e: u64 = @intCast(offs[1].integer); + if (e < b or e - b != n or 8 + hlen + e > end) return error.BadSafetensors; + try names.put(arena, kv.key_ptr.*, .{ .path = full, .off = 8 + hlen + b, .len = e - b, .dtype = dtype, .shape = shape }); + } + } + } + // every tensor's place in one buffer, 256-aligned + const Want = struct { name: []const u8, shape: []const usize, ref: *Ref }; + var w: Weights = undefined; + var wants: std.ArrayList(Want) = .empty; + try wants.append(arena, .{ .name = "patch_embed.proj.weight", .shape = &.{ hidden, 3, temporal, patch, patch }, .ref = &w.patch_w }); + try wants.append(arena, .{ .name = "patch_embed.proj.bias", .shape = &.{hidden}, .ref = &w.patch_b }); + for (&w.blocks, 0..) |*b, i| { + const fields = .{ .{ "norm1.weight", &b.norm1, &[_]usize{hidden} }, .{ "norm2.weight", &b.norm2, &[_]usize{hidden} }, .{ "attn.qkv.weight", &b.qkv_w, &[_]usize{ 3 * hidden, hidden } }, .{ "attn.qkv.bias", &b.qkv_b, &[_]usize{3 * hidden} }, .{ "attn.proj.weight", &b.proj_w, &[_]usize{ hidden, hidden } }, .{ "attn.proj.bias", &b.proj_b, &[_]usize{hidden} }, .{ "attn.q_norm.weight", &b.q_norm, &[_]usize{head_dim} }, .{ "attn.k_norm.weight", &b.k_norm, &[_]usize{head_dim} }, .{ "mlp.gate_proj.weight", &b.gate_w, &[_]usize{ mlp_inter, hidden } }, .{ "mlp.gate_proj.bias", &b.gate_b, &[_]usize{mlp_inter} }, .{ "mlp.up_proj.weight", &b.up_w, &[_]usize{ mlp_inter, hidden } }, .{ "mlp.up_proj.bias", &b.up_b, &[_]usize{mlp_inter} }, .{ "mlp.down_proj.weight", &b.down_w, &[_]usize{ hidden, mlp_inter } }, .{ "mlp.down_proj.bias", &b.down_b, &[_]usize{hidden} } }; + inline for (fields) |f| try wants.append(arena, .{ .name = try std.fmt.allocPrint(arena, "blocks.{d}.{s}", .{ i, f[0] }), .shape = f[2], .ref = f[1] }); + } + try wants.append(arena, .{ .name = "post_layernorm.weight", .shape = &.{hidden}, .ref = &w.post_norm }); + try wants.append(arena, .{ .name = "downsample.weight", .shape = &.{ out_hidden, hidden, merge, merge }, .ref = &w.down_w }); + try wants.append(arena, .{ .name = "downsample.bias", .shape = &.{out_hidden}, .ref = &w.down_b }); + try wants.append(arena, .{ .name = "merger.proj.weight", .shape = &.{ out_hidden, out_hidden }, .ref = &w.m_proj }); + try wants.append(arena, .{ .name = "merger.post_projection_norm.weight", .shape = &.{out_hidden}, .ref = &w.m_norm_w }); + try wants.append(arena, .{ .name = "merger.post_projection_norm.bias", .shape = &.{out_hidden}, .ref = &w.m_norm_b }); + try wants.append(arena, .{ .name = "merger.gate_proj.weight", .shape = &.{ merger_inter, out_hidden }, .ref = &w.m_gate }); + try wants.append(arena, .{ .name = "merger.up_proj.weight", .shape = &.{ merger_inter, out_hidden }, .ref = &w.m_up }); + try wants.append(arena, .{ .name = "merger.down_proj.weight", .shape = &.{ out_hidden, merger_inter }, .ref = &w.m_down }); + const srcs = try arena.alloc(Src, wants.items.len); + var total: usize = 0; + for (wants.items, srcs) |want, *s| { + const full = try std.fmt.allocPrint(arena, "model.visual.{s}", .{want.name}); + s.* = names.get(full) orelse { + std.log.err("glm vision: the checkpoint has no tensor {s}", .{full}); + return error.MissingTensor; + }; + if (s.dtype != .bf16 or !std.mem.eql(usize, s.shape, want.shape)) { + std.log.err("glm vision: {s} is {t} {any}; the tower reads bf16 {any}", .{ full, s.dtype, s.shape, want.shape }); + return error.UnexpectedTensor; + } + total = std.mem.alignForward(usize, total, 256) + s.len; + } + w.buf = try device.buffer(total, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked); + errdefer w.buf.deinit(); + var at: usize = 0; + for (wants.items, srcs) |want, s| { + at = std.mem.alignForward(usize, at, 256); + want.ref.* = .{ .buf = w.buf, .off = at }; + const fd = std.c.open(s.path, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = std.c.close(fd); + _ = std.c.fcntl(fd, 48, @as(c_int, 1)); // F_NOCACHE, as the language model's shards + const dst = w.buf.contents()[at..][0..s.len]; + try readAll(fd, dst, s.off); + at += s.len; + } + // downsample [O, C, 2, 2] -> [O, 2, 2, C]: a merge window's four patch rows, tap by tap + const dw: [*]u16 = @ptrCast(@alignCast(w.down_w.addr())); + const tmp = try gpa.alloc(u16, hidden * merge * merge); + defer gpa.free(tmp); + for (0..out_hidden) |o| { + const row = dw[o * hidden * 4 ..][0 .. hidden * 4]; + @memcpy(tmp, row); + for (0..hidden) |ch| for (0..4) |tap| { + row[tap * hidden + ch] = tmp[ch * 4 + tap]; + }; + } + return w; +} + +/// `e` reads images: the tower loaded beside its weights (about 0.9 GiB with scratch for `lim.image_tokens` image +/// tokens a prompt) and the rows a prompt's images fill; `config` is the checkpoint's config.json. +pub fn attach(e: *Engine, gpa: std.mem.Allocator, dir: []const u8, config: []const u8, lim: Limits) !void { + if (e.ep != null) return error.VisionOnPair; + const pj: ?mtl.MappedFile = blk: { + const pp = try std.fmt.allocPrintSentinel(gpa, "{s}/processor_config.json", .{dir}, 0); + defer gpa.free(pp); + break :blk mtl.MappedFile.open(pp) catch null; + }; + defer if (pj) |m| m.deinit(); + var cfg_arena = std.heap.ArenaAllocator.init(gpa); + defer cfg_arena.deinit(); + const vc = (try Config.parse(cfg_arena.allocator(), config, if (pj) |m| m.bytes[0..m.size] else null)) orelse return error.NoVisionTower; + e.vision = try Vision.load(gpa, e.device, dir, vc, lim.image_tokens); + e.vision_rows = try e.arena.buffer(@as(usize, lim.image_tokens) * e.c.hidden * 2); + e.vision_limits = lim; + std.log.info("glm: image tower loaded ({d:.2} GiB with scratch for {d} image tokens a prompt)", .{ @as(f64, @floatFromInt(e.vision.?.bytes)) / (1 << 30), lim.image_tokens }); +} + +test "bf16 rounds to nearest even" { + try std.testing.expectEqual(@as(u16, 0x3f80), bf16(1.0)); + try std.testing.expectEqual(@as(u16, 0x3f80), bf16(@bitCast(@as(u32, 0x3f808000)))); // a tie to even (down) + try std.testing.expectEqual(@as(u16, 0x3f82), bf16(@bitCast(@as(u32, 0x3f818000)))); // a tie to even (up) + try std.testing.expectEqual(@as(u16, 0x3f81), bf16(@bitCast(@as(u32, 0x3f808001)))); +} diff --git a/zig/src/native/glm_host.zig b/zig/src/native/glm_host.zig index e7e9ed8f4d..ca4ac66cad 100644 --- a/zig/src/native/glm_host.zig +++ b/zig/src/native/glm_host.zig @@ -16,6 +16,7 @@ pub const Host = struct { eng: *ge.Engine, slots: glm.slots.Slots, back: glm.backend.Backend, + markers: [3]u32 = undefined, // the image wrapper's tokens (--vision): begin, image, end cfg: lanes.Config, wall: lanes.backend.WallClock, core: lanes.Engine, @@ -79,8 +80,9 @@ fn learnedStates(gpa: Allocator, eng: *const ge.Engine, sl: *glm.slots.Slots, ro } /// A GLM-5.3-Flash checkpoint served: `window` tokens of cache a stream, warmed first (the pair: `speed_up`). -pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, window: u32, speed_up: ?[]const u8, streams: u32, fixed: bool, cache_gib: ?f64, learn: ?[]const u8, learn_cap: u64, learn_floor: u64) !*Host { - const eng = try ge.Engine.loadWith(gpa, dir, window + 64, speed_up, learn != null); +pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, window: u32, speed_up: ?[]const u8, streams: u32, fixed: bool, cache_gib: ?f64, learn: ?[]const u8, learn_cap: u64, learn_floor: u64, vision: ?api.VisionOpen) !*Host { + const limits: ?glm.vision.Limits = if (vision) |v| .{ .image_tokens = v.image_tokens, .max_images = v.max_images } else null; + const eng = try ge.Engine.loadWith(gpa, dir, window + 64, speed_up, learn != null, limits); errdefer eng.deinit(); var toks: [96]u32 = undefined; for (&toks, 0..) |*t, i| t.* = @intCast(1000 + i); @@ -103,7 +105,8 @@ pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, window: u32, speed_up: h.wall = .{ .io = io }; h.core = lanes.Engine.init(gpa, &h.cfg, h.back.backend(), h.wall.clock()); errdefer h.core.deinit(); - h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "glm-zig", .lanes = n, .context_window = window, .prefill_step = eng.chunk_rows, .greedy_only = true, .admit = if (h.slots.pool != null) .{ .ctx = &h.back, .fits = glm.backend.Backend.fits } else null }); + if (eng.vision) |v| h.markers = .{ v.c.image_start, v.c.image_token, v.c.image_end }; + h.host = api.LaneHost.init(gpa, io, &h.core, .{ .name = "glm-zig", .lanes = n, .context_window = window, .prefill_step = eng.chunk_rows, .greedy_only = true, .admit = if (h.slots.pool != null) .{ .ctx = &h.back, .fits = glm.backend.Backend.fits } else null, .vision = if (eng.vision) |v| .{ .ctx = h, .prepare = prepareImage, .image_token = v.c.image_token, .markers = &h.markers, .image_tokens = eng.vision_limits.image_tokens, .max_images = eng.vision_limits.max_images } else null }); h.cache = null; const budget = cacheBudget(eng, cache_gib); if (!eng.followsPeer() and budget > 0) { @@ -133,6 +136,13 @@ pub fn open(gpa: Allocator, io: std.Io, dir: []const u8, window: u32, speed_up: return h; } +/// Info.vision's prepare: an image's bytes to the tower's patches (families/glm/image.zig), on the HTTP thread. +fn prepareImage(ctx: *anyopaque, a: Allocator, bytes: []const u8, max_tokens: u32) anyerror!api.PreparedImage { + const h: *Host = @ptrCast(@alignCast(ctx)); + const p = try glm.image.prepare(a, bytes, h.eng.vision.?.c, max_tokens); + return .{ .pixels = p.pixels, .gh = p.gh, .gw = p.gw, .tokens = p.tokens, .hash = p.hash }; +} + fn follow(h: *Host) void { glm.mirror.follow(h.eng, &h.slots) catch |err| std.log.err("speed-up mode: following rank 0 ended: {s}", .{@errorName(err)}); } diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index eb02076946..a8e0283e6c 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -107,6 +107,10 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c if (dense) return openQwen27(a, gpa, io, o, problem); if (std.mem.eql(u8, o.model_type, "qwen4_exp")) return openFlashNext(a, gpa, io, o, problem); if (std.mem.eql(u8, o.model_type, "glm5_next")) return openGlm(a, gpa, io, o, problem); + if (o.vision != null) { + problem.* = try std.fmt.allocPrint(a, "--vision: the native engine reads images for GLM-5.3-Flash only, not {s} checkpoints", .{o.model_type}); + return null; + } if (std.mem.eql(u8, o.model_type, "qwen3_5")) return @import("qwen35.zig").open(a, gpa, io, o, problem); if (!std.mem.eql(u8, o.model_type, "nemotron_h")) { problem.* = try std.fmt.allocPrint(a, "the native engine has no backend for {s} checkpoints yet; the Python engine 0.6.6 may serve them: python -m pip install git+https://github.com/ashhart/TensorFold.git@v0.6.6", .{o.model_type}); @@ -337,7 +341,11 @@ fn openGlm(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]co } const pool = mtl.objc.Pool.push(); defer pool.pop(); - const h = glm.open(gpa, io, o.dir, @intCast(window), o.speed_up, o.lanes, o.lanes_fixed, o.prompt_cache_gib, o.learn, @intFromFloat(o.learn_gib * (1 << 30)), @intFromFloat(o.learn_min_free_gib * (1 << 30))) catch |e| { + if (o.vision != null and o.speed_up != null) { + problem.* = try std.fmt.allocPrint(a, "--vision is served on one Mac; the --speed-up pair reads text only", .{}); + return null; + } + const h = glm.open(gpa, io, o.dir, @intCast(window), o.speed_up, o.lanes, o.lanes_fixed, o.prompt_cache_gib, o.learn, @intFromFloat(o.learn_gib * (1 << 30)), @intFromFloat(o.learn_min_free_gib * (1 << 30)), o.vision) catch |e| { if (e == error.BadLoadLimit) problem.* = "GLM_LOAD_LIMIT_GB is not a load limit: give a number of GB above 0, or unset it for 70% of RAM" else problem.* = if (linkProblem(e)) |link| try std.fmt.allocPrint(a, "the native GLM-5.3-Flash engine cannot load {s}: {s}", .{ o.dir, link }) else try std.fmt.allocPrint(a, "the native GLM-5.3-Flash engine cannot load {s} ({s})", .{ o.dir, @errorName(e) }); return null; }; diff --git a/zig/src/server/anthropic.zig b/zig/src/server/anthropic.zig index 4821e0f3eb..41f7adb038 100644 --- a/zig/src/server/anthropic.zig +++ b/zig/src/server/anthropic.zig @@ -51,7 +51,7 @@ fn countTokens(srv: *Server, cx: *Cx, chat: Value) errors.Refused!usize { const thinking = try fields.thinkingFields(cx, chat, srv.effort_levels); const on = thinking.enable orelse srv.config.enable_thinking; const tools = try tool_specs.active(cx, chat.get("tools"), chat.get("tool_choice")); - const msgs = try messages_mod.normalize(cx, chat.get("messages"), "system", srv.needs_user_after_tool); + const msgs = try messages_mod.normalizeWith(cx, chat.get("messages"), "system", srv.needs_user_after_tool, srv.info.vision != null); const ids_ = prompt.renderIds(srv, cx, msgs, tools, on, srv.effortFor(thinking.effort), true) catch |e| switch (e) { error.Refused => return cx.other(cx.message), else => |x| return x, diff --git a/zig/src/server/chat.zig b/zig/src/server/chat.zig index 6315e32991..76c65ec12c 100644 --- a/zig/src/server/chat.zig +++ b/zig/src/server/chat.zig @@ -193,6 +193,8 @@ pub fn prepare(srv: *Server, cx: *Cx, input: Input, gone: anytype) Failure!Prepa .decode_spans = try prompt_mod.replySpans(srv, cx, rendered.ids), // from the tokens alone, for every request: cached states are keyed by tokens .tools_json = if (input.tools.len > 0) try json.stringify(a, .{ .array = @constCast(input.tools) }, .{ .ascii = false }) else "", .decide = if (input.prompt == null and !background) prompt_mod.decideOf(srv, rendered.ids) else null, + .images = rendered.images, + .cache_key = rendered.cache_key, }; try srv.checkFeatures(cx, f, input.tools.len > 0, thinking, rendered.ids, input.tools, &request); if (thinking) { @@ -279,35 +281,6 @@ pub fn release(srv: *Server, preparing: bool) void { if (preparing) _ = srv.preparing.fetchSub(1, .acq_rel); } -/// The text a stream has sent: Python's ``streamed = visible``, grown by its delta while it only grows. -const Shown = struct { - text: std.ArrayList(u8) = .empty, - chars: usize = 0, - at: ?[*]const u8 = null, // where the last shown text lay; the decode buffer only grows, so that prefix holds - extends: bool = false, - - /// Python's ``now[len(shown):]``; ``stable``: ``now`` lies in the append-only decode buffer. - fn after(s: *Shown, now: []const u8, stable: bool) []const u8 { - const sent = s.text.items; - s.extends = (stable and s.at == now.ptr and now.len >= sent.len) or std.mem.startsWith(u8, now, sent); - if (!s.extends) return reply_text.afterChars(now, s.chars); - if (stable) s.at = now.ptr; // the buffer moved but kept its bytes: the next check is free again - return now[sent.len..]; - } - - fn set(s: *Shown, a: Allocator, now: []const u8, delta: []const u8) Allocator.Error!void { - if (s.extends) { - try s.text.appendSlice(a, delta); - s.chars += reply_text.charCount(delta); - } else { - s.text = .empty; - try s.text.appendSlice(a, now); - s.chars = reply_text.charCount(now); - } - s.at = now.ptr; - } -}; - /// The token loop and the text it streams. const Generation = struct { srv: *Server, @@ -325,8 +298,8 @@ const Generation = struct { collected: std.ArrayList(u32) = .empty, visible: reply_text.Incremental = .{}, hidden: std.ArrayList(u8) = .empty, // reused for the answer without its call blocks - streamed: Shown = .{}, // what content streamed, kept apart from the decode buffer that grows under slices - streamed_reasoning: Shown = .{}, + streamed: reply_text.Shown = .{}, // what content streamed, kept apart from the decode buffer that grows under slices + streamed_reasoning: reply_text.Shown = .{}, streaming_done: bool = false, first_ns: ?i96 = null, last_ns: ?i96 = null, diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index 40ee1e0ee8..c30c26a1d4 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -49,11 +49,11 @@ pub const flags = [_]Flag{ .{ .name = "--api-key-file", .native = true }, .{ .name = "--metrics-open", .kind = .store_true, .native = true }, .{ .name = "--dashboard", .kind = .store_true, .native = true }, - .{ .name = "--vision", .kind = .store_true }, + .{ .name = "--vision", .kind = .store_true, .native = true }, .{ .name = "--vision-urls", .kind = .store_true }, .{ .name = "--vision-offload", .kind = .store_true }, - .{ .name = "--vision-max-images" }, - .{ .name = "--vision-image-tokens" }, + .{ .name = "--vision-max-images", .native = true }, + .{ .name = "--vision-image-tokens", .native = true }, .{ .name = "--context", .native = true }, .{ .name = "--speed-up", .native = true }, .{ .name = "--max-tokens", .native = true }, @@ -167,6 +167,9 @@ pub const Args = struct { master: []const u8 = "", master_port: u16 = 29551, policy: []const u8 = "", // every --policy joined by commas + vision: bool = false, // image input (GLM-5.3-Flash): data-URL image_url parts in user messages + vision_max_images: u32 = 4, + vision_image_tokens: u32 = 4096, }; /// A usage error's message (argparse's ``error:`` line); the caller exits 2. @@ -272,7 +275,15 @@ fn apply(a: Allocator, out: *Args, name: []const u8, value: ?[]const u8, u: *Usa const n = try int(u, a, name, v); if (n < 0) return fail(u, a, "argument --compact-keep: expected a token count from 0: '{s}'", .{v}); out.compact_keep = @intCast(n); - } else if (is(name, "--compact-memory")) out.compact_memory = v else if (is(name, "--slide")) out.slide = true else if (is(name, "--slide-graph")) out.slide_graph = v else if (is(name, "--parallel")) out.parallel = v else if (is(name, "--backend")) out.backend = v; + } else if (is(name, "--compact-memory")) out.compact_memory = v else if (is(name, "--slide")) out.slide = true else if (is(name, "--slide-graph")) out.slide_graph = v else if (is(name, "--parallel")) out.parallel = v else if (is(name, "--backend")) out.backend = v else if (is(name, "--vision")) out.vision = true else if (is(name, "--vision-max-images")) { + const n = try int(u, a, name, v); + if (n < 1 or n > 64) return fail(u, a, "argument --vision-max-images: expected 1 to 64 images: '{s}'", .{v}); + out.vision_max_images = @intCast(n); + } else if (is(name, "--vision-image-tokens")) { + const n = try int(u, a, name, v); + if (n < 16 or n > 16384) return fail(u, a, "argument --vision-image-tokens: expected 16 to 16384 tokens: '{s}'", .{v}); + out.vision_image_tokens = @intCast(n); + } } /// The CUDA build's --device and --segments; false for any other flag. @@ -364,6 +375,7 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args .master = args.master, .master_port = args.master_port, .policy = args.policy, + .vision = if (args.vision) .{ .image_tokens = args.vision_image_tokens, .max_images = args.vision_max_images } else null, }; } diff --git a/zig/src/server/hf_text.zig b/zig/src/server/hf_text.zig index 22e4b976dc..416593c28c 100644 --- a/zig/src/server/hf_text.zig +++ b/zig/src/server/hf_text.zig @@ -209,6 +209,7 @@ pub const HfText = struct { .template_source = sourceFn, .special = specialFn, .token_bytes = tokenBytesFn, + .encode_plain = encodePlainFn, } }; } @@ -229,6 +230,10 @@ pub const HfText = struct { return std.mem.concat(a, u32, &.{ t.prefix, ids, t.suffix }); } + fn encodePlainFn(ctx: *anyopaque, a: Allocator, s: []const u8) model_text.Error![]u32 { + return self(ctx).tok.encodePlain(a, s) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else error.Template; + } + fn decodeFn(ctx: *anyopaque, a: Allocator, ids: []const u32) Allocator.Error![]u8 { return self(ctx).tok.decode(a, ids, false) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else a.dupe(u8, "\u{fffd}"); } diff --git a/zig/src/server/images.zig b/zig/src/server/images.zig new file mode 100644 index 0000000000..ba1253ef2c --- /dev/null +++ b/zig/src/server/images.zig @@ -0,0 +1,350 @@ +//! Image input (--vision): a request's image_url parts to the engine's tower input, and the rendered prompt's +//! placeholders expanded to each image's tokens, as TensorFold 0.6.6's vision path does for GLM-5.3-Flash. +const std = @import("std"); +const api = @import("engine_api"); +const json = @import("json.zig"); +const errors = @import("errors.zig"); +const messages_mod = @import("messages.zig"); +const model_text = @import("model_text.zig"); +const Value = json.Value; +const Cx = errors.Cx; + +/// images.py's limits: an image's and a request's encoded bytes, and the visual tokens of a low-detail image. +pub const max_image_bytes = 10 << 20; +pub const max_request_bytes = 20 << 20; +const low_detail_tokens = 256; + +pub const Expanded = struct { + ids: []const u32, + history_len: usize, + images: []const api.Image, + cache_key: []const u32, +}; + +const Part = struct { url: []const u8, low: bool }; + +/// The image parts of normalized `messages`, in the order the template meets them. +fn parts(cx: *Cx, messages: Value) errors.Refused![]const Part { + var out: std.ArrayList(Part) = .empty; + if (messages != .array) return out.items; + for (messages.array) |m| { + const content = m.get("content") orelse continue; + if (content != .array) continue; + for (content.array) |p| { + const kind = p.get("type") orelse continue; + if (kind != .string or !std.mem.eql(u8, kind.string, "image_url")) continue; + const iu = p.get("image_url").?; + const detail = iu.get("detail"); + try out.append(cx.a, .{ .url = iu.get("url").?.string, .low = detail != null and std.mem.eql(u8, detail.?.string, "low") }); + } + } + return out.items; +} + +/// A data URL's bytes (base64 or percent-encoded); remote URLs are refused (the native server fetches nothing). +pub fn dataBytes(cx: *Cx, url: []const u8) errors.Refused![]const u8 { + if (std.ascii.startsWithIgnoreCase(url, "http://") or std.ascii.startsWithIgnoreCase(url, "https://")) + return cx.refuse("image URLs are not fetched by this server; send the image inline as a data: URL (data:image/png;base64,...)"); + if (!std.ascii.startsWithIgnoreCase(url, "data:")) return cx.refuse("invalid image data URL"); + const comma = std.mem.indexOfScalar(u8, url, ',') orelse return cx.refuse("invalid image data URL"); + const header = url[5..comma]; + const payload = url[comma + 1 ..]; + if (header.len > 256) return cx.refuse("invalid image data URL"); + var it = std.mem.splitScalar(u8, header, ';'); + _ = it.next(); + const enc = it.next(); + if (it.next() != null or (enc != null and !std.ascii.eqlIgnoreCase(enc.?, "base64"))) return cx.refuse("image data URL supports only optional base64 encoding"); + if (enc != null) { + if (payload.len > 4 * ((max_image_bytes + 2) / 3)) return cx.refuse("image data URL exceeds the encoded byte limit"); + const d = std.base64.standard.Decoder; + const n = d.calcSizeForSlice(payload) catch return cx.refuse("invalid image data URL encoding"); + const out = try cx.a.alloc(u8, n); + d.decode(out, payload) catch return cx.refuse("invalid image data URL encoding"); + if (out.len == 0 or out.len > max_image_bytes) return cx.refuse("image is empty or exceeds the encoded byte limit"); + return out; + } + if (payload.len > max_image_bytes * 3) return cx.refuse("image data URL exceeds the encoded byte limit"); + var out: std.ArrayList(u8) = .empty; + var i: usize = 0; + while (i < payload.len) : (i += 1) { + if (payload[i] == '%' and i + 2 < payload.len) { + const b = std.fmt.parseInt(u8, payload[i + 1 .. i + 3], 16) catch return cx.refuse("invalid image data URL encoding"); + try out.append(cx.a, b); + i += 2; + } else try out.append(cx.a, payload[i]); + } + if (out.items.len == 0 or out.items.len > max_image_bytes) return cx.refuse("image is empty or exceeds the encoded byte limit"); + return out.items; +} + +pub const Format = enum { png, jpeg, gif, webp, other }; + +/// The image's format from its first bytes, before any decoder sees it. +pub fn sniff(bytes: []const u8) Format { + if (std.mem.startsWith(u8, bytes, "\x89PNG\r\n\x1a\n")) return .png; + if (std.mem.startsWith(u8, bytes, "\xff\xd8\xff")) return .jpeg; + if (std.mem.startsWith(u8, bytes, "GIF87a") or std.mem.startsWith(u8, bytes, "GIF89a")) return .gif; + if (bytes.len >= 12 and std.mem.eql(u8, bytes[0..4], "RIFF") and std.mem.eql(u8, bytes[8..12], "WEBP")) return .webp; + return .other; +} + +fn refusal(cx: *Cx, i: usize, err: anyerror) errors.Refused { + return switch (err) { + error.OutOfMemory => error.OutOfMemory, + error.ImageTooLarge => cx.refuse(try std.fmt.allocPrint(cx.a, "image {d} exceeds the limits: 10 MB encoded, 8192 pixels a side, 16 megapixels", .{i + 1})), + error.Animated => cx.refuse(try std.fmt.allocPrint(cx.a, "image {d}: animated and multipage images are unsupported; send a single frame", .{i + 1})), + error.ImageTooSmall => cx.refuse(try std.fmt.allocPrint(cx.a, "image {d}: the visual-token budget is too small for an image", .{i + 1})), + else => cx.refuse(try std.fmt.allocPrint(cx.a, "image {d} could not be decoded; send PNG or JPEG", .{i + 1})), + }; +} + +/// A quoted marker's stand-in while the template renders: two private-use code points, the second naming the marker. +const sentinel = "\u{F0000}"; + +/// The image markers' text (Vision.markers), longest first so one never matches inside another. +pub fn markerText(text: model_text.Text, a: std.mem.Allocator, v: api.Vision) ![]const []const u8 { + var out: std.ArrayList([]const u8) = .empty; + for (v.markers) |id| { + const s = try text.tokenString(a, id); + if (s.len > 0) try out.append(a, s); + } + std.mem.sort([]const u8, out.items, {}, struct { + fn longer(_: void, x: []const u8, y: []const u8) bool { + return x.len > y.len; + } + }.longer); + return out.items; +} + +/// `messages` with each marker the text spells (a user's words, a tool's output) replaced by its sentinel, so only +/// the template's own markers, one wrapper per image part, render as the image tokens (#511). +pub fn shield(a: std.mem.Allocator, v: Value, markers: []const []const u8) !Value { + return switch (v) { + .string => |s| .{ .string = try shieldText(a, s, markers) }, + .array => |items| blk: { + const out = try a.alloc(Value, items.len); + for (items, out) |x, *o| o.* = try shield(a, x, markers); + break :blk .{ .array = out }; + }, + .object => |o| blk: { + const out = try json.newObject(a); + var it = o.iterator(); + while (it.next()) |kv| try out.put(a, kv.key_ptr.*, try shield(a, kv.value_ptr.*, markers)); + break :blk .{ .object = out }; + }, + else => v, + }; +} + +fn shieldText(a: std.mem.Allocator, s: []const u8, markers: []const []const u8) ![]const u8 { + var out: std.ArrayList(u8) = .empty; + var i: usize = 0; + var changed = false; + outer: while (i < s.len) { + for (markers, 0..) |m, k| if (std.mem.startsWith(u8, s[i..], m)) { + try out.appendSlice(a, sentinel); + try out.appendSlice(a, try std.fmt.allocPrint(a, "{u}", .{@as(u21, @intCast(0xF0001 + k))})); + i += m.len; + changed = true; + continue :outer; + }; + try out.append(a, s[i]); + i += 1; + } + return if (changed) out.items else s; +} + +/// The rendered prompt's ids: its text as the tokenizer reads it, each sentinel's marker as ordinary text tokens. +/// Added tokens split the text anyway, so a prompt without quoted markers gets exactly `encode`'s ids. +pub fn encodeShielded(text: model_text.Text, a: std.mem.Allocator, rendered: []const u8, markers: []const []const u8) model_text.Error![]u32 { + var ids: std.ArrayList(u32) = .empty; + var rest = rendered; + while (std.mem.indexOf(u8, rest, sentinel)) |at| { + const after = rest[at + sentinel.len ..]; + const cp: u21 = if (after.len >= 4) std.unicode.utf8Decode(after[0..4]) catch 0 else 0; + const k: usize = if (cp >= 0xF0001) cp - 0xF0001 else markers.len; + if (k >= markers.len) { // not one of ours: the code point is ordinary text + try ids.appendSlice(a, try text.encode(a, rest[0 .. at + sentinel.len], false)); + rest = after; + continue; + } + try ids.appendSlice(a, try text.encode(a, rest[0..at], false)); + try ids.appendSlice(a, try text.encodePlain(a, markers[k])); + rest = after[4..]; + } + try ids.appendSlice(a, try text.encode(a, rest, false)); + return ids.items; +} + +/// The prompt with each image's placeholder expanded to its tokens, the images for the engine and the prompt as the +/// cache matches it; null when the engine reads no images. Images are counted from the request's image parts; the +/// rendered placeholders are the template's alone (text that spells a marker was shielded), one per part. +pub fn expand(vision: ?api.Vision, cx: *Cx, messages: Value, ids: []const u32, history_len: usize) errors.Refused!?Expanded { + const v = vision orelse return null; + const ps = try parts(cx, messages); + var marks: usize = 0; + for (ids) |t| marks += @intFromBool(t == v.image_token); + if (marks != ps.len) return cx.fail(.server, "the chat template wrote {d} image placeholders for {d} image parts", .{ marks, ps.len }); + if (ps.len == 0) return null; + if (ps.len > v.max_images) return cx.refuse(try std.fmt.allocPrint(cx.a, "a request may hold at most {d} images (--vision-max-images); it has {d}", .{ v.max_images, ps.len })); + const budget = v.image_tokens / @as(u32, @intCast(ps.len)); + if (budget < 1) return cx.refuse("the image count exceeds the visual-token budget"); + const prepared = try cx.a.alloc(api.PreparedImage, ps.len); + var bytes_total: usize = 0; + var extra: usize = 0; + for (ps, prepared, 0..) |p, *out, i| { + const bytes = try dataBytes(cx, p.url); + bytes_total += bytes.len; + if (bytes_total > max_request_bytes) return cx.refuse("the request's images exceed 20 MB encoded"); + switch (sniff(bytes)) { // the Python frontend's formats (images.py: JPEG, PNG, WebP), WebP not yet decoded here + .png, .jpeg => {}, + .webp => return cx.refuse(try std.fmt.allocPrint(cx.a, "image {d} is WebP, which this server does not decode yet; send PNG or JPEG", .{i + 1})), + .gif, .other => return cx.refuse(try std.fmt.allocPrint(cx.a, "image {d} is not PNG or JPEG; send one of those", .{i + 1})), + } + const cap = if (p.low) @min(budget, low_detail_tokens) else budget; + out.* = v.prepare(v.ctx, cx.a, bytes, cap) catch |e| return refusal(cx, i, e); + if (out.tokens == 0 or out.tokens > cap) return cx.refuse("the image processor exceeded the per-image visual-token budget"); + extra += out.tokens - 1; + } + const n = ids.len + extra; + const out_ids = try cx.a.alloc(u32, n); + const key = try cx.a.alloc(u32, n); + const imgs = try cx.a.alloc(api.Image, ps.len); + var hist = history_len; + var at: usize = 0; + var k: usize = 0; + for (ids, 0..) |t, src| { + if (t != v.image_token) { + out_ids[at] = t; + key[at] = t; + at += 1; + continue; + } + const img = prepared[k]; + imgs[k] = .{ .pixels = img.pixels, .gh = img.gh, .gw = img.gw, .at = @intCast(at), .tokens = img.tokens }; + if (src < history_len) hist += img.tokens - 1; + for (0..img.tokens) |j| { + out_ids[at + j] = t; + // the cache's ids for this image's rows: past the vocabulary, by content and row + key[at + j] = 0x8000_0000 | @as(u32, @truncate(std.hash.Wyhash.hash(img.hash, std.mem.asBytes(&j)) & 0x7fff_ffff)); + } + at += img.tokens; + k += 1; + } + return .{ .ids = out_ids, .history_len = hist, .images = imgs, .cache_key = key }; +} + +test "data URLs decode; remote URLs are refused" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var cx: Cx = .{ .a = arena.allocator() }; + try std.testing.expectEqualStrings("hi!", try dataBytes(&cx, "data:image/png;base64,aGkh")); + try std.testing.expectEqualStrings("a b", try dataBytes(&cx, "data:text/plain,a%20b")); + try std.testing.expectError(error.Refused, dataBytes(&cx, "https://example.com/x.png")); + try std.testing.expectError(error.Refused, dataBytes(&cx, "data:image/png;base64,***")); +} + +fn fakePrepare(ctx: *anyopaque, a: std.mem.Allocator, bytes: []const u8, max_tokens: u32) anyerror!api.PreparedImage { + _ = ctx; + _ = max_tokens; + const tokens: u32 = @intCast(bytes.len - 3); // the test's images: a JPEG's first bytes, then one token a byte + return .{ .pixels = try a.alloc(f32, 0), .gh = 2, .gw = 2 * tokens, .tokens = tokens, .hash = bytes[3] }; +} + +test "placeholders expand to each image's tokens, the history and cache key with them" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var cx: Cx = .{ .a = a }; + var dummy: u8 = 0; + const v: api.Vision = .{ .ctx = &dummy, .prepare = fakePrepare, .image_token = 9, .image_tokens = 64, .max_images = 4 }; + const text = + \\[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:,%FF%D8%FFab"}}, {"type": "text", "text": "x"}]}, + \\ {"role": "assistant", "content": "y"}, + \\ {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:,%FF%D8%FFcde"}}]}] + ; + const msgs = switch (try json.parseText(a, text)) { + .ok => |m| m, + .err => return error.TestUnexpectedResult, + }; + const ids = [_]u32{ 1, 9, 2, 3, 9, 4 }; + const x = (try expand(v, &cx, msgs, &ids, 4)).?; + try std.testing.expectEqualSlices(u32, &.{ 1, 9, 9, 2, 3, 9, 9, 9, 4 }, x.ids); + try std.testing.expectEqual(@as(usize, 5), x.history_len); // the first image is in the history, the second not + try std.testing.expectEqual(@as(u32, 1), x.images[0].at); + try std.testing.expectEqual(@as(u32, 5), x.images[1].at); + try std.testing.expect(x.cache_key[1] >= 0x8000_0000 and x.cache_key[1] != x.cache_key[2]); + try std.testing.expectEqual(@as(u32, 2), x.cache_key[3]); + // a placeholder the images do not account for is refused + try std.testing.expectError(error.Refused, expand(v, &cx, msgs, &[_]u32{ 9, 9, 9 }, 0)); + _ = messages_mod; +} + +/// A test tokenizer: "<|image|>" is token 9 unless read plainly, every other byte its own id plus 100. +const MarkText = struct { + fn text() model_text.Text { + return .{ .ctx = undefined, .vtable = &.{ .encode = encode, .decode = undefined, .token_id = undefined, .token_string = tokenString, .vocab_size = undefined, .eos_ids = undefined, .render = undefined, .template_source = undefined, .encode_plain = plain } }; + } + fn encode(_: *anyopaque, a: std.mem.Allocator, s: []const u8, _: bool) model_text.Error![]u32 { + var out: std.ArrayList(u32) = .empty; + var i: usize = 0; + while (i < s.len) { + if (std.mem.startsWith(u8, s[i..], "<|image|>")) { + try out.append(a, 9); + i += 9; + } else { + try out.append(a, @as(u32, s[i]) + 100); + i += 1; + } + } + return out.items; + } + fn plain(_: *anyopaque, a: std.mem.Allocator, s: []const u8) model_text.Error![]u32 { + const out = try a.alloc(u32, s.len); + for (out, s) |*o, c| o.* = @as(u32, c) + 100; + return out; + } + fn tokenString(_: *anyopaque, a: std.mem.Allocator, id: u32) std.mem.Allocator.Error![]u8 { + return a.dupe(u8, if (id == 9) "<|image|>" else ""); + } +}; + +test "a marker the text spells stays text: only the template's markers are image tokens (#511)" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + const v: api.Vision = .{ .ctx = &dummy, .prepare = fakePrepare, .image_token = 9, .markers = &.{9}, .image_tokens = 64, .max_images = 4 }; + const markers = try markerText(MarkText.text(), a, v); + const conversation = + \\[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:,ab"}}, {"type": "text", "text": "what is <|image|>?"}]}, + \\ {"role": "tool", "content": "source: x = '<|image|>'"}] + ; + const msgs = switch (try json.parseText(a, conversation)) { + .ok => |m| m, + .err => return error.TestUnexpectedResult, + }; + const shielded = try shield(a, msgs, markers); + // the "template": one marker per image part, then each message's text + var rendered: std.ArrayList(u8) = .empty; + try rendered.appendSlice(a, "<|image|>"); + try rendered.appendSlice(a, shielded.array[0].get("content").?.array[1].get("text").?.string); + try rendered.appendSlice(a, shielded.array[1].get("content").?.string); + const ids = try encodeShielded(MarkText.text(), a, rendered.items, markers); + var marks: usize = 0; + for (ids) |t| marks += @intFromBool(t == 9); + try std.testing.expectEqual(@as(usize, 1), marks); + // the quoted markers read as their characters, and text without one is exactly encode's + try std.testing.expectEqualSlices(u32, try MarkText.encode(undefined, a, "<|image|>what is ", false), ids[0..9]); + try std.testing.expectEqual(@as(u32, '<' + 100), ids[9]); + const plain_text = "no markers here"; + try std.testing.expectEqualSlices(u32, try MarkText.encode(undefined, a, plain_text, false), try encodeShielded(MarkText.text(), a, plain_text, markers)); +} + +test "formats from their first bytes: WebP is refused by name" { + try std.testing.expectEqual(Format.png, sniff("\x89PNG\r\n\x1a\nrest")); + try std.testing.expectEqual(Format.jpeg, sniff("\xff\xd8\xff\xe0")); + try std.testing.expectEqual(Format.gif, sniff("GIF89a...")); + try std.testing.expectEqual(Format.webp, sniff("RIFF\x10\x00\x00\x00WEBPVP8 ")); + try std.testing.expectEqual(Format.other, sniff("\x00\x00\x00\x18ftypheic")); +} diff --git a/zig/src/server/messages.zig b/zig/src/server/messages.zig index 9559ed784b..48a0d77f6c 100644 --- a/zig/src/server/messages.zig +++ b/zig/src/server/messages.zig @@ -23,6 +23,36 @@ fn withField(cx: *Cx, o: *const json.Object, key: []const u8, value: Value) !*js /// ``normalize_messages`` (text only): leading system and developer text merged, later ones as ``late_system``; a template that needs a user query gains one user turn after a trailing tool run. pub fn normalize(cx: *Cx, messages: ?Value, late_system: []const u8, needs_user_after_tool: bool) errors.Refused!Value { + return normalizeWith(cx, messages, late_system, needs_user_after_tool, false); +} + +/// One image part as the template and the image path read it: {"type": "image_url", "image_url": {"url", "detail"?}}. +fn imagePart(cx: *Cx, part: Value) errors.Refused!Value { + const iu = part.get("image_url") orelse return cx.refuse("an image_url content part must contain image_url"); + var url: []const u8 = ""; + var detail: ?Value = null; + if (iu == .string) url = iu.string else if (iu == .object) { + const u = iu.get("url") orelse return cx.refuse("image_url must contain a url string"); + if (u != .string) return cx.refuse("image_url must contain a url string"); + url = u.string; + detail = iu.get("detail"); + if (detail) |d| if (d != .string or !(std.mem.eql(u8, d.string, "auto") or std.mem.eql(u8, d.string, "low") or std.mem.eql(u8, d.string, "high"))) + return cx.refuse("image_url detail must be auto, low or high"); + } else return cx.refuse("image_url must be a url string or an object with one"); + if (url.len == 0) return cx.refuse("image_url must contain a url string"); + for ([_][]const u8{ "audio", "input_audio", "video", "video_url" }) |k| if (json.truthyField(part, k)) return cx.refuse("image_url parts cannot contain other media"); + const inner = try json.newObject(cx.a); + try inner.put(cx.a, "url", .{ .string = url }); + if (detail) |d| try inner.put(cx.a, "detail", d); + const out = try json.newObject(cx.a); + try out.put(cx.a, "type", .{ .string = "image_url" }); + try out.put(cx.a, "image_url", .{ .object = inner }); + return .{ .object = out }; +} + +/// `normalize`, and with `images` (an engine that reads them) a user or tool message's image_url parts kept beside its +/// text parts, in order, for the template's image markers; a message without image parts is joined text as before. +pub fn normalizeWith(cx: *Cx, messages: ?Value, late_system: []const u8, needs_user_after_tool: bool, images: bool) errors.Refused!Value { const list = messages orelse return cx.refuse("messages must be a non-empty list"); if (list != .array or list.array.len == 0) return cx.refuse("messages must be a non-empty list"); var out: std.ArrayList(Value) = .empty; @@ -36,14 +66,29 @@ pub fn normalize(cx: *Cx, messages: ?Value, late_system: []const u8, needs_user_ var item: *json.Object = message.object; if (content != null and content.? == .array) { var text: std.ArrayList(u8) = .empty; + var parts: std.ArrayList(Value) = .empty; // with images: text and image parts in order + var has_image = false; for (content.?.array) |part| { - const typed = part == .object and part.get("type") != null and part.get("type").? == .string and std.mem.eql(u8, part.get("type").?.string, "text"); - if (!typed or fields.hasMedia(part)) return cx.refuse("this server accepts text parts only; image, audio and video inputs are unsupported"); + const kind = if (part == .object and part.get("type") != null and part.get("type").? == .string) part.get("type").?.string else ""; + if (images and std.mem.eql(u8, kind, "image_url")) { + // a tool's result carries images as agents send screenshots; the template renders them in its response + if (!std.mem.eql(u8, role.?.string, "user") and !std.mem.eql(u8, role.?.string, "tool")) return cx.refuse("image parts are accepted in user and tool messages only"); + try parts.append(cx.a, try imagePart(cx, part)); + has_image = true; + continue; + } + if (!std.mem.eql(u8, kind, "text") or fields.hasMedia(part)) return cx.refuse(if (images) "content parts must be text or image_url; audio and video inputs are unsupported" else "this server accepts text parts only; image, audio and video inputs are unsupported"); const t = part.get("text") orelse return cx.refuse("a text content part must contain a text string"); if (t != .string) return cx.refuse("a text content part must contain a text string"); try text.appendSlice(cx.a, t.string); + if (images) { + const tp = try json.newObject(cx.a); + try tp.put(cx.a, "type", .{ .string = "text" }); + try tp.put(cx.a, "text", t); + try parts.append(cx.a, .{ .object = tp }); + } } - item = try withField(cx, message.object, "content", .{ .string = text.items }); + item = try withField(cx, message.object, "content", if (has_image) Value{ .array = parts.items } else Value{ .string = text.items }); } else if (content == null or content.? == .null) { item = try withField(cx, message.object, "content", .{ .string = "" }); } else if (content.? != .string) { diff --git a/zig/src/server/model_text.zig b/zig/src/server/model_text.zig index 0b36055a7f..33b5baa78c 100644 --- a/zig/src/server/model_text.zig +++ b/zig/src/server/model_text.zig @@ -36,6 +36,8 @@ pub const Text = struct { template_source: *const fn (ctx: *anyopaque) []const u8, /// Whether an id is a special token (an added token flagged special); null: none are known. special: ?*const fn (ctx: *anyopaque, id: u32) bool = null, + /// Ids of ``text`` with no added token matched (text that spells a special token is ordinary text); null: ``encode``. + encode_plain: ?*const fn (ctx: *anyopaque, a: Allocator, text: []const u8) Error![]u32 = null, /// A token's own bytes (a partial UTF-8 sequence kept whole) for logprob rows; null: the tokenizer can't say. token_bytes: ?*const fn (ctx: *anyopaque, a: Allocator, id: u32) Allocator.Error!?[]u8 = null, }; @@ -44,6 +46,11 @@ pub const Text = struct { return t.vtable.encode(t.ctx, a, text, add_special); } + pub fn encodePlain(t: Text, a: Allocator, text: []const u8) Error![]u32 { + const f = t.vtable.encode_plain orelse return t.encode(a, text, false); + return f(t.ctx, a, text); + } + pub fn decode(t: Text, a: Allocator, ids: []const u32) Allocator.Error![]u8 { return t.vtable.decode(t.ctx, a, ids); } diff --git a/zig/src/server/openai.zig b/zig/src/server/openai.zig index 91d02cceb8..793dea3cdf 100644 --- a/zig/src/server/openai.zig +++ b/zig/src/server/openai.zig @@ -125,10 +125,10 @@ fn plan(srv: *Server, cx: *Cx, is_chat: bool, raw: Value) errors.Refused!Plan { if (srv.config.request_log) |path| request_log.append(cx.a, path, body); var input: chat.Input = .{ .fields = undefined }; if (is_chat) { - input.messages = try messages.normalize(cx, body.get("messages"), "system", srv.needs_user_after_tool); + input.messages = try messages.normalizeWith(cx, body.get("messages"), "system", srv.needs_user_after_tool, srv.info.vision != null); input.tools = try tool_specs.active(cx, body.get("tools"), body.get("tool_choice")); } else if (body.get("messages")) |m| if (m == .array and m.array.len > 0) { - input.messages = try messages.normalize(cx, m, "system", srv.needs_user_after_tool); + input.messages = try messages.normalizeWith(cx, m, "system", srv.needs_user_after_tool, srv.info.vision != null); }; if (!is_chat and input.messages.array.len == 0) input.prompt = try legacyPrompt(srv, cx, body.get("prompt")); // ``body.get("max_tokens") or body.get("max_completion_tokens")``: both are ints or None by now diff --git a/zig/src/server/prompt.zig b/zig/src/server/prompt.zig index e68363bfe7..5500803b68 100644 --- a/zig/src/server/prompt.zig +++ b/zig/src/server/prompt.zig @@ -1,15 +1,19 @@ //! Prompt ids from a request, as ``prepare_prompt`` and ``render_prompt_ids`` make them. const std = @import("std"); +const api = @import("engine_api"); const json = @import("json.zig"); const errors = @import("errors.zig"); const messages_mod = @import("messages.zig"); const model_text = @import("model_text.zig"); const chat = @import("chat.zig"); +const images = @import("images.zig"); const Server = @import("server.zig").Server; const Value = json.Value; const Cx = errors.Cx; -pub const Rendered = struct { ids: []const u32, history_len: usize = 0, rewind_len: usize = 0 }; +/// With image input, `ids` holds each image's tokens in place of its placeholder, `images` are theirs in order and +/// `cache_key` is the prompt as the cache matches it (engine_api.Request). +pub const Rendered = struct { ids: []const u32, history_len: usize = 0, rewind_len: usize = 0, images: []const api.Image = &.{}, cache_key: []const u32 = &.{} }; /// Where lessons decide to act on a reply: the row before the generation prompt's ``, else before its last. pub fn decideOf(srv: *const Server, ids: []const u32) u32 { @@ -27,7 +31,7 @@ pub const isTitle = messages_mod.isTitleRequest; /// ``render_prompt_ids``: messages normalized for this template, rendered, and a dangling ```` closed. pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, thinking: bool, effort: ?[]const u8, generation: bool) errors.Refused![]const u32 { - const normalized = try messages_mod.toolArguments(cx, try messages_mod.normalize(cx, messages, srv.late_system, srv.needs_user_after_tool)); + const normalized = try messages_mod.toolArguments(cx, try messages_mod.normalizeWith(cx, messages, srv.late_system, srv.needs_user_after_tool, srv.info.vision != null)); var problem: []const u8 = ""; const options: model_text.RenderOptions = .{ .tools = if (tools.len > 0) Value{ .array = @constCast(tools) } else null, @@ -35,7 +39,7 @@ pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, t .enable_thinking = thinking, .reasoning_effort = if (thinking) effort else null, }; - var ids = srv.text.renderIds(cx.a, normalized, options, &problem) catch |e| switch (e) { + var ids = render(srv, cx.a, normalized, options, &problem) catch |e| switch (e) { error.OutOfMemory => return error.OutOfMemory, error.Template => return cx.fail(.server, "{s}", .{problem}), }; @@ -52,6 +56,14 @@ pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, t return ids; } +/// The template's ids for `messages`; with image input, a marker the messages' text spells is ordinary text (images.zig). +fn render(srv: *Server, a: std.mem.Allocator, messages: Value, options: model_text.RenderOptions, problem: *[]const u8) model_text.Error![]u32 { + const v = srv.info.vision orelse return srv.text.renderIds(a, messages, options, problem); + const markers = try images.markerText(srv.text, a, v); + const text = try srv.text.render(a, try images.shield(a, messages, markers), options, problem); + return images.encodeShielded(srv.text, a, text, markers); +} + /// The template's generation prompt (thinking on or off): what a probe conversation renders past its history. fn generationPrompt(srv: *Server, cx: *Cx, thinking: bool) errors.Refused![]const u32 { const user = try json.newObject(cx.a); @@ -101,6 +113,9 @@ pub fn prepare(srv: *Server, cx: *Cx, input: chat.Input, thinking: bool, effort: var history_len: usize = 0; if (history.len > 0 and history.len < prompt.len and std.mem.eql(u32, prompt[0..history.len], history)) history_len = history.len; if (history_len == 0 and prompt.len > 1 and std.mem.eql(u32, history, prompt)) history_len = prompt.len - 1; // a template with no generation suffix + // --vision: each image's placeholder expanded to its tokens before the window, chunks and cache see the prompt; + // such a prompt keeps no rewind, which the expansion does not translate + if (try images.expand(srv.info.vision, cx, input.messages, prompt, history_len)) |x| return .{ .ids = x.ids, .history_len = x.history_len, .images = x.images, .cache_key = x.cache_key }; const count = if (input.messages == .array) input.messages.array.len else 0; const rewind_len = if (srv.info.prompt_cache) rewindLen(srv, cx, input.messages, count, input.tools, prompt, thinking, effort) else 0; // one more render return .{ .ids = prompt, .history_len = history_len, .rewind_len = rewind_len }; diff --git a/zig/src/server/reply_text.zig b/zig/src/server/reply_text.zig index 648a1d8cd2..991ae959ce 100644 --- a/zig/src/server/reply_text.zig +++ b/zig/src/server/reply_text.zig @@ -292,6 +292,35 @@ pub const Stops = struct { } }; +/// The text a stream has sent: Python's ``streamed = visible``, grown by its delta while it only grows. +pub const Shown = struct { + text: std.ArrayList(u8) = .empty, + chars: usize = 0, + at: ?[*]const u8 = null, // where the last shown text lay; the decode buffer only grows, so that prefix holds + extends: bool = false, + + /// Python's ``now[len(shown):]``; ``stable``: ``now`` lies in the append-only decode buffer. + pub fn after(s: *Shown, now: []const u8, stable: bool) []const u8 { + const sent = s.text.items; + s.extends = (stable and s.at == now.ptr and now.len >= sent.len) or std.mem.startsWith(u8, now, sent); + if (!s.extends) return afterChars(now, s.chars); + if (stable) s.at = now.ptr; // the buffer moved but kept its bytes: the next check is free again + return now[sent.len..]; + } + + pub fn set(s: *Shown, a: Allocator, now: []const u8, delta: []const u8) Allocator.Error!void { + if (s.extends) { + try s.text.appendSlice(a, delta); + s.chars += charCount(delta); + } else { + s.text = .empty; + try s.text.appendSlice(a, now); + s.chars = charCount(now); + } + s.at = now.ptr; + } +}; + /// Decodes only new tokens with the previous chunk as context, so characters split across tokens stay whole. pub const Incremental = struct { tokens: std.ArrayList(u32) = .empty, diff --git a/zig/src/server/root.zig b/zig/src/server/root.zig index ecc81c575a..b669d55c3a 100644 --- a/zig/src/server/root.zig +++ b/zig/src/server/root.zig @@ -38,6 +38,7 @@ test { _ = @import("process_memory.zig"); _ = @import("log.zig"); _ = @import("messages.zig"); + _ = @import("images.zig"); _ = @import("stream_preflight_test.zig"); _ = @import("logprobs_test.zig"); _ = @import("late_system_test.zig"); diff --git a/zig/tests/glm_vision.zig b/zig/tests/glm_vision.zig new file mode 100644 index 0000000000..7df41b512c --- /dev/null +++ b/zig/tests/glm_vision.zig @@ -0,0 +1,148 @@ +//! GLM-5.3-Flash's image tower against a reference dump (tools/zig/glm_vision_fixtures.py: mlx-vlm's tower). +const std = @import("std"); +const mtl = @import("metal"); +const tf = @import("tensorfold"); +const vision = tf.glm.vision; + +const usage = + \\tf-glm-vision MODEL_DIR REF_DIR [STAGE] [IMAGE] + \\REF_DIR holds meta.json (grid_thw), patches.f32 and the stage's reference: embed.f32 (STAGE embed, after the patch + \\projection), block0.f32 (STAGE block0, after the first block), blocks.f32 (STAGE blocks, after the last) or + \\features.f32 (STAGE features, the default: the tower's output rows). Writes native_STAGE.f32 there and prints the + \\largest and mean absolute differences, the reference's mean magnitude and the rows' cosine similarity. + \\IMAGE: prepare the patches from this file natively (decode, resize, pad, normalize), compare them with + \\patches.f32 value by value, and run the tower on them instead (meta.json's max_tokens is the budget). +; + +fn readFile(a: std.mem.Allocator, dir: []const u8, name: []const u8) ![]u8 { + const path = try std.fmt.allocPrintSentinel(a, "{s}/{s}", .{ dir, name }, 0); + const f = try mtl.MappedFile.open(path); + defer f.deinit(); + return a.dupe(u8, f.bytes[0..f.size]); +} + +fn bf16ToF32(x: u16) f32 { + return @bitCast(@as(u32, x) << 16); +} + +pub fn main(init: std.process.Init) !void { + const a = init.arena.allocator(); + const args = try init.minimal.args.toSlice(a); + if (args.len < 3) { + std.debug.print("{s}\n", .{usage}); + return error.Usage; + } + const dir = args[1]; + const ref = args[2]; + const stage = if (args.len > 3) args[3] else "features"; + const stop: ?u32 = if (std.mem.eql(u8, stage, "embed")) 0 else if (std.mem.eql(u8, stage, "block0")) 1 else if (std.mem.eql(u8, stage, "blocks")) vision.depth else if (std.mem.eql(u8, stage, "features")) null else return error.Usage; + + const meta = try std.json.parseFromSliceLeaky(std.json.Value, a, try readFile(a, ref, "meta.json"), .{}); + const grid = meta.object.get("grid_thw").?.array.items; + const gh: u32 = @intCast(grid[1].integer); + const gw: u32 = @intCast(grid[2].integer); + const raw = try readFile(a, ref, "patches.f32"); + var pixels = try a.alloc(f32, raw.len / 4); + @memcpy(std.mem.sliceAsBytes(pixels), raw[0 .. pixels.len * 4]); + const P = gh * gw; + const T = P / vision.patches_per_token; + + const device = try mtl.Device.init(); + const pool = mtl.objc.Pool.push(); + defer pool.pop(); + const queue = try device.queue(); + const cfg_text = try readFile(a, dir, "config.json"); + const proc_text = readFile(a, dir, "processor_config.json") catch null; + const c = (try vision.Config.parse(a, cfg_text, proc_text)) orelse return error.NoVisionConfig; + if (args.len > 4) { + const img_path = try std.fmt.allocPrintSentinel(a, "{s}", .{args[4]}, 0); + const f = try mtl.MappedFile.open(img_path); + defer f.deinit(); + const max_tokens: u32 = @intCast(meta.object.get("max_tokens").?.integer); + const t0i = std.Io.Clock.awake.now(init.io).toNanoseconds(); + const prep = try tf.glm.image.prepare(a, f.bytes[0..f.size], c, max_tokens); + const t1i = std.Io.Clock.awake.now(init.io).toNanoseconds(); + std.debug.print("prepared {s}: grid {d}x{d}, {d} tokens in {d:.1} ms\n", .{ args[4], prep.gh, prep.gw, prep.tokens, @as(f64, @floatFromInt(t1i - t0i)) / 1e6 }); + if (prep.gh != gh or prep.gw != gw) { + std.debug.print("grid differs from the reference's {d}x{d}\n", .{ gh, gw }); + return error.GridMismatch; + } + var differ: usize = 0; + var max_p: f32 = 0; + for (prep.pixels, pixels) |x, y| { + if (x != y) differ += 1; + max_p = @max(max_p, @abs(x - y)); + } + std.debug.print("patches: {d} of {d} values differ from the reference (largest {d:.6})\n", .{ differ, pixels.len, max_p }); + pixels = prep.pixels; + } + var t0 = std.Io.Clock.awake.now(init.io).toNanoseconds(); + const v = try vision.Vision.load(std.heap.page_allocator, device, dir, c, @max(T, 1024)); + defer v.deinit(); + var t1 = std.Io.Clock.awake.now(init.io).toNanoseconds(); + std.debug.print("tower loaded in {d:.2} s ({d:.2} GiB with scratch for {d} patches, attention chunks of {d} rows)\n", .{ @as(f64, @floatFromInt(t1 - t0)) / 1e9, @as(f64, @floatFromInt(v.bytes)) / (1 << 30), v.max_patches, v.chunk }); + const out_buf = try device.buffer(@as(usize, T) * vision.out_hidden * 2, mtl.ResourceOptions.shared); + defer out_buf.deinit(); + + for (0..2) |run| { // the second run is the timed one (the first compiles pipelines' state on the GPU) + const cb = queue.commandBuffer(); + const enc = cb.compute(.serial); + t0 = std.Io.Clock.awake.now(init.io).toNanoseconds(); + try v.encode(enc, pixels, gh, gw, .{ .buf = out_buf }, stop); + enc.end(); + cb.commit(); + cb.wait(); + t1 = std.Io.Clock.awake.now(init.io).toNanoseconds(); + if (cb.failure()) |msg| { + std.debug.print("command buffer failed: {s}\n", .{msg}); + return error.GpuFailed; + } + if (run == 1) std.debug.print("encode: {d} patches -> {d} tokens in {d:.1} ms\n", .{ P, T, @as(f64, @floatFromInt(t1 - t0)) / 1e6 }); + } + // the stage's rows: the residual stream for a stop, else the output rows + const width: usize = if (stop != null) vision.hidden else vision.out_hidden; + const rows: usize = if (stop != null) P else T; + const src: [*]const u16 = if (stop != null) @ptrCast(@alignCast(v.x.addr())) else @ptrCast(@alignCast(out_buf.contents())); + const got = try a.alloc(f32, rows * width); + for (got, 0..) |*g, i| g.* = bf16ToF32(src[i]); + const name = try std.fmt.allocPrint(a, "native_{s}.f32", .{stage}); + { + const path = try std.fmt.allocPrint(a, "{s}/{s}", .{ ref, name }); + const f = try std.Io.Dir.cwd().createFile(init.io, path, .{}); + defer f.close(init.io); + try f.writeStreamingAll(init.io, std.mem.sliceAsBytes(got)); + } + const want_raw = try readFile(a, ref, try std.fmt.allocPrint(a, "{s}.f32", .{stage})); + const want = try a.alloc(f32, want_raw.len / 4); + @memcpy(std.mem.sliceAsBytes(want), want_raw[0 .. want.len * 4]); + if (want.len != got.len) { + std.debug.print("size: native {d} values, reference {d}\n", .{ got.len, want.len }); + return error.SizeMismatch; + } + var max_d: f64 = 0; + var sum_d: f64 = 0; + var sum_m: f64 = 0; + var worst_cos: f64 = 1; + var mean_cos: f64 = 0; + for (0..rows) |r| { + var dot: f64 = 0; + var ng: f64 = 0; + var nw: f64 = 0; + for (0..width) |j| { + const g: f64 = got[r * width + j]; + const w: f64 = want[r * width + j]; + const d = @abs(g - w); + max_d = @max(max_d, d); + sum_d += d; + sum_m += @abs(w); + dot += g * w; + ng += g * g; + nw += w * w; + } + const cs = dot / (@sqrt(ng * nw) + 1e-30); + worst_cos = @min(worst_cos, cs); + mean_cos += cs; + } + const n: f64 = @floatFromInt(got.len); + std.debug.print("{s}: max |d| {d:.4}, mean |d| {d:.5}, mean |ref| {d:.4}, cosine mean {d:.6} worst {d:.6} ({d} rows of {d})\n", .{ stage, max_d, sum_d / n, sum_m / n, mean_cos / @as(f64, @floatFromInt(rows)), worst_cos, rows, width }); +} From 47c40985ce6a766150a25bddf0999753703101d1 Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:02:35 -0500 Subject: [PATCH 05/19] Native image input: --vision-urls fetches public HTTPS image URLs as 0.6.6 did media_fetch.zig follows images_http.py: an https URL on port 443 without credentials, a fragment, a backslash or controls; every address the host resolves to public (Python's is_global less 192.0.0.0/24 and 168.63.129.16; IPv6 global unicast only, no 6to4 or Teredo); TLS verified for the host on the address that was checked; GET with identity encoding; up to 3 redirects, each checked again; the declared media type JPEG, PNG or WebP (WebP is then refused by its bytes, as with data URLs); the bytes within what the request has left of 10 MB an image and 20 MB in all; 10 s a download and 30 s for a request's downloads. std's connect has no timeout yet: a nonblocking connect and poll bound it, the socket's send and receive timeouts bound the rest. The HTTP exchange reads from any reader, so the response rules (redirects joined and rechecked, status, types, lengths, chunking, encodings, the deadline) are tested against canned responses; the URL and address rules are tested directly. Without --vision-urls an https part is refused with the flag's name, other schemes as 0.6.6 refused them; --vision-urls needs --vision. tools/zig/vision_urls_served.py checks a served model against public hosts. --- docs/api.md | 3 +- docs/vision.md | 12 +- tools/zig/vision_served.py | 6 +- tools/zig/vision_urls_served.py | 109 +++++++ zig/src/server/cli.zig | 10 +- zig/src/server/images.zig | 104 ++++++- zig/src/server/media_fetch.zig | 497 ++++++++++++++++++++++++++++++++ zig/src/server/prompt.zig | 2 +- zig/src/server/root.zig | 1 + zig/src/server/serve.zig | 1 + zig/src/server/server.zig | 2 + 11 files changed, 730 insertions(+), 17 deletions(-) create mode 100644 tools/zig/vision_urls_served.py create mode 100644 zig/src/server/media_fetch.zig diff --git a/docs/api.md b/docs/api.md index 76a950230b..c716b914a0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -68,7 +68,8 @@ chat fields that shape the prompt (`tools`, `reasoning_effort`, `chat_template_k request, images expanded. `/detokenize` takes `tokens` and returns `prompt`. With `--vision`, supported Qwen3.5/3.8 dense checkpoints accept `image_url` content parts alongside text in user messages and in tool results (`role: "tool"`), such as an agent's screenshots. The native server does the same for -GLM-5.3-Flash on Metal, with data URLs only ([image input](vision.md#the-native-server)). +GLM-5.3-Flash on Metal, with data URLs, and public HTTPS URLs under `--vision-urls` +([image input](vision.md#the-native-server)). See [image input](vision.md) for data URLs, public image URLs, limits and cache behavior. Unsupported image input, audio, video and non-text output requests receive HTTP 400. diff --git a/docs/vision.md b/docs/vision.md index 75a37c1c58..1f12dbb43c 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -135,8 +135,16 @@ For Qwen, each image request carries its own multimodal rotary positions and con 4,096) for engines that read images; GLM-5.3-Flash on one Mac reads them so far, and other native engines refuse the flag at startup. -- Images arrive as `image_url` data URLs in user messages and tool results. Remote URLs are refused, since - `--vision-urls` isn't on the native server yet. +- Images arrive as `image_url` data URLs in user messages and tool results. With `--vision-urls` (it needs + `--vision`), a part may name a public HTTPS URL instead, fetched as 0.6.6 fetched it: port 443 without credentials + or a fragment; every address the host resolves to public (no private, loopback, link-local, carrier-grade NAT, + documentation, multicast or metadata address, and for IPv6 global unicast only, without 6to4 or Teredo); TLS + verified for the host against the system's certificates, on the address that was checked; up to 3 redirects, + each checked again; a declared `image/jpeg`, `image/png` or `image/webp`, uncompressed; 10 MB an image and 20 MB + a request across data URLs and downloads; 10 s a download and 30 s a request's downloads. The refusals are + 0.6.6's words; a TLS failure, like any other failed connection, is "image download failed or timed out". Host + names that aren't ASCII and IPv6 address literals are refused. Without the flag, an HTTPS URL is refused with the + flag's name. - PNG and JPEG are decoded by the platform's decoder (ImageIO on macOS). WebP is refused by name until a decoder exists; other formats are refused. - The limits and the processing are 0.6.6's: EXIF orientation, alpha composited over white, the GLM5-Next processor's diff --git a/tools/zig/vision_served.py b/tools/zig/vision_served.py index 49bd56bca6..c58a4b5bf6 100644 --- a/tools/zig/vision_served.py +++ b/tools/zig/vision_served.py @@ -6,7 +6,8 @@ model's image marker beside a real image is ordinary text (images come from the request's image parts, #511); drafted equals ``"draft": false`` and concurrent equals alone with images; an image conversation's next turn resumes past its image, and another image under the same placeholders resumes none of the first one's rows; and the refusals (WebP by name, other formats, an image outside a -user or tool message, too many images, remote URLs, bad data URLs, a bad detail). +user or tool message, too many images, remote URLs without --vision-urls, bad data URLs, a bad detail); tools/zig/vision_urls_served.py +checks a server started with --vision-urls. """ from __future__ import annotations @@ -151,7 +152,8 @@ def turns(url: str) -> tuple[list, list]: ("GIF", [user(image_part(data_url(gif, "image/gif")), text("hi"))], "not PNG or JPEG"), ("an image in an assistant message", [user(text("hi")), {"role": "assistant", "content": [image_part(url_a)]}, user(text("ok"))], "user and tool messages only"), ("too many images", [user(*[image_part(url_a)] * 5, text("hi"))], "at most"), - ("a remote URL", [user(image_part("https://example.com/cat.png"), text("hi"))], "data: URL"), + ("a remote URL without --vision-urls", [user(image_part("https://example.com/cat.png"), text("hi"))], "--vision-urls"), + ("a URL that is neither data nor https", [user(image_part("http://example.com/cat.png"), text("hi"))], "data URLs or public HTTPS URLs"), ("bad base64", [user(image_part("data:image/png;base64,@@@@"), text("hi"))], "encoding"), ("a bad detail", [user(image_part(url_a, "medium"), text("hi"))], "detail"), ("bytes that decode as nothing", [user(image_part(data_url(b"\x89PNG\r\n\x1a\n" + bytes(32), "image/png")), text("hi"))], "could not be decoded"), diff --git a/tools/zig/vision_urls_served.py b/tools/zig/vision_urls_served.py new file mode 100644 index 0000000000..f383eabbf6 --- /dev/null +++ b/tools/zig/vision_urls_served.py @@ -0,0 +1,109 @@ +"""Image URLs served end to end on a running ``tensorfold-native --vision --vision-urls`` (#565's URL checks). + + python tools/zig/vision_urls_served.py --url http://127.0.0.1:8000 + +Checks, against public hosts: an HTTPS image URL answers exactly as the same bytes sent as a data URL (the prompt's +tokens and the greedy reply), and so does a URL that redirects to it; a URL and a data URL in one request; and the +refusals images_http.py gave, before any token (HTTP status, content type, a host that resolves to a private +address, a name that is never public, an address literal that isn't public, TLS that doesn't verify, a scheme, port, +credentials or fragment the URL rules refuse, an unresolvable host). The fetch's own rules (redirect limits, +lengths, chunking, encodings, deadlines) are the unit tests in zig/src/server/media_fetch.zig. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import ssl +import sys +import urllib.error +import urllib.request + +IMAGE = "https://raw.githubusercontent.com/ashhart/TensorFold/main/assets/tensorfold-hero.png" +REDIRECT = "https://github.com/ashhart/TensorFold/raw/main/assets/tensorfold-hero.png" # 302 to IMAGE's host +OTHER = "https://raw.githubusercontent.com/ashhart/TensorFold/main/assets/tensorfold-cuda-banner.png" + + +def post(url: str, body: dict) -> tuple[int, dict]: + req = urllib.request.Request(url + "/v1/chat/completions", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=1800) as r: + return r.status, json.loads(r.read()) + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b"{}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--url", default="http://127.0.0.1:8000") + ap.add_argument("--tokens", type=int, default=48) + args = ap.parse_args() + with urllib.request.urlopen(args.url + "/v1/models") as r: + model = json.loads(r.read())["data"][0]["id"] + try: # python.org's macOS builds ship no CA store until "Install Certificates" runs + import certifi + tls = ssl.create_default_context(cafile=certifi.where()) + except ImportError: + tls = ssl.create_default_context() + with urllib.request.urlopen(IMAGE, timeout=30, context=tls) as r: + inline = "data:image/png;base64," + base64.b64encode(r.read()).decode() + failures = 0 + + def ask(*parts) -> tuple[int, dict]: + return post(args.url, {"model": model, "messages": [{"role": "user", "content": list(parts)}], "max_tokens": args.tokens, + "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}) + + def image(url: str) -> dict: + return {"type": "image_url", "image_url": {"url": url}} + + def text(t: str) -> dict: + return {"type": "text", "text": t} + + def check(label: str, ok: bool, detail: str = "") -> None: + nonlocal failures + failures += not ok + print(f"{'ok ' if ok else 'FAIL'} {label}{': ' + detail if detail else ''}") + + def same(label: str, a: tuple[int, dict], b: tuple[int, dict]) -> None: + ok = a[0] == 200 and b[0] == 200 and a[1]["usage"]["prompt_tokens"] == b[1]["usage"]["prompt_tokens"] and \ + a[1]["choices"][0]["message"]["content"] == b[1]["choices"][0]["message"]["content"] + detail = f"HTTP {a[0]}/{b[0]}" if a[0] != 200 or b[0] != 200 else \ + f"{a[1]['usage']['prompt_tokens']} prompt tokens, {a[1]['choices'][0]['message']['content'][:60]!r}" + if a[0] != 200 or b[0] != 200: + detail += " " + json.dumps(a[1].get("error") or b[1].get("error"))[:120] + check(label, ok, detail) + + q = text("Describe this image in one sentence.") + by_data = ask(image(inline), q) + same("an HTTPS image URL answers as its bytes in a data URL", ask(image(IMAGE), q), by_data) + same("a URL that redirects to the image answers the same", ask(image(REDIRECT), q), by_data) + two = ask(image(OTHER), image(inline), text("How do these two images differ?")) + check("a URL and a data URL in one request answer", two[0] == 200, f"HTTP {two[0]} {json.dumps(two[1].get('error', ''))[:100]}") + + refusals = [ + ("an HTTP error", "https://raw.githubusercontent.com/ashhart/TensorFold/main/assets/no-such-image.png", "image download returned HTTP 404"), + ("a page that isn't an image", "https://example.com/", "image URL content type must be JPEG, PNG or WebP"), + ("a host that resolves to a private address", "https://localtest.me/x.png", "image URLs must resolve only to public internet addresses"), + ("a private address literal", "https://169.254.169.254/latest/meta-data", "image URLs must resolve only to public internet addresses"), + ("localhost", "https://localhost/x.png", "image URLs must use public internet hosts"), + ("cloud metadata by name", "https://metadata.google.internal/x.png", "image URLs must use public internet hosts"), + ("an expired certificate", "https://expired.badssl.com/", "image download failed or timed out"), + ("a certificate for another host", "https://wrong.host.badssl.com/", "image download failed or timed out"), + ("a self-signed certificate", "https://self-signed.badssl.com/", "image download failed or timed out"), + ("plain HTTP", "http://example.com/x.png", "images require data URLs or public HTTPS URLs"), + ("another port", "https://example.com:8443/x.png", "image URL must be HTTPS on port 443, without credentials or a fragment"), + ("credentials", "https://user:pw@example.com/x.png", "image URL must be HTTPS on port 443, without credentials or a fragment"), + ("a fragment", "https://example.com/x.png#a", "image URL must be HTTPS on port 443, without credentials or a fragment"), + ("a host that never resolves", "https://no-such-host.invalid/x.png", "image host could not be resolved"), + ] + for label, url, words in refusals: + code, r = ask(image(url), text("hi")) + message = r.get("error", {}).get("message", json.dumps(r)) + check(f"refused {label}", code == 400 and message == words, f"HTTP {code} {message[:100]!r}") + print("PASS" if failures == 0 else f"FAIL: {failures}") + return 0 if failures == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index c30c26a1d4..02f7578161 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -50,7 +50,7 @@ pub const flags = [_]Flag{ .{ .name = "--metrics-open", .kind = .store_true, .native = true }, .{ .name = "--dashboard", .kind = .store_true, .native = true }, .{ .name = "--vision", .kind = .store_true, .native = true }, - .{ .name = "--vision-urls", .kind = .store_true }, + .{ .name = "--vision-urls", .kind = .store_true, .native = true }, .{ .name = "--vision-offload", .kind = .store_true }, .{ .name = "--vision-max-images", .native = true }, .{ .name = "--vision-image-tokens", .native = true }, @@ -168,6 +168,7 @@ pub const Args = struct { master_port: u16 = 29551, policy: []const u8 = "", // every --policy joined by commas vision: bool = false, // image input (GLM-5.3-Flash): data-URL image_url parts in user messages + vision_urls: bool = false, // with --vision, public HTTPS image URLs too (media_fetch.zig) vision_max_images: u32 = 4, vision_image_tokens: u32 = 4096, }; @@ -221,6 +222,7 @@ pub fn parse(a: Allocator, argv: []const []const u8, u: *Usage) error{ Usage, Ou } out.model = model orelse return fail(u, a, "the following arguments are required: model", .{}); try ranks(u, a, out); + if (out.vision_urls and !out.vision) return fail(u, a, "--vision-urls needs --vision", .{}); out.alias = alias.items; out.api_key = keys.items; return out; @@ -275,7 +277,7 @@ fn apply(a: Allocator, out: *Args, name: []const u8, value: ?[]const u8, u: *Usa const n = try int(u, a, name, v); if (n < 0) return fail(u, a, "argument --compact-keep: expected a token count from 0: '{s}'", .{v}); out.compact_keep = @intCast(n); - } else if (is(name, "--compact-memory")) out.compact_memory = v else if (is(name, "--slide")) out.slide = true else if (is(name, "--slide-graph")) out.slide_graph = v else if (is(name, "--parallel")) out.parallel = v else if (is(name, "--backend")) out.backend = v else if (is(name, "--vision")) out.vision = true else if (is(name, "--vision-max-images")) { + } else if (is(name, "--compact-memory")) out.compact_memory = v else if (is(name, "--slide")) out.slide = true else if (is(name, "--slide-graph")) out.slide_graph = v else if (is(name, "--parallel")) out.parallel = v else if (is(name, "--backend")) out.backend = v else if (is(name, "--vision")) out.vision = true else if (is(name, "--vision-urls")) out.vision_urls = true else if (is(name, "--vision-max-images")) { const n = try int(u, a, name, v); if (n < 1 or n > 64) return fail(u, a, "argument --vision-max-images: expected 1 to 64 images: '{s}'", .{v}); out.vision_max_images = @intCast(n); @@ -458,6 +460,10 @@ test "parse and capabilities share the table" { try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--compact-at", "0" }, &u)); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--compact-at", "2" }, &u)); try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--compact-keep", "-1" }, &u)); + const urls = try parse(a, &.{ "m", "--vision", "--vision-urls" }, &u); + try std.testing.expect(urls.vision and urls.vision_urls); + try std.testing.expectError(error.Usage, parse(a, &.{ "m", "--vision-urls" }, &u)); + try std.testing.expectEqualStrings("--vision-urls needs --vision", u.message); } test "--device and --segments: CUDA builds serve them, values checked" { diff --git a/zig/src/server/images.zig b/zig/src/server/images.zig index ba1253ef2c..d99b121300 100644 --- a/zig/src/server/images.zig +++ b/zig/src/server/images.zig @@ -6,6 +6,7 @@ const json = @import("json.zig"); const errors = @import("errors.zig"); const messages_mod = @import("messages.zig"); const model_text = @import("model_text.zig"); +const media_fetch = @import("media_fetch.zig"); const Value = json.Value; const Cx = errors.Cx; @@ -41,10 +42,44 @@ fn parts(cx: *Cx, messages: Value) errors.Refused![]const Part { return out.items; } -/// A data URL's bytes (base64 or percent-encoded); remote URLs are refused (the native server fetches nothing). +/// --vision-urls: how a public HTTPS image's bytes are fetched (media_fetch.zig, or a test's fixtures). +pub const Fetch = struct { + ctx: ?*anyopaque = null, + io: std.Io, + get: *const fn (ctx: ?*anyopaque, a: std.mem.Allocator, io: std.Io, url: []const u8, max_bytes: usize, deadline: std.Io.Timestamp, f: *media_fetch.Failure) media_fetch.Error![]const u8, + + /// The network, as media_fetch checks it. + pub fn network(io: std.Io) Fetch { + return .{ .io = io, .get = struct { + fn get(_: ?*anyopaque, a: std.mem.Allocator, i: std.Io, url: []const u8, max_bytes: usize, deadline: std.Io.Timestamp, f: *media_fetch.Failure) media_fetch.Error![]const u8 { + return (try media_fetch.fetch(a, i, url, max_bytes, deadline, &media_fetch.image_media, f)).data; + } + }.get }; + } +}; + +/// An image part's bytes, at most `limit`: a data URL's, or with --vision-urls (`fetch`) a public HTTPS URL's, fetched +/// before `deadline` (images.py _check_source and load_images). +pub fn sourceBytes(cx: *Cx, fetch: ?Fetch, url: []const u8, limit: usize, deadline: std.Io.Timestamp) errors.Refused![]const u8 { + if (std.ascii.startsWithIgnoreCase(url, "data:")) { + const bytes = try dataBytes(cx, url); + if (bytes.len > limit) return cx.refuse("the request's images exceed 20 MB encoded"); + return bytes; + } + if (!std.mem.startsWith(u8, url, "https://")) return cx.refuse("images require data URLs or public HTTPS URLs"); + const f = fetch orelse return cx.refuse("image URLs are off on this server; send the image as a data URL, or start the server with --vision-urls"); + if (url.len > media_fetch.max_url_chars) return cx.refuse("image URL is too long"); + var failure: media_fetch.Failure = .{}; + const bytes = f.get(f.ctx, cx.a, f.io, url, limit, deadline, &failure) catch |e| switch (e) { + error.OutOfMemory => return error.OutOfMemory, + error.Media => return cx.refuse(failure.text), + }; + if (bytes.len == 0) return cx.refuse("image is empty or exceeds the encoded byte limit"); + return bytes; +} + +/// A data URL's bytes (base64 or percent-encoded). pub fn dataBytes(cx: *Cx, url: []const u8) errors.Refused![]const u8 { - if (std.ascii.startsWithIgnoreCase(url, "http://") or std.ascii.startsWithIgnoreCase(url, "https://")) - return cx.refuse("image URLs are not fetched by this server; send the image inline as a data: URL (data:image/png;base64,...)"); if (!std.ascii.startsWithIgnoreCase(url, "data:")) return cx.refuse("invalid image data URL"); const comma = std.mem.indexOfScalar(u8, url, ',') orelse return cx.refuse("invalid image data URL"); const header = url[5..comma]; @@ -179,7 +214,7 @@ pub fn encodeShielded(text: model_text.Text, a: std.mem.Allocator, rendered: []c /// The prompt with each image's placeholder expanded to its tokens, the images for the engine and the prompt as the /// cache matches it; null when the engine reads no images. Images are counted from the request's image parts; the /// rendered placeholders are the template's alone (text that spells a marker was shielded), one per part. -pub fn expand(vision: ?api.Vision, cx: *Cx, messages: Value, ids: []const u32, history_len: usize) errors.Refused!?Expanded { +pub fn expand(vision: ?api.Vision, fetch: ?Fetch, cx: *Cx, messages: Value, ids: []const u32, history_len: usize) errors.Refused!?Expanded { const v = vision orelse return null; const ps = try parts(cx, messages); var marks: usize = 0; @@ -192,10 +227,11 @@ pub fn expand(vision: ?api.Vision, cx: *Cx, messages: Value, ids: []const u32, h const prepared = try cx.a.alloc(api.PreparedImage, ps.len); var bytes_total: usize = 0; var extra: usize = 0; + // every URL's download inside one request's seconds (images.py total_timeout_seconds) + const deadline: std.Io.Timestamp = if (fetch) |f| std.Io.Clock.awake.now(f.io).addDuration(.fromSeconds(media_fetch.total_s)) else .{ .nanoseconds = 0 }; for (ps, prepared, 0..) |p, *out, i| { - const bytes = try dataBytes(cx, p.url); + const bytes = try sourceBytes(cx, fetch, p.url, @min(max_image_bytes, max_request_bytes - bytes_total), deadline); bytes_total += bytes.len; - if (bytes_total > max_request_bytes) return cx.refuse("the request's images exceed 20 MB encoded"); switch (sniff(bytes)) { // the Python frontend's formats (images.py: JPEG, PNG, WebP), WebP not yet decoded here .png, .jpeg => {}, .webp => return cx.refuse(try std.fmt.allocPrint(cx.a, "image {d} is WebP, which this server does not decode yet; send PNG or JPEG", .{i + 1})), @@ -240,8 +276,16 @@ test "data URLs decode; remote URLs are refused" { var cx: Cx = .{ .a = arena.allocator() }; try std.testing.expectEqualStrings("hi!", try dataBytes(&cx, "data:image/png;base64,aGkh")); try std.testing.expectEqualStrings("a b", try dataBytes(&cx, "data:text/plain,a%20b")); - try std.testing.expectError(error.Refused, dataBytes(&cx, "https://example.com/x.png")); try std.testing.expectError(error.Refused, dataBytes(&cx, "data:image/png;base64,***")); + // without --vision-urls an https URL is refused with the flag's name; other schemes always + const never: std.Io.Timestamp = .{ .nanoseconds = 0 }; + try std.testing.expectError(error.Refused, sourceBytes(&cx, null, "https://example.com/x.png", max_image_bytes, never)); + try std.testing.expectEqualStrings("image URLs are off on this server; send the image as a data URL, or start the server with --vision-urls", cx.message); + for ([_][]const u8{ "http://example.com/x.png", "ftp://example.com/x.png", "HTTPS://example.com/x.png", "file:///etc/passwd" }) |u| { + try std.testing.expectError(error.Refused, sourceBytes(&cx, null, u, max_image_bytes, never)); + try std.testing.expectEqualStrings("images require data URLs or public HTTPS URLs", cx.message); + } + try std.testing.expectError(error.Refused, sourceBytes(&cx, null, "data:,abc", 2, never)); } fn fakePrepare(ctx: *anyopaque, a: std.mem.Allocator, bytes: []const u8, max_tokens: u32) anyerror!api.PreparedImage { @@ -268,7 +312,7 @@ test "placeholders expand to each image's tokens, the history and cache key with .err => return error.TestUnexpectedResult, }; const ids = [_]u32{ 1, 9, 2, 3, 9, 4 }; - const x = (try expand(v, &cx, msgs, &ids, 4)).?; + const x = (try expand(v, null, &cx, msgs, &ids, 4)).?; try std.testing.expectEqualSlices(u32, &.{ 1, 9, 9, 2, 3, 9, 9, 9, 4 }, x.ids); try std.testing.expectEqual(@as(usize, 5), x.history_len); // the first image is in the history, the second not try std.testing.expectEqual(@as(u32, 1), x.images[0].at); @@ -276,10 +320,52 @@ test "placeholders expand to each image's tokens, the history and cache key with try std.testing.expect(x.cache_key[1] >= 0x8000_0000 and x.cache_key[1] != x.cache_key[2]); try std.testing.expectEqual(@as(u32, 2), x.cache_key[3]); // a placeholder the images do not account for is refused - try std.testing.expectError(error.Refused, expand(v, &cx, msgs, &[_]u32{ 9, 9, 9 }, 0)); + try std.testing.expectError(error.Refused, expand(v, null, &cx, msgs, &[_]u32{ 9, 9, 9 }, 0)); _ = messages_mod; } +test "--vision-urls: an https part's bytes come from the fetch, within what the request has left" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var cx: Cx = .{ .a = a }; + var dummy: u8 = 0; + const v: api.Vision = .{ .ctx = &dummy, .prepare = fakePrepare, .image_token = 9, .image_tokens = 64, .max_images = 4 }; + const Pages = struct { + var limits: [4]usize = undefined; + var n: usize = 0; + fn get(_: ?*anyopaque, al: std.mem.Allocator, _: std.Io, url: []const u8, max_bytes: usize, _: std.Io.Timestamp, f: *media_fetch.Failure) media_fetch.Error![]const u8 { + limits[n] = max_bytes; + n += 1; + if (std.mem.eql(u8, url, "https://e.com/a.jpg")) return al.dupe(u8, "\xff\xd8\xffxyz"); + f.text = "image download returned HTTP 404"; + return error.Media; + } + }; + const fetch: Fetch = .{ .io = std.testing.io, .get = Pages.get }; + const text = + \\[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:,%FF%D8%FFab"}}, + \\ {"type": "image_url", "image_url": {"url": "https://e.com/a.jpg"}}]}] + ; + const msgs = switch (try json.parseText(a, text)) { + .ok => |m| m, + .err => return error.TestUnexpectedResult, + }; + const x = (try expand(v, fetch, &cx, msgs, &[_]u32{ 1, 9, 9, 2 }, 0)).?; + try std.testing.expectEqualSlices(u32, &.{ 1, 9, 9, 9, 9, 9, 2 }, x.ids); // 2 tokens, then the fetched image's 3 + try std.testing.expectEqual(@as(u32, 3), x.images[1].at); + try std.testing.expectEqual(@as(usize, max_image_bytes), Pages.limits[0]); // 5 bytes used of 20 MB: 10 MB left + // the fetch's refusal is the request's; without the fetch the URL is refused by the flag's name + const missing = switch (try json.parseText(a, "[{\"role\": \"user\", \"content\": [{\"type\": \"image_url\", \"image_url\": {\"url\": \"https://e.com/gone.png\"}}]}]")) { + .ok => |m| m, + .err => return error.TestUnexpectedResult, + }; + try std.testing.expectError(error.Refused, expand(v, fetch, &cx, missing, &[_]u32{9}, 0)); + try std.testing.expectEqualStrings("image download returned HTTP 404", cx.message); + try std.testing.expectError(error.Refused, expand(v, null, &cx, missing, &[_]u32{9}, 0)); + try std.testing.expectEqualStrings("image URLs are off on this server; send the image as a data URL, or start the server with --vision-urls", cx.message); +} + /// A test tokenizer: "<|image|>" is token 9 unless read plainly, every other byte its own id plus 100. const MarkText = struct { fn text() model_text.Text { diff --git a/zig/src/server/media_fetch.zig b/zig/src/server/media_fetch.zig new file mode 100644 index 0000000000..d642ce0b2e --- /dev/null +++ b/zig/src/server/media_fetch.zig @@ -0,0 +1,497 @@ +//! Public HTTPS images for --vision-urls, as TensorFold 0.6.6's server fetched them (vision/images_http.py): an https +//! URL on port 443 without credentials or a fragment, its host resolved and every address checked public, a TLS +//! connection verified for the host on an address that was checked, GET with identity encoding, up to 3 redirects +//! (each checked the same way), a declared media type the caller accepts, the bytes within the caller's limit, all +//! inside a deadline. +const std = @import("std"); +const Io = std.Io; +const net = std.Io.net; + +pub const Error = error{ OutOfMemory, Media }; + +/// The failure's words for the client (the request is refused with them). +pub const Failure = struct { text: []const u8 = "" }; + +/// images.py's ImageLimits: a URL's characters, the redirects followed, one download's seconds and a request's. +pub const max_url_chars = 4096; +pub const max_redirects = 3; +pub const timeout_s = 10; +pub const total_s = 30; + +/// images_http.MEDIA_TYPES: the declared types an image URL may answer with (WebP is refused after, by its bytes). +pub const image_media = [_][]const u8{ "image/jpeg", "image/png", "image/webp" }; + +fn fail(f: *Failure, text: []const u8) Error { + f.text = text; + return error.Media; +} + +/// Python ipaddress: is_global and not multicast or reserved, nor 192.0.0.0/24 or 168.63.129.16; no IPv6 address +/// that embeds an IPv4 one (mapped, 6to4, Teredo). +pub fn publicIp(a: net.IpAddress) bool { + switch (a) { + .ip4 => |v| { + const x = std.mem.readInt(u32, &v.bytes, .big); + const blocked = [_]struct { [4]u8, u5 }{ + .{ .{ 0, 0, 0, 0 }, 8 }, .{ .{ 10, 0, 0, 0 }, 8 }, .{ .{ 100, 64, 0, 0 }, 10 }, .{ .{ 127, 0, 0, 0 }, 8 }, + .{ .{ 169, 254, 0, 0 }, 16 }, .{ .{ 172, 16, 0, 0 }, 12 }, .{ .{ 192, 0, 0, 0 }, 24 }, .{ .{ 192, 0, 2, 0 }, 24 }, + .{ .{ 192, 168, 0, 0 }, 16 }, .{ .{ 198, 18, 0, 0 }, 15 }, .{ .{ 198, 51, 100, 0 }, 24 }, .{ .{ 203, 0, 113, 0 }, 24 }, + .{ .{ 224, 0, 0, 0 }, 4 }, .{ .{ 240, 0, 0, 0 }, 4 }, + }; + for (blocked) |blk| { + const mask = ~@as(u32, 0) << @intCast(32 - @as(u6, blk[1])); + if (x & mask == std.mem.readInt(u32, &blk[0], .big) & mask) return false; + } + return !std.mem.eql(u8, &v.bytes, &.{ 168, 63, 129, 16 }); + }, + .ip6 => |v| { + const b = v.bytes; + // global unicast 2000::/3 only, less documentation (2001:db8::/32), the 2001::/23 IETF block (Teredo + // 2001::/32 among it) and 6to4 (2002::/16) + if (b[0] & 0xe0 != 0x20) return false; + if (b[0] == 0x20 and b[1] == 0x01 and b[2] == 0x0d and b[3] == 0xb8) return false; + if (b[0] == 0x20 and b[1] == 0x01 and b[2] < 0x02) return false; + return !(b[0] == 0x20 and b[1] == 0x02); + }, + } +} + +/// A URL as a request takes it: the lowercased host and the percent-quoted path and query. +pub const Url = struct { host: []const u8, target: []const u8 }; + +/// images_http._url: https, a host, port 443, no credentials, fragment, backslash, whitespace or controls. A host +/// that isn't ASCII (an IDNA name) or is an IPv6 literal is refused: std has no IDNA encoder, and TLS here verifies +/// names. +pub fn parseUrl(a: std.mem.Allocator, value: []const u8, f: *Failure) Error!Url { + if (value.len > max_url_chars) return fail(f, "image URL is too long or contains whitespace/control characters"); + for (value) |c| if (c <= 32 or c == 127) return fail(f, "image URL is too long or contains whitespace/control characters"); + const bad = "image URL must be HTTPS on port 443, without credentials or a fragment"; + if (!std.ascii.startsWithIgnoreCase(value, "https://")) return fail(f, bad); + if (std.mem.indexOfAny(u8, value, "\\#") != null) return fail(f, bad); + const rest = value["https://".len..]; + const end = std.mem.indexOfAny(u8, rest, "/?") orelse rest.len; + const authority = rest[0..end]; + if (std.mem.indexOfScalar(u8, authority, '@') != null or std.mem.startsWith(u8, authority, "[")) return fail(f, bad); + var host = authority; + if (std.mem.lastIndexOfScalar(u8, authority, ':')) |c| { + const port = authority[c + 1 ..]; + if (port.len > 0 and !std.mem.eql(u8, port, "443")) return fail(f, bad); + host = authority[0..c]; + } + if (host.len == 0 or std.mem.indexOfScalar(u8, host, '%') != null) return fail(f, bad); + for (host) |c| if (c >= 128) return fail(f, bad); + const lower = try std.ascii.allocLowerString(a, host); + const name = std.mem.trimEnd(u8, lower, "."); + for ([_][]const u8{ "localhost", "metadata.google.internal", "instance-data" }) |n| if (std.mem.eql(u8, name, n)) return fail(f, "image URLs must use public internet hosts"); + // the path and query, percent-quoted as urllib.parse.quote(safe=...) leaves them + var t: std.ArrayList(u8) = .empty; + const tail = rest[end..]; + if (tail.len == 0 or tail[0] == '?') try t.append(a, '/'); + var in_query = false; + var query_at: usize = 0; + for (tail) |c| { + if (c == '?' and !in_query) { + in_query = true; + try t.append(a, '?'); + query_at = t.items.len; + continue; + } + const safe = std.ascii.isAlphanumeric(c) or std.mem.indexOfScalar(u8, "/%:@!$&'()*+,;=-._~", c) != null or (in_query and c == '?'); + if (safe) try t.append(a, c) else try t.print(a, "%{X:0>2}", .{c}); + } + if (in_query and t.items.len == query_at) t.items.len -= 1; // an empty query, as urlsplit drops it + return .{ .host = lower, .target = t.items }; +} + +/// The location a redirect names, resolved against `base` (urljoin's absolute, scheme-relative, absolute-path and +/// relative forms). +pub fn join(a: std.mem.Allocator, base: []const u8, location: []const u8) ![]const u8 { + if (std.ascii.startsWithIgnoreCase(location, "https://") or std.ascii.startsWithIgnoreCase(location, "http://")) return a.dupe(u8, location); + if (std.mem.startsWith(u8, location, "//")) return std.fmt.allocPrint(a, "https:{s}", .{location}); + const rest = base["https://".len..]; + const end = std.mem.indexOfAny(u8, rest, "/?") orelse rest.len; + const origin = base[0 .. "https://".len + end]; + if (std.mem.startsWith(u8, location, "/")) return std.fmt.allocPrint(a, "{s}{s}", .{ origin, location }); + const path = rest[end..][0 .. std.mem.indexOfScalar(u8, rest[end..], '?') orelse rest.len - end]; + const dir = if (std.mem.lastIndexOfScalar(u8, path, '/')) |s| path[0 .. s + 1] else "/"; + return std.fmt.allocPrint(a, "{s}{s}{s}", .{ origin, dir, location }); +} + +fn remaining(io: Io, deadline: Io.Timestamp, f: *Failure) Error!i64 { + const left = Io.Clock.awake.now(io).durationTo(deadline).toMilliseconds(); + if (left <= 0) return fail(f, "image download timed out"); + return left; +} + +/// A fetched body and its declared media type. +pub const Body = struct { data: []u8, media: []const u8 }; + +/// What one request gave: the body, or a redirect's location. +pub const Got = union(enum) { body: Body, redirect: []const u8 }; + +/// The bytes of `url` and its media type (one of `media_types`), at most `max_bytes`, before `deadline` and within +/// one download's seconds, redirects followed and each checked again. +pub fn fetch(a: std.mem.Allocator, io: Io, url: []const u8, max_bytes: usize, deadline: Io.Timestamp, media_types: []const []const u8, f: *Failure) Error!Body { + return fetchVia(a, io, url, max_bytes, deadline, media_types, f, Network{}); +} + +/// fetch over `via`, whose request(a, io, url, max_bytes, deadline, media_types, f) answers one checked URL. +pub fn fetchVia(a: std.mem.Allocator, io: Io, url_in: []const u8, max_bytes: usize, deadline_in: Io.Timestamp, media_types: []const []const u8, f: *Failure, via: anytype) Error!Body { + const one = Io.Clock.awake.now(io).addDuration(.fromSeconds(timeout_s)); + const deadline = if (one.nanoseconds < deadline_in.nanoseconds) one else deadline_in; + var url = url_in; + var hop: usize = 0; + while (true) : (hop += 1) { + const u = try parseUrl(a, url, f); + switch (try via.request(a, io, u, max_bytes, deadline, media_types, f)) { + .body => |b| return b, + .redirect => |location| { + if (hop == max_redirects) return fail(f, "image download has too many redirects"); + url = try join(a, url, location); + }, + } + } +} + +/// The GET a checked URL is asked with. +pub fn ask(w: *Io.Writer, u: Url, accept: []const u8) Io.Writer.Error!void { + try w.print("GET {s} HTTP/1.1\r\nHost: {s}\r\nAccept: {s}\r\nAccept-Encoding: identity\r\nUser-Agent: TensorFold-native\r\nConnection: close\r\n\r\n", .{ u.target, u.host, accept }); + try w.flush(); +} + +/// The response read from `r`: a redirect's location, or a 200's body when its encoding, declared type and length +/// pass (Content-Length or chunked, else to the end of the stream). +pub fn answer(a: std.mem.Allocator, io: Io, r: *Io.Reader, max_bytes: usize, deadline: Io.Timestamp, media_types: []const []const u8, f: *Failure) Error!Got { + const broke = "image download failed or timed out"; + const too_big = "image response exceeds the encoded byte limit"; + const status_line = r.takeDelimiterInclusive('\n') catch return fail(f, broke); + if (status_line.len < 12 or !std.mem.startsWith(u8, status_line, "HTTP/1.")) return fail(f, broke); + const status = std.fmt.parseInt(u16, status_line[9..12], 10) catch return fail(f, broke); + var location: ?[]const u8 = null; + var media: []const u8 = ""; + var length: ?usize = null; + var chunked = false; + var encoded = false; + var header_bytes: usize = 0; + while (true) { + const line = r.takeDelimiterInclusive('\n') catch return fail(f, broke); + header_bytes += line.len; + if (header_bytes > 64 * 1024) return fail(f, broke); + const l = std.mem.trimEnd(u8, line, "\r\n"); + if (l.len == 0) break; + const colon = std.mem.indexOfScalar(u8, l, ':') orelse continue; + const key = std.mem.trim(u8, l[0..colon], " \t"); + const val = std.mem.trim(u8, l[colon + 1 ..], " \t"); + if (std.ascii.eqlIgnoreCase(key, "location")) { + location = try a.dupe(u8, val); + } else if (std.ascii.eqlIgnoreCase(key, "content-type")) { + media = try std.ascii.allocLowerString(a, std.mem.trim(u8, val[0 .. std.mem.indexOfScalar(u8, val, ';') orelse val.len], " \t")); + } else if (std.ascii.eqlIgnoreCase(key, "content-length")) { + if (val.len == 0) return fail(f, too_big); + for (val) |c| if (!std.ascii.isDigit(c)) return fail(f, too_big); + length = std.fmt.parseInt(usize, val, 10) catch return fail(f, too_big); + } else if (std.ascii.eqlIgnoreCase(key, "transfer-encoding")) { + chunked = std.ascii.findIgnoreCase(val, "chunked") != null; + } else if (std.ascii.eqlIgnoreCase(key, "content-encoding") and !std.ascii.eqlIgnoreCase(val, "identity")) encoded = true; + } + _ = try remaining(io, deadline, f); + switch (status) { + 301, 302, 303, 307, 308 => return .{ .redirect = location orelse return fail(f, "image redirect has no destination") }, + 200 => {}, + else => return fail(f, try std.fmt.allocPrint(a, "image download returned HTTP {d}", .{status})), + } + if (encoded) return fail(f, "compressed HTTP image responses are unsupported"); + for (media_types) |m| { + if (std.mem.eql(u8, m, media)) break; + } else return fail(f, "image URL content type must be JPEG, PNG or WebP"); + if (length) |n| if (n > max_bytes) return fail(f, too_big); + var data: std.ArrayList(u8) = .empty; + if (chunked) { + while (true) { + _ = try remaining(io, deadline, f); + const size_line = r.takeDelimiterInclusive('\n') catch return fail(f, broke); + const hex = std.mem.trim(u8, size_line[0 .. std.mem.indexOfScalar(u8, size_line, ';') orelse size_line.len], " \t\r\n"); + const n = std.fmt.parseInt(usize, hex, 16) catch return fail(f, broke); + if (n == 0) break; + if (n > max_bytes - data.items.len) return fail(f, too_big); + try readInto(a, io, r, &data, n, deadline, f); + const crlf = r.takeDelimiterInclusive('\n') catch return fail(f, broke); + if (std.mem.trimEnd(u8, crlf, "\r\n").len != 0) return fail(f, broke); + } + } else if (length) |n| { + try readInto(a, io, r, &data, n, deadline, f); + } else while (true) { // no length: to the end of the stream (Connection: close) + _ = try remaining(io, deadline, f); + const piece = r.peekGreedy(1) catch |e| switch (e) { + error.EndOfStream => break, + else => return fail(f, broke), + }; + if (piece.len > max_bytes - data.items.len) return fail(f, too_big); + try data.appendSlice(a, piece); + r.toss(piece.len); + } + _ = try remaining(io, deadline, f); + return .{ .body = .{ .data = data.items, .media = media } }; +} + +/// Exactly `n` more bytes from `r` onto `data`, in whatever pieces the reader's buffer holds. +fn readInto(a: std.mem.Allocator, io: Io, r: *Io.Reader, data: *std.ArrayList(u8), n: usize, deadline: Io.Timestamp, f: *Failure) Error!void { + var left = n; + while (left > 0) { + _ = try remaining(io, deadline, f); + const piece = r.peekGreedy(1) catch return fail(f, "image download failed or timed out"); + const take = @min(piece.len, left); + try data.appendSlice(a, piece[0..take]); + r.toss(take); + left -= take; + } +} + +/// A TCP connection to the checked address on port 443, its connect bounded by `ms` (std's connect has no timeout +/// yet: a nonblocking connect and poll) and its sends and reads by the socket's timeouts. +fn connect(addr: net.IpAddress, ms: i64, f: *Failure) Error!net.Stream { + const c = std.c; + const failed = "image download failed or timed out"; + var storage: c.sockaddr.storage = undefined; + var len: c.socklen_t = undefined; + switch (addr) { + .ip4 => |v| { + const sa: *c.sockaddr.in = @ptrCast(@alignCast(&storage)); + sa.* = std.mem.zeroes(c.sockaddr.in); + sa.family = c.AF.INET; + sa.port = std.mem.nativeToBig(u16, 443); + sa.addr = @bitCast(v.bytes); + len = @sizeOf(c.sockaddr.in); + }, + .ip6 => |v| { + const sa: *c.sockaddr.in6 = @ptrCast(@alignCast(&storage)); + sa.* = std.mem.zeroes(c.sockaddr.in6); + sa.family = c.AF.INET6; + sa.port = std.mem.nativeToBig(u16, 443); + sa.addr = v.bytes; + len = @sizeOf(c.sockaddr.in6); + }, + } + const fd = c.socket(@as(c_uint, storage.family), c.SOCK.STREAM, 0); + if (fd < 0) return fail(f, failed); + errdefer _ = c.close(fd); + _ = c.fcntl(fd, c.F.SETFD, @as(c_int, c.FD_CLOEXEC)); + const flags = c.fcntl(fd, c.F.GETFL); + const nonblock: c_int = @bitCast(@as(u32, @bitCast(c.O{ .NONBLOCK = true }))); + if (flags < 0 or c.fcntl(fd, c.F.SETFL, flags | nonblock) < 0) return fail(f, failed); + if (c.connect(fd, @ptrCast(&storage), len) != 0) { + if (c.errno(@as(c_int, -1)) != .INPROGRESS) return fail(f, failed); + var pfd = [_]c.pollfd{.{ .fd = fd, .events = c.POLL.OUT, .revents = 0 }}; + if (c.poll(&pfd, 1, @intCast(@min(ms, std.math.maxInt(c_int)))) != 1) return fail(f, failed); + var err: c_int = 0; + var err_len: c.socklen_t = @sizeOf(c_int); + if (c.getsockopt(fd, c.SOL.SOCKET, c.SO.ERROR, &err, &err_len) != 0 or err != 0) return fail(f, failed); + } + if (c.fcntl(fd, c.F.SETFL, flags) < 0) return fail(f, failed); + const tv: c.timeval = .{ .sec = @intCast(@divTrunc(ms, 1000)), .usec = @intCast(@mod(ms, 1000) * 1000) }; + _ = c.setsockopt(fd, c.SOL.SOCKET, c.SO.RCVTIMEO, &tv, @sizeOf(c.timeval)); + _ = c.setsockopt(fd, c.SOL.SOCKET, c.SO.SNDTIMEO, &tv, @sizeOf(c.timeval)); + return .{ .socket = .{ .handle = fd, .address = addr } }; +} + +var bundle_lock: Io.RwLock = .init; +var bundle: std.crypto.Certificate.Bundle = .empty; +var bundle_loaded = false; + +/// The network: every address the host resolves to checked public, then TLS verified for the host on the first. +const Network = struct { + fn request(_: Network, a: std.mem.Allocator, io: Io, u: Url, max_bytes: usize, deadline: Io.Timestamp, media_types: []const []const u8, f: *Failure) Error!Got { + const unresolved = "image host could not be resolved"; + var first: ?net.IpAddress = null; + if (net.IpAddress.parseIp4(u.host, 443)) |literal| { + if (!publicIp(literal)) return fail(f, "image URLs must resolve only to public internet addresses"); + first = literal; + } else |_| { + const name = net.HostName.init(std.mem.trimEnd(u8, u.host, ".")) catch return fail(f, unresolved); + var results: [32]net.HostName.LookupResult = undefined; + var q: Io.Queue(net.HostName.LookupResult) = .init(&results); + name.lookup(io, &q, .{ .port = 443 }) catch return fail(f, unresolved); + while (q.getOneUncancelable(io)) |r| switch (r) { + .address => |addr| { + if (!publicIp(addr)) return fail(f, "image URLs must resolve only to public internet addresses"); + if (first == null) first = addr; + }, + .canonical_name => {}, + } else |_| {} + } + const addr = first orelse return fail(f, unresolved); + { + bundle_lock.lockUncancelable(io); + defer bundle_lock.unlock(io); + if (!bundle_loaded) { + bundle.rescan(std.heap.page_allocator, io, Io.Clock.real.now(io)) catch return fail(f, "image download failed: no CA certificates on the server"); + bundle_loaded = true; + } + } + const stream = try connect(addr, try remaining(io, deadline, f), f); + defer stream.close(io); + const tls_len = std.crypto.tls.Client.min_buffer_len; + const bufs = try a.alloc(u8, 4 * tls_len); + var sr = stream.reader(io, bufs[0..tls_len]); + var sw = stream.writer(io, bufs[tls_len .. 2 * tls_len]); + var entropy: [std.crypto.tls.Client.Options.entropy_len]u8 = undefined; + io.random(&entropy); + var tls = std.crypto.tls.Client.init(&sr.interface, &sw.interface, .{ + .host = .{ .explicit = std.mem.trimEnd(u8, u.host, ".") }, + .ca = .{ .bundle = .{ .gpa = std.heap.page_allocator, .io = io, .lock = &bundle_lock, .bundle = &bundle } }, + .read_buffer = bufs[2 * tls_len .. 3 * tls_len], + .write_buffer = bufs[3 * tls_len ..], + .entropy = &entropy, + .realtime_now = Io.Clock.real.now(io), + .allow_truncation_attacks = true, // HTTP's own length and chunk framing end the body + }) catch return fail(f, "image download failed or timed out"); // images_http.py: an ssl.SSLError is an OSError + const accept = try std.mem.join(a, ", ", media_types); + ask(&tls.writer, u, accept) catch return fail(f, "image download failed or timed out"); + sw.interface.flush() catch return fail(f, "image download failed or timed out"); + return answer(a, io, &tls.reader, max_bytes, deadline, media_types, f); + } +}; + +test "public addresses as Python's ipaddress checks them" { + const ip4 = struct { + fn of(b: [4]u8) net.IpAddress { + return .{ .ip4 = .{ .bytes = b, .port = 443 } }; + } + }.of; + const ip6 = struct { + fn of(text: []const u8) net.IpAddress { + return net.IpAddress.parseIp6(text, 443) catch unreachable; + } + }.of; + for ([_][4]u8{ .{ 8, 8, 8, 8 }, .{ 151, 101, 1, 69 }, .{ 1, 1, 1, 1 } }) |b| try std.testing.expect(publicIp(ip4(b))); + for ([_][4]u8{ .{ 10, 1, 2, 3 }, .{ 127, 0, 0, 1 }, .{ 169, 254, 169, 254 }, .{ 172, 20, 0, 1 }, .{ 192, 168, 1, 1 }, .{ 100, 100, 0, 1 }, .{ 0, 0, 0, 0 }, .{ 224, 0, 0, 1 }, .{ 255, 255, 255, 255 }, .{ 168, 63, 129, 16 }, .{ 192, 0, 0, 9 }, .{ 198, 18, 0, 1 }, .{ 203, 0, 113, 5 } }) |b| + try std.testing.expect(!publicIp(ip4(b))); + try std.testing.expect(publicIp(ip6("2606:4700::1111"))); + for ([_][]const u8{ "::1", "::", "fe80::1", "fc00::1", "ff02::1", "::ffff:8.8.8.8", "2002:808:808::1", "2001::1", "2001:db8::1", "64:ff9b::808:808" }) |t| + try std.testing.expect(!publicIp(ip6(t))); +} + +test "URLs as images_http._url takes them" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var f: Failure = .{}; + const ok = try parseUrl(a, "https://Example.COM:443/img/c.png?x=%20&y=\xc3\xa4?z", &f); + try std.testing.expectEqualStrings("example.com", ok.host); + try std.testing.expectEqualStrings("/img/c.png?x=%20&y=%C3%A4?z", ok.target); + try std.testing.expectEqualStrings("/", (try parseUrl(a, "https://example.com", &f)).target); + try std.testing.expectEqualStrings("/", (try parseUrl(a, "https://example.com?", &f)).target); + try std.testing.expectEqualStrings("/a??", (try parseUrl(a, "https://example.com/a??", &f)).target); + try std.testing.expectEqualStrings("/%5Ba%5D%22%3C", (try parseUrl(a, "https://example.com/[a]\"<", &f)).target); + const cases = [_]struct { []const u8, []const u8 }{ + .{ "https://example.com/a b.png", "image URL is too long or contains whitespace/control characters" }, + .{ "http://example.com/x.png", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://user:pw@example.com/x", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://example.com:8443/x", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://example.com/x#frag", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://example.com\\@evil/x", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https:///x", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://[::1]/x", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://localhost/x", "image URLs must use public internet hosts" }, + .{ "https://LOCALHOST./x", "image URLs must use public internet hosts" }, + .{ "https://metadata.google.internal/x", "image URLs must use public internet hosts" }, + }; + for (cases) |c| { + try std.testing.expectError(error.Media, parseUrl(a, c[0], &f)); + try std.testing.expectEqualStrings(c[1], f.text); + } + const long = try a.alloc(u8, max_url_chars + 1); + @memcpy(long[0.."https://e.com/".len], "https://e.com/"); + @memset(long["https://e.com/".len..], 'a'); + try std.testing.expectError(error.Media, parseUrl(a, long, &f)); + try std.testing.expectEqualStrings("https://e.com/b/c.png", try join(a, "https://e.com/b/a.png?q", "c.png")); + try std.testing.expectEqualStrings("https://e.com/z", try join(a, "https://e.com/b/a.png", "/z")); + try std.testing.expectEqualStrings("https://cdn.e.com/z", try join(a, "https://e.com/b/a.png", "//cdn.e.com/z")); + try std.testing.expectEqualStrings("http://e.com/z", try join(a, "https://e.com/b", "http://e.com/z")); +} + +/// Canned responses by host and path, for the HTTP rules without a network. +const Fixtures = struct { + pages: []const struct { []const u8, []const u8 }, + asked: *std.ArrayList(u8), + + fn request(self: Fixtures, a: std.mem.Allocator, io: Io, u: Url, max_bytes: usize, deadline: Io.Timestamp, media_types: []const []const u8, f: *Failure) Error!Got { + var w: Io.Writer.Allocating = .init(a); + ask(&w.writer, u, try std.mem.join(a, ", ", media_types)) catch return error.OutOfMemory; + try self.asked.appendSlice(a, w.written()); + const key = try std.fmt.allocPrint(a, "{s}{s}", .{ u.host, u.target }); + for (self.pages) |p| if (std.mem.eql(u8, p[0], key)) { + var r: Io.Reader = .fixed(p[1]); + return answer(a, io, &r, max_bytes, deadline, media_types, f); + }; + return fail(f, "image host could not be resolved"); + } +}; + +test "responses as images_http._request reads them: redirects, types, lengths, encodings" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + const io = std.testing.io; + var asked: std.ArrayList(u8) = .empty; + const png = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 5\r\n\r\n\x89PNG!"; + const fx: Fixtures = .{ .asked = &asked, .pages = &.{ + .{ "e.com/a.png", png }, + .{ "e.com/chunked", "HTTP/1.1 200 OK\r\ncontent-type: Image/JPEG; q=1\r\nTransfer-Encoding: chunked\r\n\r\n3;x\r\n\xff\xd8\xff\r\n2\r\nab\r\n0\r\n\r\n" }, + .{ "e.com/eof", "HTTP/1.0 200 OK\r\nContent-Type: image/jpeg\r\n\r\n\xff\xd8\xffxyz" }, + .{ "e.com/r1", "HTTP/1.1 302 Found\r\nLocation: /r2\r\n\r\n" }, + .{ "e.com/r2", "HTTP/1.1 301 Moved\r\nLocation: https://cdn.e.com/b/x\r\n\r\n" }, + .{ "cdn.e.com/b/x", "HTTP/1.1 307 Temporary\r\nLocation: y.png\r\n\r\n" }, + .{ "cdn.e.com/b/y.png", png }, + .{ "e.com/loop", "HTTP/1.1 302 Found\r\nLocation: /loop\r\n\r\n" }, + .{ "e.com/plain", "HTTP/1.1 302 Found\r\nLocation: http://e.com/a.png\r\n\r\n" }, + .{ "e.com/inner", "HTTP/1.1 302 Found\r\nLocation: https://localhost/a.png\r\n\r\n" }, + .{ "e.com/nowhere", "HTTP/1.1 302 Found\r\n\r\n" }, + .{ "e.com/404", "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n" }, + .{ "e.com/gz", "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Encoding: gzip\r\nContent-Length: 1\r\n\r\nx" }, + .{ "e.com/html", "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 1\r\n\r\nx" }, + .{ "e.com/big", "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 11\r\n\r\n01234567890" }, + .{ "e.com/bigchunk", "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nTransfer-Encoding: chunked\r\n\r\n6\r\n012345\r\n6\r\n678901\r\n0\r\n\r\n" }, + .{ "e.com/bigeof", "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\n\r\n01234567890" }, + .{ "e.com/badlen", "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: +5\r\n\r\n12345" }, + .{ "e.com/short", "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 9\r\n\r\n12345" }, + .{ "e.com/junk", "SSH-2.0-OpenSSH\r\n\r\n" }, + } }; + const far = Io.Clock.awake.now(io).addDuration(.fromSeconds(60)); + var f: Failure = .{}; + const one = try fetchVia(a, io, "https://e.com/a.png", 10, far, &image_media, &f, fx); + try std.testing.expectEqualStrings("\x89PNG!", one.data); + try std.testing.expectEqualStrings("image/png", one.media); + try std.testing.expectEqualStrings("GET /a.png HTTP/1.1\r\nHost: e.com\r\nAccept: image/jpeg, image/png, image/webp\r\nAccept-Encoding: identity\r\nUser-Agent: TensorFold-native\r\nConnection: close\r\n\r\n", asked.items); + const ch = try fetchVia(a, io, "https://e.com/chunked", 10, far, &image_media, &f, fx); + try std.testing.expectEqualStrings("\xff\xd8\xffab", ch.data); + try std.testing.expectEqualStrings("image/jpeg", ch.media); + try std.testing.expectEqualStrings("\xff\xd8\xffxyz", (try fetchVia(a, io, "https://e.com/eof", 10, far, &image_media, &f, fx)).data); + // three redirects, each joined and checked again, reach the image; a fourth is refused + asked.clearRetainingCapacity(); + try std.testing.expectEqualStrings("\x89PNG!", (try fetchVia(a, io, "https://e.com/r1", 10, far, &image_media, &f, fx)).data); + try std.testing.expectEqual(@as(usize, 4), std.mem.count(u8, asked.items, "GET ")); + try std.testing.expect(std.mem.indexOf(u8, asked.items, "GET /b/y.png HTTP/1.1\r\nHost: cdn.e.com\r\n") != null); + const refusals = [_]struct { []const u8, []const u8 }{ + .{ "https://e.com/loop", "image download has too many redirects" }, + .{ "https://e.com/plain", "image URL must be HTTPS on port 443, without credentials or a fragment" }, + .{ "https://e.com/inner", "image URLs must use public internet hosts" }, + .{ "https://e.com/nowhere", "image redirect has no destination" }, + .{ "https://e.com/404", "image download returned HTTP 404" }, + .{ "https://e.com/gz", "compressed HTTP image responses are unsupported" }, + .{ "https://e.com/html", "image URL content type must be JPEG, PNG or WebP" }, + .{ "https://e.com/big", "image response exceeds the encoded byte limit" }, + .{ "https://e.com/bigchunk", "image response exceeds the encoded byte limit" }, + .{ "https://e.com/bigeof", "image response exceeds the encoded byte limit" }, + .{ "https://e.com/badlen", "image response exceeds the encoded byte limit" }, + .{ "https://e.com/short", "image download failed or timed out" }, + .{ "https://e.com/junk", "image download failed or timed out" }, + }; + for (refusals) |c| { + try std.testing.expectError(error.Media, fetchVia(a, io, c[0], 10, far, &image_media, &f, fx)); + try std.testing.expectEqualStrings(c[1], f.text); + } + // a deadline already passed refuses before the body + try std.testing.expectError(error.Media, fetchVia(a, io, "https://e.com/a.png", 10, Io.Clock.awake.now(io), &image_media, &f, fx)); + try std.testing.expectEqualStrings("image download timed out", f.text); +} diff --git a/zig/src/server/prompt.zig b/zig/src/server/prompt.zig index 5500803b68..df2a7024c6 100644 --- a/zig/src/server/prompt.zig +++ b/zig/src/server/prompt.zig @@ -115,7 +115,7 @@ pub fn prepare(srv: *Server, cx: *Cx, input: chat.Input, thinking: bool, effort: if (history_len == 0 and prompt.len > 1 and std.mem.eql(u32, history, prompt)) history_len = prompt.len - 1; // a template with no generation suffix // --vision: each image's placeholder expanded to its tokens before the window, chunks and cache see the prompt; // such a prompt keeps no rewind, which the expansion does not translate - if (try images.expand(srv.info.vision, cx, input.messages, prompt, history_len)) |x| return .{ .ids = x.ids, .history_len = x.history_len, .images = x.images, .cache_key = x.cache_key }; + if (try images.expand(srv.info.vision, if (srv.config.vision_urls) images.Fetch.network(srv.io) else null, cx, input.messages, prompt, history_len)) |x| return .{ .ids = x.ids, .history_len = x.history_len, .images = x.images, .cache_key = x.cache_key }; const count = if (input.messages == .array) input.messages.array.len else 0; const rewind_len = if (srv.info.prompt_cache) rewindLen(srv, cx, input.messages, count, input.tools, prompt, thinking, effort) else 0; // one more render return .{ .ids = prompt, .history_len = history_len, .rewind_len = rewind_len }; diff --git a/zig/src/server/root.zig b/zig/src/server/root.zig index b669d55c3a..36186aea81 100644 --- a/zig/src/server/root.zig +++ b/zig/src/server/root.zig @@ -39,6 +39,7 @@ test { _ = @import("log.zig"); _ = @import("messages.zig"); _ = @import("images.zig"); + _ = @import("media_fetch.zig"); _ = @import("stream_preflight_test.zig"); _ = @import("logprobs_test.zig"); _ = @import("late_system_test.zig"); diff --git a/zig/src/server/serve.zig b/zig/src/server/serve.zig index 967c0979b1..31aa5bed83 100644 --- a/zig/src/server/serve.zig +++ b/zig/src/server/serve.zig @@ -116,6 +116,7 @@ pub fn run(gpa: Allocator, io: std.Io, args: cli.Args, s: Setup) u8 { .keep_warm_s = args.keep_warm, .default_sampling = s.sampling, .use_drafts = !args.no_drafts, + .vision_urls = args.vision_urls, .seed_salt = salt, .request_log = env(s, "TENSORFOLD_REQUEST_LOG"), .dashboard = args.dashboard, diff --git a/zig/src/server/server.zig b/zig/src/server/server.zig index 255b02a2b6..a8dd67e581 100644 --- a/zig/src/server/server.zig +++ b/zig/src/server/server.zig @@ -45,6 +45,8 @@ pub const Config = struct { /// Sampling when a request names none (generation_config.json and the serve flags), or null for greedy. default_sampling: ?Value = null, use_drafts: bool = true, + /// --vision-urls: image parts may name public HTTPS URLs, fetched before admission (media_fetch.zig). + vision_urls: bool = false, seed_salt: i64 = 0, /// TENSORFOLD_REQUEST_LOG: where chat and completion bodies are appended; null: nowhere. request_log: ?[]const u8 = null, From fa22819fb1557ddd8adf73543dc4efe1a6475a90 Mon Sep 17 00:00:00 2001 From: BobClawblaw Date: Mon, 12 Oct 2026 02:33:44 +0000 Subject: [PATCH 06/19] Qwen3.5 vision tower on CUDA: qwen_vision.cu kernels, the Tower, and its check against 0.6.6 Row cosine against 0.6.6's flash-attention tower on nine fixture images: mean 0.9997-0.99998, min 0.981-0.99998; 0.6.6's own math-vs-flash attention gives min 0.9895 on the same shapes image. --- tools/zig/qwen_vision_fixtures.py | 103 ++++++ zig/build/cuda.zig | 1 + zig/kernels/cuda/qwen_vision.cu | 208 +++++++++++++ zig/src/cli/qwen27_cuda.zig | 73 +++++ zig/src/cuda/kernels.zig | 1 + zig/src/families/qwen27/cuda.zig | 2 + zig/src/families/qwen27/cuda/vision.zig | 397 ++++++++++++++++++++++++ 7 files changed, 785 insertions(+) create mode 100644 tools/zig/qwen_vision_fixtures.py create mode 100644 zig/kernels/cuda/qwen_vision.cu create mode 100644 zig/src/families/qwen27/cuda/vision.zig diff --git a/tools/zig/qwen_vision_fixtures.py b/tools/zig/qwen_vision_fixtures.py new file mode 100644 index 0000000000..5f989a42ef --- /dev/null +++ b/tools/zig/qwen_vision_fixtures.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Fixtures for the native Qwen vision path, from TensorFold 0.6.6's --vision frontend and tower on CUDA. + +For a fixed image set: each image's file, its decoded RGB (images.py), its patches and grid (QwenImageProcessor's +processor at 0.6.6's per-image cap), and the tower's merged rows (QwenCudaVision, bf16, flash attention). The Zig +checks compare their own decode, patches and rows against these: ``tf-qwen27-cuda vision MODEL --fixtures DIR``. +""" + +from __future__ import annotations + +import argparse +import io +import json +from pathlib import Path + +import numpy as np + + +def images(): + """(name, encoded bytes) for the set: PNG layouts, sizes from tiny to large, an EXIF-rotated JPEG.""" + from PIL import Image, ImageDraw + + out = [] + shapes = Image.new("RGB", (517, 389), "white") + d = ImageDraw.Draw(shapes) + d.ellipse((30, 40, 230, 240), fill=(220, 20, 20)) + d.rectangle((280, 60, 490, 260), fill=(20, 60, 220)) + d.text((60, 300), "ORANGE 7351", fill="black") + rng = np.random.default_rng(7) + y, x = np.mgrid[0:768, 0:1024] + photo = np.stack([(x / 4) % 256, (y / 3) % 256, ((x + y) / 7) % 256], -1) + rng.normal(0, 12, (768, 1024, 3)) + photo = Image.fromarray(np.clip(photo, 0, 255).astype(np.uint8)) + large = photo.resize((2000, 1500)) + tiny = shapes.resize((20, 15)) + logo = Image.new("RGBA", (96, 96), (0, 0, 0, 0)) + ImageDraw.Draw(logo).ellipse((8, 8, 88, 88), fill=(30, 160, 90, 200)) + palette = shapes.convert("P", palette=Image.Palette.ADAPTIVE, colors=16) + palette.info["transparency"] = 0 + gray = photo.convert("L") + + def png(img, **kw): + b = io.BytesIO() + img.save(b, "PNG", **kw) + return b.getvalue() + + def jpeg(img, exif=None): + b = io.BytesIO() + img.save(b, "JPEG", quality=90, **({"exif": exif} if exif else {})) + return b.getvalue() + + exif = Image.Exif() + exif[0x0112] = 6 # rotate 90 on display + out += [("shapes.png", png(shapes)), ("photo.png", png(photo)), ("large.png", png(large)), + ("tiny.png", png(tiny)), ("logo.png", png(logo)), ("palette.png", png(palette)), ("gray.png", png(gray)), + ("photo.jpg", jpeg(photo)), ("rotated.jpg", jpeg(shapes, exif.tobytes()))] + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", required=True) + ap.add_argument("--out", required=True) + a = ap.parse_args() + import torch + from torch.nn.attention import SDPBackend, sdpa_kernel + from tensorfold.vision.images import _decode, DEFAULT_LIMITS + from tensorfold.vision.qwen_cuda import QwenCudaVision, TOKENS_PER_IMAGE + + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + vision = QwenCudaVision(a.model, torch.device("cuda:0")) + proc = vision.frontend.processor + factor = vision.config["patch_size"] * vision.config["spatial_merge_size"] + cap = min(int(getattr(proc, "max_pixels", TOKENS_PER_IMAGE * factor**2)), TOKENS_PER_IMAGE * factor**2) + floor = min(int(getattr(proc, "min_pixels", factor**2)), cap) + entries = [] + for name, data in images(): + stem = name.replace(".", "_") + (out / name).write_bytes(data) + decoded = _decode(data, "auto", DEFAULT_LIMITS, DEFAULT_LIMITS.max_pixels) + pil = decoded.to_pil() + rgb = np.asarray(pil, dtype=np.uint8) + (out / f"{stem}.rgb").write_bytes(rgb.tobytes()) + result = proc(images=[pil], max_pixels=cap, min_pixels=floor) + pixels = np.asarray(result["pixel_values"], dtype=np.float32) + t, h, w = (int(v) for v in np.asarray(result["image_grid_thw"])[0]) + (out / f"{stem}.f32").write_bytes(pixels.tobytes()) + with torch.inference_mode(), sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]): + feats = vision.tower(torch.tensor(pixels, dtype=torch.bfloat16, device="cuda:0"), + grid_thw=torch.tensor([[t, h, w]], device="cuda:0"), return_dict=True).pooler_output + feats = feats.to(torch.bfloat16).contiguous().cpu() + (out / f"{stem}.bf16").write_bytes(feats.view(torch.int16).numpy().tobytes()) + entries.append({"name": name, "file": name, "rgb": f"{stem}.rgb", "size": [rgb.shape[1], rgb.shape[0]], + "grid": [h, w], "pixels": f"{stem}.f32", "features": f"{stem}.bf16", + "tokens": h * w // 4}) + print(f"{name}: {rgb.shape[1]}x{rgb.shape[0]} -> grid {h}x{w}, {h * w // 4} tokens", flush=True) + (out / "manifest.json").write_text(json.dumps({"images": entries, "cap_pixels": cap, "min_pixels": floor}, + indent=1) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/zig/build/cuda.zig b/zig/build/cuda.zig index bac51ae59f..188de6ea75 100644 --- a/zig/build/cuda.zig +++ b/zig/build/cuda.zig @@ -30,6 +30,7 @@ const kernels = [_]Kernel{ .{ .name = "sample", .flags = &.{ "-O3", "--fmad=false", "--ftz=false" } }, // ours: the Metal engine's keyed draws .{ .name = "train", .flags = &.{"-O3"} }, // ours: Sliding Weights' learning step (train.metal's kernels) .{ .name = "qwen_ops", .flags = &.{"-O3"} }, // ours: the 27B's batched state copies, taps, drafter norms and rotary + .{ .name = "qwen_vision", .flags = &.{ "-O3", "--fmad=false" } }, // ours: the Qwen3.5 vision tower's glue and attention .{ .name = "nemotron_norms", .flags = glue }, // ours, each with a host reference (glue_ref.zig): Nemotron's glue .{ .name = "nemotron_route", .flags = glue }, .{ .name = "nemotron_mamba", .flags = glue }, diff --git a/zig/kernels/cuda/qwen_vision.cu b/zig/kernels/cuda/qwen_vision.cu new file mode 100644 index 0000000000..9b1a73db9d --- /dev/null +++ b/zig/kernels/cuda/qwen_vision.cu @@ -0,0 +1,208 @@ +// The Qwen3.5 vision tower's kernels (transformers' Qwen3_5VisionModel, as 0.6.6's --vision runs it in bf16). +// The tower's Linears run on cuBLASLt with fp32 output; each epilogue here adds the bias and rounds to bf16 once, +// where torch's bf16 Linear rounds, before the op that follows. +#include +#include + +__device__ __forceinline__ float bf(__nv_bfloat16 v) { return __bfloat162float(v); } +__device__ __forceinline__ float rnd(float v) { return __bfloat162float(__float2bfloat16_rn(v)); } + +// Patch embedding plus the resampled position grid: x = bf16(acc + b) + bf16(sum_t pos[idx_t] * w_t). +extern "C" __global__ void __launch_bounds__(256) tf_qv_embed(const float* __restrict__ acc, + const __nv_bfloat16* __restrict__ bias, + const __nv_bfloat16* __restrict__ pos, + const int* __restrict__ idx, + const float* __restrict__ wts, + __nv_bfloat16* __restrict__ out, int D) { + const long long row = blockIdx.x; + const int* ri = idx + row * 4; + const float* rw = wts + row * 4; + for (int i = threadIdx.x; i < D; i += blockDim.x) { + const float h = rnd(acc[row * D + i] + bf(bias[i])); + float p = 0.0f; + for (int t = 0; t < 4; ++t) p += bf(pos[(long long)ri[t] * D + i]) * rw[t]; + out[row * D + i] = __float2bfloat16_rn(h + rnd(p)); + } +} + +// LayerNorm of bf16 rows in fp32 (mean, then the biased variance about it), bf16 out; a block a row. +extern "C" __global__ void __launch_bounds__(256) tf_qv_layernorm(const __nv_bfloat16* __restrict__ x, + const __nv_bfloat16* __restrict__ w, + const __nv_bfloat16* __restrict__ b, + __nv_bfloat16* __restrict__ y, int D, float eps) { + __shared__ float part[32]; + __shared__ float stat; + const long long row = blockIdx.x; + const __nv_bfloat16* xr = x + row * D; + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, warps = blockDim.x >> 5; + float s = 0.0f; + for (int i = threadIdx.x; i < D; i += blockDim.x) s += bf(xr[i]); + for (int o = 16; o; o >>= 1) s += __shfl_xor_sync(0xffffffffu, s, o); + if (lane == 0) part[warp] = s; + __syncthreads(); + if (threadIdx.x == 0) { + float t = 0.0f; + for (int i = 0; i < warps; ++i) t += part[i]; + stat = t / D; + } + __syncthreads(); + const float mean = stat; + float v = 0.0f; + for (int i = threadIdx.x; i < D; i += blockDim.x) { + const float d = bf(xr[i]) - mean; + v += d * d; + } + for (int o = 16; o; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o); + __syncthreads(); + if (lane == 0) part[warp] = v; + __syncthreads(); + if (threadIdx.x == 0) { + float t = 0.0f; + for (int i = 0; i < warps; ++i) t += part[i]; + stat = rsqrtf(t / D + eps); + } + __syncthreads(); + const float r = stat; + for (int i = threadIdx.x; i < D; i += blockDim.x) + y[row * D + i] = __float2bfloat16_rn((bf(xr[i]) - mean) * r * bf(w[i]) + bf(b[i])); +} + +// qkv = bf16(acc + b) split into q, k, v [heads][rows][hd]; q and k rotated by the axial 2-D rotary in fp32 +// (apply_rotary_pos_emb_vision): angle j of a head is h * inv[j % half'] for the first quarter's frequencies and +// w * inv[...] for the second, repeated over both halves; rotate_half pairs j with j +- hd/2. +extern "C" __global__ void __launch_bounds__(128) tf_qv_qkv_rope(const float* __restrict__ acc, + const __nv_bfloat16* __restrict__ bias, + const int* __restrict__ hw, + const float* __restrict__ inv, + __nv_bfloat16* __restrict__ q, + __nv_bfloat16* __restrict__ k, + __nv_bfloat16* __restrict__ v, int rows, int heads, + int hd) { + const long long row = blockIdx.x; + const int head = blockIdx.y; + const int D = heads * hd; + const int half = hd / 2, quarter = hd / 4; + const float ph = static_cast(hw[row * 2]), pw = static_cast(hw[row * 2 + 1]); + const float* a = acc + row * 3 * D; + const long long o = (static_cast(head) * rows + row) * hd; + for (int j = threadIdx.x; j < hd; j += blockDim.x) { + const int c = head * hd + j; + const int pj = j < half ? j + half : j - half; + const int pc = head * hd + pj; + const int jj = j % half; + const float ang = jj < quarter ? ph * inv[jj] : pw * inv[jj - quarter]; + const float cs = cosf(ang), sn = sinf(ang); + const float sign = j < half ? -1.0f : 1.0f; + const float qx = rnd(a[c] + bf(bias[c])), qp = rnd(a[pc] + bf(bias[pc])); + const float kx = rnd(a[D + c] + bf(bias[D + c])), kp = rnd(a[D + pc] + bf(bias[D + pc])); + q[o + j] = __float2bfloat16_rn(qx * cs + sign * qp * sn); + k[o + j] = __float2bfloat16_rn(kx * cs + sign * kp * sn); + v[o + j] = __float2bfloat16_rn(a[2 * D + c] + bf(bias[2 * D + c])); + } +} + +// Bidirectional attention within one image: rows [s0, s0 + n) of q, k, v [heads][rows][HD] attend to each other; +// out [rows][heads * HD]. A thread a query row with an online softmax in fp32; keys and values in 64-row tiles. +template +__device__ void qv_attention(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ k, + const __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ out, int rows, + int heads, int s0, int n, float scale) { + constexpr int T = 64; + __shared__ float ks[T][HD]; + __shared__ float vs[T][HD]; + const int head = blockIdx.y; + const int qi = blockIdx.x * T + threadIdx.x; + const bool live = qi < n; + const long long base = static_cast(head) * rows + s0; + float qr[HD], acc[HD]; + float m = -INFINITY, l = 0.0f; + if (live) { + const __nv_bfloat16* qp = q + (base + qi) * HD; + for (int d = 0; d < HD; ++d) { + qr[d] = bf(qp[d]) * scale; + acc[d] = 0.0f; + } + } + for (int t0 = 0; t0 < n; t0 += T) { + const int count = min(T, n - t0); + __syncthreads(); + for (int i = threadIdx.x; i < count * HD; i += blockDim.x) { + const int r = i / HD, d = i % HD; + ks[r][d] = bf(k[(base + t0 + r) * HD + d]); + vs[r][d] = bf(v[(base + t0 + r) * HD + d]); + } + __syncthreads(); + if (!live) continue; + for (int r = 0; r < count; ++r) { + float s = 0.0f; +#pragma unroll + for (int d = 0; d < HD; ++d) s += qr[d] * ks[r][d]; + const float mn = fmaxf(m, s); + const float corr = __expf(m - mn); + const float p = __expf(s - mn); + l = l * corr + p; +#pragma unroll + for (int d = 0; d < HD; ++d) acc[d] = acc[d] * corr + p * vs[r][d]; + m = mn; + } + } + if (!live) return; + const float il = 1.0f / l; + __nv_bfloat16* op = out + static_cast(s0 + qi) * heads * HD + head * HD; + for (int d = 0; d < HD; ++d) op[d] = __float2bfloat16_rn(acc[d] * il); +} + +extern "C" __global__ void __launch_bounds__(64) tf_qv_attention72(const __nv_bfloat16* __restrict__ q, + const __nv_bfloat16* __restrict__ k, + const __nv_bfloat16* __restrict__ v, + __nv_bfloat16* __restrict__ out, int rows, + int heads, int s0, int n, float scale) { + qv_attention<72>(q, k, v, out, rows, heads, s0, n, scale); +} + +// y = gelu(bf16(acc + b)), the tanh form (gelu_pytorch_tanh) or the erf form (nn.GELU), computed in fp32, bf16 out. +extern "C" __global__ void __launch_bounds__(256) tf_qv_bias_gelu(const float* __restrict__ acc, + const __nv_bfloat16* __restrict__ bias, + __nv_bfloat16* __restrict__ y, int D, int tanh_form) { + const long long row = blockIdx.x; + for (int i = threadIdx.x; i < D; i += blockDim.x) { + const float x = rnd(acc[row * D + i] + bf(bias[i])); + float g; + if (tanh_form) { + const float kBeta = 0.7978845608028654f; // sqrt(2 / pi) + g = 0.5f * x * (1.0f + tanhf(kBeta * (x + 0.044715f * x * x * x))); + } else { + g = 0.5f * x * (1.0f + erff(x * 0.7071067811865476f)); + } + y[row * D + i] = __float2bfloat16_rn(g); + } +} + +// x += bf16(acc + b), the residual add in bf16 (both operands bf16, the sum rounded). +extern "C" __global__ void __launch_bounds__(256) tf_qv_bias_residual(const float* __restrict__ acc, + const __nv_bfloat16* __restrict__ bias, + __nv_bfloat16* __restrict__ x, int D) { + const long long row = blockIdx.x; + for (int i = threadIdx.x; i < D; i += blockDim.x) { + const float y = rnd(acc[row * D + i] + bf(bias[i])); + x[row * D + i] = __float2bfloat16_rn(bf(x[row * D + i]) + y); + } +} + +// y = bf16(acc + b): the merger's last Linear. +extern "C" __global__ void __launch_bounds__(256) tf_qv_bias_cast(const float* __restrict__ acc, + const __nv_bfloat16* __restrict__ bias, + __nv_bfloat16* __restrict__ y, int D) { + const long long row = blockIdx.x; + for (int i = threadIdx.x; i < D; i += blockDim.x) y[row * D + i] = __float2bfloat16_rn(acc[row * D + i] + bf(bias[i])); +} + +// Image rows into a prompt chunk's embeddings: row r of `x` (stride D) takes feature row src[r] where src[r] >= 0. +extern "C" __global__ void __launch_bounds__(256) tf_qv_splice(__nv_bfloat16* __restrict__ x, + const __nv_bfloat16* __restrict__ features, + const int* __restrict__ src, int D) { + const long long row = blockIdx.x; + const int s = src[row]; + if (s < 0) return; + for (int i = threadIdx.x; i < D; i += blockDim.x) x[row * D + i] = features[static_cast(s) * D + i]; +} diff --git a/zig/src/cli/qwen27_cuda.zig b/zig/src/cli/qwen27_cuda.zig index a92416ce7b..dfdfe785f9 100644 --- a/zig/src/cli/qwen27_cuda.zig +++ b/zig/src/cli/qwen27_cuda.zig @@ -16,6 +16,8 @@ const usage = \\ Greedy shared rounds of one row a stream over every prompt, timed. \\ tf-qwen27-cuda multi MODEL --kernels DIR --drafter DIR --prompts multi.json \\ capture_multi.py's drafted shared rounds: each round's windows and every reply against the capture's. + \\ tf-qwen27-cuda vision MODEL --fixtures DIR + \\ The image tower on qwen_vision_fixtures.py's patches: each image's rows against 0.6.6's (cosine, max error). \\ ; @@ -40,6 +42,8 @@ pub fn main(init: std.process.Init) !u8 { const rounds = args.len >= 2 and std.mem.eql(u8, args[1], "rounds"); const drafting = args.len >= 2 and std.mem.eql(u8, args[1], "draft"); const multi = args.len >= 2 and std.mem.eql(u8, args[1], "multi"); + if (args.len >= 5 and std.mem.eql(u8, args[1], "vision") and std.mem.eql(u8, args[3], "--fixtures")) + return checkVision(gpa, arena, init.io, args[2], args[4]); if (args.len < 3 or !(std.mem.eql(u8, args[1], "run") or windows or rounds or drafting or multi)) { std.debug.print("{s}", .{usage}); return 2; @@ -457,3 +461,72 @@ fn runMulti(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.E std.debug.print("{d} rounds (python {d}), {d} windows differ from Python's, {d} replies differ; {d:.1} tok/s\n", .{ round, want_rounds.len, differ, bad, @as(f64, @floatFromInt(produced)) / secs }); return if (bad == 0 and round == want_rounds.len) 0 else 1; } + +/// The tower against qwen_vision_fixtures.py's manifest: per image, the features' row cosines and largest error. +fn checkVision(gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, model: []const u8, dir: []const u8) !u8 { + const V = qwen.vision; + const manifest = try readJson(arena, io, try std.fs.path.join(arena, &.{ dir, "manifest.json" })); + var driver = try cuda.Driver.open(); + defer driver.close(); + var ctx = try cuda.Context.init(&driver, 0); + defer ctx.deinit(); + var stream = try cuda.Stream.init(&driver, true); + defer stream.deinit(); + const gpu: V.Gpu = .{ .d = &driver, .s = stream, .discrete = try ctx.attribute(.integrated) == 0 }; + const clock = std.Io.Clock.awake; + var t0 = clock.now(io).toNanoseconds(); + const tower = try V.Tower.load(gpa, io, gpu, model); + defer tower.deinit(); + std.debug.print("tower loaded in {d:.1}s, {d:.2} GiB\n", .{ @as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t0)) / 1e9, @as(f64, @floatFromInt(tower.weight_bytes)) / (1 << 30) }); + var worst: f64 = 1; + for (manifest.object.get("images").?.array.items) |img| { + const name = img.object.get("name").?.string; + const grid = img.object.get("grid").?.array.items; + const g: V.Grid = .{ .h = @intCast(grid[0].integer), .w = @intCast(grid[1].integer) }; + const px_bytes = try std.Io.Dir.cwd().readFileAlloc(io, try std.fs.path.join(arena, &.{ dir, img.object.get("pixels").?.string }), arena, .limited(1 << 30)); + const want_bytes = try std.Io.Dir.cwd().readFileAlloc(io, try std.fs.path.join(arena, &.{ dir, img.object.get("features").?.string }), arena, .limited(1 << 30)); + const px = try arena.alloc(f32, px_bytes.len / 4); + @memcpy(std.mem.sliceAsBytes(px), px_bytes[0 .. px.len * 4]); + const rows = @as(usize, g.h) * g.w / 4; + var out = try cuda.DeviceBuffer.alloc(&driver, rows * V.out_hidden * 2); + defer out.free(); + t0 = clock.now(io).toNanoseconds(); + try tower.encode(gpu, px, &.{g}, out.ptr); + const ms = @as(f64, @floatFromInt(clock.now(io).toNanoseconds() - t0)) / 1e6; + const got = try arena.alloc(u16, rows * V.out_hidden); + try driver.check(driver.api.cuMemcpyDtoH_v2(@ptrCast(got.ptr), out.ptr, got.len * 2), "cuMemcpyDtoH"); + const want: []align(1) const u16 = std.mem.bytesAsSlice(u16, want_bytes); + if (want.len != got.len) { + std.debug.print("{s}: {d} features, 0.6.6 has {d}\n", .{ name, got.len, want.len }); + return 1; + } + var min_cos: f64 = 1; + var sum_cos: f64 = 0; + var max_err: f64 = 0; + var equal: usize = 0; + for (0..rows) |r| { + var dot: f64 = 0; + var na: f64 = 0; + var nb: f64 = 0; + for (0..V.out_hidden) |c| { + const a = bf(got[r * V.out_hidden + c]); + const b = bf(want[r * V.out_hidden + c]); + if (got[r * V.out_hidden + c] == want[r * V.out_hidden + c]) equal += 1; + dot += a * b; + na += a * a; + nb += b * b; + max_err = @max(max_err, @abs(a - b)); + } + const cs = dot / @sqrt(na * nb); + min_cos = @min(min_cos, cs); + sum_cos += cs; + } + worst = @min(worst, min_cos); + std.debug.print("{s}: grid {d}x{d}, {d} rows in {d:.1} ms; row cosine min {d:.5} mean {d:.5}; max error {d:.4}; {d:.1}% values equal\n", .{ name, g.h, g.w, rows, ms, min_cos, sum_cos / @as(f64, @floatFromInt(rows)), max_err, 100 * @as(f64, @floatFromInt(equal)) / @as(f64, @floatFromInt(got.len)) }); + } + return if (worst > 0.98) 0 else 1; +} + +fn bf(v: u16) f64 { + return @as(f32, @bitCast(@as(u32, v) << 16)); +} diff --git a/zig/src/cuda/kernels.zig b/zig/src/cuda/kernels.zig index 2e93589095..56dc43cf2e 100644 --- a/zig/src/cuda/kernels.zig +++ b/zig/src/cuda/kernels.zig @@ -28,6 +28,7 @@ pub const affine4_pack: []const u8 = if (available) &Blob("fatbin_affine4_pack") pub const fp8_experts: []const u8 = if (available) &Blob("fatbin_fp8_experts").bytes else &.{}; pub const nvfp4_experts: []const u8 = if (available) &Blob("fatbin_nvfp4_experts").bytes else &.{}; pub const qwen_ops: []const u8 = if (available) &Blob("fatbin_qwen_ops").bytes else &.{}; +pub const qwen_vision: []const u8 = if (available) &Blob("fatbin_qwen_vision").bytes else &.{}; pub const sample: []const u8 = if (available) &Blob("fatbin_sample").bytes else &.{}; pub const train: []const u8 = if (available) &Blob("fatbin_train").bytes else &.{}; pub const qmmf: []const u8 = if (available) &Blob("fatbin_qmmf").bytes else &.{}; diff --git a/zig/src/families/qwen27/cuda.zig b/zig/src/families/qwen27/cuda.zig index 3532c69cde..746c12d658 100644 --- a/zig/src/families/qwen27/cuda.zig +++ b/zig/src/families/qwen27/cuda.zig @@ -14,6 +14,7 @@ pub const native = @import("cuda/native.zig"); pub const draft = @import("cuda/draft.zig"); pub const lone = @import("cuda/lone.zig"); pub const drafts = @import("cuda/drafts.zig"); +pub const vision = @import("cuda/vision.zig"); test { _ = config; @@ -30,4 +31,5 @@ test { _ = draft; _ = lone; _ = native; + _ = vision; } diff --git a/zig/src/families/qwen27/cuda/vision.zig b/zig/src/families/qwen27/cuda/vision.zig new file mode 100644 index 0000000000..1305d9b3d6 --- /dev/null +++ b/zig/src/families/qwen27/cuda/vision.zig @@ -0,0 +1,397 @@ +//! The Qwen3.5 vision tower on CUDA (transformers' Qwen3_5VisionModel, as 0.6.6's --vision runs it in bf16): patch +//! embedding, the learned position grid resampled bilinearly to each image, 27 pre-norm blocks of axial 2-D rotary +//! attention within each image, and the 2x2 merger to the language model's width. Linears on cuBLASLt (fp32 out), +//! the rest in qwen_vision.cu. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const nemotron = @import("nemotron"); +const source = nemotron.source; + +const lt = cuda.cublaslt; +const prefix = "vision_tower."; + +/// vision_config as the 27B's checkpoint gives it; another tower is refused. +pub const hidden = 1152; +pub const heads = 16; +pub const head_dim = hidden / heads; // 72 +pub const depth = 27; +pub const intermediate = 4304; +pub const out_hidden = 5120; +pub const patch = 16; +pub const temporal = 2; +pub const merge = 2; +pub const channels = 3; +pub const patch_width = channels * temporal * patch * patch; // 1536 +pub const grid_side = 48; // sqrt(num_position_embeddings 2304) +pub const merged = hidden * merge * merge; // 4608 +pub const eps: f32 = 1e-6; +/// One tower call's patches: a 4,096-token image's (0.6.6's MAX_PATCHES). +pub const max_patches = 16384; +const workspace_bytes = 32 << 20; + +/// The device, stream and memory kind the tower runs on (the engine's, or a checker's own). +pub const Gpu = struct { + d: *const cuda.Driver, + s: cuda.Stream, + discrete: bool, + + fn upload(g: Gpu, dst: u64, bytes: []const u8) !void { + if (bytes.len == 0) return; + try g.d.check(g.d.api.cuMemcpyHtoDAsync_v2(dst, bytes.ptr, bytes.len, g.s.handle), "cuMemcpyHtoDAsync"); + } +}; + +/// One image's patch grid (rows and columns of 16x16 patches, both even). +pub const Grid = struct { h: u32, w: u32 }; + +const Block = struct { norm1_w: u64, norm1_b: u64, qkv_w: u64, qkv_b: u64, proj_w: u64, proj_b: u64, norm2_w: u64, norm2_b: u64, fc1_w: u64, fc1_b: u64, fc2_w: u64, fc2_b: u64 }; + +const Fns = struct { + embed: cuda.Function, + layernorm: cuda.Function, + qkv_rope: cuda.Function, + attention: cuda.Function, + bias_gelu: cuda.Function, + bias_residual: cuda.Function, + bias_cast: cuda.Function, + splice: cuda.Function, +}; + +/// The image tower's weights on the GPU and its kernels; `encode` turns patches into language-width rows. +pub const Tower = struct { + gpa: std.mem.Allocator, + d: *const cuda.Driver, + mod: cuda.Module, + f: Fns, + lib: lt.Library, + buffers: std.ArrayList(cuda.DeviceBuffer) = .empty, + patch_w: u64 = 0, + patch_b: u64 = 0, + pos: u64 = 0, + blocks: [depth]Block = undefined, + merger_norm_w: u64 = 0, + merger_norm_b: u64 = 0, + fc1_w: u64 = 0, + fc1_b: u64 = 0, + fc2_w: u64 = 0, + fc2_b: u64 = 0, + inv: u64 = 0, + weight_bytes: usize = 0, + + /// The checkpoint's vision_tower tensors (bf16), every one used; the patch convolution's weight as rows of + /// [channel, frame, y, x], the order the patches come in. + pub fn load(gpa: std.mem.Allocator, io: std.Io, ops: Gpu, dir: []const u8) !*Tower { + const t = try gpa.create(Tower); + errdefer gpa.destroy(t); + t.* = .{ .gpa = gpa, .d = ops.d, .mod = undefined, .f = undefined, .lib = undefined }; + t.lib = try lt.Library.open(); + errdefer t.lib.close(); + t.mod = try cuda.Module.load(ops.d, cuda.kernels.qwen_vision); + errdefer t.mod.unload(); + t.f = .{ + .embed = try t.mod.function("tf_qv_embed"), + .layernorm = try t.mod.function("tf_qv_layernorm"), + .qkv_rope = try t.mod.function("tf_qv_qkv_rope"), + .attention = try t.mod.function("tf_qv_attention72"), + .bias_gelu = try t.mod.function("tf_qv_bias_gelu"), + .bias_residual = try t.mod.function("tf_qv_bias_residual"), + .bias_cast = try t.mod.function("tf_qv_bias_cast"), + .splice = try t.mod.function("tf_qv_splice"), + }; + errdefer t.freeBuffers(); + var src = try source.Source.init(gpa, .{ .d = ops.d, .s = ops.s, .discrete = ops.discrete }); + defer src.deinit(); + var ck = try core.Checkpoint.openModelPrefix(gpa, io, dir, prefix); + defer ck.close(); + try src.add(&ck); + const L = struct { + t: *Tower, + ck: *core.Checkpoint, + src: *source.Source, + fn get(l: @This(), name: []const u8, shape: []const usize) !u64 { + var buf: [96]u8 = undefined; + const tn = try l.ck.expect(try std.fmt.bufPrint(&buf, prefix ++ "{s}", .{name}), .bf16, shape); + const ptr = try l.t.alloc(tn.bytes.len); + try l.src.upload(ptr, tn.bytes); + return ptr; + } + }; + const l: L = .{ .t = t, .ck = &ck, .src = &src }; + { // [hidden, T, P, P, C] as stored (MLX) to [hidden, C, T, P, P] + const tn = try ck.expect(prefix ++ "patch_embed.proj.weight", .bf16, &.{ hidden, temporal, patch, patch, channels }); + const host = try gpa.alloc(u16, hidden * patch_width); + defer gpa.free(host); + const raw = try gpa.alloc(u8, tn.bytes.len); + defer gpa.free(raw); + try src.read(raw, tn.bytes); + const stored: []align(1) const u16 = std.mem.bytesAsSlice(u16, raw); + for (0..hidden) |o| for (0..channels) |ch| for (0..temporal) |tt| for (0..patch) |y| for (0..patch) |x| { + host[o * patch_width + ((ch * temporal + tt) * patch + y) * patch + x] = stored[(((o * temporal + tt) * patch + y) * patch + x) * channels + ch]; + }; + t.patch_w = try t.alloc(host.len * 2); + try src.upload(t.patch_w, std.mem.sliceAsBytes(host)); + try ops.s.synchronize(); + } + t.patch_b = try l.get("patch_embed.proj.bias", &.{hidden}); + t.pos = try l.get("pos_embed.weight", &.{ grid_side * grid_side, hidden }); + for (&t.blocks, 0..) |*b, i| { + var pb: [32]u8 = undefined; + const p = try std.fmt.bufPrint(&pb, "blocks.{d}.", .{i}); + var nb: [96]u8 = undefined; + const n = struct { + fn f(buf: []u8, a: []const u8, s: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "{s}{s}", .{ a, s }); + } + }.f; + b.norm1_w = try l.get(try n(&nb, p, "norm1.weight"), &.{hidden}); + b.norm1_b = try l.get(try n(&nb, p, "norm1.bias"), &.{hidden}); + b.qkv_w = try l.get(try n(&nb, p, "attn.qkv.weight"), &.{ 3 * hidden, hidden }); + b.qkv_b = try l.get(try n(&nb, p, "attn.qkv.bias"), &.{3 * hidden}); + b.proj_w = try l.get(try n(&nb, p, "attn.proj.weight"), &.{ hidden, hidden }); + b.proj_b = try l.get(try n(&nb, p, "attn.proj.bias"), &.{hidden}); + b.norm2_w = try l.get(try n(&nb, p, "norm2.weight"), &.{hidden}); + b.norm2_b = try l.get(try n(&nb, p, "norm2.bias"), &.{hidden}); + b.fc1_w = try l.get(try n(&nb, p, "mlp.linear_fc1.weight"), &.{ intermediate, hidden }); + b.fc1_b = try l.get(try n(&nb, p, "mlp.linear_fc1.bias"), &.{intermediate}); + b.fc2_w = try l.get(try n(&nb, p, "mlp.linear_fc2.weight"), &.{ hidden, intermediate }); + b.fc2_b = try l.get(try n(&nb, p, "mlp.linear_fc2.bias"), &.{hidden}); + } + t.merger_norm_w = try l.get("merger.norm.weight", &.{hidden}); + t.merger_norm_b = try l.get("merger.norm.bias", &.{hidden}); + t.fc1_w = try l.get("merger.linear_fc1.weight", &.{ merged, merged }); + t.fc1_b = try l.get("merger.linear_fc1.bias", &.{merged}); + t.fc2_w = try l.get("merger.linear_fc2.weight", &.{ out_hidden, merged }); + t.fc2_b = try l.get("merger.linear_fc2.bias", &.{out_hidden}); + if (ck.unused() != 0) return error.UnusedVisionTensors; + var inv: [head_dim / 4]f32 = undefined; + invFreq(&inv); + t.inv = try t.alloc(@sizeOf(@TypeOf(inv))); + try src.flush(); + try ops.upload(t.inv, std.mem.asBytes(&inv)); + try ops.s.synchronize(); + return t; + } + + pub fn deinit(t: *Tower) void { + t.freeBuffers(); + t.mod.unload(); + t.lib.close(); + t.gpa.destroy(t); + } + + fn freeBuffers(t: *Tower) void { + for (t.buffers.items) |*b| b.free(); + t.buffers.deinit(t.gpa); + } + + fn alloc(t: *Tower, len: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(t.d, len); + try t.buffers.append(t.gpa, b); + t.weight_bytes += len; + return b.ptr; + } + + /// The scratch one tower call of `n` patches needs, in bytes (the patches' bf16 copy included). + pub fn scratchBytes(n: usize) usize { + return n * patch_width * 2 + // patches + 2 * n * hidden * 2 + // x, h + n * @max(3 * hidden, intermediate) * 4 + // fp32 Linear out + 3 * n * hidden * 2 + // q, k, v + n * hidden * 2 + // attention out + n * @max(intermediate, hidden) * 2 + // gelu rows + n * (2 + 4) * 4 + n * 4 * 4 + // hw, idx, weights + workspace_bytes; + } + + /// Images' patches (fp32 rows of `patch_width`, each image's in its grid's merge order) to their merged rows + /// (bf16 [sum(h * w) / 4, out_hidden]) at `out`; every image attends only to itself. + pub fn encode(t: *Tower, ops: Gpu, pixels: []const f32, grids: []const Grid, out: u64) !void { + var n: usize = 0; + for (grids) |g| { + if (g.h == 0 or g.w == 0 or g.h % merge != 0 or g.w % merge != 0) return error.BadImageGrid; + n += @as(usize, g.h) * g.w; + } + if (n == 0 or n > max_patches or pixels.len != n * patch_width) return error.BadImagePatches; + const gpa = t.gpa; + // host side: bf16 patches, each patch's (row, column) and its 4 position-grid taps + const px16 = try gpa.alloc(u16, n * patch_width); + defer gpa.free(px16); + for (px16, pixels) |*o, v| o.* = bf16(v); + const hw = try gpa.alloc(i32, n * 2); + defer gpa.free(hw); + const idx = try gpa.alloc(i32, n * 4); + defer gpa.free(idx); + const wts = try gpa.alloc(f32, n * 4); + defer gpa.free(wts); + var at: usize = 0; + for (grids) |g| { + positions(g, hw[at * 2 ..][0 .. @as(usize, g.h) * g.w * 2], idx[at * 4 ..][0 .. @as(usize, g.h) * g.w * 4], wts[at * 4 ..][0 .. @as(usize, g.h) * g.w * 4]); + at += @as(usize, g.h) * g.w; + } + var scratch = try cuda.DeviceBuffer.alloc(t.d, scratchBytes(n)); + defer scratch.free(); + var cur: u64 = scratch.ptr; + const take = struct { + fn f(p: *u64, len: usize) u64 { + const r = p.*; + p.* += std.mem.alignForward(usize, len, 256); + return r; + } + }.f; + const d_px = take(&cur, n * patch_width * 2); + const x = take(&cur, n * hidden * 2); + const h = take(&cur, n * hidden * 2); + const acc = take(&cur, n * @max(3 * hidden, intermediate) * 4); + const q = take(&cur, n * hidden * 2); + const k = take(&cur, n * hidden * 2); + const v = take(&cur, n * hidden * 2); + const att = take(&cur, n * hidden * 2); + const mid = take(&cur, n * @max(intermediate, hidden) * 2); + const d_hw = take(&cur, n * 2 * 4); + const d_idx = take(&cur, n * 4 * 4); + const d_wts = take(&cur, n * 4 * 4); + const ws = cur; + const ws_len = scratch.ptr + scratch.len - cur; + try ops.upload(d_px, std.mem.sliceAsBytes(px16)); + try ops.upload(d_hw, std.mem.sliceAsBytes(hw)); + try ops.upload(d_idx, std.mem.sliceAsBytes(idx)); + try ops.upload(d_wts, std.mem.sliceAsBytes(wts)); + + var mm: Gemms = .{ .lib = &t.lib }; + defer mm.deinit(); + const m4 = n / (merge * merge); + try mm.run(n, hidden, patch_width, d_px, t.patch_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.embed, n, 1, 256, .{ acc, t.patch_b, t.pos, d_idx, d_wts, x, int(hidden) }); + const scale: f32 = 1.0 / @sqrt(@as(f32, head_dim)); + for (t.blocks) |b| { + try t.go(ops, t.f.layernorm, n, 1, 256, .{ x, b.norm1_w, b.norm1_b, h, int(hidden), eps }); + try mm.run(n, 3 * hidden, hidden, h, b.qkv_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.qkv_rope, n, heads, 128, .{ acc, b.qkv_b, d_hw, t.inv, q, k, v, int(n), int(heads), int(head_dim) }); + var s0: usize = 0; + for (grids) |g| { + const len = @as(usize, g.h) * g.w; + try t.go(ops, t.f.attention, (len + 63) / 64, heads, 64, .{ q, k, v, att, int(n), int(heads), int(s0), int(len), scale }); + s0 += len; + } + try mm.run(n, hidden, hidden, att, b.proj_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.bias_residual, n, 1, 256, .{ acc, b.proj_b, x, int(hidden) }); + try t.go(ops, t.f.layernorm, n, 1, 256, .{ x, b.norm2_w, b.norm2_b, h, int(hidden), eps }); + try mm.run(n, intermediate, hidden, h, b.fc1_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.bias_gelu, n, 1, 256, .{ acc, b.fc1_b, mid, int(intermediate), int(1) }); + try mm.run(n, hidden, intermediate, mid, b.fc2_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.bias_residual, n, 1, 256, .{ acc, b.fc2_b, x, int(hidden) }); + } + // merger: LayerNorm over 1152, four merge-ordered rows read as one of 4608 + try t.go(ops, t.f.layernorm, n, 1, 256, .{ x, t.merger_norm_w, t.merger_norm_b, h, int(hidden), eps }); + try mm.run(m4, merged, merged, h, t.fc1_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.bias_gelu, m4, 1, 256, .{ acc, t.fc1_b, mid, int(merged), int(0) }); + try mm.run(m4, out_hidden, merged, mid, t.fc2_w, acc, ws, ws_len, ops.s); + try t.go(ops, t.f.bias_cast, m4, 1, 256, .{ acc, t.fc2_b, out, int(out_hidden) }); + try ops.s.synchronize(); + } + + /// A prompt chunk's embeddings `x` (rows bf16 of the language width) take image rows: row r takes `features` + /// row src[r] (device int32) where src[r] >= 0. + pub fn splice(t: *Tower, ops: Gpu, x: u64, features: u64, src: u64, rows: usize) !void { + try t.go(ops, t.f.splice, rows, 1, 256, .{ x, features, src, int(out_hidden) }); + } + + fn go(t: *Tower, ops: Gpu, f: cuda.Function, gx: usize, gy: usize, block: u32, args: anytype) !void { + _ = t; + var a: cuda.Args = .{}; + inline for (args) |v| a.add(v); + try cuda.launch.launch(f, .{ .grid = .{ .x = @intCast(gx), .y = @intCast(gy) }, .block = .{ .x = block } }, ops.s, &a); + } +}; + +/// cuBLASLt plans for the shapes one tower call runs, made on first use. +const Gemms = struct { + lib: *const lt.Library, + plans: [8]?struct { m: usize, n: usize, k: usize, lin: lt.Linear } = @splat(null), + + fn run(g: *Gemms, m: usize, n: usize, k: usize, x: u64, w: u64, out: u64, ws: u64, ws_len: usize, s: cuda.Stream) !void { + for (&g.plans) |*p| { + if (p.*) |*have| { + if (have.m == m and have.n == n and have.k == k) return have.lin.run(x, w, out, ws, ws_len, s.handle); + continue; + } + p.* = .{ .m = m, .n = n, .k = k, .lin = try lt.Linear.init(g.lib, m, n, k, .f32, ws_len) }; + return p.*.?.lin.run(x, w, out, ws, ws_len, s.handle); + } + return error.TooManyShapes; + } + + fn deinit(g: *Gemms) void { + for (&g.plans) |*p| if (p.*) |*have| have.lin.deinit(); + } +}; + +fn int(x: usize) c_int { + return @intCast(x); +} + +/// fp32 to bf16, round to nearest even (torch's .to(bfloat16)). +fn bf16(v: f32) u16 { + const b: u32 = @bitCast(v); + if (std.math.isNan(v)) return 0x7fc0; + return @intCast((b + 0x7fff + ((b >> 16) & 1)) >> 16); +} + +/// The axial rotary's frequencies: 1 / 10000 ** (i / (head_dim / 2)) for even i, in fp32 as torch computes them. +pub fn invFreq(out: []f32) void { + const dim: f32 = @floatFromInt(head_dim / 2); + for (out, 0..) |*f, j| { + const e: f32 = @as(f32, @floatFromInt(2 * j)) / dim; + f.* = 1.0 / std.math.pow(f32, 10000.0, e); + } +} + +/// One image's patches in merge order: each patch's (row, column) for the rotary, and its 4 bilinear taps (with +/// align_corners) into the 48x48 learned grid with their fp32 weights, as transformers' vision_utils computes them. +pub fn positions(g: Grid, hw: []i32, idx: []i32, wts: []f32) void { + const bw = g.w / merge; + const n = @as(usize, g.h) * g.w; + for (0..n) |i| { + const in_col: u32 = @intCast(i % merge); + const in_row: u32 = @intCast((i / merge) % merge); + const block_col: u32 = @intCast((i / (merge * merge)) % bw); + const block_row: u32 = @intCast(i / (merge * merge * bw)); + const r = block_row * merge + in_row; + const c = block_col * merge + in_col; + hw[i * 2] = @intCast(r); + hw[i * 2 + 1] = @intCast(c); + const ht = taps(r, g.h); + const wt = taps(c, g.w); + for (0..2) |a| for (0..2) |b| { + idx[i * 4 + a * 2 + b] = ht.i[a] * grid_side + wt.i[b]; + wts[i * 4 + a * 2 + b] = ht.w[a] * wt.w[b]; + }; + } +} + +const Taps = struct { i: [2]i32, w: [2]f32 }; + +fn taps(index: u32, size: u32) Taps { + const side_m1: f32 = grid_side - 1; + const denom: f32 = @floatFromInt(@max(size, 2) - 1); + const src = @as(f32, @floatFromInt(index)) * side_m1 / denom; + const fl = @floor(src); + var out: Taps = undefined; + for (0..2) |o| { + const raw: i32 = @as(i32, @intFromFloat(fl)) + @as(i32, @intCast(o)); + out.i[o] = std.math.clamp(raw, 0, grid_side - 1); + const dist = @abs(src - fl - @as(f32, @floatFromInt(o))); + out.w[o] = @max(1.0 - dist, 0.0); + } + return out; +} + +test "bilinear taps match align_corners resampling at the ends and middle" { + const a = taps(0, 10); + try std.testing.expectEqual(@as(i32, 0), a.i[0]); + try std.testing.expectEqual(@as(f32, 1), a.w[0]); + const b = taps(9, 10); + try std.testing.expectEqual(@as(i32, 47), b.i[0]); + try std.testing.expectEqual(@as(f32, 1), b.w[0]); +} From 407cb2847de6c312dd7477ef626a153df4025f2d Mon Sep 17 00:00:00 2001 From: BobClawblaw Date: Mon, 12 Oct 2026 02:37:38 +0000 Subject: [PATCH 07/19] Qwen image preprocessing on Linux: stb_image decode, EXIF, alpha, smart_resize, PIL bicubic, patches PNG decode and patches equal 0.6.6's bit for bit on seven fixtures; JPEGs differ in 1-6% of bytes (stb's IDCT against libjpeg-turbo's). --- zig/build/cuda.zig | 3 + zig/src/cli/qwen27_cuda.zig | 17 + zig/src/families/qwen27/cuda.zig | 2 + zig/src/families/qwen27/cuda/image.zig | 354 ++ zig/vendor/stb/COMMIT | 1 + zig/vendor/stb/stb_image.c | 9 + zig/vendor/stb/stb_image.h | 7988 ++++++++++++++++++++++++ 7 files changed, 8374 insertions(+) create mode 100644 zig/src/families/qwen27/cuda/image.zig create mode 100644 zig/vendor/stb/COMMIT create mode 100644 zig/vendor/stb/stb_image.c create mode 100644 zig/vendor/stb/stb_image.h diff --git a/zig/build/cuda.zig b/zig/build/cuda.zig index 188de6ea75..50e669ae28 100644 --- a/zig/build/cuda.zig +++ b/zig/build/cuda.zig @@ -94,6 +94,9 @@ fn family(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin qwen27.addImport("core", core); qwen27.addImport("lanes", lanes); qwen27.addImport("nemotron", nemotron); // the checkpoint Source and torch ops the CUDA families share + // images' PNG and JPEG decoding on Linux (stb_image, vendored) + qwen27.addCSourceFile(.{ .file = b.path("zig/vendor/stb/stb_image.c"), .flags = &.{ "-O2", "-fno-sanitize=undefined" } }); + qwen27.addIncludePath(b.path("zig/vendor/stb")); return .{ .core = core, .lanes = lanes, .nemotron = nemotron, .qwen27 = qwen27, .tokenizer = tokenizer, .heat = heat }; } diff --git a/zig/src/cli/qwen27_cuda.zig b/zig/src/cli/qwen27_cuda.zig index dfdfe785f9..d1d0e879ad 100644 --- a/zig/src/cli/qwen27_cuda.zig +++ b/zig/src/cli/qwen27_cuda.zig @@ -487,6 +487,23 @@ fn checkVision(gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, mod const want_bytes = try std.Io.Dir.cwd().readFileAlloc(io, try std.fs.path.join(arena, &.{ dir, img.object.get("features").?.string }), arena, .limited(1 << 30)); const px = try arena.alloc(f32, px_bytes.len / 4); @memcpy(std.mem.sliceAsBytes(px), px_bytes[0 .. px.len * 4]); + { // our decode and patches from the image's file, against 0.6.6's + const file = try std.Io.Dir.cwd().readFileAlloc(io, try std.fs.path.join(arena, &.{ dir, img.object.get("file").?.string }), arena, .limited(1 << 30)); + const want_rgb = try std.Io.Dir.cwd().readFileAlloc(io, try std.fs.path.join(arena, &.{ dir, img.object.get("rgb").?.string }), arena, .limited(1 << 30)); + const rgb = try qwen.image.decode(gpa, file); + defer rgb.deinit(gpa); + var px_diff: usize = 0; + if (rgb.px.len == want_rgb.len) { + for (rgb.px, want_rgb) |a, b| px_diff += @intFromBool(a != b); + } else px_diff = std.math.maxInt(usize); + const prep = try qwen.image.prepare(gpa, file, 4096); + defer gpa.free(prep.pixels); + var patch_diff: usize = 0; + if (prep.gh == g.h and prep.gw == g.w and prep.pixels.len == px.len) { + for (prep.pixels, px) |a, b| patch_diff += @intFromBool(@as(u32, @bitCast(a)) != @as(u32, @bitCast(b))); + } else patch_diff = std.math.maxInt(usize); + std.debug.print("{s}: decode {d}x{d}, {d} of {d} bytes differ from PIL's; patches grid {d}x{d}, {d} of {d} values differ\n", .{ name, rgb.w, rgb.h, px_diff, want_rgb.len, prep.gh, prep.gw, patch_diff, px.len }); + } const rows = @as(usize, g.h) * g.w / 4; var out = try cuda.DeviceBuffer.alloc(&driver, rows * V.out_hidden * 2); defer out.free(); diff --git a/zig/src/families/qwen27/cuda.zig b/zig/src/families/qwen27/cuda.zig index 746c12d658..650f1748ef 100644 --- a/zig/src/families/qwen27/cuda.zig +++ b/zig/src/families/qwen27/cuda.zig @@ -15,6 +15,7 @@ pub const draft = @import("cuda/draft.zig"); pub const lone = @import("cuda/lone.zig"); pub const drafts = @import("cuda/drafts.zig"); pub const vision = @import("cuda/vision.zig"); +pub const image = @import("cuda/image.zig"); test { _ = config; @@ -32,4 +33,5 @@ test { _ = lone; _ = native; _ = vision; + _ = image; } diff --git a/zig/src/families/qwen27/cuda/image.zig b/zig/src/families/qwen27/cuda/image.zig new file mode 100644 index 0000000000..0546e66e8c --- /dev/null +++ b/zig/src/families/qwen27/cuda/image.zig @@ -0,0 +1,354 @@ +//! An image's bytes to Qwen3.5 vision patches as TensorFold 0.6.6's --vision makes them on CUDA: decoded as its +//! vision/images.py does with PIL (stb_image here: the stored values, no colour management; EXIF orientation applied; +//! alpha composited over white with PIL's integer blend), then transformers' Qwen2VLImageProcessorPil at the caps +//! 0.6.6 passes it: smart_resize to multiples of 32 within [32^2, tokens * 32^2] pixels, PIL's 8-bit bicubic +//! resampler, x / 255 in float64 rounded to float32, (x - 0.5) / 0.5, and 16x16 patches in 2x2 merge order with each +//! patch's frame given twice. PNG patches equal 0.6.6's bit for bit; JPEG decoders may differ by a level. +const std = @import("std"); +const vision = @import("vision.zig"); + +extern fn stbi_load_from_memory(buffer: [*]const u8, len: c_int, x: *c_int, y: *c_int, comp: *c_int, req_comp: c_int) ?[*]u8; +extern fn stbi_load_16_from_memory(buffer: [*]const u8, len: c_int, x: *c_int, y: *c_int, comp: *c_int, req_comp: c_int) ?[*]u16; +extern fn stbi_is_16_bit_from_memory(buffer: [*]const u8, len: c_int) c_int; +extern fn stbi_info_from_memory(buffer: [*]const u8, len: c_int, x: *c_int, y: *c_int, comp: *c_int) c_int; +extern fn stbi_image_free(p: ?*anyopaque) void; + +pub const Rgb = struct { + w: u32, + h: u32, + px: []u8, // row-major RGB, 3 bytes a pixel + + pub fn deinit(r: Rgb, gpa: std.mem.Allocator) void { + gpa.free(r.px); + } +}; + +/// One image ready for the tower: its patches [gh * gw, 1536] in merge order, its grid and its tokens (gh * gw / 4). +pub const Prepared = struct { pixels: []f32, gh: u32, gw: u32, tokens: u32, hash: u64 }; + +pub const Error = error{ BadImage, ImageTooLarge, Animated, ImageTooSmall, OutOfMemory }; + +/// images.py's ImageLimits: sides and decoded pixels (the server bounds the encoded bytes). +pub const max_side = 8192; +pub const max_pixels = 16 << 20; +const factor = vision.patch * vision.merge; // 32 + +/// Bytes to patches for at most `max_tokens` tokens (0.6.6's per-image cap: the request's budget shared, 4096 at most). +pub fn prepare(gpa: std.mem.Allocator, bytes: []const u8, max_tokens: u32) Error!Prepared { + const img = try decode(gpa, bytes); + defer img.deinit(gpa); + return prepareRgb(gpa, img, std.hash.Wyhash.hash(0x51e7, bytes), max_tokens); +} + +pub fn prepareRgb(gpa: std.mem.Allocator, img: Rgb, hash: u64, max_tokens: u32) Error!Prepared { + const tokens = @min(max_tokens, vision.max_patches / (vision.merge * vision.merge)); + if (tokens == 0) return error.ImageTooSmall; + const size = try smartResize(img.h, img.w, factor * factor, @as(u64, tokens) * factor * factor); + const resized = try resize(gpa, img, size.w, size.h); + defer resized.deinit(gpa); + const gh = size.h / vision.patch; + const gw = size.w / vision.patch; + return .{ .pixels = try patchify(gpa, resized), .gh = gh, .gw = gw, .tokens = gh * gw / (vision.merge * vision.merge), .hash = hash }; +} + +pub const Size = struct { h: u32, w: u32 }; + +/// transformers' smart_resize: both sides multiples of `factor` (Python's round, half to even), the pixels within +/// [min_pixels, max_pixels], the aspect ratio kept as well as that allows. +pub fn smartResize(height: u32, width: u32, min_pixels: u64, max_pixels_: u64) Error!Size { + const h: f64 = @floatFromInt(height); + const w: f64 = @floatFromInt(width); + if (@max(h, w) / @min(h, w) > 200) return error.BadImage; + const f: f64 = factor; + var hb: f64 = roundEven(h / f) * f; + var wb: f64 = roundEven(w / f) * f; + if (hb * wb > @as(f64, @floatFromInt(max_pixels_))) { + const beta = @sqrt(h * w / @as(f64, @floatFromInt(max_pixels_))); + hb = @max(f, @floor(h / beta / f) * f); + wb = @max(f, @floor(w / beta / f) * f); + } else if (hb * wb < @as(f64, @floatFromInt(min_pixels))) { + const beta = @sqrt(@as(f64, @floatFromInt(min_pixels)) / (h * w)); + hb = @ceil(h * beta / f) * f; + wb = @ceil(w * beta / f) * f; + } + return .{ .h = @intFromFloat(hb), .w = @intFromFloat(wb) }; +} + +fn roundEven(x: f64) f64 { + const r = @round(x); + return if (@abs(x - @trunc(x)) == 0.5 and @mod(r, 2) != 0) r - std.math.sign(x) else r; +} + +/// The resized image scaled by 1/255 (in float64, then float32), normalized by mean 0.5 and std 0.5, cut into +/// 16x16 patches in 2x2 merge order, each patch [channel, frame (twice), row, column]. +pub fn patchify(gpa: std.mem.Allocator, img: Rgb) Error![]f32 { + const P = vision.patch; + const M = vision.merge; + const T = vision.temporal; + const gh = img.h / P; + const gw = img.w / P; + const out = try gpa.alloc(f32, @as(usize, gh) * gw * vision.patch_width); + var lut: [256]f32 = undefined; + for (&lut, 0..) |*v, i| { + const scaled: f32 = @floatCast(@as(f64, @floatFromInt(i)) * (1.0 / 255.0)); + v.* = (scaled - 0.5) / 0.5; + } + var row: usize = 0; + for (0..gh / M) |by| for (0..gw / M) |bx| for (0..M) |my| for (0..M) |mx| { + const y0 = (by * M + my) * P; + const x0 = (bx * M + mx) * P; + const dst = out[row * vision.patch_width ..][0..vision.patch_width]; + for (0..3) |ch| for (0..T) |t| for (0..P) |dy| for (0..P) |dx| { + dst[((ch * T + t) * P + dy) * P + dx] = lut[img.px[((y0 + dy) * img.w + x0 + dx) * 3 + ch]]; + }; + row += 1; + }; + return out; +} + +// ---- decoding (stb_image), EXIF orientation and alpha, as PIL gives images.py its RGB ---- + +/// A single-frame PNG or JPEG as 8-bit RGB: upright by its EXIF orientation, alpha composited over white, 16-bit +/// gray clipped to 255 and other 16-bit layouts cut to their high byte (what PIL's convert("RGB") gives). +pub fn decode(gpa: std.mem.Allocator, bytes: []const u8) Error!Rgb { + if (bytes.len < 8 or bytes.len > std.math.maxInt(c_int)) return error.BadImage; + const n: c_int = @intCast(bytes.len); + if (std.mem.startsWith(u8, bytes, "\x89PNG\r\n\x1a\n") and pngAnimated(bytes)) return error.Animated; + var w: c_int = 0; + var h: c_int = 0; + var comp: c_int = 0; + if (stbi_info_from_memory(bytes.ptr, n, &w, &h, &comp) == 0) return error.BadImage; + if (w <= 0 or h <= 0) return error.BadImage; + if (w > max_side or h > max_side or @as(u64, @intCast(w)) * @as(u64, @intCast(h)) > max_pixels) return error.ImageTooLarge; + const W: usize = @intCast(w); + const H: usize = @intCast(h); + const out = try gpa.alloc(u8, W * H * 3); + errdefer gpa.free(out); + if (stbi_is_16_bit_from_memory(bytes.ptr, n) != 0 and comp == 1) { + const p = stbi_load_16_from_memory(bytes.ptr, n, &w, &h, &comp, 0) orelse return error.BadImage; + defer stbi_image_free(p); + for (0..W * H) |i| @memset(out[i * 3 ..][0..3], @intCast(@min(p[i], 255))); + } else { + const p = stbi_load_from_memory(bytes.ptr, n, &w, &h, &comp, 0) orelse return error.BadImage; + defer stbi_image_free(p); + const c: usize = @intCast(comp); + for (0..W * H) |i| { + const s = p[i * c ..][0..c]; + const px = out[i * 3 ..][0..3]; + switch (c) { + 1 => @memset(px, s[0]), + 2 => @memset(px, overWhite(s[0], s[1])), + 3 => @memcpy(px, s[0..3]), + 4 => for (0..3) |k| { + px[k] = overWhite(s[k], s[3]); + }, + else => return error.BadImage, + } + } + } + const img: Rgb = .{ .w = @intCast(W), .h = @intCast(H), .px = out }; + const o = orientation(bytes); + if (o < 2 or o > 8) return img; + defer img.deinit(gpa); + return orient(gpa, img, o); +} + +/// An APNG (an acTL chunk before the image data) has more than one frame: images.py refuses it. +fn pngAnimated(bytes: []const u8) bool { + var at: usize = 8; + while (at + 8 <= bytes.len) { + const len = std.mem.readInt(u32, bytes[at..][0..4], .big); + const kind = bytes[at + 4 ..][0..4]; + if (std.mem.eql(u8, kind, "acTL")) return true; + if (std.mem.eql(u8, kind, "IDAT")) return false; + at += 12 + @as(usize, len); + } + return false; +} + +/// The EXIF orientation (tag 0x0112 of IFD0) a JPEG's APP1 or a PNG's eXIf chunk carries; 1 when none. +pub fn orientation(bytes: []const u8) u32 { + if (std.mem.startsWith(u8, bytes, "\x89PNG\r\n\x1a\n")) { + var at: usize = 8; + while (at + 12 <= bytes.len) { + const len: usize = std.mem.readInt(u32, bytes[at..][0..4], .big); + if (at + 12 + len > bytes.len) break; + if (std.mem.eql(u8, bytes[at + 4 ..][0..4], "eXIf")) return tiffOrientation(bytes[at + 8 ..][0..len]); + at += 12 + len; + } + return 1; + } + var at: usize = 2; + while (at + 4 <= bytes.len and bytes[at] == 0xff) { + const marker = bytes[at + 1]; + if (marker == 0xd9 or marker == 0xda) break; // end of image, start of scan + const len: usize = std.mem.readInt(u16, bytes[at + 2 ..][0..2], .big); + if (len < 2 or at + 2 + len > bytes.len) break; + const seg = bytes[at + 4 .. at + 2 + len]; + if (marker == 0xe1 and std.mem.startsWith(u8, seg, "Exif\x00\x00")) return tiffOrientation(seg[6..]); + at += 2 + len; + } + return 1; +} + +fn tiffOrientation(t: []const u8) u32 { + if (t.len < 8) return 1; + const endian: std.builtin.Endian = if (std.mem.eql(u8, t[0..2], "II")) .little else if (std.mem.eql(u8, t[0..2], "MM")) .big else return 1; + if (std.mem.readInt(u16, t[2..4], endian) != 42) return 1; + const ifd: usize = std.mem.readInt(u32, t[4..8], endian); + if (ifd + 2 > t.len) return 1; + const count: usize = std.mem.readInt(u16, t[ifd..][0..2], endian); + for (0..count) |i| { + const e = ifd + 2 + i * 12; + if (e + 12 > t.len) return 1; + if (std.mem.readInt(u16, t[e..][0..2], endian) != 0x0112) continue; + if (std.mem.readInt(u16, t[e + 2 ..][0..2], endian) != 3) return 1; // SHORT + return std.mem.readInt(u16, t[e + 8 ..][0..2], endian); + } + return 1; +} + +// The functions below are the ones GLM's image.zig (families/glm) has for its Metal path: the same PIL arithmetic, +// kept here because the CUDA module cannot import that family's files. + +/// PIL's exif_transpose: orientation 2-8 as FLIP_LEFT_RIGHT, ROTATE_180, FLIP_TOP_BOTTOM, TRANSPOSE, ROTATE_270, +/// TRANSVERSE, ROTATE_90; each output pixel read from its source pixel. +pub fn orient(gpa: std.mem.Allocator, src: Rgb, o: u32) Error!Rgb { + const w: usize = src.w; + const h: usize = src.h; + const swap = o >= 5; + const ow = if (swap) h else w; + const oh = if (swap) w else h; + const out = try gpa.alloc(u8, ow * oh * 3); + for (0..oh) |y| for (0..ow) |x| { + const s: [2]usize = switch (o) { + 2 => .{ w - 1 - x, y }, + 3 => .{ w - 1 - x, h - 1 - y }, + 4 => .{ x, h - 1 - y }, + 5 => .{ y, x }, + 6 => .{ y, h - 1 - x }, + 7 => .{ w - 1 - y, h - 1 - x }, + else => .{ w - 1 - y, x }, + }; + @memcpy(out[(y * ow + x) * 3 ..][0..3], src.px[(s[1] * w + s[0]) * 3 ..][0..3]); + }; + return .{ .w = @intCast(ow), .h = @intCast(oh), .px = out }; +} + +/// PIL's paste of a pixel with alpha `a` onto white: DIV255(255 * (255 - a) + c * a). +fn overWhite(c: u8, a: u8) u8 { + const t: u32 = 255 * (255 - @as(u32, a)) + @as(u32, c) * a + 128; + return @intCast(((t >> 8) + t) >> 8); +} + +// ---- PIL's ImagingResample for 8-bit images (Resample.c), bicubic (a = -0.5) ---- + +const precision_bits = 32 - 8 - 2; + +fn bicubic(x0: f64) f64 { + const a = -0.5; + const x = @abs(x0); + if (x < 1.0) return ((a + 2.0) * x - (a + 3.0)) * x * x + 1; + if (x < 2.0) return (((x - 5) * x + 8) * x - 4) * a; + return 0; +} + +const Coeffs = struct { bounds: []u32, kk: []i32, ksize: usize }; + +/// precompute_coeffs and normalize_coeffs_8bpc: each output pixel's first input pixel, count and fixed-point weights. +fn coeffs(gpa: std.mem.Allocator, in_size: u32, out_size: u32) Error!Coeffs { + const scale = @as(f64, @floatFromInt(in_size)) / @as(f64, @floatFromInt(out_size)); + const filterscale = @max(scale, 1.0); + const support = 2.0 * filterscale; + const ksize: usize = @as(usize, @intFromFloat(@ceil(support))) * 2 + 1; + const bounds = try gpa.alloc(u32, @as(usize, out_size) * 2); + errdefer gpa.free(bounds); + const kk = try gpa.alloc(i32, @as(usize, out_size) * ksize); + errdefer gpa.free(kk); + var pre = try gpa.alloc(f64, ksize); + defer gpa.free(pre); + const ss = 1.0 / filterscale; + for (0..out_size) |xx| { + const center = (@as(f64, @floatFromInt(xx)) + 0.5) * scale; + var xmin_i: i64 = @intFromFloat(center - support + 0.5); // (int) truncates toward zero, as C does + if (xmin_i < 0) xmin_i = 0; + var xmax_i: i64 = @intFromFloat(center + support + 0.5); + if (xmax_i > in_size) xmax_i = in_size; + xmax_i -= xmin_i; + const xmin: usize = @intCast(xmin_i); + const xmax: usize = @intCast(xmax_i); + var ww: f64 = 0; + for (0..xmax) |x| { + const w = bicubic((@as(f64, @floatFromInt(x + xmin)) - center + 0.5) * ss); + pre[x] = w; + ww += w; + } + for (0..ksize) |x| { + const k = if (x < xmax) (if (ww != 0) pre[x] / ww else pre[x]) else 0; + const v = k * @as(f64, @floatFromInt(@as(i64, 1) << precision_bits)); + kk[xx * ksize + x] = @intFromFloat(if (v < 0) v - 0.5 else v + 0.5); // C's (int) of the rounded value + } + bounds[xx * 2] = @intCast(xmin); + bounds[xx * 2 + 1] = @intCast(xmax); + } + return .{ .bounds = bounds, .kk = kk, .ksize = ksize }; +} + +fn clip8(v: i64) u8 { + const s = v >> precision_bits; + return if (s < 0) 0 else if (s > 255) 255 else @intCast(s); +} + +/// Image.resize((w, h), BICUBIC) on an 8-bit RGB image: horizontal pass, then vertical, each rounded to 8 bits. +pub fn resize(gpa: std.mem.Allocator, src: Rgb, w: u32, h: u32) Error!Rgb { + var cur = src; + var owned = false; + errdefer if (owned) cur.deinit(gpa); + if (w != src.w) { + const c = try coeffs(gpa, src.w, w); + defer gpa.free(c.bounds); + defer gpa.free(c.kk); + const out = try gpa.alloc(u8, @as(usize, w) * cur.h * 3); + for (0..cur.h) |y| for (0..w) |xx| { + const xmin = c.bounds[xx * 2]; + const xmax = c.bounds[xx * 2 + 1]; + const k = c.kk[xx * c.ksize ..]; + for (0..3) |ch| { + var ss: i64 = 1 << (precision_bits - 1); + for (0..xmax) |x| ss += @as(i64, cur.px[(y * cur.w + x + xmin) * 3 + ch]) * k[x]; + out[(y * w + xx) * 3 + ch] = clip8(ss); + } + }; + if (owned) cur.deinit(gpa); + cur = .{ .w = w, .h = cur.h, .px = out }; + owned = true; + } + if (h != src.h) { + const c = try coeffs(gpa, src.h, h); + defer gpa.free(c.bounds); + defer gpa.free(c.kk); + const out = try gpa.alloc(u8, @as(usize, cur.w) * h * 3); + for (0..h) |yy| { + const ymin = c.bounds[yy * 2]; + const ymax = c.bounds[yy * 2 + 1]; + const k = c.kk[yy * c.ksize ..]; + for (0..cur.w) |x| for (0..3) |ch| { + var ss: i64 = 1 << (precision_bits - 1); + for (0..ymax) |y| ss += @as(i64, cur.px[((y + ymin) * cur.w + x) * 3 + ch]) * k[y]; + out[(yy * cur.w + x) * 3 + ch] = clip8(ss); + }; + } + if (owned) cur.deinit(gpa); + cur = .{ .w = cur.w, .h = h, .px = out }; + owned = true; + } + if (!owned) cur = .{ .w = src.w, .h = src.h, .px = try gpa.dupe(u8, src.px) }; + return cur; +} + +test "smart_resize rounds half to even and respects the pixel bounds" { + try std.testing.expectEqual(Size{ .h = 32, .w = 64 }, try smartResize(15, 20, 1024, 4096 * 1024)); + try std.testing.expectEqual(Size{ .h = 384, .w = 512 }, try smartResize(389, 517, 1024, 4096 * 1024)); + try std.testing.expectEqual(@as(f64, 2), roundEven(2.5)); + try std.testing.expectEqual(@as(f64, 2), roundEven(1.5)); +} diff --git a/zig/vendor/stb/COMMIT b/zig/vendor/stb/COMMIT new file mode 100644 index 0000000000..7b40c30ab3 --- /dev/null +++ b/zig/vendor/stb/COMMIT @@ -0,0 +1 @@ +2c980bb59875b0d32144a71867fbdebb2f77cd20 diff --git a/zig/vendor/stb/stb_image.c b/zig/vendor/stb/stb_image.c new file mode 100644 index 0000000000..488630f836 --- /dev/null +++ b/zig/vendor/stb/stb_image.c @@ -0,0 +1,9 @@ +// stb_image (v2.30, public domain or MIT; the commit in COMMIT): PNG and JPEG from memory, for the Linux image path. +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_PNG +#define STBI_ONLY_JPEG +#define STBI_NO_STDIO +#define STBI_NO_LINEAR +#define STBI_NO_HDR +#define STBI_MAX_DIMENSIONS 8192 +#include "stb_image.h" diff --git a/zig/vendor/stb/stb_image.h b/zig/vendor/stb/stb_image.h new file mode 100644 index 0000000000..9eedabedc4 --- /dev/null +++ b/zig/vendor/stb/stb_image.h @@ -0,0 +1,7988 @@ +/* stb_image - v2.30 - public domain image loader - http://nothings.org/stb + no warranty implied; use at your own risk + + Do this: + #define STB_IMAGE_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + + // i.e. it should look like this: + #include ... + #include ... + #include ... + #define STB_IMAGE_IMPLEMENTATION + #include "stb_image.h" + + You can #define STBI_ASSERT(x) before the #include to avoid using assert.h. + And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free + + + QUICK NOTES: + Primarily of interest to game developers and other people who can + avoid problematic images and only need the trivial interface + + JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib) + PNG 1/2/4/8/16-bit-per-channel + + TGA (not sure what subset, if a subset) + BMP non-1bpp, non-RLE + PSD (composited view only, no extra channels, 8/16 bit-per-channel) + + GIF (*comp always reports as 4-channel) + HDR (radiance rgbE format) + PIC (Softimage PIC) + PNM (PPM and PGM binary only) + + Animated GIF still needs a proper API, but here's one way to do it: + http://gist.github.com/urraka/685d9a6340b26b830d49 + + - decode from memory or through FILE (define STBI_NO_STDIO to remove code) + - decode from arbitrary I/O callbacks + - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON) + + Full documentation under "DOCUMENTATION" below. + + +LICENSE + + See end of file for license information. + +RECENT REVISION HISTORY: + + 2.30 (2024-05-31) avoid erroneous gcc warning + 2.29 (2023-05-xx) optimizations + 2.28 (2023-01-29) many error fixes, security errors, just tons of stuff + 2.27 (2021-07-11) document stbi_info better, 16-bit PNM support, bug fixes + 2.26 (2020-07-13) many minor fixes + 2.25 (2020-02-02) fix warnings + 2.24 (2020-02-02) fix warnings; thread-local failure_reason and flip_vertically + 2.23 (2019-08-11) fix clang static analysis warning + 2.22 (2019-03-04) gif fixes, fix warnings + 2.21 (2019-02-25) fix typo in comment + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings + 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes + 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64 + RGB-format JPEG; remove white matting in PSD; + allocate large structures on the stack; + correct channel count for PNG & BMP + 2.10 (2016-01-22) avoid warning introduced in 2.09 + 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED + + See end of file for full revision history. + + + ============================ Contributors ========================= + + Image formats Extensions, features + Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info) + Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info) + Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG) + Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks) + Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG) + Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip) + Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD) + github:urraka (animated gif) Junggon Kim (PNM comments) + Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA) + socks-the-fox (16-bit PNG) + Jeremy Sawicki (handle all ImageNet JPGs) + Optimizations & bugfixes Mikhail Morozov (1-bit BMP) + Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query) + Arseny Kapoulkine Simon Breuss (16-bit PNM) + John-Mark Allen + Carmelo J Fdez-Aguera + + Bug & warning fixes + Marc LeBlanc David Woo Guillaume George Martins Mozeiko + Christpher Lloyd Jerry Jansson Joseph Thomson Blazej Dariusz Roszkowski + Phil Jordan Dave Moore Roy Eltham + Hayaki Saito Nathan Reed Won Chun + Luke Graham Johan Duparc Nick Verigakis the Horde3D community + Thomas Ruf Ronny Chevalier github:rlyeh + Janez Zemva John Bartholomew Michal Cichon github:romigrou + Jonathan Blow Ken Hamada Tero Hanninen github:svdijk + Eugene Golushkov Laurent Gomila Cort Stratton github:snagar + Aruelien Pocheville Sergio Gonzalez Thibault Reuille github:Zelex + Cass Everitt Ryamond Barbiero github:grim210 + Paul Du Bois Engin Manap Aldo Culquicondor github:sammyhw + Philipp Wiesemann Dale Weiler Oriol Ferrer Mesia github:phprus + Josh Tobin Neil Bickford Matthew Gregan github:poppolopoppo + Julian Raschke Gregory Mullen Christian Floisand github:darealshinji + Baldur Karlsson Kevin Schmidt JR Smith github:Michaelangel007 + Brad Weinberger Matvey Cherevko github:mosra + Luca Sas Alexander Veselov Zack Middleton [reserved] + Ryan C. Gordon [reserved] [reserved] + DO NOT ADD YOUR NAME HERE + + Jacko Dirks + + To add your name to the credits, pick a random blank space in the middle and fill it. + 80% of merge conflicts on stb PRs are due to people adding their name at the end + of the credits. +*/ + +#ifndef STBI_INCLUDE_STB_IMAGE_H +#define STBI_INCLUDE_STB_IMAGE_H + +// DOCUMENTATION +// +// Limitations: +// - no 12-bit-per-channel JPEG +// - no JPEGs with arithmetic coding +// - GIF always returns *comp=4 +// +// Basic usage (see HDR discussion below for HDR usage): +// int x,y,n; +// unsigned char *data = stbi_load(filename, &x, &y, &n, 0); +// // ... process data if not NULL ... +// // ... x = width, y = height, n = # 8-bit components per pixel ... +// // ... replace '0' with '1'..'4' to force that many components per pixel +// // ... but 'n' will always be the number that it would have been if you said 0 +// stbi_image_free(data); +// +// Standard parameters: +// int *x -- outputs image width in pixels +// int *y -- outputs image height in pixels +// int *channels_in_file -- outputs # of image components in image file +// int desired_channels -- if non-zero, # of image components requested in result +// +// The return value from an image loader is an 'unsigned char *' which points +// to the pixel data, or NULL on an allocation failure or if the image is +// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels, +// with each pixel consisting of N interleaved 8-bit components; the first +// pixel pointed to is top-left-most in the image. There is no padding between +// image scanlines or between pixels, regardless of format. The number of +// components N is 'desired_channels' if desired_channels is non-zero, or +// *channels_in_file otherwise. If desired_channels is non-zero, +// *channels_in_file has the number of components that _would_ have been +// output otherwise. E.g. if you set desired_channels to 4, you will always +// get RGBA output, but you can check *channels_in_file to see if it's trivially +// opaque because e.g. there were only 3 channels in the source image. +// +// An output image with N components has the following components interleaved +// in this order in each pixel: +// +// N=#comp components +// 1 grey +// 2 grey, alpha +// 3 red, green, blue +// 4 red, green, blue, alpha +// +// If image loading fails for any reason, the return value will be NULL, +// and *x, *y, *channels_in_file will be unchanged. The function +// stbi_failure_reason() can be queried for an extremely brief, end-user +// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS +// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly +// more user-friendly ones. +// +// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized. +// +// To query the width, height and component count of an image without having to +// decode the full file, you can use the stbi_info family of functions: +// +// int x,y,n,ok; +// ok = stbi_info(filename, &x, &y, &n); +// // returns ok=1 and sets x, y, n if image is a supported format, +// // 0 otherwise. +// +// Note that stb_image pervasively uses ints in its public API for sizes, +// including sizes of memory buffers. This is now part of the API and thus +// hard to change without causing breakage. As a result, the various image +// loaders all have certain limits on image size; these differ somewhat +// by format but generally boil down to either just under 2GB or just under +// 1GB. When the decoded image would be larger than this, stb_image decoding +// will fail. +// +// Additionally, stb_image will reject image files that have any of their +// dimensions set to a larger value than the configurable STBI_MAX_DIMENSIONS, +// which defaults to 2**24 = 16777216 pixels. Due to the above memory limit, +// the only way to have an image with such dimensions load correctly +// is for it to have a rather extreme aspect ratio. Either way, the +// assumption here is that such larger images are likely to be malformed +// or malicious. If you do need to load an image with individual dimensions +// larger than that, and it still fits in the overall size limit, you can +// #define STBI_MAX_DIMENSIONS on your own to be something larger. +// +// =========================================================================== +// +// UNICODE: +// +// If compiling for Windows and you wish to use Unicode filenames, compile +// with +// #define STBI_WINDOWS_UTF8 +// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert +// Windows wchar_t filenames to utf8. +// +// =========================================================================== +// +// Philosophy +// +// stb libraries are designed with the following priorities: +// +// 1. easy to use +// 2. easy to maintain +// 3. good performance +// +// Sometimes I let "good performance" creep up in priority over "easy to maintain", +// and for best performance I may provide less-easy-to-use APIs that give higher +// performance, in addition to the easy-to-use ones. Nevertheless, it's important +// to keep in mind that from the standpoint of you, a client of this library, +// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all. +// +// Some secondary priorities arise directly from the first two, some of which +// provide more explicit reasons why performance can't be emphasized. +// +// - Portable ("ease of use") +// - Small source code footprint ("easy to maintain") +// - No dependencies ("ease of use") +// +// =========================================================================== +// +// I/O callbacks +// +// I/O callbacks allow you to read from arbitrary sources, like packaged +// files or some other source. Data read from callbacks are processed +// through a small internal buffer (currently 128 bytes) to try to reduce +// overhead. +// +// The three functions you must define are "read" (reads some bytes of data), +// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end). +// +// =========================================================================== +// +// SIMD support +// +// The JPEG decoder will try to automatically use SIMD kernels on x86 when +// supported by the compiler. For ARM Neon support, you must explicitly +// request it. +// +// (The old do-it-yourself SIMD API is no longer supported in the current +// code.) +// +// On x86, SSE2 will automatically be used when available based on a run-time +// test; if not, the generic C versions are used as a fall-back. On ARM targets, +// the typical path is to have separate builds for NEON and non-NEON devices +// (at least this is true for iOS and Android). Therefore, the NEON support is +// toggled by a build flag: define STBI_NEON to get NEON loops. +// +// If for some reason you do not want to use any of SIMD code, or if +// you have issues compiling it, you can disable it entirely by +// defining STBI_NO_SIMD. +// +// =========================================================================== +// +// HDR image support (disable by defining STBI_NO_HDR) +// +// stb_image supports loading HDR images in general, and currently the Radiance +// .HDR file format specifically. You can still load any file through the existing +// interface; if you attempt to load an HDR file, it will be automatically remapped +// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1; +// both of these constants can be reconfigured through this interface: +// +// stbi_hdr_to_ldr_gamma(2.2f); +// stbi_hdr_to_ldr_scale(1.0f); +// +// (note, do not use _inverse_ constants; stbi_image will invert them +// appropriately). +// +// Additionally, there is a new, parallel interface for loading files as +// (linear) floats to preserve the full dynamic range: +// +// float *data = stbi_loadf(filename, &x, &y, &n, 0); +// +// If you load LDR images through this interface, those images will +// be promoted to floating point values, run through the inverse of +// constants corresponding to the above: +// +// stbi_ldr_to_hdr_scale(1.0f); +// stbi_ldr_to_hdr_gamma(2.2f); +// +// Finally, given a filename (or an open file or memory block--see header +// file for details) containing image data, you can query for the "most +// appropriate" interface to use (that is, whether the image is HDR or +// not), using: +// +// stbi_is_hdr(char *filename); +// +// =========================================================================== +// +// iPhone PNG support: +// +// We optionally support converting iPhone-formatted PNGs (which store +// premultiplied BGRA) back to RGB, even though they're internally encoded +// differently. To enable this conversion, call +// stbi_convert_iphone_png_to_rgb(1). +// +// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per +// pixel to remove any premultiplied alpha *only* if the image file explicitly +// says there's premultiplied data (currently only happens in iPhone images, +// and only if iPhone convert-to-rgb processing is on). +// +// =========================================================================== +// +// ADDITIONAL CONFIGURATION +// +// - You can suppress implementation of any of the decoders to reduce +// your code footprint by #defining one or more of the following +// symbols before creating the implementation. +// +// STBI_NO_JPEG +// STBI_NO_PNG +// STBI_NO_BMP +// STBI_NO_PSD +// STBI_NO_TGA +// STBI_NO_GIF +// STBI_NO_HDR +// STBI_NO_PIC +// STBI_NO_PNM (.ppm and .pgm) +// +// - You can request *only* certain decoders and suppress all other ones +// (this will be more forward-compatible, as addition of new decoders +// doesn't require you to disable them explicitly): +// +// STBI_ONLY_JPEG +// STBI_ONLY_PNG +// STBI_ONLY_BMP +// STBI_ONLY_PSD +// STBI_ONLY_TGA +// STBI_ONLY_GIF +// STBI_ONLY_HDR +// STBI_ONLY_PIC +// STBI_ONLY_PNM (.ppm and .pgm) +// +// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still +// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB +// +// - If you define STBI_MAX_DIMENSIONS, stb_image will reject images greater +// than that size (in either width or height) without further processing. +// This is to let programs in the wild set an upper bound to prevent +// denial-of-service attacks on untrusted data, as one could generate a +// valid image of gigantic dimensions and force stb_image to allocate a +// huge block of memory and spend disproportionate time decoding it. By +// default this is set to (1 << 24), which is 16777216, but that's still +// very big. + +#ifndef STBI_NO_STDIO +#include +#endif // STBI_NO_STDIO + +#define STBI_VERSION 1 + +enum +{ + STBI_default = 0, // only used for desired_channels + + STBI_grey = 1, + STBI_grey_alpha = 2, + STBI_rgb = 3, + STBI_rgb_alpha = 4 +}; + +#include +typedef unsigned char stbi_uc; +typedef unsigned short stbi_us; + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef STBIDEF +#ifdef STB_IMAGE_STATIC +#define STBIDEF static +#else +#define STBIDEF extern +#endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// PRIMARY API - works on images of any type +// + +// +// load image by filename, open file, or memory buffer +// + +typedef struct +{ + int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read + void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative + int (*eof) (void *user); // returns nonzero if we are at end of file/data +} stbi_io_callbacks; + +//////////////////////////////////// +// +// 8-bits-per-channel interface +// + +STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +// for stbi_load_from_file, file pointer is left pointing immediately after image +#endif + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +#endif + +#ifdef STBI_WINDOWS_UTF8 +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif + +//////////////////////////////////// +// +// 16-bits-per-channel interface +// + +STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +#endif + +//////////////////////////////////// +// +// float-per-channel interface +// +#ifndef STBI_NO_LINEAR + STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + + #ifndef STBI_NO_STDIO + STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); + #endif +#endif + +#ifndef STBI_NO_HDR + STBIDEF void stbi_hdr_to_ldr_gamma(float gamma); + STBIDEF void stbi_hdr_to_ldr_scale(float scale); +#endif // STBI_NO_HDR + +#ifndef STBI_NO_LINEAR + STBIDEF void stbi_ldr_to_hdr_gamma(float gamma); + STBIDEF void stbi_ldr_to_hdr_scale(float scale); +#endif // STBI_NO_LINEAR + +// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user); +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len); +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename); +STBIDEF int stbi_is_hdr_from_file(FILE *f); +#endif // STBI_NO_STDIO + + +// get a VERY brief reason for failure +// on most compilers (and ALL modern mainstream compilers) this is threadsafe +STBIDEF const char *stbi_failure_reason (void); + +// free the loaded image -- this is just free() +STBIDEF void stbi_image_free (void *retval_from_stbi_load); + +// get image dimensions & components without fully decoding +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len); +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user); + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit (char const *filename); +STBIDEF int stbi_is_16_bit_from_file(FILE *f); +#endif + + + +// for image formats that explicitly notate that they have premultiplied alpha, +// we just return the colors as stored in the file. set this flag to force +// unpremultiplication. results are undefined if the unpremultiply overflow. +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply); + +// indicate whether we should process iphone images back to canonical format, +// or just pass them through "as-is" +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert); + +// flip the image vertically, so the first pixel in the output array is the bottom left +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip); + +// as above, but only applies to images loaded on the thread that calls the function +// this function is only available if your compiler supports thread-local variables; +// calling it will fail to link if your compiler doesn't +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply); +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert); +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip); + +// ZLIB client - used by PNG, available for other purposes + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen); +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header); +STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + +STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + + +#ifdef __cplusplus +} +#endif + +// +// +//// end header file ///////////////////////////////////////////////////// +#endif // STBI_INCLUDE_STB_IMAGE_H + +#ifdef STB_IMAGE_IMPLEMENTATION + +#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \ + || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \ + || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \ + || defined(STBI_ONLY_ZLIB) + #ifndef STBI_ONLY_JPEG + #define STBI_NO_JPEG + #endif + #ifndef STBI_ONLY_PNG + #define STBI_NO_PNG + #endif + #ifndef STBI_ONLY_BMP + #define STBI_NO_BMP + #endif + #ifndef STBI_ONLY_PSD + #define STBI_NO_PSD + #endif + #ifndef STBI_ONLY_TGA + #define STBI_NO_TGA + #endif + #ifndef STBI_ONLY_GIF + #define STBI_NO_GIF + #endif + #ifndef STBI_ONLY_HDR + #define STBI_NO_HDR + #endif + #ifndef STBI_ONLY_PIC + #define STBI_NO_PIC + #endif + #ifndef STBI_ONLY_PNM + #define STBI_NO_PNM + #endif +#endif + +#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB) +#define STBI_NO_ZLIB +#endif + + +#include +#include // ptrdiff_t on osx +#include +#include +#include + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +#include // ldexp, pow +#endif + +#ifndef STBI_NO_STDIO +#include +#endif + +#ifndef STBI_ASSERT +#include +#define STBI_ASSERT(x) assert(x) +#endif + +#ifdef __cplusplus +#define STBI_EXTERN extern "C" +#else +#define STBI_EXTERN extern +#endif + + +#ifndef _MSC_VER + #ifdef __cplusplus + #define stbi_inline inline + #else + #define stbi_inline + #endif +#else + #define stbi_inline __forceinline +#endif + +#ifndef STBI_NO_THREAD_LOCALS + #if defined(__cplusplus) && __cplusplus >= 201103L + #define STBI_THREAD_LOCAL thread_local + #elif defined(__GNUC__) && __GNUC__ < 5 + #define STBI_THREAD_LOCAL __thread + #elif defined(_MSC_VER) + #define STBI_THREAD_LOCAL __declspec(thread) + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) + #define STBI_THREAD_LOCAL _Thread_local + #endif + + #ifndef STBI_THREAD_LOCAL + #if defined(__GNUC__) + #define STBI_THREAD_LOCAL __thread + #endif + #endif +#endif + +#if defined(_MSC_VER) || defined(__SYMBIAN32__) +typedef unsigned short stbi__uint16; +typedef signed short stbi__int16; +typedef unsigned int stbi__uint32; +typedef signed int stbi__int32; +#else +#include +typedef uint16_t stbi__uint16; +typedef int16_t stbi__int16; +typedef uint32_t stbi__uint32; +typedef int32_t stbi__int32; +#endif + +// should produce compiler error if size is wrong +typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1]; + +#ifdef _MSC_VER +#define STBI_NOTUSED(v) (void)(v) +#else +#define STBI_NOTUSED(v) (void)sizeof(v) +#endif + +#ifdef _MSC_VER +#define STBI_HAS_LROTL +#endif + +#ifdef STBI_HAS_LROTL + #define stbi_lrot(x,y) _lrotl(x,y) +#else + #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (-(y) & 31))) +#endif + +#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED)) +// ok +#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)." +#endif + +#ifndef STBI_MALLOC +#define STBI_MALLOC(sz) malloc(sz) +#define STBI_REALLOC(p,newsz) realloc(p,newsz) +#define STBI_FREE(p) free(p) +#endif + +#ifndef STBI_REALLOC_SIZED +#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz) +#endif + +// x86/x64 detection +#if defined(__x86_64__) || defined(_M_X64) +#define STBI__X64_TARGET +#elif defined(__i386) || defined(_M_IX86) +#define STBI__X86_TARGET +#endif + +#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD) +// gcc doesn't support sse2 intrinsics unless you compile with -msse2, +// which in turn means it gets to use SSE2 everywhere. This is unfortunate, +// but previous attempts to provide the SSE2 functions with runtime +// detection caused numerous issues. The way architecture extensions are +// exposed in GCC/Clang is, sadly, not really suited for one-file libs. +// New behavior: if compiled with -msse2, we use SSE2 without any +// detection; if not, we don't use it at all. +#define STBI_NO_SIMD +#endif + +#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD) +// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET +// +// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the +// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant. +// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not +// simultaneously enabling "-mstackrealign". +// +// See https://github.com/nothings/stb/issues/81 for more information. +// +// So default to no SSE2 on 32-bit MinGW. If you've read this far and added +// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2. +#define STBI_NO_SIMD +#endif + +#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET)) +#define STBI_SSE2 +#include + +#ifdef _MSC_VER + +#if _MSC_VER >= 1400 // not VC6 +#include // __cpuid +static int stbi__cpuid3(void) +{ + int info[4]; + __cpuid(info,1); + return info[3]; +} +#else +static int stbi__cpuid3(void) +{ + int res; + __asm { + mov eax,1 + cpuid + mov res,edx + } + return res; +} +#endif + +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + int info3 = stbi__cpuid3(); + return ((info3 >> 26) & 1) != 0; +} +#endif + +#else // assume GCC-style if not VC++ +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + // If we're even attempting to compile this on GCC/Clang, that means + // -msse2 is on, which means the compiler is allowed to use SSE2 + // instructions at will, and so are we. + return 1; +} +#endif + +#endif +#endif + +// ARM NEON +#if defined(STBI_NO_SIMD) && defined(STBI_NEON) +#undef STBI_NEON +#endif + +#ifdef STBI_NEON +#include +#ifdef _MSC_VER +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name +#else +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) +#endif +#endif + +#ifndef STBI_SIMD_ALIGN +#define STBI_SIMD_ALIGN(type, name) type name +#endif + +#ifndef STBI_MAX_DIMENSIONS +#define STBI_MAX_DIMENSIONS (1 << 24) +#endif + +/////////////////////////////////////////////// +// +// stbi__context struct and start_xxx functions + +// stbi__context structure is our basic context used by all images, so it +// contains all the IO context, plus some basic image information +typedef struct +{ + stbi__uint32 img_x, img_y; + int img_n, img_out_n; + + stbi_io_callbacks io; + void *io_user_data; + + int read_from_callbacks; + int buflen; + stbi_uc buffer_start[128]; + int callback_already_read; + + stbi_uc *img_buffer, *img_buffer_end; + stbi_uc *img_buffer_original, *img_buffer_original_end; +} stbi__context; + + +static void stbi__refill_buffer(stbi__context *s); + +// initialize a memory-decode context +static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len) +{ + s->io.read = NULL; + s->read_from_callbacks = 0; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer; + s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len; +} + +// initialize a callback-based context +static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user) +{ + s->io = *c; + s->io_user_data = user; + s->buflen = sizeof(s->buffer_start); + s->read_from_callbacks = 1; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = s->buffer_start; + stbi__refill_buffer(s); + s->img_buffer_original_end = s->img_buffer_end; +} + +#ifndef STBI_NO_STDIO + +static int stbi__stdio_read(void *user, char *data, int size) +{ + return (int) fread(data,1,size,(FILE*) user); +} + +static void stbi__stdio_skip(void *user, int n) +{ + int ch; + fseek((FILE*) user, n, SEEK_CUR); + ch = fgetc((FILE*) user); /* have to read a byte to reset feof()'s flag */ + if (ch != EOF) { + ungetc(ch, (FILE *) user); /* push byte back onto stream if valid. */ + } +} + +static int stbi__stdio_eof(void *user) +{ + return feof((FILE*) user) || ferror((FILE *) user); +} + +static stbi_io_callbacks stbi__stdio_callbacks = +{ + stbi__stdio_read, + stbi__stdio_skip, + stbi__stdio_eof, +}; + +static void stbi__start_file(stbi__context *s, FILE *f) +{ + stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f); +} + +//static void stop_file(stbi__context *s) { } + +#endif // !STBI_NO_STDIO + +static void stbi__rewind(stbi__context *s) +{ + // conceptually rewind SHOULD rewind to the beginning of the stream, + // but we just rewind to the beginning of the initial buffer, because + // we only use it after doing 'test', which only ever looks at at most 92 bytes + s->img_buffer = s->img_buffer_original; + s->img_buffer_end = s->img_buffer_original_end; +} + +enum +{ + STBI_ORDER_RGB, + STBI_ORDER_BGR +}; + +typedef struct +{ + int bits_per_channel; + int num_channels; + int channel_order; +} stbi__result_info; + +#ifndef STBI_NO_JPEG +static int stbi__jpeg_test(stbi__context *s); +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNG +static int stbi__png_test(stbi__context *s); +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__png_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_BMP +static int stbi__bmp_test(stbi__context *s); +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_TGA +static int stbi__tga_test(stbi__context *s); +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s); +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc); +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__psd_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_HDR +static int stbi__hdr_test(stbi__context *s); +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_test(stbi__context *s); +static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_GIF +static int stbi__gif_test(stbi__context *s); +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNM +static int stbi__pnm_test(stbi__context *s); +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__pnm_is16(stbi__context *s); +#endif + +static +#ifdef STBI_THREAD_LOCAL +STBI_THREAD_LOCAL +#endif +const char *stbi__g_failure_reason; + +STBIDEF const char *stbi_failure_reason(void) +{ + return stbi__g_failure_reason; +} + +#ifndef STBI_NO_FAILURE_STRINGS +static int stbi__err(const char *str) +{ + stbi__g_failure_reason = str; + return 0; +} +#endif + +static void *stbi__malloc(size_t size) +{ + return STBI_MALLOC(size); +} + +// stb_image uses ints pervasively, including for offset calculations. +// therefore the largest decoded image size we can support with the +// current code, even on 64-bit targets, is INT_MAX. this is not a +// significant limitation for the intended use case. +// +// we do, however, need to make sure our size calculations don't +// overflow. hence a few helper functions for size calculations that +// multiply integers together, making sure that they're non-negative +// and no overflow occurs. + +// return 1 if the sum is valid, 0 on overflow. +// negative terms are considered invalid. +static int stbi__addsizes_valid(int a, int b) +{ + if (b < 0) return 0; + // now 0 <= b <= INT_MAX, hence also + // 0 <= INT_MAX - b <= INTMAX. + // And "a + b <= INT_MAX" (which might overflow) is the + // same as a <= INT_MAX - b (no overflow) + return a <= INT_MAX - b; +} + +// returns 1 if the product is valid, 0 on overflow. +// negative factors are considered invalid. +static int stbi__mul2sizes_valid(int a, int b) +{ + if (a < 0 || b < 0) return 0; + if (b == 0) return 1; // mul-by-0 is always safe + // portable way to check for no overflows in a*b + return a <= INT_MAX/b; +} + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow +static int stbi__mad2sizes_valid(int a, int b, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add); +} +#endif + +// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow +static int stbi__mad3sizes_valid(int a, int b, int c, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__addsizes_valid(a*b*c, add); +} + +// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add); +} +#endif + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// mallocs with size overflow checking +static void *stbi__malloc_mad2(int a, int b, int add) +{ + if (!stbi__mad2sizes_valid(a, b, add)) return NULL; + return stbi__malloc(a*b + add); +} +#endif + +static void *stbi__malloc_mad3(int a, int b, int c, int add) +{ + if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL; + return stbi__malloc(a*b*c + add); +} + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static void *stbi__malloc_mad4(int a, int b, int c, int d, int add) +{ + if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL; + return stbi__malloc(a*b*c*d + add); +} +#endif + +// returns 1 if the sum of two signed ints is valid (between -2^31 and 2^31-1 inclusive), 0 on overflow. +static int stbi__addints_valid(int a, int b) +{ + if ((a >= 0) != (b >= 0)) return 1; // a and b have different signs, so no overflow + if (a < 0 && b < 0) return a >= INT_MIN - b; // same as a + b >= INT_MIN; INT_MIN - b cannot overflow since b < 0. + return a <= INT_MAX - b; +} + +// returns 1 if the product of two ints fits in a signed short, 0 on overflow. +static int stbi__mul2shorts_valid(int a, int b) +{ + if (b == 0 || b == -1) return 1; // multiplication by 0 is always 0; check for -1 so SHRT_MIN/b doesn't overflow + if ((a >= 0) == (b >= 0)) return a <= SHRT_MAX/b; // product is positive, so similar to mul2sizes_valid + if (b < 0) return a <= SHRT_MIN / b; // same as a * b >= SHRT_MIN + return a >= SHRT_MIN / b; +} + +// stbi__err - error +// stbi__errpf - error returning pointer to float +// stbi__errpuc - error returning pointer to unsigned char + +#ifdef STBI_NO_FAILURE_STRINGS + #define stbi__err(x,y) 0 +#elif defined(STBI_FAILURE_USERMSG) + #define stbi__err(x,y) stbi__err(y) +#else + #define stbi__err(x,y) stbi__err(x) +#endif + +#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL)) +#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL)) + +STBIDEF void stbi_image_free(void *retval_from_stbi_load) +{ + STBI_FREE(retval_from_stbi_load); +} + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp); +#endif + +#ifndef STBI_NO_HDR +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp); +#endif + +static int stbi__vertically_flip_on_load_global = 0; + +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_global = flag_true_if_should_flip; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__vertically_flip_on_load stbi__vertically_flip_on_load_global +#else +static STBI_THREAD_LOCAL int stbi__vertically_flip_on_load_local, stbi__vertically_flip_on_load_set; + +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_local = flag_true_if_should_flip; + stbi__vertically_flip_on_load_set = 1; +} + +#define stbi__vertically_flip_on_load (stbi__vertically_flip_on_load_set \ + ? stbi__vertically_flip_on_load_local \ + : stbi__vertically_flip_on_load_global) +#endif // STBI_THREAD_LOCAL + +static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields + ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed + ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order + ri->num_channels = 0; + + // test the formats with a very explicit header first (at least a FOURCC + // or distinctive magic number first) + #ifndef STBI_NO_PNG + if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_BMP + if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_GIF + if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PSD + if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc); + #else + STBI_NOTUSED(bpc); + #endif + #ifndef STBI_NO_PIC + if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri); + #endif + + // then the formats that can end up attempting to load with just 1 or 2 + // bytes matching expectations; these are prone to false positives, so + // try them later + #ifndef STBI_NO_JPEG + if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PNM + if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri); + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri); + return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp); + } + #endif + + #ifndef STBI_NO_TGA + // test tga last because it's a crappy test! + if (stbi__tga_test(s)) + return stbi__tga_load(s,x,y,comp,req_comp, ri); + #endif + + return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt"); +} + +static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi_uc *reduced; + + reduced = (stbi_uc *) stbi__malloc(img_len); + if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling + + STBI_FREE(orig); + return reduced; +} + +static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi__uint16 *enlarged; + + enlarged = (stbi__uint16 *) stbi__malloc(img_len*2); + if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff + + STBI_FREE(orig); + return enlarged; +} + +static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel) +{ + int row; + size_t bytes_per_row = (size_t)w * bytes_per_pixel; + stbi_uc temp[2048]; + stbi_uc *bytes = (stbi_uc *)image; + + for (row = 0; row < (h>>1); row++) { + stbi_uc *row0 = bytes + row*bytes_per_row; + stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row; + // swap row0 with row1 + size_t bytes_left = bytes_per_row; + while (bytes_left) { + size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp); + memcpy(temp, row0, bytes_copy); + memcpy(row0, row1, bytes_copy); + memcpy(row1, temp, bytes_copy); + row0 += bytes_copy; + row1 += bytes_copy; + bytes_left -= bytes_copy; + } + } +} + +#ifndef STBI_NO_GIF +static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel) +{ + int slice; + int slice_size = w * h * bytes_per_pixel; + + stbi_uc *bytes = (stbi_uc *)image; + for (slice = 0; slice < z; ++slice) { + stbi__vertical_flip(bytes, w, h, bytes_per_pixel); + bytes += slice_size; + } +} +#endif + +static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 8) { + result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 8; + } + + // @TODO: move stbi__convert_format to here + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc)); + } + + return (unsigned char *) result; +} + +static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 16) { + result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 16; + } + + // @TODO: move stbi__convert_format16 to here + // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16)); + } + + return (stbi__uint16 *) result; +} + +#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR) +static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp) +{ + if (stbi__vertically_flip_on_load && result != NULL) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(float)); + } +} +#endif + +#ifndef STBI_NO_STDIO + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); +#endif + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbi__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + + +STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + unsigned char *result; + if (!f) return stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__uint16 *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + stbi__uint16 *result; + if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file_16(f,x,y,comp,req_comp); + fclose(f); + return result; +} + + +#endif //!STBI_NO_STDIO + +STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_mem(&s,buffer,len); + + result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp); + if (stbi__vertically_flip_on_load) { + stbi__vertical_flip_slices( result, *x, *y, *z, *comp ); + } + + return result; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *data; + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + stbi__result_info ri; + float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri); + if (hdr_data) + stbi__float_postprocess(hdr_data,x,y,comp,req_comp); + return hdr_data; + } + #endif + data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp); + if (data) + return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp); + return stbi__errpf("unknown image type", "Image not of any known type, or corrupt"); +} + +STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_STDIO +STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + float *result; + FILE *f = stbi__fopen(filename, "rb"); + if (!f) return stbi__errpf("can't fopen", "Unable to open file"); + result = stbi_loadf_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_file(&s,f); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} +#endif // !STBI_NO_STDIO + +#endif // !STBI_NO_LINEAR + +// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is +// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always +// reports false! + +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(buffer); + STBI_NOTUSED(len); + return 0; + #endif +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result=0; + if (f) { + result = stbi_is_hdr_from_file(f); + fclose(f); + } + return result; +} + +STBIDEF int stbi_is_hdr_from_file(FILE *f) +{ + #ifndef STBI_NO_HDR + long pos = ftell(f); + int res; + stbi__context s; + stbi__start_file(&s,f); + res = stbi__hdr_test(&s); + fseek(f, pos, SEEK_SET); + return res; + #else + STBI_NOTUSED(f); + return 0; + #endif +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(clbk); + STBI_NOTUSED(user); + return 0; + #endif +} + +#ifndef STBI_NO_LINEAR +static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f; + +STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; } +STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; } +#endif + +static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f; + +STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; } +STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; } + + +////////////////////////////////////////////////////////////////////////////// +// +// Common code used by all image loaders +// + +enum +{ + STBI__SCAN_load=0, + STBI__SCAN_type, + STBI__SCAN_header +}; + +static void stbi__refill_buffer(stbi__context *s) +{ + int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen); + s->callback_already_read += (int) (s->img_buffer - s->img_buffer_original); + if (n == 0) { + // at end of file, treat same as if from memory, but need to handle case + // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file + s->read_from_callbacks = 0; + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start+1; + *s->img_buffer = 0; + } else { + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start + n; + } +} + +stbi_inline static stbi_uc stbi__get8(stbi__context *s) +{ + if (s->img_buffer < s->img_buffer_end) + return *s->img_buffer++; + if (s->read_from_callbacks) { + stbi__refill_buffer(s); + return *s->img_buffer++; + } + return 0; +} + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_HDR) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +stbi_inline static int stbi__at_eof(stbi__context *s) +{ + if (s->io.read) { + if (!(s->io.eof)(s->io_user_data)) return 0; + // if feof() is true, check if buffer = end + // special case: we've only got the special 0 character at the end + if (s->read_from_callbacks == 0) return 1; + } + + return s->img_buffer >= s->img_buffer_end; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) +// nothing +#else +static void stbi__skip(stbi__context *s, int n) +{ + if (n == 0) return; // already there! + if (n < 0) { + s->img_buffer = s->img_buffer_end; + return; + } + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + s->img_buffer = s->img_buffer_end; + (s->io.skip)(s->io_user_data, n - blen); + return; + } + } + s->img_buffer += n; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_TGA) && defined(STBI_NO_HDR) && defined(STBI_NO_PNM) +// nothing +#else +static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n) +{ + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + int res, count; + + memcpy(buffer, s->img_buffer, blen); + + count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen); + res = (count == (n-blen)); + s->img_buffer = s->img_buffer_end; + return res; + } + } + + if (s->img_buffer+n <= s->img_buffer_end) { + memcpy(buffer, s->img_buffer, n); + s->img_buffer += n; + return 1; + } else + return 0; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static int stbi__get16be(stbi__context *s) +{ + int z = stbi__get8(s); + return (z << 8) + stbi__get8(s); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static stbi__uint32 stbi__get32be(stbi__context *s) +{ + stbi__uint32 z = stbi__get16be(s); + return (z << 16) + stbi__get16be(s); +} +#endif + +#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) +// nothing +#else +static int stbi__get16le(stbi__context *s) +{ + int z = stbi__get8(s); + return z + (stbi__get8(s) << 8); +} +#endif + +#ifndef STBI_NO_BMP +static stbi__uint32 stbi__get32le(stbi__context *s) +{ + stbi__uint32 z = stbi__get16le(s); + z += (stbi__uint32)stbi__get16le(s) << 16; + return z; +} +#endif + +#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +////////////////////////////////////////////////////////////////////////////// +// +// generic converter from built-in img_n to req_comp +// individual types do this automatically as much as possible (e.g. jpeg +// does all cases internally since it needs to colorspace convert anyway, +// and it never has alpha, so very few cases ). png can automatically +// interleave an alpha=255 channel, but falls back to this for other cases +// +// assume data buffer is malloced, so malloc a new one and free that one +// only failure mode is malloc failing + +static stbi_uc stbi__compute_y(int r, int g, int b) +{ + return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + unsigned char *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0); + if (good == NULL) { + STBI_FREE(data); + return stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + unsigned char *src = data + j * x * img_n ; + unsigned char *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 stbi__compute_y_16(int r, int g, int b) +{ + return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + stbi__uint16 *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2); + if (good == NULL) { + STBI_FREE(data); + return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + stbi__uint16 *src = data + j * x * img_n ; + stbi__uint16 *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return (stbi__uint16*) stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp) +{ + int i,k,n; + float *output; + if (!data) return NULL; + output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale); + } + } + if (n < comp) { + for (i=0; i < x*y; ++i) { + output[i*comp + n] = data[i*comp + n]/255.0f; + } + } + STBI_FREE(data); + return output; +} +#endif + +#ifndef STBI_NO_HDR +#define stbi__float2int(x) ((int) (x)) +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp) +{ + int i,k,n; + stbi_uc *output; + if (!data) return NULL; + output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + if (k < comp) { + float z = data[i*comp+k] * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + } + STBI_FREE(data); + return output; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// "baseline" JPEG/JFIF decoder +// +// simple implementation +// - doesn't support delayed output of y-dimension +// - simple interface (only one output format: 8-bit interleaved RGB) +// - doesn't try to recover corrupt jpegs +// - doesn't allow partial loading, loading multiple at once +// - still fast on x86 (copying globals into locals doesn't help x86) +// - allocates lots of intermediate memory (full size of all components) +// - non-interleaved case requires this anyway +// - allows good upsampling (see next) +// high-quality +// - upsampled channels are bilinearly interpolated, even across blocks +// - quality integer IDCT derived from IJG's 'slow' +// performance +// - fast huffman; reasonable integer IDCT +// - some SIMD kernels for common paths on targets with SSE2/NEON +// - uses a lot of intermediate memory, could cache poorly + +#ifndef STBI_NO_JPEG + +// huffman decoding acceleration +#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache + +typedef struct +{ + stbi_uc fast[1 << FAST_BITS]; + // weirdly, repacking this into AoS is a 10% speed loss, instead of a win + stbi__uint16 code[256]; + stbi_uc values[256]; + stbi_uc size[257]; + unsigned int maxcode[18]; + int delta[17]; // old 'firstsymbol' - old 'firstcode' +} stbi__huffman; + +typedef struct +{ + stbi__context *s; + stbi__huffman huff_dc[4]; + stbi__huffman huff_ac[4]; + stbi__uint16 dequant[4][64]; + stbi__int16 fast_ac[4][1 << FAST_BITS]; + +// sizes for components, interleaved MCUs + int img_h_max, img_v_max; + int img_mcu_x, img_mcu_y; + int img_mcu_w, img_mcu_h; + +// definition of jpeg image component + struct + { + int id; + int h,v; + int tq; + int hd,ha; + int dc_pred; + + int x,y,w2,h2; + stbi_uc *data; + void *raw_data, *raw_coeff; + stbi_uc *linebuf; + short *coeff; // progressive only + int coeff_w, coeff_h; // number of 8x8 coefficient blocks + } img_comp[4]; + + stbi__uint32 code_buffer; // jpeg entropy-coded buffer + int code_bits; // number of valid bits + unsigned char marker; // marker seen while filling entropy buffer + int nomore; // flag if we saw a marker so must stop + + int progressive; + int spec_start; + int spec_end; + int succ_high; + int succ_low; + int eob_run; + int jfif; + int app14_color_transform; // Adobe APP14 tag + int rgb; + + int scan_n, order[4]; + int restart_interval, todo; + +// kernels + void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]); + void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step); + stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs); +} stbi__jpeg; + +static int stbi__build_huffman(stbi__huffman *h, int *count) +{ + int i,j,k=0; + unsigned int code; + // build size list for each symbol (from JPEG spec) + for (i=0; i < 16; ++i) { + for (j=0; j < count[i]; ++j) { + h->size[k++] = (stbi_uc) (i+1); + if(k >= 257) return stbi__err("bad size list","Corrupt JPEG"); + } + } + h->size[k] = 0; + + // compute actual symbols (from jpeg spec) + code = 0; + k = 0; + for(j=1; j <= 16; ++j) { + // compute delta to add to code to compute symbol id + h->delta[j] = k - code; + if (h->size[k] == j) { + while (h->size[k] == j) + h->code[k++] = (stbi__uint16) (code++); + if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG"); + } + // compute largest code + 1 for this size, preshifted as needed later + h->maxcode[j] = code << (16-j); + code <<= 1; + } + h->maxcode[j] = 0xffffffff; + + // build non-spec acceleration table; 255 is flag for not-accelerated + memset(h->fast, 255, 1 << FAST_BITS); + for (i=0; i < k; ++i) { + int s = h->size[i]; + if (s <= FAST_BITS) { + int c = h->code[i] << (FAST_BITS-s); + int m = 1 << (FAST_BITS-s); + for (j=0; j < m; ++j) { + h->fast[c+j] = (stbi_uc) i; + } + } + } + return 1; +} + +// build a table that decodes both magnitude and value of small ACs in +// one go. +static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h) +{ + int i; + for (i=0; i < (1 << FAST_BITS); ++i) { + stbi_uc fast = h->fast[i]; + fast_ac[i] = 0; + if (fast < 255) { + int rs = h->values[fast]; + int run = (rs >> 4) & 15; + int magbits = rs & 15; + int len = h->size[fast]; + + if (magbits && len + magbits <= FAST_BITS) { + // magnitude code followed by receive_extend code + int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits); + int m = 1 << (magbits - 1); + if (k < m) k += (~0U << magbits) + 1; + // if the result is small enough, we can fit it in fast_ac table + if (k >= -128 && k <= 127) + fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits)); + } + } + } +} + +static void stbi__grow_buffer_unsafe(stbi__jpeg *j) +{ + do { + unsigned int b = j->nomore ? 0 : stbi__get8(j->s); + if (b == 0xff) { + int c = stbi__get8(j->s); + while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes + if (c != 0) { + j->marker = (unsigned char) c; + j->nomore = 1; + return; + } + } + j->code_buffer |= b << (24 - j->code_bits); + j->code_bits += 8; + } while (j->code_bits <= 24); +} + +// (1 << n) - 1 +static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535}; + +// decode a jpeg huffman value from the bitstream +stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h) +{ + unsigned int temp; + int c,k; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + // look at the top FAST_BITS and determine what symbol ID it is, + // if the code is <= FAST_BITS + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + k = h->fast[c]; + if (k < 255) { + int s = h->size[k]; + if (s > j->code_bits) + return -1; + j->code_buffer <<= s; + j->code_bits -= s; + return h->values[k]; + } + + // naive test is to shift the code_buffer down so k bits are + // valid, then test against maxcode. To speed this up, we've + // preshifted maxcode left so that it has (16-k) 0s at the + // end; in other words, regardless of the number of bits, it + // wants to be compared against something shifted to have 16; + // that way we don't need to shift inside the loop. + temp = j->code_buffer >> 16; + for (k=FAST_BITS+1 ; ; ++k) + if (temp < h->maxcode[k]) + break; + if (k == 17) { + // error! code not found + j->code_bits -= 16; + return -1; + } + + if (k > j->code_bits) + return -1; + + // convert the huffman code to the symbol id + c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k]; + if(c < 0 || c >= 256) // symbol id out of bounds! + return -1; + STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]); + + // convert the id to a symbol + j->code_bits -= k; + j->code_buffer <<= k; + return h->values[c]; +} + +// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j); + if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing + + sgn = j->code_buffer >> 31; // sign bit always in MSB; 0 if MSB clear (positive), 1 if MSB set (negative) + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k + (stbi__jbias[n] & (sgn - 1)); +} + +// get some unsigned bits +stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n) +{ + unsigned int k; + if (j->code_bits < n) stbi__grow_buffer_unsafe(j); + if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k; +} + +stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j) +{ + unsigned int k; + if (j->code_bits < 1) stbi__grow_buffer_unsafe(j); + if (j->code_bits < 1) return 0; // ran out of bits from stream, return 0s intead of continuing + k = j->code_buffer; + j->code_buffer <<= 1; + --j->code_bits; + return k & 0x80000000; +} + +// given a value that's at position X in the zigzag stream, +// where does it appear in the 8x8 matrix coded as row-major? +static const stbi_uc stbi__jpeg_dezigzag[64+15] = +{ + 0, 1, 8, 16, 9, 2, 3, 10, + 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, + 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, + 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63, + // let corrupt input sample past end + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63 +}; + +// decode one 64-entry block-- +static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant) +{ + int diff,dc,k; + int t; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("bad huffman code","Corrupt JPEG"); + + // 0 all the ac values now so we can do it 32-bits at a time + memset(data,0,64*sizeof(data[0])); + + diff = t ? stbi__extend_receive(j, t) : 0; + if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta","Corrupt JPEG"); + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + if (!stbi__mul2shorts_valid(dc, dequant[0])) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + data[0] = (short) (dc * dequant[0]); + + // decode AC components, see JPEG spec + k = 1; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available"); + j->code_buffer <<= s; + j->code_bits -= s; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * dequant[zig]); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (rs != 0xf0) break; // end block + k += 16; + } else { + k += r; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]); + } + } + } while (k < 64); + return 1; +} + +static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b) +{ + int diff,dc; + int t; + if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + if (j->succ_high == 0) { + // first scan for DC coefficient, must be first + memset(data,0,64*sizeof(data[0])); // 0 all the ac values now + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + diff = t ? stbi__extend_receive(j, t) : 0; + + if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta", "Corrupt JPEG"); + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + if (!stbi__mul2shorts_valid(dc, 1 << j->succ_low)) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + data[0] = (short) (dc * (1 << j->succ_low)); + } else { + // refinement scan for DC coefficient + if (stbi__jpeg_get_bit(j)) + data[0] += (short) (1 << j->succ_low); + } + return 1; +} + +// @OPTIMIZE: store non-zigzagged during the decode passes, +// and only de-zigzag when dequantizing +static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac) +{ + int k; + if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->succ_high == 0) { + int shift = j->succ_low; + + if (j->eob_run) { + --j->eob_run; + return 1; + } + + k = j->spec_start; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available"); + j->code_buffer <<= s; + j->code_bits -= s; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * (1 << shift)); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r); + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + --j->eob_run; + break; + } + k += 16; + } else { + k += r; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * (1 << shift)); + } + } + } while (k <= j->spec_end); + } else { + // refinement scan for these AC coefficients + + short bit = (short) (1 << j->succ_low); + + if (j->eob_run) { + --j->eob_run; + for (k = j->spec_start; k <= j->spec_end; ++k) { + short *p = &data[stbi__jpeg_dezigzag[k]]; + if (*p != 0) + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } + } else { + k = j->spec_start; + do { + int r,s; + int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r) - 1; + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + r = 64; // force end of block + } else { + // r=15 s=0 should write 16 0s, so we just do + // a run of 15 0s and then write s (which is 0), + // so we don't have to do anything special here + } + } else { + if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG"); + // sign bit + if (stbi__jpeg_get_bit(j)) + s = bit; + else + s = -bit; + } + + // advance by r + while (k <= j->spec_end) { + short *p = &data[stbi__jpeg_dezigzag[k++]]; + if (*p != 0) { + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } else { + if (r == 0) { + *p = (short) s; + break; + } + --r; + } + } + } while (k <= j->spec_end); + } + } + return 1; +} + +// take a -128..127 value and stbi__clamp it and convert to 0..255 +stbi_inline static stbi_uc stbi__clamp(int x) +{ + // trick to use a single test to catch both cases + if ((unsigned int) x > 255) { + if (x < 0) return 0; + if (x > 255) return 255; + } + return (stbi_uc) x; +} + +#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5))) +#define stbi__fsh(x) ((x) * 4096) + +// derived from jidctint -- DCT_ISLOW +#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \ + int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \ + p2 = s2; \ + p3 = s6; \ + p1 = (p2+p3) * stbi__f2f(0.5411961f); \ + t2 = p1 + p3*stbi__f2f(-1.847759065f); \ + t3 = p1 + p2*stbi__f2f( 0.765366865f); \ + p2 = s0; \ + p3 = s4; \ + t0 = stbi__fsh(p2+p3); \ + t1 = stbi__fsh(p2-p3); \ + x0 = t0+t3; \ + x3 = t0-t3; \ + x1 = t1+t2; \ + x2 = t1-t2; \ + t0 = s7; \ + t1 = s5; \ + t2 = s3; \ + t3 = s1; \ + p3 = t0+t2; \ + p4 = t1+t3; \ + p1 = t0+t3; \ + p2 = t1+t2; \ + p5 = (p3+p4)*stbi__f2f( 1.175875602f); \ + t0 = t0*stbi__f2f( 0.298631336f); \ + t1 = t1*stbi__f2f( 2.053119869f); \ + t2 = t2*stbi__f2f( 3.072711026f); \ + t3 = t3*stbi__f2f( 1.501321110f); \ + p1 = p5 + p1*stbi__f2f(-0.899976223f); \ + p2 = p5 + p2*stbi__f2f(-2.562915447f); \ + p3 = p3*stbi__f2f(-1.961570560f); \ + p4 = p4*stbi__f2f(-0.390180644f); \ + t3 += p1+p4; \ + t2 += p2+p3; \ + t1 += p2+p4; \ + t0 += p1+p3; + +static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64]) +{ + int i,val[64],*v=val; + stbi_uc *o; + short *d = data; + + // columns + for (i=0; i < 8; ++i,++d, ++v) { + // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing + if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0 + && d[40]==0 && d[48]==0 && d[56]==0) { + // no shortcut 0 seconds + // (1|2|3|4|5|6|7)==0 0 seconds + // all separate -0.047 seconds + // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds + int dcterm = d[0]*4; + v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm; + } else { + STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56]) + // constants scaled things up by 1<<12; let's bring them back + // down, but keep 2 extra bits of precision + x0 += 512; x1 += 512; x2 += 512; x3 += 512; + v[ 0] = (x0+t3) >> 10; + v[56] = (x0-t3) >> 10; + v[ 8] = (x1+t2) >> 10; + v[48] = (x1-t2) >> 10; + v[16] = (x2+t1) >> 10; + v[40] = (x2-t1) >> 10; + v[24] = (x3+t0) >> 10; + v[32] = (x3-t0) >> 10; + } + } + + for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) { + // no fast case since the first 1D IDCT spread components out + STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7]) + // constants scaled things up by 1<<12, plus we had 1<<2 from first + // loop, plus horizontal and vertical each scale by sqrt(8) so together + // we've got an extra 1<<3, so 1<<17 total we need to remove. + // so we want to round that, which means adding 0.5 * 1<<17, + // aka 65536. Also, we'll end up with -128 to 127 that we want + // to encode as 0..255 by adding 128, so we'll add that before the shift + x0 += 65536 + (128<<17); + x1 += 65536 + (128<<17); + x2 += 65536 + (128<<17); + x3 += 65536 + (128<<17); + // tried computing the shifts into temps, or'ing the temps to see + // if any were out of range, but that was slower + o[0] = stbi__clamp((x0+t3) >> 17); + o[7] = stbi__clamp((x0-t3) >> 17); + o[1] = stbi__clamp((x1+t2) >> 17); + o[6] = stbi__clamp((x1-t2) >> 17); + o[2] = stbi__clamp((x2+t1) >> 17); + o[5] = stbi__clamp((x2-t1) >> 17); + o[3] = stbi__clamp((x3+t0) >> 17); + o[4] = stbi__clamp((x3-t0) >> 17); + } +} + +#ifdef STBI_SSE2 +// sse2 integer IDCT. not the fastest possible implementation but it +// produces bit-identical results to the generic C version so it's +// fully "transparent". +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + // This is constructed to match our regular (generic) integer IDCT exactly. + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i tmp; + + // dot product constant: even elems=x, odd elems=y + #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y)) + + // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit) + // out(1) = c1[even]*x + c1[odd]*y + #define dct_rot(out0,out1, x,y,c0,c1) \ + __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \ + __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \ + __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \ + __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \ + __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \ + __m128i out1##_h = _mm_madd_epi16(c0##hi, c1) + + // out = in << 12 (in 16-bit, out 32-bit) + #define dct_widen(out, in) \ + __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \ + __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4) + + // wide add + #define dct_wadd(out, a, b) \ + __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_add_epi32(a##_h, b##_h) + + // wide sub + #define dct_wsub(out, a, b) \ + __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_sub_epi32(a##_h, b##_h) + + // butterfly a/b, add bias, then shift by "s" and pack + #define dct_bfly32o(out0, out1, a,b,bias,s) \ + { \ + __m128i abiased_l = _mm_add_epi32(a##_l, bias); \ + __m128i abiased_h = _mm_add_epi32(a##_h, bias); \ + dct_wadd(sum, abiased, b); \ + dct_wsub(dif, abiased, b); \ + out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \ + out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \ + } + + // 8-bit interleave step (for transposes) + #define dct_interleave8(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi8(a, b); \ + b = _mm_unpackhi_epi8(tmp, b) + + // 16-bit interleave step (for transposes) + #define dct_interleave16(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi16(a, b); \ + b = _mm_unpackhi_epi16(tmp, b) + + #define dct_pass(bias,shift) \ + { \ + /* even part */ \ + dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \ + __m128i sum04 = _mm_add_epi16(row0, row4); \ + __m128i dif04 = _mm_sub_epi16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \ + dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \ + __m128i sum17 = _mm_add_epi16(row1, row7); \ + __m128i sum35 = _mm_add_epi16(row3, row5); \ + dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \ + dct_wadd(x4, y0o, y4o); \ + dct_wadd(x5, y1o, y5o); \ + dct_wadd(x6, y2o, y5o); \ + dct_wadd(x7, y3o, y4o); \ + dct_bfly32o(row0,row7, x0,x7,bias,shift); \ + dct_bfly32o(row1,row6, x1,x6,bias,shift); \ + dct_bfly32o(row2,row5, x2,x5,bias,shift); \ + dct_bfly32o(row3,row4, x3,x4,bias,shift); \ + } + + __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f)); + __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f)); + __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f)); + __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f)); + __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f)); + __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f)); + __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f)); + __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f)); + + // rounding biases in column/row passes, see stbi__idct_block for explanation. + __m128i bias_0 = _mm_set1_epi32(512); + __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17)); + + // load + row0 = _mm_load_si128((const __m128i *) (data + 0*8)); + row1 = _mm_load_si128((const __m128i *) (data + 1*8)); + row2 = _mm_load_si128((const __m128i *) (data + 2*8)); + row3 = _mm_load_si128((const __m128i *) (data + 3*8)); + row4 = _mm_load_si128((const __m128i *) (data + 4*8)); + row5 = _mm_load_si128((const __m128i *) (data + 5*8)); + row6 = _mm_load_si128((const __m128i *) (data + 6*8)); + row7 = _mm_load_si128((const __m128i *) (data + 7*8)); + + // column pass + dct_pass(bias_0, 10); + + { + // 16bit 8x8 transpose pass 1 + dct_interleave16(row0, row4); + dct_interleave16(row1, row5); + dct_interleave16(row2, row6); + dct_interleave16(row3, row7); + + // transpose pass 2 + dct_interleave16(row0, row2); + dct_interleave16(row1, row3); + dct_interleave16(row4, row6); + dct_interleave16(row5, row7); + + // transpose pass 3 + dct_interleave16(row0, row1); + dct_interleave16(row2, row3); + dct_interleave16(row4, row5); + dct_interleave16(row6, row7); + } + + // row pass + dct_pass(bias_1, 17); + + { + // pack + __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7 + __m128i p1 = _mm_packus_epi16(row2, row3); + __m128i p2 = _mm_packus_epi16(row4, row5); + __m128i p3 = _mm_packus_epi16(row6, row7); + + // 8bit 8x8 transpose pass 1 + dct_interleave8(p0, p2); // a0e0a1e1... + dct_interleave8(p1, p3); // c0g0c1g1... + + // transpose pass 2 + dct_interleave8(p0, p1); // a0c0e0g0... + dct_interleave8(p2, p3); // b0d0f0h0... + + // transpose pass 3 + dct_interleave8(p0, p2); // a0b0c0d0... + dct_interleave8(p1, p3); // a4b4c4d4... + + // store + _mm_storel_epi64((__m128i *) out, p0); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p2); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p1); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p3); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e)); + } + +#undef dct_const +#undef dct_rot +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_interleave8 +#undef dct_interleave16 +#undef dct_pass +} + +#endif // STBI_SSE2 + +#ifdef STBI_NEON + +// NEON integer IDCT. should produce bit-identical +// results to the generic C version. +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + int16x8_t row0, row1, row2, row3, row4, row5, row6, row7; + + int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f)); + int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f)); + int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f)); + int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f)); + int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f)); + int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f)); + int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f)); + int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f)); + int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f)); + int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f)); + int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f)); + int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f)); + +#define dct_long_mul(out, inq, coeff) \ + int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff) + +#define dct_long_mac(out, acc, inq, coeff) \ + int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff) + +#define dct_widen(out, inq) \ + int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \ + int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12) + +// wide add +#define dct_wadd(out, a, b) \ + int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vaddq_s32(a##_h, b##_h) + +// wide sub +#define dct_wsub(out, a, b) \ + int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vsubq_s32(a##_h, b##_h) + +// butterfly a/b, then shift using "shiftop" by "s" and pack +#define dct_bfly32o(out0,out1, a,b,shiftop,s) \ + { \ + dct_wadd(sum, a, b); \ + dct_wsub(dif, a, b); \ + out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \ + out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \ + } + +#define dct_pass(shiftop, shift) \ + { \ + /* even part */ \ + int16x8_t sum26 = vaddq_s16(row2, row6); \ + dct_long_mul(p1e, sum26, rot0_0); \ + dct_long_mac(t2e, p1e, row6, rot0_1); \ + dct_long_mac(t3e, p1e, row2, rot0_2); \ + int16x8_t sum04 = vaddq_s16(row0, row4); \ + int16x8_t dif04 = vsubq_s16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + int16x8_t sum15 = vaddq_s16(row1, row5); \ + int16x8_t sum17 = vaddq_s16(row1, row7); \ + int16x8_t sum35 = vaddq_s16(row3, row5); \ + int16x8_t sum37 = vaddq_s16(row3, row7); \ + int16x8_t sumodd = vaddq_s16(sum17, sum35); \ + dct_long_mul(p5o, sumodd, rot1_0); \ + dct_long_mac(p1o, p5o, sum17, rot1_1); \ + dct_long_mac(p2o, p5o, sum35, rot1_2); \ + dct_long_mul(p3o, sum37, rot2_0); \ + dct_long_mul(p4o, sum15, rot2_1); \ + dct_wadd(sump13o, p1o, p3o); \ + dct_wadd(sump24o, p2o, p4o); \ + dct_wadd(sump23o, p2o, p3o); \ + dct_wadd(sump14o, p1o, p4o); \ + dct_long_mac(x4, sump13o, row7, rot3_0); \ + dct_long_mac(x5, sump24o, row5, rot3_1); \ + dct_long_mac(x6, sump23o, row3, rot3_2); \ + dct_long_mac(x7, sump14o, row1, rot3_3); \ + dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \ + dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \ + dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \ + dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \ + } + + // load + row0 = vld1q_s16(data + 0*8); + row1 = vld1q_s16(data + 1*8); + row2 = vld1q_s16(data + 2*8); + row3 = vld1q_s16(data + 3*8); + row4 = vld1q_s16(data + 4*8); + row5 = vld1q_s16(data + 5*8); + row6 = vld1q_s16(data + 6*8); + row7 = vld1q_s16(data + 7*8); + + // add DC bias + row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0)); + + // column pass + dct_pass(vrshrn_n_s32, 10); + + // 16bit 8x8 transpose + { +// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively. +// whether compilers actually get this is another story, sadly. +#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); } +#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); } + + // pass 1 + dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6 + dct_trn16(row2, row3); + dct_trn16(row4, row5); + dct_trn16(row6, row7); + + // pass 2 + dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4 + dct_trn32(row1, row3); + dct_trn32(row4, row6); + dct_trn32(row5, row7); + + // pass 3 + dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0 + dct_trn64(row1, row5); + dct_trn64(row2, row6); + dct_trn64(row3, row7); + +#undef dct_trn16 +#undef dct_trn32 +#undef dct_trn64 + } + + // row pass + // vrshrn_n_s32 only supports shifts up to 16, we need + // 17. so do a non-rounding shift of 16 first then follow + // up with a rounding shift by 1. + dct_pass(vshrn_n_s32, 16); + + { + // pack and round + uint8x8_t p0 = vqrshrun_n_s16(row0, 1); + uint8x8_t p1 = vqrshrun_n_s16(row1, 1); + uint8x8_t p2 = vqrshrun_n_s16(row2, 1); + uint8x8_t p3 = vqrshrun_n_s16(row3, 1); + uint8x8_t p4 = vqrshrun_n_s16(row4, 1); + uint8x8_t p5 = vqrshrun_n_s16(row5, 1); + uint8x8_t p6 = vqrshrun_n_s16(row6, 1); + uint8x8_t p7 = vqrshrun_n_s16(row7, 1); + + // again, these can translate into one instruction, but often don't. +#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); } +#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); } + + // sadly can't use interleaved stores here since we only write + // 8 bytes to each scan line! + + // 8x8 8-bit transpose pass 1 + dct_trn8_8(p0, p1); + dct_trn8_8(p2, p3); + dct_trn8_8(p4, p5); + dct_trn8_8(p6, p7); + + // pass 2 + dct_trn8_16(p0, p2); + dct_trn8_16(p1, p3); + dct_trn8_16(p4, p6); + dct_trn8_16(p5, p7); + + // pass 3 + dct_trn8_32(p0, p4); + dct_trn8_32(p1, p5); + dct_trn8_32(p2, p6); + dct_trn8_32(p3, p7); + + // store + vst1_u8(out, p0); out += out_stride; + vst1_u8(out, p1); out += out_stride; + vst1_u8(out, p2); out += out_stride; + vst1_u8(out, p3); out += out_stride; + vst1_u8(out, p4); out += out_stride; + vst1_u8(out, p5); out += out_stride; + vst1_u8(out, p6); out += out_stride; + vst1_u8(out, p7); + +#undef dct_trn8_8 +#undef dct_trn8_16 +#undef dct_trn8_32 + } + +#undef dct_long_mul +#undef dct_long_mac +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_pass +} + +#endif // STBI_NEON + +#define STBI__MARKER_none 0xff +// if there's a pending marker from the entropy stream, return that +// otherwise, fetch from the stream and get a marker. if there's no +// marker, return 0xff, which is never a valid marker value +static stbi_uc stbi__get_marker(stbi__jpeg *j) +{ + stbi_uc x; + if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; } + x = stbi__get8(j->s); + if (x != 0xff) return STBI__MARKER_none; + while (x == 0xff) + x = stbi__get8(j->s); // consume repeated 0xff fill bytes + return x; +} + +// in each scan, we'll have scan_n components, and the order +// of the components is specified by order[] +#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7) + +// after a restart interval, stbi__jpeg_reset the entropy decoder and +// the dc prediction +static void stbi__jpeg_reset(stbi__jpeg *j) +{ + j->code_bits = 0; + j->code_buffer = 0; + j->nomore = 0; + j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0; + j->marker = STBI__MARKER_none; + j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff; + j->eob_run = 0; + // no more than 1<<31 MCUs if no restart_interal? that's plenty safe, + // since we don't even allow 1<<30 pixels +} + +static int stbi__parse_entropy_coded_data(stbi__jpeg *z) +{ + stbi__jpeg_reset(z); + if (!z->progressive) { + if (z->scan_n == 1) { + int i,j; + STBI_SIMD_ALIGN(short, data[64]); + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + // if it's NOT a restart, then just bail, so we get corrupt data + // rather than no data + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + STBI_SIMD_ALIGN(short, data[64]); + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x)*8; + int y2 = (j*z->img_comp[n].v + y)*8; + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data); + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } else { + if (z->scan_n == 1) { + int i,j; + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + if (z->spec_start == 0) { + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } else { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha])) + return 0; + } + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x); + int y2 = (j*z->img_comp[n].v + y); + short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w); + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } +} + +static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant) +{ + int i; + for (i=0; i < 64; ++i) + data[i] *= dequant[i]; +} + +static void stbi__jpeg_finish(stbi__jpeg *z) +{ + if (z->progressive) { + // dequantize and idct the data + int i,j,n; + for (n=0; n < z->s->img_n; ++n) { + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]); + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + } + } + } + } +} + +static int stbi__process_marker(stbi__jpeg *z, int m) +{ + int L; + switch (m) { + case STBI__MARKER_none: // no marker found + return stbi__err("expected marker","Corrupt JPEG"); + + case 0xDD: // DRI - specify restart interval + if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG"); + z->restart_interval = stbi__get16be(z->s); + return 1; + + case 0xDB: // DQT - define quantization table + L = stbi__get16be(z->s)-2; + while (L > 0) { + int q = stbi__get8(z->s); + int p = q >> 4, sixteen = (p != 0); + int t = q & 15,i; + if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG"); + if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG"); + + for (i=0; i < 64; ++i) + z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s)); + L -= (sixteen ? 129 : 65); + } + return L==0; + + case 0xC4: // DHT - define huffman table + L = stbi__get16be(z->s)-2; + while (L > 0) { + stbi_uc *v; + int sizes[16],i,n=0; + int q = stbi__get8(z->s); + int tc = q >> 4; + int th = q & 15; + if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG"); + for (i=0; i < 16; ++i) { + sizes[i] = stbi__get8(z->s); + n += sizes[i]; + } + if(n > 256) return stbi__err("bad DHT header","Corrupt JPEG"); // Loop over i < n would write past end of values! + L -= 17; + if (tc == 0) { + if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0; + v = z->huff_dc[th].values; + } else { + if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0; + v = z->huff_ac[th].values; + } + for (i=0; i < n; ++i) + v[i] = stbi__get8(z->s); + if (tc != 0) + stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th); + L -= n; + } + return L==0; + } + + // check for comment block or APP blocks + if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) { + L = stbi__get16be(z->s); + if (L < 2) { + if (m == 0xFE) + return stbi__err("bad COM len","Corrupt JPEG"); + else + return stbi__err("bad APP len","Corrupt JPEG"); + } + L -= 2; + + if (m == 0xE0 && L >= 5) { // JFIF APP0 segment + static const unsigned char tag[5] = {'J','F','I','F','\0'}; + int ok = 1; + int i; + for (i=0; i < 5; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 5; + if (ok) + z->jfif = 1; + } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment + static const unsigned char tag[6] = {'A','d','o','b','e','\0'}; + int ok = 1; + int i; + for (i=0; i < 6; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 6; + if (ok) { + stbi__get8(z->s); // version + stbi__get16be(z->s); // flags0 + stbi__get16be(z->s); // flags1 + z->app14_color_transform = stbi__get8(z->s); // color transform + L -= 6; + } + } + + stbi__skip(z->s, L); + return 1; + } + + return stbi__err("unknown marker","Corrupt JPEG"); +} + +// after we see SOS +static int stbi__process_scan_header(stbi__jpeg *z) +{ + int i; + int Ls = stbi__get16be(z->s); + z->scan_n = stbi__get8(z->s); + if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG"); + if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG"); + for (i=0; i < z->scan_n; ++i) { + int id = stbi__get8(z->s), which; + int q = stbi__get8(z->s); + for (which = 0; which < z->s->img_n; ++which) + if (z->img_comp[which].id == id) + break; + if (which == z->s->img_n) return 0; // no match + z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG"); + z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG"); + z->order[i] = which; + } + + { + int aa; + z->spec_start = stbi__get8(z->s); + z->spec_end = stbi__get8(z->s); // should be 63, but might be 0 + aa = stbi__get8(z->s); + z->succ_high = (aa >> 4); + z->succ_low = (aa & 15); + if (z->progressive) { + if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13) + return stbi__err("bad SOS", "Corrupt JPEG"); + } else { + if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG"); + if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG"); + z->spec_end = 63; + } + } + + return 1; +} + +static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why) +{ + int i; + for (i=0; i < ncomp; ++i) { + if (z->img_comp[i].raw_data) { + STBI_FREE(z->img_comp[i].raw_data); + z->img_comp[i].raw_data = NULL; + z->img_comp[i].data = NULL; + } + if (z->img_comp[i].raw_coeff) { + STBI_FREE(z->img_comp[i].raw_coeff); + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].coeff = 0; + } + if (z->img_comp[i].linebuf) { + STBI_FREE(z->img_comp[i].linebuf); + z->img_comp[i].linebuf = NULL; + } + } + return why; +} + +static int stbi__process_frame_header(stbi__jpeg *z, int scan) +{ + stbi__context *s = z->s; + int Lf,p,i,q, h_max=1,v_max=1,c; + Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG + p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline + s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG + s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + c = stbi__get8(s); + if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG"); + s->img_n = c; + for (i=0; i < c; ++i) { + z->img_comp[i].data = NULL; + z->img_comp[i].linebuf = NULL; + } + + if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG"); + + z->rgb = 0; + for (i=0; i < s->img_n; ++i) { + static const unsigned char rgb[3] = { 'R', 'G', 'B' }; + z->img_comp[i].id = stbi__get8(s); + if (s->img_n == 3 && z->img_comp[i].id == rgb[i]) + ++z->rgb; + q = stbi__get8(s); + z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG"); + z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG"); + z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG"); + } + + if (scan != STBI__SCAN_load) return 1; + + if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode"); + + for (i=0; i < s->img_n; ++i) { + if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h; + if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v; + } + + // check that plane subsampling factors are integer ratios; our resamplers can't deal with fractional ratios + // and I've never seen a non-corrupted JPEG file actually use them + for (i=0; i < s->img_n; ++i) { + if (h_max % z->img_comp[i].h != 0) return stbi__err("bad H","Corrupt JPEG"); + if (v_max % z->img_comp[i].v != 0) return stbi__err("bad V","Corrupt JPEG"); + } + + // compute interleaved mcu info + z->img_h_max = h_max; + z->img_v_max = v_max; + z->img_mcu_w = h_max * 8; + z->img_mcu_h = v_max * 8; + // these sizes can't be more than 17 bits + z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w; + z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h; + + for (i=0; i < s->img_n; ++i) { + // number of effective pixels (e.g. for non-interleaved MCU) + z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max; + z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max; + // to simplify generation, we'll allocate enough memory to decode + // the bogus oversized data from using interleaved MCUs and their + // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't + // discard the extra data until colorspace conversion + // + // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier) + // so these muls can't overflow with 32-bit ints (which we require) + z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8; + z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8; + z->img_comp[i].coeff = 0; + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].linebuf = NULL; + z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15); + if (z->img_comp[i].raw_data == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + // align blocks for idct using mmx/sse + z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15); + if (z->progressive) { + // w2, h2 are multiples of 8 (see above) + z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8; + z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8; + z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15); + if (z->img_comp[i].raw_coeff == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15); + } + } + + return 1; +} + +// use comparisons since in some cases we handle more than one case (e.g. SOF) +#define stbi__DNL(x) ((x) == 0xdc) +#define stbi__SOI(x) ((x) == 0xd8) +#define stbi__EOI(x) ((x) == 0xd9) +#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2) +#define stbi__SOS(x) ((x) == 0xda) + +#define stbi__SOF_progressive(x) ((x) == 0xc2) + +static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan) +{ + int m; + z->jfif = 0; + z->app14_color_transform = -1; // valid values are 0,1,2 + z->marker = STBI__MARKER_none; // initialize cached marker to empty + m = stbi__get_marker(z); + if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG"); + if (scan == STBI__SCAN_type) return 1; + m = stbi__get_marker(z); + while (!stbi__SOF(m)) { + if (!stbi__process_marker(z,m)) return 0; + m = stbi__get_marker(z); + while (m == STBI__MARKER_none) { + // some files have extra padding after their blocks, so ok, we'll scan + if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG"); + m = stbi__get_marker(z); + } + } + z->progressive = stbi__SOF_progressive(m); + if (!stbi__process_frame_header(z, scan)) return 0; + return 1; +} + +static stbi_uc stbi__skip_jpeg_junk_at_end(stbi__jpeg *j) +{ + // some JPEGs have junk at end, skip over it but if we find what looks + // like a valid marker, resume there + while (!stbi__at_eof(j->s)) { + stbi_uc x = stbi__get8(j->s); + while (x == 0xff) { // might be a marker + if (stbi__at_eof(j->s)) return STBI__MARKER_none; + x = stbi__get8(j->s); + if (x != 0x00 && x != 0xff) { + // not a stuffed zero or lead-in to another marker, looks + // like an actual marker, return it + return x; + } + // stuffed zero has x=0 now which ends the loop, meaning we go + // back to regular scan loop. + // repeated 0xff keeps trying to read the next byte of the marker. + } + } + return STBI__MARKER_none; +} + +// decode image to YCbCr format +static int stbi__decode_jpeg_image(stbi__jpeg *j) +{ + int m; + for (m = 0; m < 4; m++) { + j->img_comp[m].raw_data = NULL; + j->img_comp[m].raw_coeff = NULL; + } + j->restart_interval = 0; + if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0; + m = stbi__get_marker(j); + while (!stbi__EOI(m)) { + if (stbi__SOS(m)) { + if (!stbi__process_scan_header(j)) return 0; + if (!stbi__parse_entropy_coded_data(j)) return 0; + if (j->marker == STBI__MARKER_none ) { + j->marker = stbi__skip_jpeg_junk_at_end(j); + // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0 + } + m = stbi__get_marker(j); + if (STBI__RESTART(m)) + m = stbi__get_marker(j); + } else if (stbi__DNL(m)) { + int Ld = stbi__get16be(j->s); + stbi__uint32 NL = stbi__get16be(j->s); + if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG"); + if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG"); + m = stbi__get_marker(j); + } else { + if (!stbi__process_marker(j, m)) return 1; + m = stbi__get_marker(j); + } + } + if (j->progressive) + stbi__jpeg_finish(j); + return 1; +} + +// static jfif-centered resampling (across block boundaries) + +typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1, + int w, int hs); + +#define stbi__div4(x) ((stbi_uc) ((x) >> 2)) + +static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + STBI_NOTUSED(out); + STBI_NOTUSED(in_far); + STBI_NOTUSED(w); + STBI_NOTUSED(hs); + return in_near; +} + +static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples vertically for every one in input + int i; + STBI_NOTUSED(hs); + for (i=0; i < w; ++i) + out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2); + return out; +} + +static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples horizontally for every one in input + int i; + stbi_uc *input = in_near; + + if (w == 1) { + // if only one sample, can't do any interpolation + out[0] = out[1] = input[0]; + return out; + } + + out[0] = input[0]; + out[1] = stbi__div4(input[0]*3 + input[1] + 2); + for (i=1; i < w-1; ++i) { + int n = 3*input[i]+2; + out[i*2+0] = stbi__div4(n+input[i-1]); + out[i*2+1] = stbi__div4(n+input[i+1]); + } + out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2); + out[i*2+1] = input[w-1]; + + STBI_NOTUSED(in_far); + STBI_NOTUSED(hs); + + return out; +} + +#define stbi__div16(x) ((stbi_uc) ((x) >> 4)) + +static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i,t0,t1; + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + out[0] = stbi__div4(t1+2); + for (i=1; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i=0,t0,t1; + + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + // process groups of 8 pixels for as long as we can. + // note we can't handle the last pixel in a row in this loop + // because we need to handle the filter boundary conditions. + for (; i < ((w-1) & ~7); i += 8) { +#if defined(STBI_SSE2) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + __m128i zero = _mm_setzero_si128(); + __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i)); + __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i)); + __m128i farw = _mm_unpacklo_epi8(farb, zero); + __m128i nearw = _mm_unpacklo_epi8(nearb, zero); + __m128i diff = _mm_sub_epi16(farw, nearw); + __m128i nears = _mm_slli_epi16(nearw, 2); + __m128i curr = _mm_add_epi16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + __m128i prv0 = _mm_slli_si128(curr, 2); + __m128i nxt0 = _mm_srli_si128(curr, 2); + __m128i prev = _mm_insert_epi16(prv0, t1, 0); + __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + __m128i bias = _mm_set1_epi16(8); + __m128i curs = _mm_slli_epi16(curr, 2); + __m128i prvd = _mm_sub_epi16(prev, curr); + __m128i nxtd = _mm_sub_epi16(next, curr); + __m128i curb = _mm_add_epi16(curs, bias); + __m128i even = _mm_add_epi16(prvd, curb); + __m128i odd = _mm_add_epi16(nxtd, curb); + + // interleave even and odd pixels, then undo scaling. + __m128i int0 = _mm_unpacklo_epi16(even, odd); + __m128i int1 = _mm_unpackhi_epi16(even, odd); + __m128i de0 = _mm_srli_epi16(int0, 4); + __m128i de1 = _mm_srli_epi16(int1, 4); + + // pack and write output + __m128i outv = _mm_packus_epi16(de0, de1); + _mm_storeu_si128((__m128i *) (out + i*2), outv); +#elif defined(STBI_NEON) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + uint8x8_t farb = vld1_u8(in_far + i); + uint8x8_t nearb = vld1_u8(in_near + i); + int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb)); + int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2)); + int16x8_t curr = vaddq_s16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + int16x8_t prv0 = vextq_s16(curr, curr, 7); + int16x8_t nxt0 = vextq_s16(curr, curr, 1); + int16x8_t prev = vsetq_lane_s16(t1, prv0, 0); + int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + int16x8_t curs = vshlq_n_s16(curr, 2); + int16x8_t prvd = vsubq_s16(prev, curr); + int16x8_t nxtd = vsubq_s16(next, curr); + int16x8_t even = vaddq_s16(curs, prvd); + int16x8_t odd = vaddq_s16(curs, nxtd); + + // undo scaling and round, then store with even/odd phases interleaved + uint8x8x2_t o; + o.val[0] = vqrshrun_n_s16(even, 4); + o.val[1] = vqrshrun_n_s16(odd, 4); + vst2_u8(out + i*2, o); +#endif + + // "previous" value for next iter + t1 = 3*in_near[i+7] + in_far[i+7]; + } + + t0 = t1; + t1 = 3*in_near[i] + in_far[i]; + out[i*2] = stbi__div16(3*t1 + t0 + 8); + + for (++i; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} +#endif + +static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // resample with nearest-neighbor + int i,j; + STBI_NOTUSED(in_far); + for (i=0; i < w; ++i) + for (j=0; j < hs; ++j) + out[i*hs+j] = in_near[i]; + return out; +} + +// this is a reduced-precision calculation of YCbCr-to-RGB introduced +// to make sure the code produces the same results in both SIMD and scalar +#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8) +static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step) +{ + int i; + for (i=0; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step) +{ + int i = 0; + +#ifdef STBI_SSE2 + // step == 3 is pretty ugly on the final interleave, and i'm not convinced + // it's useful in practice (you wouldn't use it for textures, for example). + // so just accelerate step == 4 case. + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + __m128i signflip = _mm_set1_epi8(-0x80); + __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f)); + __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f)); + __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f)); + __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f)); + __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128); + __m128i xw = _mm_set1_epi16(255); // alpha channel + + for (; i+7 < count; i += 8) { + // load + __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i)); + __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i)); + __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i)); + __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128 + __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128 + + // unpack to short (and left-shift cr, cb by 8) + __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes); + __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased); + __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased); + + // color transform + __m128i yws = _mm_srli_epi16(yw, 4); + __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw); + __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw); + __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1); + __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1); + __m128i rws = _mm_add_epi16(cr0, yws); + __m128i gwt = _mm_add_epi16(cb0, yws); + __m128i bws = _mm_add_epi16(yws, cb1); + __m128i gws = _mm_add_epi16(gwt, cr1); + + // descale + __m128i rw = _mm_srai_epi16(rws, 4); + __m128i bw = _mm_srai_epi16(bws, 4); + __m128i gw = _mm_srai_epi16(gws, 4); + + // back to byte, set up for transpose + __m128i brb = _mm_packus_epi16(rw, bw); + __m128i gxb = _mm_packus_epi16(gw, xw); + + // transpose to interleave channels + __m128i t0 = _mm_unpacklo_epi8(brb, gxb); + __m128i t1 = _mm_unpackhi_epi8(brb, gxb); + __m128i o0 = _mm_unpacklo_epi16(t0, t1); + __m128i o1 = _mm_unpackhi_epi16(t0, t1); + + // store + _mm_storeu_si128((__m128i *) (out + 0), o0); + _mm_storeu_si128((__m128i *) (out + 16), o1); + out += 32; + } + } +#endif + +#ifdef STBI_NEON + // in this version, step=3 support would be easy to add. but is there demand? + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + uint8x8_t signflip = vdup_n_u8(0x80); + int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f)); + int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f)); + int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f)); + int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f)); + + for (; i+7 < count; i += 8) { + // load + uint8x8_t y_bytes = vld1_u8(y + i); + uint8x8_t cr_bytes = vld1_u8(pcr + i); + uint8x8_t cb_bytes = vld1_u8(pcb + i); + int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip)); + int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip)); + + // expand to s16 + int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4)); + int16x8_t crw = vshll_n_s8(cr_biased, 7); + int16x8_t cbw = vshll_n_s8(cb_biased, 7); + + // color transform + int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0); + int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0); + int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1); + int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1); + int16x8_t rws = vaddq_s16(yws, cr0); + int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1); + int16x8_t bws = vaddq_s16(yws, cb1); + + // undo scaling, round, convert to byte + uint8x8x4_t o; + o.val[0] = vqrshrun_n_s16(rws, 4); + o.val[1] = vqrshrun_n_s16(gws, 4); + o.val[2] = vqrshrun_n_s16(bws, 4); + o.val[3] = vdup_n_u8(255); + + // store, interleaving r/g/b/a + vst4_u8(out, o); + out += 8*4; + } + } +#endif + + for (; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} +#endif + +// set up the kernels +static void stbi__setup_jpeg(stbi__jpeg *j) +{ + j->idct_block_kernel = stbi__idct_block; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2; + +#ifdef STBI_SSE2 + if (stbi__sse2_available()) { + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; + } +#endif + +#ifdef STBI_NEON + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; +#endif +} + +// clean up the temporary component buffers +static void stbi__cleanup_jpeg(stbi__jpeg *j) +{ + stbi__free_jpeg_components(j, j->s->img_n, 0); +} + +typedef struct +{ + resample_row_func resample; + stbi_uc *line0,*line1; + int hs,vs; // expansion factor in each axis + int w_lores; // horizontal pixels pre-expansion + int ystep; // how far through vertical expansion we are + int ypos; // which pre-expansion row we're on +} stbi__resample; + +// fast 0..255 * 0..255 => 0..255 rounded multiplication +static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y) +{ + unsigned int t = x*y + 128; + return (stbi_uc) ((t + (t >>8)) >> 8); +} + +static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp) +{ + int n, decode_n, is_rgb; + z->s->img_n = 0; // make stbi__cleanup_jpeg safe + + // validate req_comp + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + + // load a jpeg image from whichever source, but leave in YCbCr format + if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; } + + // determine actual number of components to generate + n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1; + + is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif)); + + if (z->s->img_n == 3 && n < 3 && !is_rgb) + decode_n = 1; + else + decode_n = z->s->img_n; + + // nothing to do if no components requested; check this now to avoid + // accessing uninitialized coutput[0] later + if (decode_n <= 0) { stbi__cleanup_jpeg(z); return NULL; } + + // resample and color-convert + { + int k; + unsigned int i,j; + stbi_uc *output; + stbi_uc *coutput[4] = { NULL, NULL, NULL, NULL }; + + stbi__resample res_comp[4]; + + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + + // allocate line buffer big enough for upsampling off the edges + // with upsample factor of 4 + z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3); + if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + r->hs = z->img_h_max / z->img_comp[k].h; + r->vs = z->img_v_max / z->img_comp[k].v; + r->ystep = r->vs >> 1; + r->w_lores = (z->s->img_x + r->hs-1) / r->hs; + r->ypos = 0; + r->line0 = r->line1 = z->img_comp[k].data; + + if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1; + else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2; + else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2; + else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel; + else r->resample = stbi__resample_row_generic; + } + + // can't error after this so, this is safe + output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1); + if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + // now go ahead and resample + for (j=0; j < z->s->img_y; ++j) { + stbi_uc *out = output + n * z->s->img_x * j; + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + int y_bot = r->ystep >= (r->vs >> 1); + coutput[k] = r->resample(z->img_comp[k].linebuf, + y_bot ? r->line1 : r->line0, + y_bot ? r->line0 : r->line1, + r->w_lores, r->hs); + if (++r->ystep >= r->vs) { + r->ystep = 0; + r->line0 = r->line1; + if (++r->ypos < z->img_comp[k].y) + r->line1 += z->img_comp[k].w2; + } + } + if (n >= 3) { + stbi_uc *y = coutput[0]; + if (z->s->img_n == 3) { + if (is_rgb) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = y[i]; + out[1] = coutput[1][i]; + out[2] = coutput[2][i]; + out[3] = 255; + out += n; + } + } else { + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else if (z->s->img_n == 4) { + if (z->app14_color_transform == 0) { // CMYK + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(coutput[0][i], m); + out[1] = stbi__blinn_8x8(coutput[1][i], m); + out[2] = stbi__blinn_8x8(coutput[2][i], m); + out[3] = 255; + out += n; + } + } else if (z->app14_color_transform == 2) { // YCCK + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(255 - out[0], m); + out[1] = stbi__blinn_8x8(255 - out[1], m); + out[2] = stbi__blinn_8x8(255 - out[2], m); + out += n; + } + } else { // YCbCr + alpha? Ignore the fourth channel for now + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else + for (i=0; i < z->s->img_x; ++i) { + out[0] = out[1] = out[2] = y[i]; + out[3] = 255; // not used if n==3 + out += n; + } + } else { + if (is_rgb) { + if (n == 1) + for (i=0; i < z->s->img_x; ++i) + *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + else { + for (i=0; i < z->s->img_x; ++i, out += 2) { + out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + out[1] = 255; + } + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 0) { + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + stbi_uc r = stbi__blinn_8x8(coutput[0][i], m); + stbi_uc g = stbi__blinn_8x8(coutput[1][i], m); + stbi_uc b = stbi__blinn_8x8(coutput[2][i], m); + out[0] = stbi__compute_y(r, g, b); + out[1] = 255; + out += n; + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 2) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]); + out[1] = 255; + out += n; + } + } else { + stbi_uc *y = coutput[0]; + if (n == 1) + for (i=0; i < z->s->img_x; ++i) out[i] = y[i]; + else + for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; } + } + } + } + stbi__cleanup_jpeg(z); + *out_x = z->s->img_x; + *out_y = z->s->img_y; + if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output + return output; + } +} + +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + unsigned char* result; + stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__errpuc("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + STBI_NOTUSED(ri); + j->s = s; + stbi__setup_jpeg(j); + result = load_jpeg_image(j, x,y,comp,req_comp); + STBI_FREE(j); + return result; +} + +static int stbi__jpeg_test(stbi__context *s) +{ + int r; + stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__err("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + j->s = s; + stbi__setup_jpeg(j); + r = stbi__decode_jpeg_header(j, STBI__SCAN_type); + stbi__rewind(s); + STBI_FREE(j); + return r; +} + +static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp) +{ + if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) { + stbi__rewind( j->s ); + return 0; + } + if (x) *x = j->s->img_x; + if (y) *y = j->s->img_y; + if (comp) *comp = j->s->img_n >= 3 ? 3 : 1; + return 1; +} + +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp) +{ + int result; + stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg))); + if (!j) return stbi__err("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + j->s = s; + result = stbi__jpeg_info_raw(j, x, y, comp); + STBI_FREE(j); + return result; +} +#endif + +// public domain zlib decode v0.2 Sean Barrett 2006-11-18 +// simple implementation +// - all input must be provided in an upfront buffer +// - all output is written to a single output buffer (can malloc/realloc) +// performance +// - fast huffman + +#ifndef STBI_NO_ZLIB + +// fast-way is faster to check than jpeg huffman, but slow way is slower +#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables +#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1) +#define STBI__ZNSYMS 288 // number of symbols in literal/length alphabet + +// zlib-style huffman encoding +// (jpegs packs from left, zlib from right, so can't share code) +typedef struct +{ + stbi__uint16 fast[1 << STBI__ZFAST_BITS]; + stbi__uint16 firstcode[16]; + int maxcode[17]; + stbi__uint16 firstsymbol[16]; + stbi_uc size[STBI__ZNSYMS]; + stbi__uint16 value[STBI__ZNSYMS]; +} stbi__zhuffman; + +stbi_inline static int stbi__bitreverse16(int n) +{ + n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1); + n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2); + n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4); + n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8); + return n; +} + +stbi_inline static int stbi__bit_reverse(int v, int bits) +{ + STBI_ASSERT(bits <= 16); + // to bit reverse n bits, reverse 16 and shift + // e.g. 11 bits, bit reverse and shift away 5 + return stbi__bitreverse16(v) >> (16-bits); +} + +static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num) +{ + int i,k=0; + int code, next_code[16], sizes[17]; + + // DEFLATE spec for generating codes + memset(sizes, 0, sizeof(sizes)); + memset(z->fast, 0, sizeof(z->fast)); + for (i=0; i < num; ++i) + ++sizes[sizelist[i]]; + sizes[0] = 0; + for (i=1; i < 16; ++i) + if (sizes[i] > (1 << i)) + return stbi__err("bad sizes", "Corrupt PNG"); + code = 0; + for (i=1; i < 16; ++i) { + next_code[i] = code; + z->firstcode[i] = (stbi__uint16) code; + z->firstsymbol[i] = (stbi__uint16) k; + code = (code + sizes[i]); + if (sizes[i]) + if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG"); + z->maxcode[i] = code << (16-i); // preshift for inner loop + code <<= 1; + k += sizes[i]; + } + z->maxcode[16] = 0x10000; // sentinel + for (i=0; i < num; ++i) { + int s = sizelist[i]; + if (s) { + int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s]; + stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i); + z->size [c] = (stbi_uc ) s; + z->value[c] = (stbi__uint16) i; + if (s <= STBI__ZFAST_BITS) { + int j = stbi__bit_reverse(next_code[s],s); + while (j < (1 << STBI__ZFAST_BITS)) { + z->fast[j] = fastv; + j += (1 << s); + } + } + ++next_code[s]; + } + } + return 1; +} + +// zlib-from-memory implementation for PNG reading +// because PNG allows splitting the zlib stream arbitrarily, +// and it's annoying structurally to have PNG call ZLIB call PNG, +// we require PNG read all the IDATs and combine them into a single +// memory buffer + +typedef struct +{ + stbi_uc *zbuffer, *zbuffer_end; + int num_bits; + int hit_zeof_once; + stbi__uint32 code_buffer; + + char *zout; + char *zout_start; + char *zout_end; + int z_expandable; + + stbi__zhuffman z_length, z_distance; +} stbi__zbuf; + +stbi_inline static int stbi__zeof(stbi__zbuf *z) +{ + return (z->zbuffer >= z->zbuffer_end); +} + +stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z) +{ + return stbi__zeof(z) ? 0 : *z->zbuffer++; +} + +static void stbi__fill_bits(stbi__zbuf *z) +{ + do { + if (z->code_buffer >= (1U << z->num_bits)) { + z->zbuffer = z->zbuffer_end; /* treat this as EOF so we fail. */ + return; + } + z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits; + z->num_bits += 8; + } while (z->num_bits <= 24); +} + +stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n) +{ + unsigned int k; + if (z->num_bits < n) stbi__fill_bits(z); + k = z->code_buffer & ((1 << n) - 1); + z->code_buffer >>= n; + z->num_bits -= n; + return k; +} + +static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s,k; + // not resolved by fast table, so compute it the slow way + // use jpeg approach, which requires MSbits at top + k = stbi__bit_reverse(a->code_buffer, 16); + for (s=STBI__ZFAST_BITS+1; ; ++s) + if (k < z->maxcode[s]) + break; + if (s >= 16) return -1; // invalid code! + // code size is s, so: + b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s]; + if (b >= STBI__ZNSYMS) return -1; // some data was corrupt somewhere! + if (z->size[b] != s) return -1; // was originally an assert, but report failure instead. + a->code_buffer >>= s; + a->num_bits -= s; + return z->value[b]; +} + +stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s; + if (a->num_bits < 16) { + if (stbi__zeof(a)) { + if (!a->hit_zeof_once) { + // This is the first time we hit eof, insert 16 extra padding btis + // to allow us to keep going; if we actually consume any of them + // though, that is invalid data. This is caught later. + a->hit_zeof_once = 1; + a->num_bits += 16; // add 16 implicit zero bits + } else { + // We already inserted our extra 16 padding bits and are again + // out, this stream is actually prematurely terminated. + return -1; + } + } else { + stbi__fill_bits(a); + } + } + b = z->fast[a->code_buffer & STBI__ZFAST_MASK]; + if (b) { + s = b >> 9; + a->code_buffer >>= s; + a->num_bits -= s; + return b & 511; + } + return stbi__zhuffman_decode_slowpath(a, z); +} + +static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes +{ + char *q; + unsigned int cur, limit, old_limit; + z->zout = zout; + if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG"); + cur = (unsigned int) (z->zout - z->zout_start); + limit = old_limit = (unsigned) (z->zout_end - z->zout_start); + if (UINT_MAX - cur < (unsigned) n) return stbi__err("outofmem", "Out of memory"); + while (cur + n > limit) { + if(limit > UINT_MAX / 2) return stbi__err("outofmem", "Out of memory"); + limit *= 2; + } + q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit); + STBI_NOTUSED(old_limit); + if (q == NULL) return stbi__err("outofmem", "Out of memory"); + z->zout_start = q; + z->zout = q + cur; + z->zout_end = q + limit; + return 1; +} + +static const int stbi__zlength_base[31] = { + 3,4,5,6,7,8,9,10,11,13, + 15,17,19,23,27,31,35,43,51,59, + 67,83,99,115,131,163,195,227,258,0,0 }; + +static const int stbi__zlength_extra[31]= +{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; + +static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, +257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; + +static const int stbi__zdist_extra[32] = +{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +static int stbi__parse_huffman_block(stbi__zbuf *a) +{ + char *zout = a->zout; + for(;;) { + int z = stbi__zhuffman_decode(a, &a->z_length); + if (z < 256) { + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes + if (zout >= a->zout_end) { + if (!stbi__zexpand(a, zout, 1)) return 0; + zout = a->zout; + } + *zout++ = (char) z; + } else { + stbi_uc *p; + int len,dist; + if (z == 256) { + a->zout = zout; + if (a->hit_zeof_once && a->num_bits < 16) { + // The first time we hit zeof, we inserted 16 extra zero bits into our bit + // buffer so the decoder can just do its speculative decoding. But if we + // actually consumed any of those bits (which is the case when num_bits < 16), + // the stream actually read past the end so it is malformed. + return stbi__err("unexpected end","Corrupt PNG"); + } + return 1; + } + if (z >= 286) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, length codes 286 and 287 must not appear in compressed data + z -= 257; + len = stbi__zlength_base[z]; + if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]); + z = stbi__zhuffman_decode(a, &a->z_distance); + if (z < 0 || z >= 30) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, distance codes 30 and 31 must not appear in compressed data + dist = stbi__zdist_base[z]; + if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]); + if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG"); + if (len > a->zout_end - zout) { + if (!stbi__zexpand(a, zout, len)) return 0; + zout = a->zout; + } + p = (stbi_uc *) (zout - dist); + if (dist == 1) { // run of one byte; common in images. + stbi_uc v = *p; + if (len) { do *zout++ = v; while (--len); } + } else { + if (len) { do *zout++ = *p++; while (--len); } + } + } + } +} + +static int stbi__compute_huffman_codes(stbi__zbuf *a) +{ + static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; + stbi__zhuffman z_codelength; + stbi_uc lencodes[286+32+137];//padding for maximum single op + stbi_uc codelength_sizes[19]; + int i,n; + + int hlit = stbi__zreceive(a,5) + 257; + int hdist = stbi__zreceive(a,5) + 1; + int hclen = stbi__zreceive(a,4) + 4; + int ntot = hlit + hdist; + + memset(codelength_sizes, 0, sizeof(codelength_sizes)); + for (i=0; i < hclen; ++i) { + int s = stbi__zreceive(a,3); + codelength_sizes[length_dezigzag[i]] = (stbi_uc) s; + } + if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0; + + n = 0; + while (n < ntot) { + int c = stbi__zhuffman_decode(a, &z_codelength); + if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG"); + if (c < 16) + lencodes[n++] = (stbi_uc) c; + else { + stbi_uc fill = 0; + if (c == 16) { + c = stbi__zreceive(a,2)+3; + if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG"); + fill = lencodes[n-1]; + } else if (c == 17) { + c = stbi__zreceive(a,3)+3; + } else if (c == 18) { + c = stbi__zreceive(a,7)+11; + } else { + return stbi__err("bad codelengths", "Corrupt PNG"); + } + if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG"); + memset(lencodes+n, fill, c); + n += c; + } + } + if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG"); + if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0; + return 1; +} + +static int stbi__parse_uncompressed_block(stbi__zbuf *a) +{ + stbi_uc header[4]; + int len,nlen,k; + if (a->num_bits & 7) + stbi__zreceive(a, a->num_bits & 7); // discard + // drain the bit-packed data into header + k = 0; + while (a->num_bits > 0) { + header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check + a->code_buffer >>= 8; + a->num_bits -= 8; + } + if (a->num_bits < 0) return stbi__err("zlib corrupt","Corrupt PNG"); + // now fill header the normal way + while (k < 4) + header[k++] = stbi__zget8(a); + len = header[1] * 256 + header[0]; + nlen = header[3] * 256 + header[2]; + if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG"); + if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG"); + if (a->zout + len > a->zout_end) + if (!stbi__zexpand(a, a->zout, len)) return 0; + memcpy(a->zout, a->zbuffer, len); + a->zbuffer += len; + a->zout += len; + return 1; +} + +static int stbi__parse_zlib_header(stbi__zbuf *a) +{ + int cmf = stbi__zget8(a); + int cm = cmf & 15; + /* int cinfo = cmf >> 4; */ + int flg = stbi__zget8(a); + if (stbi__zeof(a)) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png + if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png + // window = 1 << (8 + cinfo)... but who cares, we fully buffer output + return 1; +} + +static const stbi_uc stbi__zdefault_length[STBI__ZNSYMS] = +{ + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8 +}; +static const stbi_uc stbi__zdefault_distance[32] = +{ + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 +}; +/* +Init algorithm: +{ + int i; // use <= to match clearly with spec + for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8; + for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9; + for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7; + for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8; + + for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5; +} +*/ + +static int stbi__parse_zlib(stbi__zbuf *a, int parse_header) +{ + int final, type; + if (parse_header) + if (!stbi__parse_zlib_header(a)) return 0; + a->num_bits = 0; + a->code_buffer = 0; + a->hit_zeof_once = 0; + do { + final = stbi__zreceive(a,1); + type = stbi__zreceive(a,2); + if (type == 0) { + if (!stbi__parse_uncompressed_block(a)) return 0; + } else if (type == 3) { + return 0; + } else { + if (type == 1) { + // use fixed code lengths + if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , STBI__ZNSYMS)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0; + } else { + if (!stbi__compute_huffman_codes(a)) return 0; + } + if (!stbi__parse_huffman_block(a)) return 0; + } + } while (!final); + return 1; +} + +static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header) +{ + a->zout_start = obuf; + a->zout = obuf; + a->zout_end = obuf + olen; + a->z_expandable = exp; + + return stbi__parse_zlib(a, parse_header); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, 1)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen) +{ + return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 1)) + return (int) (a.zout - a.zout_start); + else + return -1; +} + +STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(16384); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer+len; + if (stbi__do_zlib(&a, p, 16384, 1, 0)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 0)) + return (int) (a.zout - a.zout_start); + else + return -1; +} +#endif + +// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18 +// simple implementation +// - only 8-bit samples +// - no CRC checking +// - allocates lots of intermediate memory +// - avoids problem of streaming data between subsystems +// - avoids explicit window management +// performance +// - uses stb_zlib, a PD zlib implementation with fast huffman decoding + +#ifndef STBI_NO_PNG +typedef struct +{ + stbi__uint32 length; + stbi__uint32 type; +} stbi__pngchunk; + +static stbi__pngchunk stbi__get_chunk_header(stbi__context *s) +{ + stbi__pngchunk c; + c.length = stbi__get32be(s); + c.type = stbi__get32be(s); + return c; +} + +static int stbi__check_png_header(stbi__context *s) +{ + static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 }; + int i; + for (i=0; i < 8; ++i) + if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG"); + return 1; +} + +typedef struct +{ + stbi__context *s; + stbi_uc *idata, *expanded, *out; + int depth; +} stbi__png; + + +enum { + STBI__F_none=0, + STBI__F_sub=1, + STBI__F_up=2, + STBI__F_avg=3, + STBI__F_paeth=4, + // synthetic filter used for first scanline to avoid needing a dummy row of 0s + STBI__F_avg_first +}; + +static stbi_uc first_row_filter[5] = +{ + STBI__F_none, + STBI__F_sub, + STBI__F_none, + STBI__F_avg_first, + STBI__F_sub // Paeth with b=c=0 turns out to be equivalent to sub +}; + +static int stbi__paeth(int a, int b, int c) +{ + // This formulation looks very different from the reference in the PNG spec, but is + // actually equivalent and has favorable data dependencies and admits straightforward + // generation of branch-free code, which helps performance significantly. + int thresh = c*3 - (a + b); + int lo = a < b ? a : b; + int hi = a < b ? b : a; + int t0 = (hi <= thresh) ? lo : c; + int t1 = (thresh <= lo) ? hi : t0; + return t1; +} + +static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 }; + +// adds an extra all-255 alpha channel +// dest == src is legal +// img_n must be 1 or 3 +static void stbi__create_png_alpha_expand8(stbi_uc *dest, stbi_uc *src, stbi__uint32 x, int img_n) +{ + int i; + // must process data backwards since we allow dest==src + if (img_n == 1) { + for (i=x-1; i >= 0; --i) { + dest[i*2+1] = 255; + dest[i*2+0] = src[i]; + } + } else { + STBI_ASSERT(img_n == 3); + for (i=x-1; i >= 0; --i) { + dest[i*4+3] = 255; + dest[i*4+2] = src[i*3+2]; + dest[i*4+1] = src[i*3+1]; + dest[i*4+0] = src[i*3+0]; + } + } +} + +// create the png data from post-deflated data +static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color) +{ + int bytes = (depth == 16 ? 2 : 1); + stbi__context *s = a->s; + stbi__uint32 i,j,stride = x*out_n*bytes; + stbi__uint32 img_len, img_width_bytes; + stbi_uc *filter_buf; + int all_ok = 1; + int k; + int img_n = s->img_n; // copy it into a local for later + + int output_bytes = out_n*bytes; + int filter_bytes = img_n*bytes; + int width = x; + + STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1); + a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into + if (!a->out) return stbi__err("outofmem", "Out of memory"); + + // note: error exits here don't need to clean up a->out individually, + // stbi__do_png always does on error. + if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG"); + img_width_bytes = (((img_n * x * depth) + 7) >> 3); + if (!stbi__mad2sizes_valid(img_width_bytes, y, img_width_bytes)) return stbi__err("too large", "Corrupt PNG"); + img_len = (img_width_bytes + 1) * y; + + // we used to check for exact match between raw_len and img_len on non-interlaced PNGs, + // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros), + // so just check for raw_len < img_len always. + if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG"); + + // Allocate two scan lines worth of filter workspace buffer. + filter_buf = (stbi_uc *) stbi__malloc_mad2(img_width_bytes, 2, 0); + if (!filter_buf) return stbi__err("outofmem", "Out of memory"); + + // Filtering for low-bit-depth images + if (depth < 8) { + filter_bytes = 1; + width = img_width_bytes; + } + + for (j=0; j < y; ++j) { + // cur/prior filter buffers alternate + stbi_uc *cur = filter_buf + (j & 1)*img_width_bytes; + stbi_uc *prior = filter_buf + (~j & 1)*img_width_bytes; + stbi_uc *dest = a->out + stride*j; + int nk = width * filter_bytes; + int filter = *raw++; + + // check filter type + if (filter > 4) { + all_ok = stbi__err("invalid filter","Corrupt PNG"); + break; + } + + // if first row, use special filter that doesn't sample previous row + if (j == 0) filter = first_row_filter[filter]; + + // perform actual filtering + switch (filter) { + case STBI__F_none: + memcpy(cur, raw, nk); + break; + case STBI__F_sub: + memcpy(cur, raw, filter_bytes); + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); + break; + case STBI__F_up: + for (k = 0; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + prior[k]); + break; + case STBI__F_avg: + for (k = 0; k < filter_bytes; ++k) + cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); + break; + case STBI__F_paeth: + for (k = 0; k < filter_bytes; ++k) + cur[k] = STBI__BYTECAST(raw[k] + prior[k]); // prior[k] == stbi__paeth(0,prior[k],0) + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes], prior[k], prior[k-filter_bytes])); + break; + case STBI__F_avg_first: + memcpy(cur, raw, filter_bytes); + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); + break; + } + + raw += nk; + + // expand decoded bits in cur to dest, also adding an extra alpha channel if desired + if (depth < 8) { + stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range + stbi_uc *in = cur; + stbi_uc *out = dest; + stbi_uc inb = 0; + stbi__uint32 nsmp = x*img_n; + + // expand bits to bytes first + if (depth == 4) { + for (i=0; i < nsmp; ++i) { + if ((i & 1) == 0) inb = *in++; + *out++ = scale * (inb >> 4); + inb <<= 4; + } + } else if (depth == 2) { + for (i=0; i < nsmp; ++i) { + if ((i & 3) == 0) inb = *in++; + *out++ = scale * (inb >> 6); + inb <<= 2; + } + } else { + STBI_ASSERT(depth == 1); + for (i=0; i < nsmp; ++i) { + if ((i & 7) == 0) inb = *in++; + *out++ = scale * (inb >> 7); + inb <<= 1; + } + } + + // insert alpha=255 values if desired + if (img_n != out_n) + stbi__create_png_alpha_expand8(dest, dest, x, img_n); + } else if (depth == 8) { + if (img_n == out_n) + memcpy(dest, cur, x*img_n); + else + stbi__create_png_alpha_expand8(dest, cur, x, img_n); + } else if (depth == 16) { + // convert the image data from big-endian to platform-native + stbi__uint16 *dest16 = (stbi__uint16*)dest; + stbi__uint32 nsmp = x*img_n; + + if (img_n == out_n) { + for (i = 0; i < nsmp; ++i, ++dest16, cur += 2) + *dest16 = (cur[0] << 8) | cur[1]; + } else { + STBI_ASSERT(img_n+1 == out_n); + if (img_n == 1) { + for (i = 0; i < x; ++i, dest16 += 2, cur += 2) { + dest16[0] = (cur[0] << 8) | cur[1]; + dest16[1] = 0xffff; + } + } else { + STBI_ASSERT(img_n == 3); + for (i = 0; i < x; ++i, dest16 += 4, cur += 6) { + dest16[0] = (cur[0] << 8) | cur[1]; + dest16[1] = (cur[2] << 8) | cur[3]; + dest16[2] = (cur[4] << 8) | cur[5]; + dest16[3] = 0xffff; + } + } + } + } + } + + STBI_FREE(filter_buf); + if (!all_ok) return 0; + + return 1; +} + +static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced) +{ + int bytes = (depth == 16 ? 2 : 1); + int out_bytes = out_n * bytes; + stbi_uc *final; + int p; + if (!interlaced) + return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color); + + // de-interlacing + final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0); + if (!final) return stbi__err("outofmem", "Out of memory"); + for (p=0; p < 7; ++p) { + int xorig[] = { 0,4,0,2,0,1,0 }; + int yorig[] = { 0,0,4,0,2,0,1 }; + int xspc[] = { 8,8,4,4,2,2,1 }; + int yspc[] = { 8,8,8,4,4,2,2 }; + int i,j,x,y; + // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1 + x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p]; + y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p]; + if (x && y) { + stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y; + if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) { + STBI_FREE(final); + return 0; + } + for (j=0; j < y; ++j) { + for (i=0; i < x; ++i) { + int out_y = j*yspc[p]+yorig[p]; + int out_x = i*xspc[p]+xorig[p]; + memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes, + a->out + (j*x+i)*out_bytes, out_bytes); + } + } + STBI_FREE(a->out); + image_data += img_len; + image_data_len -= img_len; + } + } + a->out = final; + + return 1; +} + +static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + // compute color-based transparency, assuming we've + // already got 255 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i=0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 255); + p += 2; + } + } else { + for (i=0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi__uint16 *p = (stbi__uint16*) z->out; + + // compute color-based transparency, assuming we've + // already got 65535 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i = 0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 65535); + p += 2; + } + } else { + for (i = 0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n) +{ + stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y; + stbi_uc *p, *temp_out, *orig = a->out; + + p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0); + if (p == NULL) return stbi__err("outofmem", "Out of memory"); + + // between here and free(out) below, exitting would leak + temp_out = p; + + if (pal_img_n == 3) { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p += 3; + } + } else { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p[3] = palette[n+3]; + p += 4; + } + } + STBI_FREE(a->out); + a->out = temp_out; + + STBI_NOTUSED(len); + + return 1; +} + +static int stbi__unpremultiply_on_load_global = 0; +static int stbi__de_iphone_flag_global = 0; + +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_global = flag_true_if_should_unpremultiply; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_global = flag_true_if_should_convert; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__unpremultiply_on_load stbi__unpremultiply_on_load_global +#define stbi__de_iphone_flag stbi__de_iphone_flag_global +#else +static STBI_THREAD_LOCAL int stbi__unpremultiply_on_load_local, stbi__unpremultiply_on_load_set; +static STBI_THREAD_LOCAL int stbi__de_iphone_flag_local, stbi__de_iphone_flag_set; + +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_local = flag_true_if_should_unpremultiply; + stbi__unpremultiply_on_load_set = 1; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_local = flag_true_if_should_convert; + stbi__de_iphone_flag_set = 1; +} + +#define stbi__unpremultiply_on_load (stbi__unpremultiply_on_load_set \ + ? stbi__unpremultiply_on_load_local \ + : stbi__unpremultiply_on_load_global) +#define stbi__de_iphone_flag (stbi__de_iphone_flag_set \ + ? stbi__de_iphone_flag_local \ + : stbi__de_iphone_flag_global) +#endif // STBI_THREAD_LOCAL + +static void stbi__de_iphone(stbi__png *z) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + if (s->img_out_n == 3) { // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 3; + } + } else { + STBI_ASSERT(s->img_out_n == 4); + if (stbi__unpremultiply_on_load) { + // convert bgr to rgb and unpremultiply + for (i=0; i < pixel_count; ++i) { + stbi_uc a = p[3]; + stbi_uc t = p[0]; + if (a) { + stbi_uc half = a / 2; + p[0] = (p[2] * 255 + half) / a; + p[1] = (p[1] * 255 + half) / a; + p[2] = ( t * 255 + half) / a; + } else { + p[0] = p[2]; + p[2] = t; + } + p += 4; + } + } else { + // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 4; + } + } + } +} + +#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d)) + +static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp) +{ + stbi_uc palette[1024], pal_img_n=0; + stbi_uc has_trans=0, tc[3]={0}; + stbi__uint16 tc16[3]; + stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0; + int first=1,k,interlace=0, color=0, is_iphone=0; + stbi__context *s = z->s; + + z->expanded = NULL; + z->idata = NULL; + z->out = NULL; + + if (!stbi__check_png_header(s)) return 0; + + if (scan == STBI__SCAN_type) return 1; + + for (;;) { + stbi__pngchunk c = stbi__get_chunk_header(s); + switch (c.type) { + case STBI__PNG_TYPE('C','g','B','I'): + is_iphone = 1; + stbi__skip(s, c.length); + break; + case STBI__PNG_TYPE('I','H','D','R'): { + int comp,filter; + if (!first) return stbi__err("multiple IHDR","Corrupt PNG"); + first = 0; + if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG"); + s->img_x = stbi__get32be(s); + s->img_y = stbi__get32be(s); + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only"); + color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG"); + comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG"); + filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG"); + interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG"); + if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG"); + if (!pal_img_n) { + s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0); + if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode"); + } else { + // if paletted, then pal_n is our final components, and + // img_n is # components to decompress/filter. + s->img_n = 1; + if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG"); + } + // even with SCAN_header, have to scan to see if we have a tRNS + break; + } + + case STBI__PNG_TYPE('P','L','T','E'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG"); + pal_len = c.length / 3; + if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG"); + for (i=0; i < pal_len; ++i) { + palette[i*4+0] = stbi__get8(s); + palette[i*4+1] = stbi__get8(s); + palette[i*4+2] = stbi__get8(s); + palette[i*4+3] = 255; + } + break; + } + + case STBI__PNG_TYPE('t','R','N','S'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG"); + if (pal_img_n) { + if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; } + if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG"); + if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG"); + pal_img_n = 4; + for (i=0; i < c.length; ++i) + palette[i*4+3] = stbi__get8(s); + } else { + if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG"); + if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG"); + has_trans = 1; + // non-paletted with tRNS = constant alpha. if header-scanning, we can stop now. + if (scan == STBI__SCAN_header) { ++s->img_n; return 1; } + if (z->depth == 16) { + for (k = 0; k < s->img_n && k < 3; ++k) // extra loop test to suppress false GCC warning + tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is + } else { + for (k = 0; k < s->img_n && k < 3; ++k) + tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger + } + } + break; + } + + case STBI__PNG_TYPE('I','D','A','T'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG"); + if (scan == STBI__SCAN_header) { + // header scan definitely stops at first IDAT + if (pal_img_n) + s->img_n = pal_img_n; + return 1; + } + if (c.length > (1u << 30)) return stbi__err("IDAT size limit", "IDAT section larger than 2^30 bytes"); + if ((int)(ioff + c.length) < (int)ioff) return 0; + if (ioff + c.length > idata_limit) { + stbi__uint32 idata_limit_old = idata_limit; + stbi_uc *p; + if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096; + while (ioff + c.length > idata_limit) + idata_limit *= 2; + STBI_NOTUSED(idata_limit_old); + p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory"); + z->idata = p; + } + if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG"); + ioff += c.length; + break; + } + + case STBI__PNG_TYPE('I','E','N','D'): { + stbi__uint32 raw_len, bpl; + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (scan != STBI__SCAN_load) return 1; + if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG"); + // initial guess for decoded data size to avoid unnecessary reallocs + bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component + raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */; + z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone); + if (z->expanded == NULL) return 0; // zlib should set error + STBI_FREE(z->idata); z->idata = NULL; + if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans) + s->img_out_n = s->img_n+1; + else + s->img_out_n = s->img_n; + if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0; + if (has_trans) { + if (z->depth == 16) { + if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0; + } else { + if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0; + } + } + if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2) + stbi__de_iphone(z); + if (pal_img_n) { + // pal_img_n == 3 or 4 + s->img_n = pal_img_n; // record the actual colors we had + s->img_out_n = pal_img_n; + if (req_comp >= 3) s->img_out_n = req_comp; + if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n)) + return 0; + } else if (has_trans) { + // non-paletted image with tRNS -> source image has (constant) alpha + ++s->img_n; + } + STBI_FREE(z->expanded); z->expanded = NULL; + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + return 1; + } + + default: + // if critical, fail + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if ((c.type & (1 << 29)) == 0) { + #ifndef STBI_NO_FAILURE_STRINGS + // not threadsafe + static char invalid_chunk[] = "XXXX PNG chunk not known"; + invalid_chunk[0] = STBI__BYTECAST(c.type >> 24); + invalid_chunk[1] = STBI__BYTECAST(c.type >> 16); + invalid_chunk[2] = STBI__BYTECAST(c.type >> 8); + invalid_chunk[3] = STBI__BYTECAST(c.type >> 0); + #endif + return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type"); + } + stbi__skip(s, c.length); + break; + } + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + } +} + +static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri) +{ + void *result=NULL; + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) { + if (p->depth <= 8) + ri->bits_per_channel = 8; + else if (p->depth == 16) + ri->bits_per_channel = 16; + else + return stbi__errpuc("bad bits_per_channel", "PNG not supported: unsupported color depth"); + result = p->out; + p->out = NULL; + if (req_comp && req_comp != p->s->img_out_n) { + if (ri->bits_per_channel == 8) + result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + else + result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + p->s->img_out_n = req_comp; + if (result == NULL) return result; + } + *x = p->s->img_x; + *y = p->s->img_y; + if (n) *n = p->s->img_n; + } + STBI_FREE(p->out); p->out = NULL; + STBI_FREE(p->expanded); p->expanded = NULL; + STBI_FREE(p->idata); p->idata = NULL; + + return result; +} + +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi__png p; + p.s = s; + return stbi__do_png(&p, x,y,comp,req_comp, ri); +} + +static int stbi__png_test(stbi__context *s) +{ + int r; + r = stbi__check_png_header(s); + stbi__rewind(s); + return r; +} + +static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp) +{ + if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) { + stbi__rewind( p->s ); + return 0; + } + if (x) *x = p->s->img_x; + if (y) *y = p->s->img_y; + if (comp) *comp = p->s->img_n; + return 1; +} + +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__png p; + p.s = s; + return stbi__png_info_raw(&p, x, y, comp); +} + +static int stbi__png_is16(stbi__context *s) +{ + stbi__png p; + p.s = s; + if (!stbi__png_info_raw(&p, NULL, NULL, NULL)) + return 0; + if (p.depth != 16) { + stbi__rewind(p.s); + return 0; + } + return 1; +} +#endif + +// Microsoft/Windows BMP image + +#ifndef STBI_NO_BMP +static int stbi__bmp_test_raw(stbi__context *s) +{ + int r; + int sz; + if (stbi__get8(s) != 'B') return 0; + if (stbi__get8(s) != 'M') return 0; + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + stbi__get32le(s); // discard data offset + sz = stbi__get32le(s); + r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124); + return r; +} + +static int stbi__bmp_test(stbi__context *s) +{ + int r = stbi__bmp_test_raw(s); + stbi__rewind(s); + return r; +} + + +// returns 0..31 for the highest set bit +static int stbi__high_bit(unsigned int z) +{ + int n=0; + if (z == 0) return -1; + if (z >= 0x10000) { n += 16; z >>= 16; } + if (z >= 0x00100) { n += 8; z >>= 8; } + if (z >= 0x00010) { n += 4; z >>= 4; } + if (z >= 0x00004) { n += 2; z >>= 2; } + if (z >= 0x00002) { n += 1;/* >>= 1;*/ } + return n; +} + +static int stbi__bitcount(unsigned int a) +{ + a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2 + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4 + a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits + a = (a + (a >> 8)); // max 16 per 8 bits + a = (a + (a >> 16)); // max 32 per 8 bits + return a & 0xff; +} + +// extract an arbitrarily-aligned N-bit value (N=bits) +// from v, and then make it 8-bits long and fractionally +// extend it to full full range. +static int stbi__shiftsigned(unsigned int v, int shift, int bits) +{ + static unsigned int mul_table[9] = { + 0, + 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/, + 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/, + }; + static unsigned int shift_table[9] = { + 0, 0,0,1,0,2,4,6,0, + }; + if (shift < 0) + v <<= -shift; + else + v >>= shift; + STBI_ASSERT(v < 256); + v >>= (8-bits); + STBI_ASSERT(bits >= 0 && bits <= 8); + return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits]; +} + +typedef struct +{ + int bpp, offset, hsz; + unsigned int mr,mg,mb,ma, all_a; + int extra_read; +} stbi__bmp_data; + +static int stbi__bmp_set_mask_defaults(stbi__bmp_data *info, int compress) +{ + // BI_BITFIELDS specifies masks explicitly, don't override + if (compress == 3) + return 1; + + if (compress == 0) { + if (info->bpp == 16) { + info->mr = 31u << 10; + info->mg = 31u << 5; + info->mb = 31u << 0; + } else if (info->bpp == 32) { + info->mr = 0xffu << 16; + info->mg = 0xffu << 8; + info->mb = 0xffu << 0; + info->ma = 0xffu << 24; + info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0 + } else { + // otherwise, use defaults, which is all-0 + info->mr = info->mg = info->mb = info->ma = 0; + } + return 1; + } + return 0; // error +} + +static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info) +{ + int hsz; + if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP"); + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + info->offset = stbi__get32le(s); + info->hsz = hsz = stbi__get32le(s); + info->mr = info->mg = info->mb = info->ma = 0; + info->extra_read = 14; + + if (info->offset < 0) return stbi__errpuc("bad BMP", "bad BMP"); + + if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown"); + if (hsz == 12) { + s->img_x = stbi__get16le(s); + s->img_y = stbi__get16le(s); + } else { + s->img_x = stbi__get32le(s); + s->img_y = stbi__get32le(s); + } + if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP"); + info->bpp = stbi__get16le(s); + if (hsz != 12) { + int compress = stbi__get32le(s); + if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE"); + if (compress >= 4) return stbi__errpuc("BMP JPEG/PNG", "BMP type not supported: unsupported compression"); // this includes PNG/JPEG modes + if (compress == 3 && info->bpp != 16 && info->bpp != 32) return stbi__errpuc("bad BMP", "bad BMP"); // bitfields requires 16 or 32 bits/pixel + stbi__get32le(s); // discard sizeof + stbi__get32le(s); // discard hres + stbi__get32le(s); // discard vres + stbi__get32le(s); // discard colorsused + stbi__get32le(s); // discard max important + if (hsz == 40 || hsz == 56) { + if (hsz == 56) { + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + } + if (info->bpp == 16 || info->bpp == 32) { + if (compress == 0) { + stbi__bmp_set_mask_defaults(info, compress); + } else if (compress == 3) { + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->extra_read += 12; + // not documented, but generated by photoshop and handled by mspaint + if (info->mr == info->mg && info->mg == info->mb) { + // ?!?!? + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else { + // V4/V5 header + int i; + if (hsz != 108 && hsz != 124) + return stbi__errpuc("bad BMP", "bad BMP"); + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->ma = stbi__get32le(s); + if (compress != 3) // override mr/mg/mb unless in BI_BITFIELDS mode, as per docs + stbi__bmp_set_mask_defaults(info, compress); + stbi__get32le(s); // discard color space + for (i=0; i < 12; ++i) + stbi__get32le(s); // discard color space parameters + if (hsz == 124) { + stbi__get32le(s); // discard rendering intent + stbi__get32le(s); // discard offset of profile data + stbi__get32le(s); // discard size of profile data + stbi__get32le(s); // discard reserved + } + } + } + return (void *) 1; +} + + +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + unsigned int mr=0,mg=0,mb=0,ma=0, all_a; + stbi_uc pal[256][4]; + int psize=0,i,j,width; + int flip_vertically, pad, target; + stbi__bmp_data info; + STBI_NOTUSED(ri); + + info.all_a = 255; + if (stbi__bmp_parse_header(s, &info) == NULL) + return NULL; // error code already set + + flip_vertically = ((int) s->img_y) > 0; + s->img_y = abs((int) s->img_y); + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + mr = info.mr; + mg = info.mg; + mb = info.mb; + ma = info.ma; + all_a = info.all_a; + + if (info.hsz == 12) { + if (info.bpp < 24) + psize = (info.offset - info.extra_read - 24) / 3; + } else { + if (info.bpp < 16) + psize = (info.offset - info.extra_read - info.hsz) >> 2; + } + if (psize == 0) { + // accept some number of extra bytes after the header, but if the offset points either to before + // the header ends or implies a large amount of extra data, reject the file as malformed + int bytes_read_so_far = s->callback_already_read + (int)(s->img_buffer - s->img_buffer_original); + int header_limit = 1024; // max we actually read is below 256 bytes currently. + int extra_data_limit = 256*4; // what ordinarily goes here is a palette; 256 entries*4 bytes is its max size. + if (bytes_read_so_far <= 0 || bytes_read_so_far > header_limit) { + return stbi__errpuc("bad header", "Corrupt BMP"); + } + // we established that bytes_read_so_far is positive and sensible. + // the first half of this test rejects offsets that are either too small positives, or + // negative, and guarantees that info.offset >= bytes_read_so_far > 0. this in turn + // ensures the number computed in the second half of the test can't overflow. + if (info.offset < bytes_read_so_far || info.offset - bytes_read_so_far > extra_data_limit) { + return stbi__errpuc("bad offset", "Corrupt BMP"); + } else { + stbi__skip(s, info.offset - bytes_read_so_far); + } + } + + if (info.bpp == 24 && ma == 0xff000000) + s->img_n = 3; + else + s->img_n = ma ? 4 : 3; + if (req_comp && req_comp >= 3) // we can directly decode 3 or 4 + target = req_comp; + else + target = s->img_n; // if they want monochrome, we'll post-convert + + // sanity-check size + if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0)) + return stbi__errpuc("too large", "Corrupt BMP"); + + out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (info.bpp < 16) { + int z=0; + if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); } + for (i=0; i < psize; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + if (info.hsz != 12) stbi__get8(s); + pal[i][3] = 255; + } + stbi__skip(s, info.offset - info.extra_read - info.hsz - psize * (info.hsz == 12 ? 3 : 4)); + if (info.bpp == 1) width = (s->img_x + 7) >> 3; + else if (info.bpp == 4) width = (s->img_x + 1) >> 1; + else if (info.bpp == 8) width = s->img_x; + else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); } + pad = (-width)&3; + if (info.bpp == 1) { + for (j=0; j < (int) s->img_y; ++j) { + int bit_offset = 7, v = stbi__get8(s); + for (i=0; i < (int) s->img_x; ++i) { + int color = (v>>bit_offset)&0x1; + out[z++] = pal[color][0]; + out[z++] = pal[color][1]; + out[z++] = pal[color][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + if((--bit_offset) < 0) { + bit_offset = 7; + v = stbi__get8(s); + } + } + stbi__skip(s, pad); + } + } else { + for (j=0; j < (int) s->img_y; ++j) { + for (i=0; i < (int) s->img_x; i += 2) { + int v=stbi__get8(s),v2=0; + if (info.bpp == 4) { + v2 = v & 15; + v >>= 4; + } + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + v = (info.bpp == 8) ? stbi__get8(s) : v2; + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + } + stbi__skip(s, pad); + } + } + } else { + int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0; + int z = 0; + int easy=0; + stbi__skip(s, info.offset - info.extra_read - info.hsz); + if (info.bpp == 24) width = 3 * s->img_x; + else if (info.bpp == 16) width = 2*s->img_x; + else /* bpp = 32 and pad = 0 */ width=0; + pad = (-width) & 3; + if (info.bpp == 24) { + easy = 1; + } else if (info.bpp == 32) { + if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000) + easy = 2; + } + if (!easy) { + if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + // right shift amt to put high bit in position #7 + rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr); + gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg); + bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb); + ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma); + if (rcount > 8 || gcount > 8 || bcount > 8 || acount > 8) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + } + for (j=0; j < (int) s->img_y; ++j) { + if (easy) { + for (i=0; i < (int) s->img_x; ++i) { + unsigned char a; + out[z+2] = stbi__get8(s); + out[z+1] = stbi__get8(s); + out[z+0] = stbi__get8(s); + z += 3; + a = (easy == 2 ? stbi__get8(s) : 255); + all_a |= a; + if (target == 4) out[z++] = a; + } + } else { + int bpp = info.bpp; + for (i=0; i < (int) s->img_x; ++i) { + stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s)); + unsigned int a; + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount)); + a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255); + all_a |= a; + if (target == 4) out[z++] = STBI__BYTECAST(a); + } + } + stbi__skip(s, pad); + } + } + + // if alpha channel is all 0s, replace with all 255s + if (target == 4 && all_a == 0) + for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4) + out[i] = 255; + + if (flip_vertically) { + stbi_uc t; + for (j=0; j < (int) s->img_y>>1; ++j) { + stbi_uc *p1 = out + j *s->img_x*target; + stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target; + for (i=0; i < (int) s->img_x*target; ++i) { + t = p1[i]; p1[i] = p2[i]; p2[i] = t; + } + } + } + + if (req_comp && req_comp != target) { + out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + return out; +} +#endif + +// Targa Truevision - TGA +// by Jonathan Dummer +#ifndef STBI_NO_TGA +// returns STBI_rgb or whatever, 0 on error +static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16) +{ + // only RGB or RGBA (incl. 16bit) or grey allowed + if (is_rgb16) *is_rgb16 = 0; + switch(bits_per_pixel) { + case 8: return STBI_grey; + case 16: if(is_grey) return STBI_grey_alpha; + // fallthrough + case 15: if(is_rgb16) *is_rgb16 = 1; + return STBI_rgb; + case 24: // fallthrough + case 32: return bits_per_pixel/8; + default: return 0; + } +} + +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp) +{ + int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp; + int sz, tga_colormap_type; + stbi__get8(s); // discard Offset + tga_colormap_type = stbi__get8(s); // colormap type + if( tga_colormap_type > 1 ) { + stbi__rewind(s); + return 0; // only RGB or indexed allowed + } + tga_image_type = stbi__get8(s); // image type + if ( tga_colormap_type == 1 ) { // colormapped (paletted) image + if (tga_image_type != 1 && tga_image_type != 9) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip image x and y origin + tga_colormap_bpp = sz; + } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE + if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) { + stbi__rewind(s); + return 0; // only RGB or grey allowed, +/- RLE + } + stbi__skip(s,9); // skip colormap specification and image x/y origin + tga_colormap_bpp = 0; + } + tga_w = stbi__get16le(s); + if( tga_w < 1 ) { + stbi__rewind(s); + return 0; // test width + } + tga_h = stbi__get16le(s); + if( tga_h < 1 ) { + stbi__rewind(s); + return 0; // test height + } + tga_bits_per_pixel = stbi__get8(s); // bits per pixel + stbi__get8(s); // ignore alpha bits + if (tga_colormap_bpp != 0) { + if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) { + // when using a colormap, tga_bits_per_pixel is the size of the indexes + // I don't think anything but 8 or 16bit indexes makes sense + stbi__rewind(s); + return 0; + } + tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL); + } else { + tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL); + } + if(!tga_comp) { + stbi__rewind(s); + return 0; + } + if (x) *x = tga_w; + if (y) *y = tga_h; + if (comp) *comp = tga_comp; + return 1; // seems to have passed everything +} + +static int stbi__tga_test(stbi__context *s) +{ + int res = 0; + int sz, tga_color_type; + stbi__get8(s); // discard Offset + tga_color_type = stbi__get8(s); // color type + if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed + sz = stbi__get8(s); // image type + if ( tga_color_type == 1 ) { // colormapped (paletted) image + if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9 + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + stbi__skip(s,4); // skip image x and y origin + } else { // "normal" image w/o colormap + if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE + stbi__skip(s,9); // skip colormap specification and image x/y origin + } + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height + sz = stbi__get8(s); // bits per pixel + if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + + res = 1; // if we got this far, everything's good and we can return 1 instead of 0 + +errorEnd: + stbi__rewind(s); + return res; +} + +// read 16bit value and convert to 24bit RGB +static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out) +{ + stbi__uint16 px = (stbi__uint16)stbi__get16le(s); + stbi__uint16 fiveBitMask = 31; + // we have 3 channels with 5bits each + int r = (px >> 10) & fiveBitMask; + int g = (px >> 5) & fiveBitMask; + int b = px & fiveBitMask; + // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later + out[0] = (stbi_uc)((r * 255)/31); + out[1] = (stbi_uc)((g * 255)/31); + out[2] = (stbi_uc)((b * 255)/31); + + // some people claim that the most significant bit might be used for alpha + // (possibly if an alpha-bit is set in the "image descriptor byte") + // but that only made 16bit test images completely translucent.. + // so let's treat all 15 and 16bit TGAs as RGB with no alpha. +} + +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + // read in the TGA header stuff + int tga_offset = stbi__get8(s); + int tga_indexed = stbi__get8(s); + int tga_image_type = stbi__get8(s); + int tga_is_RLE = 0; + int tga_palette_start = stbi__get16le(s); + int tga_palette_len = stbi__get16le(s); + int tga_palette_bits = stbi__get8(s); + int tga_x_origin = stbi__get16le(s); + int tga_y_origin = stbi__get16le(s); + int tga_width = stbi__get16le(s); + int tga_height = stbi__get16le(s); + int tga_bits_per_pixel = stbi__get8(s); + int tga_comp, tga_rgb16=0; + int tga_inverted = stbi__get8(s); + // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?) + // image data + unsigned char *tga_data; + unsigned char *tga_palette = NULL; + int i, j; + unsigned char raw_data[4] = {0}; + int RLE_count = 0; + int RLE_repeating = 0; + int read_next_pixel = 1; + STBI_NOTUSED(ri); + STBI_NOTUSED(tga_x_origin); // @TODO + STBI_NOTUSED(tga_y_origin); // @TODO + + if (tga_height > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (tga_width > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // do a tiny bit of precessing + if ( tga_image_type >= 8 ) + { + tga_image_type -= 8; + tga_is_RLE = 1; + } + tga_inverted = 1 - ((tga_inverted >> 5) & 1); + + // If I'm paletted, then I'll use the number of bits from the palette + if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16); + else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16); + + if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency + return stbi__errpuc("bad format", "Can't find out TGA pixelformat"); + + // tga info + *x = tga_width; + *y = tga_height; + if (comp) *comp = tga_comp; + + if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0)) + return stbi__errpuc("too large", "Corrupt TGA"); + + tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0); + if (!tga_data) return stbi__errpuc("outofmem", "Out of memory"); + + // skip to the data's starting position (offset usually = 0) + stbi__skip(s, tga_offset ); + + if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) { + for (i=0; i < tga_height; ++i) { + int row = tga_inverted ? tga_height -i - 1 : i; + stbi_uc *tga_row = tga_data + row*tga_width*tga_comp; + stbi__getn(s, tga_row, tga_width * tga_comp); + } + } else { + // do I need to load a palette? + if ( tga_indexed) + { + if (tga_palette_len == 0) { /* you have to have at least one entry! */ + STBI_FREE(tga_data); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + + // any data to skip? (offset usually = 0) + stbi__skip(s, tga_palette_start ); + // load the palette + tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0); + if (!tga_palette) { + STBI_FREE(tga_data); + return stbi__errpuc("outofmem", "Out of memory"); + } + if (tga_rgb16) { + stbi_uc *pal_entry = tga_palette; + STBI_ASSERT(tga_comp == STBI_rgb); + for (i=0; i < tga_palette_len; ++i) { + stbi__tga_read_rgb16(s, pal_entry); + pal_entry += tga_comp; + } + } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) { + STBI_FREE(tga_data); + STBI_FREE(tga_palette); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + } + // load the data + for (i=0; i < tga_width * tga_height; ++i) + { + // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk? + if ( tga_is_RLE ) + { + if ( RLE_count == 0 ) + { + // yep, get the next byte as a RLE command + int RLE_cmd = stbi__get8(s); + RLE_count = 1 + (RLE_cmd & 127); + RLE_repeating = RLE_cmd >> 7; + read_next_pixel = 1; + } else if ( !RLE_repeating ) + { + read_next_pixel = 1; + } + } else + { + read_next_pixel = 1; + } + // OK, if I need to read a pixel, do it now + if ( read_next_pixel ) + { + // load however much data we did have + if ( tga_indexed ) + { + // read in index, then perform the lookup + int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s); + if ( pal_idx >= tga_palette_len ) { + // invalid index + pal_idx = 0; + } + pal_idx *= tga_comp; + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = tga_palette[pal_idx+j]; + } + } else if(tga_rgb16) { + STBI_ASSERT(tga_comp == STBI_rgb); + stbi__tga_read_rgb16(s, raw_data); + } else { + // read in the data raw + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = stbi__get8(s); + } + } + // clear the reading flag for the next pixel + read_next_pixel = 0; + } // end of reading a pixel + + // copy data + for (j = 0; j < tga_comp; ++j) + tga_data[i*tga_comp+j] = raw_data[j]; + + // in case we're in RLE mode, keep counting down + --RLE_count; + } + // do I need to invert the image? + if ( tga_inverted ) + { + for (j = 0; j*2 < tga_height; ++j) + { + int index1 = j * tga_width * tga_comp; + int index2 = (tga_height - 1 - j) * tga_width * tga_comp; + for (i = tga_width * tga_comp; i > 0; --i) + { + unsigned char temp = tga_data[index1]; + tga_data[index1] = tga_data[index2]; + tga_data[index2] = temp; + ++index1; + ++index2; + } + } + } + // clear my palette, if I had one + if ( tga_palette != NULL ) + { + STBI_FREE( tga_palette ); + } + } + + // swap RGB - if the source data was RGB16, it already is in the right order + if (tga_comp >= 3 && !tga_rgb16) + { + unsigned char* tga_pixel = tga_data; + for (i=0; i < tga_width * tga_height; ++i) + { + unsigned char temp = tga_pixel[0]; + tga_pixel[0] = tga_pixel[2]; + tga_pixel[2] = temp; + tga_pixel += tga_comp; + } + } + + // convert to target component count + if (req_comp && req_comp != tga_comp) + tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height); + + // the things I do to get rid of an error message, and yet keep + // Microsoft's C compilers happy... [8^( + tga_palette_start = tga_palette_len = tga_palette_bits = + tga_x_origin = tga_y_origin = 0; + STBI_NOTUSED(tga_palette_start); + // OK, done + return tga_data; +} +#endif + +// ************************************************************************************************* +// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s) +{ + int r = (stbi__get32be(s) == 0x38425053); + stbi__rewind(s); + return r; +} + +static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount) +{ + int count, nleft, len; + + count = 0; + while ((nleft = pixelCount - count) > 0) { + len = stbi__get8(s); + if (len == 128) { + // No-op. + } else if (len < 128) { + // Copy next len+1 bytes literally. + len++; + if (len > nleft) return 0; // corrupt data + count += len; + while (len) { + *p = stbi__get8(s); + p += 4; + len--; + } + } else if (len > 128) { + stbi_uc val; + // Next -len+1 bytes in the dest are replicated from next source byte. + // (Interpret len as a negative 8-bit int.) + len = 257 - len; + if (len > nleft) return 0; // corrupt data + val = stbi__get8(s); + count += len; + while (len) { + *p = val; + p += 4; + len--; + } + } + } + + return 1; +} + +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + int pixelCount; + int channelCount, compression; + int channel, i; + int bitdepth; + int w,h; + stbi_uc *out; + STBI_NOTUSED(ri); + + // Check identifier + if (stbi__get32be(s) != 0x38425053) // "8BPS" + return stbi__errpuc("not PSD", "Corrupt PSD image"); + + // Check file type version. + if (stbi__get16be(s) != 1) + return stbi__errpuc("wrong version", "Unsupported version of PSD image"); + + // Skip 6 reserved bytes. + stbi__skip(s, 6 ); + + // Read the number of channels (R, G, B, A, etc). + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) + return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image"); + + // Read the rows and columns of the image. + h = stbi__get32be(s); + w = stbi__get32be(s); + + if (h > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (w > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // Make sure the depth is 8 bits. + bitdepth = stbi__get16be(s); + if (bitdepth != 8 && bitdepth != 16) + return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit"); + + // Make sure the color mode is RGB. + // Valid options are: + // 0: Bitmap + // 1: Grayscale + // 2: Indexed color + // 3: RGB color + // 4: CMYK color + // 7: Multichannel + // 8: Duotone + // 9: Lab color + if (stbi__get16be(s) != 3) + return stbi__errpuc("wrong color format", "PSD is not in RGB color format"); + + // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.) + stbi__skip(s,stbi__get32be(s) ); + + // Skip the image resources. (resolution, pen tool paths, etc) + stbi__skip(s, stbi__get32be(s) ); + + // Skip the reserved data. + stbi__skip(s, stbi__get32be(s) ); + + // Find out if the data is compressed. + // Known values: + // 0: no compression + // 1: RLE compressed + compression = stbi__get16be(s); + if (compression > 1) + return stbi__errpuc("bad compression", "PSD has an unknown compression format"); + + // Check size + if (!stbi__mad3sizes_valid(4, w, h, 0)) + return stbi__errpuc("too large", "Corrupt PSD"); + + // Create the destination image. + + if (!compression && bitdepth == 16 && bpc == 16) { + out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0); + ri->bits_per_channel = 16; + } else + out = (stbi_uc *) stbi__malloc(4 * w*h); + + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + pixelCount = w*h; + + // Initialize the data to zero. + //memset( out, 0, pixelCount * 4 ); + + // Finally, the image data. + if (compression) { + // RLE as used by .PSD and .TIFF + // Loop until you get the number of unpacked bytes you are expecting: + // Read the next source byte into n. + // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally. + // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times. + // Else if n is 128, noop. + // Endloop + + // The RLE-compressed data is preceded by a 2-byte data count for each row in the data, + // which we're going to just skip. + stbi__skip(s, h * channelCount * 2 ); + + // Read the RLE data by channel. + for (channel = 0; channel < 4; channel++) { + stbi_uc *p; + + p = out+channel; + if (channel >= channelCount) { + // Fill this channel with default data. + for (i = 0; i < pixelCount; i++, p += 4) + *p = (channel == 3 ? 255 : 0); + } else { + // Read the RLE data. + if (!stbi__psd_decode_rle(s, p, pixelCount)) { + STBI_FREE(out); + return stbi__errpuc("corrupt", "bad RLE data"); + } + } + } + + } else { + // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...) + // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image. + + // Read the data by channel. + for (channel = 0; channel < 4; channel++) { + if (channel >= channelCount) { + // Fill this channel with default data. + if (bitdepth == 16 && bpc == 16) { + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + stbi__uint16 val = channel == 3 ? 65535 : 0; + for (i = 0; i < pixelCount; i++, q += 4) + *q = val; + } else { + stbi_uc *p = out+channel; + stbi_uc val = channel == 3 ? 255 : 0; + for (i = 0; i < pixelCount; i++, p += 4) + *p = val; + } + } else { + if (ri->bits_per_channel == 16) { // output bpc + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + for (i = 0; i < pixelCount; i++, q += 4) + *q = (stbi__uint16) stbi__get16be(s); + } else { + stbi_uc *p = out+channel; + if (bitdepth == 16) { // input bpc + for (i = 0; i < pixelCount; i++, p += 4) + *p = (stbi_uc) (stbi__get16be(s) >> 8); + } else { + for (i = 0; i < pixelCount; i++, p += 4) + *p = stbi__get8(s); + } + } + } + } + } + + // remove weird white matte from PSD + if (channelCount >= 4) { + if (ri->bits_per_channel == 16) { + for (i=0; i < w*h; ++i) { + stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i; + if (pixel[3] != 0 && pixel[3] != 65535) { + float a = pixel[3] / 65535.0f; + float ra = 1.0f / a; + float inv_a = 65535.0f * (1 - ra); + pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a); + pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a); + pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a); + } + } + } else { + for (i=0; i < w*h; ++i) { + unsigned char *pixel = out + 4*i; + if (pixel[3] != 0 && pixel[3] != 255) { + float a = pixel[3] / 255.0f; + float ra = 1.0f / a; + float inv_a = 255.0f * (1 - ra); + pixel[0] = (unsigned char) (pixel[0]*ra + inv_a); + pixel[1] = (unsigned char) (pixel[1]*ra + inv_a); + pixel[2] = (unsigned char) (pixel[2]*ra + inv_a); + } + } + } + } + + // convert to desired output format + if (req_comp && req_comp != 4) { + if (ri->bits_per_channel == 16) + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h); + else + out = stbi__convert_format(out, 4, req_comp, w, h); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + if (comp) *comp = 4; + *y = h; + *x = w; + + return out; +} +#endif + +// ************************************************************************************************* +// Softimage PIC loader +// by Tom Seddon +// +// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format +// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/ + +#ifndef STBI_NO_PIC +static int stbi__pic_is4(stbi__context *s,const char *str) +{ + int i; + for (i=0; i<4; ++i) + if (stbi__get8(s) != (stbi_uc)str[i]) + return 0; + + return 1; +} + +static int stbi__pic_test_core(stbi__context *s) +{ + int i; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) + return 0; + + for(i=0;i<84;++i) + stbi__get8(s); + + if (!stbi__pic_is4(s,"PICT")) + return 0; + + return 1; +} + +typedef struct +{ + stbi_uc size,type,channel; +} stbi__pic_packet; + +static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest) +{ + int mask=0x80, i; + + for (i=0; i<4; ++i, mask>>=1) { + if (channel & mask) { + if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short"); + dest[i]=stbi__get8(s); + } + } + + return dest; +} + +static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src) +{ + int mask=0x80,i; + + for (i=0;i<4; ++i, mask>>=1) + if (channel&mask) + dest[i]=src[i]; +} + +static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result) +{ + int act_comp=0,num_packets=0,y,chained; + stbi__pic_packet packets[10]; + + // this will (should...) cater for even some bizarre stuff like having data + // for the same channel in multiple packets. + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return stbi__errpuc("bad format","too many packets"); + + packet = &packets[num_packets++]; + + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + + act_comp |= packet->channel; + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)"); + if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp"); + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel? + + for(y=0; ytype) { + default: + return stbi__errpuc("bad format","packet has bad compression type"); + + case 0: {//uncompressed + int x; + + for(x=0;xchannel,dest)) + return 0; + break; + } + + case 1://Pure RLE + { + int left=width, i; + + while (left>0) { + stbi_uc count,value[4]; + + count=stbi__get8(s); + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)"); + + if (count > left) + count = (stbi_uc) left; + + if (!stbi__readval(s,packet->channel,value)) return 0; + + for(i=0; ichannel,dest,value); + left -= count; + } + } + break; + + case 2: {//Mixed RLE + int left=width; + while (left>0) { + int count = stbi__get8(s), i; + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)"); + + if (count >= 128) { // Repeated + stbi_uc value[4]; + + if (count==128) + count = stbi__get16be(s); + else + count -= 127; + if (count > left) + return stbi__errpuc("bad file","scanline overrun"); + + if (!stbi__readval(s,packet->channel,value)) + return 0; + + for(i=0;ichannel,dest,value); + } else { // Raw + ++count; + if (count>left) return stbi__errpuc("bad file","scanline overrun"); + + for(i=0;ichannel,dest)) + return 0; + } + left-=count; + } + break; + } + } + } + } + + return result; +} + +static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri) +{ + stbi_uc *result; + int i, x,y, internal_comp; + STBI_NOTUSED(ri); + + if (!comp) comp = &internal_comp; + + for (i=0; i<92; ++i) + stbi__get8(s); + + x = stbi__get16be(s); + y = stbi__get16be(s); + + if (y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)"); + if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode"); + + stbi__get32be(s); //skip `ratio' + stbi__get16be(s); //skip `fields' + stbi__get16be(s); //skip `pad' + + // intermediate buffer is RGBA + result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0); + if (!result) return stbi__errpuc("outofmem", "Out of memory"); + memset(result, 0xff, x*y*4); + + if (!stbi__pic_load_core(s,x,y,comp, result)) { + STBI_FREE(result); + result=0; + } + *px = x; + *py = y; + if (req_comp == 0) req_comp = *comp; + result=stbi__convert_format(result,4,req_comp,x,y); + + return result; +} + +static int stbi__pic_test(stbi__context *s) +{ + int r = stbi__pic_test_core(s); + stbi__rewind(s); + return r; +} +#endif + +// ************************************************************************************************* +// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb + +#ifndef STBI_NO_GIF +typedef struct +{ + stbi__int16 prefix; + stbi_uc first; + stbi_uc suffix; +} stbi__gif_lzw; + +typedef struct +{ + int w,h; + stbi_uc *out; // output buffer (always 4 components) + stbi_uc *background; // The current "background" as far as a gif is concerned + stbi_uc *history; + int flags, bgindex, ratio, transparent, eflags; + stbi_uc pal[256][4]; + stbi_uc lpal[256][4]; + stbi__gif_lzw codes[8192]; + stbi_uc *color_table; + int parse, step; + int lflags; + int start_x, start_y; + int max_x, max_y; + int cur_x, cur_y; + int line_size; + int delay; +} stbi__gif; + +static int stbi__gif_test_raw(stbi__context *s) +{ + int sz; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0; + sz = stbi__get8(s); + if (sz != '9' && sz != '7') return 0; + if (stbi__get8(s) != 'a') return 0; + return 1; +} + +static int stbi__gif_test(stbi__context *s) +{ + int r = stbi__gif_test_raw(s); + stbi__rewind(s); + return r; +} + +static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp) +{ + int i; + for (i=0; i < num_entries; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + pal[i][3] = transp == i ? 0 : 255; + } +} + +static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info) +{ + stbi_uc version; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') + return stbi__err("not GIF", "Corrupt GIF"); + + version = stbi__get8(s); + if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF"); + if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF"); + + stbi__g_failure_reason = ""; + g->w = stbi__get16le(s); + g->h = stbi__get16le(s); + g->flags = stbi__get8(s); + g->bgindex = stbi__get8(s); + g->ratio = stbi__get8(s); + g->transparent = -1; + + if (g->w > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (g->h > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + + if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments + + if (is_info) return 1; + + if (g->flags & 0x80) + stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1); + + return 1; +} + +static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif)); + if (!g) return stbi__err("outofmem", "Out of memory"); + if (!stbi__gif_header(s, g, comp, 1)) { + STBI_FREE(g); + stbi__rewind( s ); + return 0; + } + if (x) *x = g->w; + if (y) *y = g->h; + STBI_FREE(g); + return 1; +} + +static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code) +{ + stbi_uc *p, *c; + int idx; + + // recurse to decode the prefixes, since the linked-list is backwards, + // and working backwards through an interleaved image would be nasty + if (g->codes[code].prefix >= 0) + stbi__out_gif_code(g, g->codes[code].prefix); + + if (g->cur_y >= g->max_y) return; + + idx = g->cur_x + g->cur_y; + p = &g->out[idx]; + g->history[idx / 4] = 1; + + c = &g->color_table[g->codes[code].suffix * 4]; + if (c[3] > 128) { // don't render transparent pixels; + p[0] = c[2]; + p[1] = c[1]; + p[2] = c[0]; + p[3] = c[3]; + } + g->cur_x += 4; + + if (g->cur_x >= g->max_x) { + g->cur_x = g->start_x; + g->cur_y += g->step; + + while (g->cur_y >= g->max_y && g->parse > 0) { + g->step = (1 << g->parse) * g->line_size; + g->cur_y = g->start_y + (g->step >> 1); + --g->parse; + } + } +} + +static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g) +{ + stbi_uc lzw_cs; + stbi__int32 len, init_code; + stbi__uint32 first; + stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear; + stbi__gif_lzw *p; + + lzw_cs = stbi__get8(s); + if (lzw_cs > 12) return NULL; + clear = 1 << lzw_cs; + first = 1; + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + bits = 0; + valid_bits = 0; + for (init_code = 0; init_code < clear; init_code++) { + g->codes[init_code].prefix = -1; + g->codes[init_code].first = (stbi_uc) init_code; + g->codes[init_code].suffix = (stbi_uc) init_code; + } + + // support no starting clear code + avail = clear+2; + oldcode = -1; + + len = 0; + for(;;) { + if (valid_bits < codesize) { + if (len == 0) { + len = stbi__get8(s); // start new block + if (len == 0) + return g->out; + } + --len; + bits |= (stbi__int32) stbi__get8(s) << valid_bits; + valid_bits += 8; + } else { + stbi__int32 code = bits & codemask; + bits >>= codesize; + valid_bits -= codesize; + // @OPTIMIZE: is there some way we can accelerate the non-clear path? + if (code == clear) { // clear code + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + avail = clear + 2; + oldcode = -1; + first = 0; + } else if (code == clear + 1) { // end of stream code + stbi__skip(s, len); + while ((len = stbi__get8(s)) > 0) + stbi__skip(s,len); + return g->out; + } else if (code <= avail) { + if (first) { + return stbi__errpuc("no clear code", "Corrupt GIF"); + } + + if (oldcode >= 0) { + p = &g->codes[avail++]; + if (avail > 8192) { + return stbi__errpuc("too many codes", "Corrupt GIF"); + } + + p->prefix = (stbi__int16) oldcode; + p->first = g->codes[oldcode].first; + p->suffix = (code == avail) ? p->first : g->codes[code].first; + } else if (code == avail) + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + + stbi__out_gif_code(g, (stbi__uint16) code); + + if ((avail & codemask) == 0 && avail <= 0x0FFF) { + codesize++; + codemask = (1 << codesize) - 1; + } + + oldcode = code; + } else { + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + } + } + } +} + +// this function is designed to support animated gifs, although stb_image doesn't support it +// two back is the image from two frames ago, used for a very specific disposal format +static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back) +{ + int dispose; + int first_frame; + int pi; + int pcount; + STBI_NOTUSED(req_comp); + + // on first frame, any non-written pixels get the background colour (non-transparent) + first_frame = 0; + if (g->out == 0) { + if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header + if (!stbi__mad3sizes_valid(4, g->w, g->h, 0)) + return stbi__errpuc("too large", "GIF image is too large"); + pcount = g->w * g->h; + g->out = (stbi_uc *) stbi__malloc(4 * pcount); + g->background = (stbi_uc *) stbi__malloc(4 * pcount); + g->history = (stbi_uc *) stbi__malloc(pcount); + if (!g->out || !g->background || !g->history) + return stbi__errpuc("outofmem", "Out of memory"); + + // image is treated as "transparent" at the start - ie, nothing overwrites the current background; + // background colour is only used for pixels that are not rendered first frame, after that "background" + // color refers to the color that was there the previous frame. + memset(g->out, 0x00, 4 * pcount); + memset(g->background, 0x00, 4 * pcount); // state of the background (starts transparent) + memset(g->history, 0x00, pcount); // pixels that were affected previous frame + first_frame = 1; + } else { + // second frame - how do we dispose of the previous one? + dispose = (g->eflags & 0x1C) >> 2; + pcount = g->w * g->h; + + if ((dispose == 3) && (two_back == 0)) { + dispose = 2; // if I don't have an image to revert back to, default to the old background + } + + if (dispose == 3) { // use previous graphic + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 ); + } + } + } else if (dispose == 2) { + // restore what was changed last frame to background before that frame; + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 ); + } + } + } else { + // This is a non-disposal case eithe way, so just + // leave the pixels as is, and they will become the new background + // 1: do not dispose + // 0: not specified. + } + + // background is what out is after the undoing of the previou frame; + memcpy( g->background, g->out, 4 * g->w * g->h ); + } + + // clear my history; + memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame + + for (;;) { + int tag = stbi__get8(s); + switch (tag) { + case 0x2C: /* Image Descriptor */ + { + stbi__int32 x, y, w, h; + stbi_uc *o; + + x = stbi__get16le(s); + y = stbi__get16le(s); + w = stbi__get16le(s); + h = stbi__get16le(s); + if (((x + w) > (g->w)) || ((y + h) > (g->h))) + return stbi__errpuc("bad Image Descriptor", "Corrupt GIF"); + + g->line_size = g->w * 4; + g->start_x = x * 4; + g->start_y = y * g->line_size; + g->max_x = g->start_x + w * 4; + g->max_y = g->start_y + h * g->line_size; + g->cur_x = g->start_x; + g->cur_y = g->start_y; + + // if the width of the specified rectangle is 0, that means + // we may not see *any* pixels or the image is malformed; + // to make sure this is caught, move the current y down to + // max_y (which is what out_gif_code checks). + if (w == 0) + g->cur_y = g->max_y; + + g->lflags = stbi__get8(s); + + if (g->lflags & 0x40) { + g->step = 8 * g->line_size; // first interlaced spacing + g->parse = 3; + } else { + g->step = g->line_size; + g->parse = 0; + } + + if (g->lflags & 0x80) { + stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1); + g->color_table = (stbi_uc *) g->lpal; + } else if (g->flags & 0x80) { + g->color_table = (stbi_uc *) g->pal; + } else + return stbi__errpuc("missing color table", "Corrupt GIF"); + + o = stbi__process_gif_raster(s, g); + if (!o) return NULL; + + // if this was the first frame, + pcount = g->w * g->h; + if (first_frame && (g->bgindex > 0)) { + // if first frame, any pixel not drawn to gets the background color + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi] == 0) { + g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be; + memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 ); + } + } + } + + return o; + } + + case 0x21: // Comment Extension. + { + int len; + int ext = stbi__get8(s); + if (ext == 0xF9) { // Graphic Control Extension. + len = stbi__get8(s); + if (len == 4) { + g->eflags = stbi__get8(s); + g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths. + + // unset old transparent + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 255; + } + if (g->eflags & 0x01) { + g->transparent = stbi__get8(s); + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 0; + } + } else { + // don't need transparent + stbi__skip(s, 1); + g->transparent = -1; + } + } else { + stbi__skip(s, len); + break; + } + } + while ((len = stbi__get8(s)) != 0) { + stbi__skip(s, len); + } + break; + } + + case 0x3B: // gif stream termination code + return (stbi_uc *) s; // using '1' causes warning on some compilers + + default: + return stbi__errpuc("unknown code", "Corrupt GIF"); + } + } +} + +static void *stbi__load_gif_main_outofmem(stbi__gif *g, stbi_uc *out, int **delays) +{ + STBI_FREE(g->out); + STBI_FREE(g->history); + STBI_FREE(g->background); + + if (out) STBI_FREE(out); + if (delays && *delays) STBI_FREE(*delays); + return stbi__errpuc("outofmem", "Out of memory"); +} + +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + if (stbi__gif_test(s)) { + int layers = 0; + stbi_uc *u = 0; + stbi_uc *out = 0; + stbi_uc *two_back = 0; + stbi__gif g; + int stride; + int out_size = 0; + int delays_size = 0; + + STBI_NOTUSED(out_size); + STBI_NOTUSED(delays_size); + + memset(&g, 0, sizeof(g)); + if (delays) { + *delays = 0; + } + + do { + u = stbi__gif_load_next(s, &g, comp, req_comp, two_back); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + + if (u) { + *x = g.w; + *y = g.h; + ++layers; + stride = g.w * g.h * 4; + + if (out) { + void *tmp = (stbi_uc*) STBI_REALLOC_SIZED( out, out_size, layers * stride ); + if (!tmp) + return stbi__load_gif_main_outofmem(&g, out, delays); + else { + out = (stbi_uc*) tmp; + out_size = layers * stride; + } + + if (delays) { + int *new_delays = (int*) STBI_REALLOC_SIZED( *delays, delays_size, sizeof(int) * layers ); + if (!new_delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + *delays = new_delays; + delays_size = layers * sizeof(int); + } + } else { + out = (stbi_uc*)stbi__malloc( layers * stride ); + if (!out) + return stbi__load_gif_main_outofmem(&g, out, delays); + out_size = layers * stride; + if (delays) { + *delays = (int*) stbi__malloc( layers * sizeof(int) ); + if (!*delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + delays_size = layers * sizeof(int); + } + } + memcpy( out + ((layers - 1) * stride), u, stride ); + if (layers >= 2) { + two_back = out - 2 * stride; + } + + if (delays) { + (*delays)[layers - 1U] = g.delay; + } + } + } while (u != 0); + + // free temp buffer; + STBI_FREE(g.out); + STBI_FREE(g.history); + STBI_FREE(g.background); + + // do the final conversion after loading everything; + if (req_comp && req_comp != 4) + out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h); + + *z = layers; + return out; + } else { + return stbi__errpuc("not GIF", "Image was not as a gif type."); + } +} + +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *u = 0; + stbi__gif g; + memset(&g, 0, sizeof(g)); + STBI_NOTUSED(ri); + + u = stbi__gif_load_next(s, &g, comp, req_comp, 0); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + if (u) { + *x = g.w; + *y = g.h; + + // moved conversion to after successful load so that the same + // can be done for multiple frames. + if (req_comp && req_comp != 4) + u = stbi__convert_format(u, 4, req_comp, g.w, g.h); + } else if (g.out) { + // if there was an error and we allocated an image buffer, free it! + STBI_FREE(g.out); + } + + // free buffers needed for multiple frame loading; + STBI_FREE(g.history); + STBI_FREE(g.background); + + return u; +} + +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp) +{ + return stbi__gif_info_raw(s,x,y,comp); +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR loader +// originally by Nicolas Schulz +#ifndef STBI_NO_HDR +static int stbi__hdr_test_core(stbi__context *s, const char *signature) +{ + int i; + for (i=0; signature[i]; ++i) + if (stbi__get8(s) != signature[i]) + return 0; + stbi__rewind(s); + return 1; +} + +static int stbi__hdr_test(stbi__context* s) +{ + int r = stbi__hdr_test_core(s, "#?RADIANCE\n"); + stbi__rewind(s); + if(!r) { + r = stbi__hdr_test_core(s, "#?RGBE\n"); + stbi__rewind(s); + } + return r; +} + +#define STBI__HDR_BUFLEN 1024 +static char *stbi__hdr_gettoken(stbi__context *z, char *buffer) +{ + int len=0; + char c = '\0'; + + c = (char) stbi__get8(z); + + while (!stbi__at_eof(z) && c != '\n') { + buffer[len++] = c; + if (len == STBI__HDR_BUFLEN-1) { + // flush to end of line + while (!stbi__at_eof(z) && stbi__get8(z) != '\n') + ; + break; + } + c = (char) stbi__get8(z); + } + + buffer[len] = 0; + return buffer; +} + +static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp) +{ + if ( input[3] != 0 ) { + float f1; + // Exponent + f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8)); + if (req_comp <= 2) + output[0] = (input[0] + input[1] + input[2]) * f1 / 3; + else { + output[0] = input[0] * f1; + output[1] = input[1] * f1; + output[2] = input[2] * f1; + } + if (req_comp == 2) output[1] = 1; + if (req_comp == 4) output[3] = 1; + } else { + switch (req_comp) { + case 4: output[3] = 1; /* fallthrough */ + case 3: output[0] = output[1] = output[2] = 0; + break; + case 2: output[1] = 1; /* fallthrough */ + case 1: output[0] = 0; + break; + } + } +} + +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int width, height; + stbi_uc *scanline; + float *hdr_data; + int len; + unsigned char count, value; + int i, j, k, c1,c2, z; + const char *headerToken; + STBI_NOTUSED(ri); + + // Check identifier + headerToken = stbi__hdr_gettoken(s,buffer); + if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0) + return stbi__errpf("not HDR", "Corrupt HDR image"); + + // Parse header + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format"); + + // Parse width and height + // can't use sscanf() if we're not using stdio! + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + height = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + width = (int) strtol(token, NULL, 10); + + if (height > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + if (width > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + + *x = width; + *y = height; + + if (comp) *comp = 3; + if (req_comp == 0) req_comp = 3; + + if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0)) + return stbi__errpf("too large", "HDR image is too large"); + + // Read data + hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0); + if (!hdr_data) + return stbi__errpf("outofmem", "Out of memory"); + + // Load image data + // image data is stored as some number of sca + if ( width < 8 || width >= 32768) { + // Read flat data + for (j=0; j < height; ++j) { + for (i=0; i < width; ++i) { + stbi_uc rgbe[4]; + main_decode_loop: + stbi__getn(s, rgbe, 4); + stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp); + } + } + } else { + // Read RLE-encoded data + scanline = NULL; + + for (j = 0; j < height; ++j) { + c1 = stbi__get8(s); + c2 = stbi__get8(s); + len = stbi__get8(s); + if (c1 != 2 || c2 != 2 || (len & 0x80)) { + // not run-length encoded, so we have to actually use THIS data as a decoded + // pixel (note this can't be a valid pixel--one of RGB must be >= 128) + stbi_uc rgbe[4]; + rgbe[0] = (stbi_uc) c1; + rgbe[1] = (stbi_uc) c2; + rgbe[2] = (stbi_uc) len; + rgbe[3] = (stbi_uc) stbi__get8(s); + stbi__hdr_convert(hdr_data, rgbe, req_comp); + i = 1; + j = 0; + STBI_FREE(scanline); + goto main_decode_loop; // yes, this makes no sense + } + len <<= 8; + len |= stbi__get8(s); + if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); } + if (scanline == NULL) { + scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0); + if (!scanline) { + STBI_FREE(hdr_data); + return stbi__errpf("outofmem", "Out of memory"); + } + } + + for (k = 0; k < 4; ++k) { + int nleft; + i = 0; + while ((nleft = width - i) > 0) { + count = stbi__get8(s); + if (count > 128) { + // Run + value = stbi__get8(s); + count -= 128; + if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = value; + } else { + // Dump + if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = stbi__get8(s); + } + } + } + for (i=0; i < width; ++i) + stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp); + } + if (scanline) + STBI_FREE(scanline); + } + + return hdr_data; +} + +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int dummy; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (stbi__hdr_test(s) == 0) { + stbi__rewind( s ); + return 0; + } + + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) { + stbi__rewind( s ); + return 0; + } + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *y = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *x = (int) strtol(token, NULL, 10); + *comp = 3; + return 1; +} +#endif // STBI_NO_HDR + +#ifndef STBI_NO_BMP +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp) +{ + void *p; + stbi__bmp_data info; + + info.all_a = 255; + p = stbi__bmp_parse_header(s, &info); + if (p == NULL) { + stbi__rewind( s ); + return 0; + } + if (x) *x = s->img_x; + if (y) *y = s->img_y; + if (comp) { + if (info.bpp == 24 && info.ma == 0xff000000) + *comp = 3; + else + *comp = info.ma ? 4 : 3; + } + return 1; +} +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp) +{ + int channelCount, dummy, depth; + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + *y = stbi__get32be(s); + *x = stbi__get32be(s); + depth = stbi__get16be(s); + if (depth != 8 && depth != 16) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 3) { + stbi__rewind( s ); + return 0; + } + *comp = 4; + return 1; +} + +static int stbi__psd_is16(stbi__context *s) +{ + int channelCount, depth; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + STBI_NOTUSED(stbi__get32be(s)); + STBI_NOTUSED(stbi__get32be(s)); + depth = stbi__get16be(s); + if (depth != 16) { + stbi__rewind( s ); + return 0; + } + return 1; +} +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp) +{ + int act_comp=0,num_packets=0,chained,dummy; + stbi__pic_packet packets[10]; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) { + stbi__rewind(s); + return 0; + } + + stbi__skip(s, 88); + + *x = stbi__get16be(s); + *y = stbi__get16be(s); + if (stbi__at_eof(s)) { + stbi__rewind( s); + return 0; + } + if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) { + stbi__rewind( s ); + return 0; + } + + stbi__skip(s, 8); + + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return 0; + + packet = &packets[num_packets++]; + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + act_comp |= packet->channel; + + if (stbi__at_eof(s)) { + stbi__rewind( s ); + return 0; + } + if (packet->size != 8) { + stbi__rewind( s ); + return 0; + } + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); + + return 1; +} +#endif + +// ************************************************************************************************* +// Portable Gray Map and Portable Pixel Map loader +// by Ken Miller +// +// PGM: http://netpbm.sourceforge.net/doc/pgm.html +// PPM: http://netpbm.sourceforge.net/doc/ppm.html +// +// Known limitations: +// Does not support comments in the header section +// Does not support ASCII image data (formats P2 and P3) + +#ifndef STBI_NO_PNM + +static int stbi__pnm_test(stbi__context *s) +{ + char p, t; + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind( s ); + return 0; + } + return 1; +} + +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + STBI_NOTUSED(ri); + + ri->bits_per_channel = stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n); + if (ri->bits_per_channel == 0) + return 0; + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + + if (!stbi__mad4sizes_valid(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0)) + return stbi__errpuc("too large", "PNM too large"); + + out = (stbi_uc *) stbi__malloc_mad4(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (!stbi__getn(s, out, s->img_n * s->img_x * s->img_y * (ri->bits_per_channel / 8))) { + STBI_FREE(out); + return stbi__errpuc("bad PNM", "PNM file truncated"); + } + + if (req_comp && req_comp != s->img_n) { + if (ri->bits_per_channel == 16) { + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, s->img_n, req_comp, s->img_x, s->img_y); + } else { + out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y); + } + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + return out; +} + +static int stbi__pnm_isspace(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; +} + +static void stbi__pnm_skip_whitespace(stbi__context *s, char *c) +{ + for (;;) { + while (!stbi__at_eof(s) && stbi__pnm_isspace(*c)) + *c = (char) stbi__get8(s); + + if (stbi__at_eof(s) || *c != '#') + break; + + while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' ) + *c = (char) stbi__get8(s); + } +} + +static int stbi__pnm_isdigit(char c) +{ + return c >= '0' && c <= '9'; +} + +static int stbi__pnm_getinteger(stbi__context *s, char *c) +{ + int value = 0; + + while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) { + value = value*10 + (*c - '0'); + *c = (char) stbi__get8(s); + if((value > 214748364) || (value == 214748364 && *c > '7')) + return stbi__err("integer parse overflow", "Parsing an integer in the PPM header overflowed a 32-bit int"); + } + + return value; +} + +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp) +{ + int maxv, dummy; + char c, p, t; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + stbi__rewind(s); + + // Get identifier + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind(s); + return 0; + } + + *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm + + c = (char) stbi__get8(s); + stbi__pnm_skip_whitespace(s, &c); + + *x = stbi__pnm_getinteger(s, &c); // read width + if(*x == 0) + return stbi__err("invalid width", "PPM image header had zero or overflowing width"); + stbi__pnm_skip_whitespace(s, &c); + + *y = stbi__pnm_getinteger(s, &c); // read height + if (*y == 0) + return stbi__err("invalid width", "PPM image header had zero or overflowing width"); + stbi__pnm_skip_whitespace(s, &c); + + maxv = stbi__pnm_getinteger(s, &c); // read max value + if (maxv > 65535) + return stbi__err("max value > 65535", "PPM image supports only 8-bit and 16-bit images"); + else if (maxv > 255) + return 16; + else + return 8; +} + +static int stbi__pnm_is16(stbi__context *s) +{ + if (stbi__pnm_info(s, NULL, NULL, NULL) == 16) + return 1; + return 0; +} +#endif + +static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp) +{ + #ifndef STBI_NO_JPEG + if (stbi__jpeg_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNG + if (stbi__png_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_GIF + if (stbi__gif_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_BMP + if (stbi__bmp_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PIC + if (stbi__pic_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_info(s, x, y, comp)) return 1; + #endif + + // test tga last because it's a crappy test! + #ifndef STBI_NO_TGA + if (stbi__tga_info(s, x, y, comp)) + return 1; + #endif + return stbi__err("unknown image type", "Image not of any known type, or corrupt"); +} + +static int stbi__is_16_main(stbi__context *s) +{ + #ifndef STBI_NO_PNG + if (stbi__png_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_is16(s)) return 1; + #endif + return 0; +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_info_from_file(f, x, y, comp); + fclose(f); + return result; +} + +STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__info_main(&s,x,y,comp); + fseek(f,pos,SEEK_SET); + return r; +} + +STBIDEF int stbi_is_16_bit(char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_is_16_bit_from_file(f); + fclose(f); + return result; +} + +STBIDEF int stbi_is_16_bit_from_file(FILE *f) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__is_16_main(&s); + fseek(f,pos,SEEK_SET); + return r; +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__is_16_main(&s); +} + +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__is_16_main(&s); +} + +#endif // STB_IMAGE_IMPLEMENTATION + +/* + revision history: + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug + 1-bit BMP + *_is_16_bit api + avoid warnings + 2.16 (2017-07-23) all functions have 16-bit variants; + STBI_NO_STDIO works again; + compilation fixes; + fix rounding in unpremultiply; + optimize vertical flip; + disable raw_len validation; + documentation fixes + 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode; + warning fixes; disable run-time SSE detection on gcc; + uniform handling of optional "return" values; + thread-safe initialization of zlib tables + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) allocate large structures on the stack + remove white matting for transparent PSD + fix reported channel count for PNG & BMP + re-enable SSE2 in non-gcc 64-bit + support RGB-formatted JPEG + read 16-bit PNGs (only as 8-bit) + 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED + 2.09 (2016-01-16) allow comments in PNM files + 16-bit-per-pixel TGA (not bit-per-component) + info() for TGA could break due to .hdr handling + info() for BMP to shares code instead of sloppy parse + can use STBI_REALLOC_SIZED if allocator doesn't support realloc + code cleanup + 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA + 2.07 (2015-09-13) fix compiler warnings + partial animated GIF support + limited 16-bpc PSD support + #ifdef unused functions + bug with < 92 byte PIC,PNM,HDR,TGA + 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value + 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning + 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit + 2.03 (2015-04-12) extra corruption checking (mmozeiko) + stbi_set_flip_vertically_on_load (nguillemot) + fix NEON support; fix mingw support + 2.02 (2015-01-19) fix incorrect assert, fix warning + 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2 + 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG + 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg) + progressive JPEG (stb) + PGM/PPM support (Ken Miller) + STBI_MALLOC,STBI_REALLOC,STBI_FREE + GIF bugfix -- seemingly never worked + STBI_NO_*, STBI_ONLY_* + 1.48 (2014-12-14) fix incorrectly-named assert() + 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb) + optimize PNG (ryg) + fix bug in interlaced PNG with user-specified channel count (stb) + 1.46 (2014-08-26) + fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG + 1.45 (2014-08-16) + fix MSVC-ARM internal compiler error by wrapping malloc + 1.44 (2014-08-07) + various warning fixes from Ronny Chevalier + 1.43 (2014-07-15) + fix MSVC-only compiler problem in code changed in 1.42 + 1.42 (2014-07-09) + don't define _CRT_SECURE_NO_WARNINGS (affects user code) + fixes to stbi__cleanup_jpeg path + added STBI_ASSERT to avoid requiring assert.h + 1.41 (2014-06-25) + fix search&replace from 1.36 that messed up comments/error messages + 1.40 (2014-06-22) + fix gcc struct-initialization warning + 1.39 (2014-06-15) + fix to TGA optimization when req_comp != number of components in TGA; + fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite) + add support for BMP version 5 (more ignored fields) + 1.38 (2014-06-06) + suppress MSVC warnings on integer casts truncating values + fix accidental rename of 'skip' field of I/O + 1.37 (2014-06-04) + remove duplicate typedef + 1.36 (2014-06-03) + convert to header file single-file library + if de-iphone isn't set, load iphone images color-swapped instead of returning NULL + 1.35 (2014-05-27) + various warnings + fix broken STBI_SIMD path + fix bug where stbi_load_from_file no longer left file pointer in correct place + fix broken non-easy path for 32-bit BMP (possibly never used) + TGA optimization by Arseny Kapoulkine + 1.34 (unknown) + use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case + 1.33 (2011-07-14) + make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements + 1.32 (2011-07-13) + support for "info" function for all supported filetypes (SpartanJ) + 1.31 (2011-06-20) + a few more leak fixes, bug in PNG handling (SpartanJ) + 1.30 (2011-06-11) + added ability to load files via callbacks to accomidate custom input streams (Ben Wenger) + removed deprecated format-specific test/load functions + removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway + error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha) + fix inefficiency in decoding 32-bit BMP (David Woo) + 1.29 (2010-08-16) + various warning fixes from Aurelien Pocheville + 1.28 (2010-08-01) + fix bug in GIF palette transparency (SpartanJ) + 1.27 (2010-08-01) + cast-to-stbi_uc to fix warnings + 1.26 (2010-07-24) + fix bug in file buffering for PNG reported by SpartanJ + 1.25 (2010-07-17) + refix trans_data warning (Won Chun) + 1.24 (2010-07-12) + perf improvements reading from files on platforms with lock-heavy fgetc() + minor perf improvements for jpeg + deprecated type-specific functions so we'll get feedback if they're needed + attempt to fix trans_data warning (Won Chun) + 1.23 fixed bug in iPhone support + 1.22 (2010-07-10) + removed image *writing* support + stbi_info support from Jetro Lauha + GIF support from Jean-Marc Lienher + iPhone PNG-extensions from James Brown + warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva) + 1.21 fix use of 'stbi_uc' in header (reported by jon blow) + 1.20 added support for Softimage PIC, by Tom Seddon + 1.19 bug in interlaced PNG corruption check (found by ryg) + 1.18 (2008-08-02) + fix a threading bug (local mutable static) + 1.17 support interlaced PNG + 1.16 major bugfix - stbi__convert_format converted one too many pixels + 1.15 initialize some fields for thread safety + 1.14 fix threadsafe conversion bug + header-file-only version (#define STBI_HEADER_FILE_ONLY before including) + 1.13 threadsafe + 1.12 const qualifiers in the API + 1.11 Support installable IDCT, colorspace conversion routines + 1.10 Fixes for 64-bit (don't use "unsigned long") + optimized upsampling by Fabian "ryg" Giesen + 1.09 Fix format-conversion for PSD code (bad global variables!) + 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz + 1.07 attempt to fix C++ warning/errors again + 1.06 attempt to fix C++ warning/errors again + 1.05 fix TGA loading to return correct *comp and use good luminance calc + 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free + 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR + 1.02 support for (subset of) HDR files, float interface for preferred access to them + 1.01 fix bug: possible bug in handling right-side up bmps... not sure + fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all + 1.00 interface to zlib that skips zlib header + 0.99 correct handling of alpha in palette + 0.98 TGA loader by lonesock; dynamically add loaders (untested) + 0.97 jpeg errors on too large a file; also catch another malloc failure + 0.96 fix detection of invalid v value - particleman@mollyrocket forum + 0.95 during header scan, seek to markers in case of padding + 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same + 0.93 handle jpegtran output; verbose errors + 0.92 read 4,8,16,24,32-bit BMP files of several formats + 0.91 output 24-bit Windows 3.0 BMP files + 0.90 fix a few more warnings; bump version number to approach 1.0 + 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd + 0.60 fix compiling as c++ + 0.59 fix warnings: merge Dave Moore's -Wall fixes + 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian + 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available + 0.56 fix bug: zlib uncompressed mode len vs. nlen + 0.55 fix bug: restart_interval not initialized to 0 + 0.54 allow NULL for 'int *comp' + 0.53 fix bug in png 3->4; speedup png decoding + 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments + 0.51 obey req_comp requests, 1-component jpegs return as 1-component, + on 'test' only check type, not whether we support this variant + 0.50 (2006-11-19) + first released version +*/ + + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ From ff7cc227a517e03d70a32dc5b973b32d99320de7 Mon Sep 17 00:00:00 2001 From: BobClawblaw Date: Mon, 12 Oct 2026 03:00:21 +0000 Subject: [PATCH 08/19] Native image input for Qwen3.8-27B on CUDA: tower rows spliced into the prompt pass, M-RoPE positions - Media (vision.zig): the request's images encoded, each prompt row's feature row, and get_rope_index's three positions; rope_delta kept on the stream for decode and verify positions (the drafter's stay plain, as 0.6.6's do) - Forward.chunk: image rows take the tower's rows; a chunk holding them runs attn_prep with three position rows (qwen_vision.cu), others the captured kernel at their shifted 1-D positions - native/cuda.zig: --vision for families that serve it (Info.vision from the family's prepareImage and markers), the tower's call scratch kept out of the streams' room; other CUDA families refuse --vision - JPEGs through libturbojpeg when it loads (PIL's decoder): patches equal 0.6.6's on every fixture, PNG and JPEG - aot: a launch with no exact captured variant takes one that assumes less of its arguments; the 27B's attention plan segments start on 16-byte boundaries, as the Python engine's tensors do Checked on an RTX 5090 against 0.6.6: prompt token counts equal on five image requests, three replies word for word, text exactness unchanged (serial 4/4, windows 54/54, drafted 4/4, shared rounds 6/6), concurrent text replies equal solo beside an image request. --- zig/kernels/cuda/qwen_vision.cu | 43 ++++++++ zig/src/cuda/aot.zig | 33 +++++- zig/src/families/qwen27/cuda/engine.zig | 9 ++ zig/src/families/qwen27/cuda/forward.zig | 24 +++- zig/src/families/qwen27/cuda/image.zig | 68 +++++++++++- zig/src/families/qwen27/cuda/lanes.zig | 13 ++- zig/src/families/qwen27/cuda/lone.zig | 10 ++ zig/src/families/qwen27/cuda/native.zig | 32 +++++- zig/src/families/qwen27/cuda/round.zig | 34 ++++-- zig/src/families/qwen27/cuda/state.zig | 9 +- zig/src/families/qwen27/cuda/vision.zig | 134 +++++++++++++++++++++++ zig/src/native/cuda.zig | 27 ++++- 12 files changed, 408 insertions(+), 28 deletions(-) diff --git a/zig/kernels/cuda/qwen_vision.cu b/zig/kernels/cuda/qwen_vision.cu index 9b1a73db9d..fe3999a5b3 100644 --- a/zig/kernels/cuda/qwen_vision.cu +++ b/zig/kernels/cuda/qwen_vision.cu @@ -206,3 +206,46 @@ extern "C" __global__ void __launch_bounds__(256) tf_qv_splice(__nv_bfloat16* __ if (s < 0) return; for (int i = threadIdx.x; i < D; i += blockDim.x) x[row * D + i] = features[static_cast(s) * D + i]; } + +// glue._attn_prep with MROPE (0.6.6's Triton kernel, three position rows): each head's q (from the [q | gate] rows) +// or k RMS-normed to bf16, then rotate-half over the first 2 * HALF dims, frequency i taking the t, h or w position +// by Qwen's interleaved sections (axis 1 where i % 3 == 1 and i < 3 * hsec, axis 2 where i % 3 == 2 and +// i < 3 * wsec, else axis 0). A block of D threads a (row, head); pos is [3][rows]. +extern "C" __global__ void __launch_bounds__(256) tf_qv_attn_prep_mrope( + const __nv_bfloat16* __restrict__ qg, const __nv_bfloat16* __restrict__ kv, const __nv_bfloat16* __restrict__ qn, + const __nv_bfloat16* __restrict__ kn, const int* __restrict__ pos, const float* __restrict__ inv, + __nv_bfloat16* __restrict__ qout, __nv_bfloat16* __restrict__ kout, float eps, int H, int HKV, int D, int HALF, + int rows, int hsec, int wsec) { + __shared__ float part[8]; + __shared__ float vals[256]; + const int row = blockIdx.x; + const int head = blockIdx.y; + const int d = threadIdx.x; + const bool is_q = head < H; + const __nv_bfloat16* src = is_q ? qg + (static_cast(row) * H + head) * 2 * D + : kv + (static_cast(row) * HKV + head - H) * D; + const __nv_bfloat16* wn = is_q ? qn : kn; + const float x = bf(src[d]); + float s = x * x; + for (int o = 16; o; o >>= 1) s += __shfl_xor_sync(0xffffffffu, s, o); + if ((d & 31) == 0) part[d >> 5] = s; + __syncthreads(); + float tot = 0.0f; + for (int i = 0; i < (D >> 5); ++i) tot += part[i]; + const float r = 1.0f / sqrtf(tot / D + eps); + const float xn = rnd(x * r * bf(wn[d])); + vals[d] = xn; + __syncthreads(); + float out = xn; + if (d < 2 * HALF) { + const int i = d < HALF ? d : d - HALF; + const int axis = (i % 3 == 1 && i < 3 * hsec) ? 1 : ((i % 3 == 2 && i < 3 * wsec) ? 2 : 0); + const float ang = static_cast(pos[axis * rows + row]) * inv[i]; + const float cs = cosf(ang), sn = sinf(ang); + const float xp = vals[d < HALF ? d + HALF : d - HALF]; + out = d < HALF ? xn * cs - xp * sn : xn * cs + xp * sn; + } + __nv_bfloat16* dst = is_q ? qout + (static_cast(row) * H + head) * D + : kout + (static_cast(row) * HKV + head - H) * D; + dst[d] = __float2bfloat16_rn(out); +} diff --git a/zig/src/cuda/aot.zig b/zig/src/cuda/aot.zig index b8e58d2e8e..924893dc2e 100644 --- a/zig/src/cuda/aot.zig +++ b/zig/src/cuda/aot.zig @@ -122,7 +122,13 @@ pub const Set = struct { pub fn find(self: *const Set, function: []const u8, args: []const Arg, consts: []const Const) !*const Variant { for (self.variants) |*v| { if (!std.mem.eql(u8, v.spec.@"fn", function)) continue; - if (matches(v.spec, args, consts)) return v; + if (matches(v.spec, args, consts, .exact)) return v; + } + // a variant that assumes less of an argument than this launch gives (no 16-divisibility it has, or a runtime + // int where Triton would fold a 1) computes the same for it: Triton specializes for speed, not for values + for (self.variants) |*v| { + if (!std.mem.eql(u8, v.spec.@"fn", function)) continue; + if (matches(v.spec, args, consts, .general)) return v; } std.log.err("no captured Triton variant of {s} for this launch:", .{function}); for (args) |a| switch (a.value) { @@ -167,7 +173,9 @@ fn lookup(args: []const Arg, name: []const u8) ?Arg { return null; } -fn matches(k: KernelJson, args: []const Arg, consts: []const Const) bool { +const Fit = enum { exact, general }; + +fn matches(k: KernelJson, args: []const Arg, consts: []const Const, fit: Fit) bool { for (consts) |c| { const got = k.consts.map.get(c.name) orelse return false; if (c.int) |x| if (got.int == null or got.int.? != x) return false; @@ -188,12 +196,20 @@ fn matches(k: KernelJson, args: []const Arg, consts: []const Const) bool { } const p = param.?; if (!std.mem.eql(u8, p.type, "i32")) return false; - if (!p.nospec and x == 1) return false; - if (p.div16 != (!p.nospec and @mod(x, 16) == 0)) return false; + const div = !p.nospec and @mod(x, 16) == 0; + switch (fit) { + .exact => if ((!p.nospec and x == 1) or p.div16 != div) return false, + .general => if (p.div16 and !div) return false, + } }, .ptr => |x| { const p = param orelse return false; - if (!std.mem.eql(u8, p.type, x.ty) or p.div16 != (x.addr % 16 == 0)) return false; + if (!std.mem.eql(u8, p.type, x.ty)) return false; + const div = x.addr % 16 == 0; + switch (fit) { + .exact => if (p.div16 != div) return false, + .general => if (p.div16 and !div) return false, + } }, .f32 => { const p = param orelse return false; @@ -201,7 +217,12 @@ fn matches(k: KernelJson, args: []const Arg, consts: []const Const) bool { }, .u64 => |x| { const p = param orelse return false; - if (!std.mem.eql(u8, p.type, "u64") or p.div16 != (x % 16 == 0)) return false; + if (!std.mem.eql(u8, p.type, "u64")) return false; + const div = x % 16 == 0; + switch (fit) { + .exact => if (p.div16 != div) return false, + .general => if (p.div16 and !div) return false, + } }, } runtime += 1; diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig index 52576d7cd3..daff994c44 100644 --- a/zig/src/families/qwen27/cuda/engine.zig +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -6,6 +6,7 @@ const c = @import("shape.zig"); const kern = @import("kernels.zig"); const tri = @import("triton.zig"); const st = @import("state.zig"); +const vision = @import("vision.zig"); const wts = @import("weights.zig"); const Forward = @import("forward.zig").Forward; const Part = @import("forward.zig").Part; @@ -73,15 +74,23 @@ pub const Engine = struct { /// prefill_state from an empty state: the prompt in even chunks of at most `prompt_rows`, its last row normed. pub fn prefill(e: *Engine, seq: *st.Seq, prompt: []const u32) !void { + return e.prefillMedia(seq, prompt, null); + } + + /// prefill with an image prompt's rows and positions (null: text); its decode offset stays on the stream. + pub fn prefillMedia(e: *Engine, seq: *st.Seq, prompt: []const u32, media: ?*const vision.Media) !void { if (prompt.len == 0) return error.EmptyPrompt; if (prompt.len >= seq.capacity) return error.PromptTooLong; try seq.reset(e.ops(), e.w.g); + seq.media = media; + defer seq.media = null; var f = e.forward(); const n = (prompt.len + prompt_rows - 1) / prompt_rows; for (0..n) |j| { const b = chunkBounds(0, prompt.len, prompt_rows, j); try f.chunk(prompt[b[0]..b[1]], seq, j == n - 1); } + if (media) |m| seq.rope_delta = m.delta; } /// decode.serial_decode, greedy: one-row windows from `pending` until `count` tokens or an end token. diff --git a/zig/src/families/qwen27/cuda/forward.zig b/zig/src/families/qwen27/cuda/forward.zig index 4e4964be4e..3777e8f6aa 100644 --- a/zig/src/families/qwen27/cuda/forward.zig +++ b/zig/src/families/qwen27/cuda/forward.zig @@ -7,6 +7,7 @@ const tri = @import("triton.zig"); const st = @import("state.zig"); const wts = @import("weights.zig"); const rnd = @import("round.zig"); +const vision = @import("vision.zig"); const bf = 2; pub const Part = rnd.Part; @@ -41,6 +42,10 @@ pub const Forward = struct { return .{ .heads = f.g().query_heads, .kv_heads = f.g().kv_heads, .dim = c.head_dim, .half = c.rotary_dim / 2 }; } + fn gpu(f: *const Forward) vision.Gpu { + return .{ .d = f.ops.k.d, .s = f.ops.s, .discrete = f.ops.k.discrete }; + } + fn upload32(f: *Forward, dst: u64, values: []const i32) !void { try f.ops.upload(dst, std.mem.sliceAsBytes(values)); try f.ops.s.synchronize(); // the host copy must outlive the async upload @@ -225,7 +230,8 @@ pub const Forward = struct { defer f.gpa.free(host); for (ids, 0..) |tok, r| { host[r] = @intCast(tok); - host[W + r] = @intCast(p0 + r); + // an image prompt's text rows: their three positions are equal, so one row of them is theirs + host[W + r] = if (seq.media) |m| m.pos[p0 + r] else @intCast(p0 + r); for (0..c.conv_taps) |j| host[2 * W + r * c.conv_taps + j] = @intCast(r + j); } try f.upload32(s.ids, host[0..W]); @@ -233,6 +239,17 @@ pub const Forward = struct { try f.upload32(s.windows, host[2 * W ..]); } try f.t.embed(s.ids, w.embed.w, w.embed.s, w.embed.b, s.x, W, g_.hidden); + // rows of an image take the tower's rows, and the chunk's attention their three positions + const media: ?*const vision.Media = if (seq.media) |m| (if (m.hasImage(p0, W)) m else null) else null; + if (media) |m| { + const n = m.src.len; + const host = try f.gpa.alloc(i32, 4 * W); + defer f.gpa.free(host); + @memcpy(host[0..W], m.src[p0..][0..W]); + for (0..3) |a| @memcpy(host[W + a * W ..][0..W], m.pos[a * n + p0 ..][0..W]); + try f.upload32(m.dev.ptr, host); + try m.tower.splice(f.gpu(), s.x, m.features.ptr, m.dev.ptr, W); + } var x = s.x; var pending: ?u64 = null; const kv = g_.kvInner() * bf; @@ -263,7 +280,10 @@ pub const Forward = struct { try f.ops.prefillDense(s.y, a.q, s.qg, W); try f.ops.prefillDense(s.y, a.k, s.key, W); try f.ops.prefillDense(s.y, a.v, seq.values[i] + p0 * kv, W); - try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, seq.keys[i] + p0 * kv, W, f.attnShape(), c.eps); + if (media) |m| { + const sh = f.attnShape(); + try m.tower.attnPrep(f.gpu(), s.qg, s.key, a.q_norm, a.k_norm, m.dev.ptr + W * 4, w.inv_freq, s.q_rot, seq.keys[i] + p0 * kv, W, .{ .heads = sh.heads, .kv_heads = sh.kv_heads, .dim = sh.dim, .half = sh.half }, c.eps); + } else try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, seq.keys[i] + p0 * kv, W, f.attnShape(), c.eps); const scale: f32 = @floatCast(std.math.pow(f64, c.head_dim, -0.5)); try f.ops.prefillAttention(s.q_rot, seq.keys[i], seq.values[i], s.attn, p0, W, g_.query_heads, g_.kv_heads, scale); try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); diff --git a/zig/src/families/qwen27/cuda/image.zig b/zig/src/families/qwen27/cuda/image.zig index 0546e66e8c..736e6960c1 100644 --- a/zig/src/families/qwen27/cuda/image.zig +++ b/zig/src/families/qwen27/cuda/image.zig @@ -3,7 +3,8 @@ //! alpha composited over white with PIL's integer blend), then transformers' Qwen2VLImageProcessorPil at the caps //! 0.6.6 passes it: smart_resize to multiples of 32 within [32^2, tokens * 32^2] pixels, PIL's 8-bit bicubic //! resampler, x / 255 in float64 rounded to float32, (x - 0.5) / 0.5, and 16x16 patches in 2x2 merge order with each -//! patch's frame given twice. PNG patches equal 0.6.6's bit for bit; JPEG decoders may differ by a level. +//! patch's frame given twice. JPEGs go through libjpeg-turbo (PIL's decoder) when libturbojpeg.so.0 loads, else +//! stb_image, whose IDCT may differ from PIL's by a level. const std = @import("std"); const vision = @import("vision.zig"); @@ -108,12 +109,71 @@ pub fn patchify(gpa: std.mem.Allocator, img: Rgb) Error![]f32 { // ---- decoding (stb_image), EXIF orientation and alpha, as PIL gives images.py its RGB ---- +/// TurboJPEG's decompression API (libturbojpeg.so.0), opened on first use; null when the library is absent. +const Turbo = struct { + init: *const fn () callconv(.c) ?*anyopaque, + header: *const fn (?*anyopaque, [*]const u8, c_ulong, *c_int, *c_int, *c_int, *c_int) callconv(.c) c_int, + decompress: *const fn (?*anyopaque, [*]const u8, c_ulong, [*]u8, c_int, c_int, c_int, c_int, c_int) callconv(.c) c_int, + destroy: *const fn (?*anyopaque) callconv(.c) c_int, + + var state = std.atomic.Value(u8).init(0); // 0: not opened, 1: opening, 2: done (api set or not) + var api: ?Turbo = null; + var lib: ?std.DynLib = null; + + fn load() void { + var l = std.DynLib.open("libturbojpeg.so.0") catch return; + api = .{ + .init = l.lookup(@FieldType(Turbo, "init"), "tjInitDecompress") orelse return l.close(), + .header = l.lookup(@FieldType(Turbo, "header"), "tjDecompressHeader3") orelse return l.close(), + .decompress = l.lookup(@FieldType(Turbo, "decompress"), "tjDecompress2") orelse return l.close(), + .destroy = l.lookup(@FieldType(Turbo, "destroy"), "tjDestroy") orelse return l.close(), + }; + lib = l; + } + + fn get() ?Turbo { + if (state.cmpxchgStrong(0, 1, .acquire, .acquire) == null) { + load(); + state.store(2, .release); + } + while (state.load(.acquire) != 2) std.atomic.spinLoopHint(); + return api; + } +}; + +const tjpf_rgb = 0; +const tjcs_cmyk = 3; +const tjcs_ycck = 4; + +/// A JPEG through libjpeg-turbo as PIL decodes it (accurate integer IDCT, fancy upsampling) into RGB rows; null +/// when TurboJPEG is absent or the image is CMYK (PIL converts those itself). +fn turboJpeg(gpa: std.mem.Allocator, bytes: []const u8) Error!?Rgb { + const t = Turbo.get() orelse return null; + const h = t.init() orelse return null; + defer _ = t.destroy(h); + var w: c_int = 0; + var ht: c_int = 0; + var sub: c_int = 0; + var cs: c_int = 0; + if (t.header(h, bytes.ptr, bytes.len, &w, &ht, &sub, &cs) != 0) return error.BadImage; + if (cs == tjcs_cmyk or cs == tjcs_ycck) return null; + if (w <= 0 or ht <= 0) return error.BadImage; + if (w > max_side or ht > max_side or @as(u64, @intCast(w)) * @as(u64, @intCast(ht)) > max_pixels) return error.ImageTooLarge; + const W: usize = @intCast(w); + const H: usize = @intCast(ht); + const out = try gpa.alloc(u8, W * H * 3); + errdefer gpa.free(out); + if (t.decompress(h, bytes.ptr, bytes.len, out.ptr, w, w * 3, ht, tjpf_rgb, 0) != 0) return error.BadImage; + return .{ .w = @intCast(W), .h = @intCast(H), .px = out }; +} + /// A single-frame PNG or JPEG as 8-bit RGB: upright by its EXIF orientation, alpha composited over white, 16-bit /// gray clipped to 255 and other 16-bit layouts cut to their high byte (what PIL's convert("RGB") gives). pub fn decode(gpa: std.mem.Allocator, bytes: []const u8) Error!Rgb { if (bytes.len < 8 or bytes.len > std.math.maxInt(c_int)) return error.BadImage; const n: c_int = @intCast(bytes.len); if (std.mem.startsWith(u8, bytes, "\x89PNG\r\n\x1a\n") and pngAnimated(bytes)) return error.Animated; + if (std.mem.startsWith(u8, bytes, "\xff\xd8\xff")) if (try turboJpeg(gpa, bytes)) |img| return upright(gpa, img, bytes); var w: c_int = 0; var h: c_int = 0; var comp: c_int = 0; @@ -146,7 +206,11 @@ pub fn decode(gpa: std.mem.Allocator, bytes: []const u8) Error!Rgb { } } } - const img: Rgb = .{ .w = @intCast(W), .h = @intCast(H), .px = out }; + return upright(gpa, .{ .w = @intCast(W), .h = @intCast(H), .px = out }, bytes); +} + +/// The decoded image turned upright by the EXIF orientation its bytes carry (it takes `img`). +fn upright(gpa: std.mem.Allocator, img: Rgb, bytes: []const u8) Error!Rgb { const o = orientation(bytes); if (o < 2 or o > 8) return img; defer img.deinit(gpa); diff --git a/zig/src/families/qwen27/cuda/lanes.zig b/zig/src/families/qwen27/cuda/lanes.zig index 8a5f86fad8..d88d43baf6 100644 --- a/zig/src/families/qwen27/cuda/lanes.zig +++ b/zig/src/families/qwen27/cuda/lanes.zig @@ -13,6 +13,7 @@ const dr = @import("draft.zig"); const drafts = @import("drafts.zig"); const lone = @import("lone.zig"); const trees = @import("tree.zig"); +const vision = @import("vision.zig"); const be = lanes.backend; /// The row counts calibrate times (calibration_rows without the tile steps a GB10 skips), to 16 a stream. @@ -41,6 +42,7 @@ pub const Cuda = struct { values: []f64 = &.{}, pending: Pending = .{}, drafter: ?*dr.DFlash2 = null, // DFlash2, when its checkpoint is pulled and drafts are on + tower: ?*vision.Tower = null, // the image tower, with --vision drafting: std.AutoHashMapUnmanaged(*lanes.Stream, *drafts.Stream) = .empty, costs: [calibration.len + 1]lanes.config.Cost = undefined, // the forward's ms by a round's rows (calibrate) cost_count: usize = 0, @@ -228,11 +230,18 @@ pub const Cuda = struct { /// prefill_state into an opened stream: the prompt in chunks, a drafting stream's last window of taps absorbed. fn prefillOpened(x: *Cuda, s: *lanes.Stream, seq: *st.Seq) !void { + // an image prompt: the tower's rows and the prompt's three positions for its pass (EncodedVision) + const media: ?*vision.Media = if (s.images.len == 0) null else blk: { + const tower = x.tower orelse return error.VisionDisabled; + const gpu: vision.Gpu = .{ .d = x.e.ops().k.d, .s = x.e.ops().s, .discrete = x.e.ops().k.discrete }; + break :blk try vision.Media.init(x.gpa, tower, gpu, s.prompt(), s.images, prompt_rows); + }; + defer if (media) |m| m.deinit(); if (x.drafter) |d| if (s.drafts) { const ds = try x.draftingOf(s); - return lone.prefill(x.e, seq, &ds.ctx, d, s.prompt()); + return lone.prefillMedia(x.e, seq, &ds.ctx, d, s.prompt(), media); }; - try x.e.prefill(seq, s.prompt()); + try x.e.prefillMedia(seq, s.prompt(), media); } /// prefill_batch: prompts in shared passes of at most a chunk's rows (a longer one alone, in its chunks). diff --git a/zig/src/families/qwen27/cuda/lone.zig b/zig/src/families/qwen27/cuda/lone.zig index 0397ea706f..6aa91e3fe3 100644 --- a/zig/src/families/qwen27/cuda/lone.zig +++ b/zig/src/families/qwen27/cuda/lone.zig @@ -72,7 +72,17 @@ pub const Stats = struct { rounds: usize = 0, drafted: usize = 0, accepted: usiz /// prefill_state with a drafter: the prompt in even chunks, its last `window` rows' taps into the context. pub fn prefill(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, prompt: []const u32) !void { + return prefillMedia(e, seq, ctx, d, prompt, null); +} + +/// prefill with an image prompt's rows and positions (null: text); its decode offset stays on the stream. +pub fn prefillMedia(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, prompt: []const u32, media: ?*const @import("vision.zig").Media) !void { try seq.reset(e.ops(), e.w.g); + seq.media = media; + defer seq.media = null; + defer if (media) |m| { + seq.rope_delta = m.delta; + }; ctx.len = 0; ctx.end = 0; const tap_from = prompt.len -| dl.window; diff --git a/zig/src/families/qwen27/cuda/native.zig b/zig/src/families/qwen27/cuda/native.zig index 6e3f362fdf..8b8fdd1ea0 100644 --- a/zig/src/families/qwen27/cuda/native.zig +++ b/zig/src/families/qwen27/cuda/native.zig @@ -8,6 +8,8 @@ const st = @import("state.zig"); const engine = @import("engine.zig"); const Lanes = @import("lanes.zig").Cuda; const dr = @import("draft.zig"); +const vision = @import("vision.zig"); +const image = @import("image.zig"); pub const model_type = "qwen3_5"; pub const formats: []const []const u8 = &.{"mlx-q4g64"}; @@ -15,7 +17,19 @@ pub const default_context: i64 = 32768; pub const max_segments: u32 = 1; pub const prompt_rows: u32 = engine.prompt_rows; -pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, drafter: ?[]const u8 = null }; +pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, drafter: ?[]const u8 = null, vision: ?Vision = null }; + +/// --vision's limits: the visual tokens a request's images share, and its images. +pub const Vision = struct { image_tokens: u32, max_images: u32 }; + +/// The image placeholder the chat template writes once an image, and the tokens it wraps it in. +pub const image_token = vision.image_token; +pub const image_markers = [_]u32{ vision.vision_start, vision.image_token, vision.vision_end }; + +/// Info.vision's prepare: an image's bytes to the tower's patches (image.zig), on the HTTP thread. +pub fn prepareImage(a: std.mem.Allocator, bytes: []const u8, max_tokens: u32) !image.Prepared { + return image.prepare(a, bytes, max_tokens); +} /// A lone drafted stream's own driver (none here: one stream decodes in the lane core too, as under --parallel). pub const LoneRun = *const fn (ctx: *anyopaque, s: *lanes.Stream, hooks: *anyopaque, committed: *const fn (*anyopaque) void, yield: *const fn (*anyopaque) bool) anyerror!bool; @@ -31,12 +45,14 @@ pub const Loaded = struct { free_streams: u32 = 0, /// What a --slide trainer keeps out of the streams' room (none: the 27B does not learn on CUDA yet). learn_bytes: usize = 0, + /// What the image tower keeps out of the streams' room (--vision): one call's scratch and a request's rows. + vision_bytes: usize = 0, ctx: *anyopaque, deinit: *const fn (*anyopaque) void, lone: ?LoneRun = null, }; -const Owned = struct { gpa: std.mem.Allocator, e: *engine.Engine, lanes: Lanes, draft: ?*dr.DFlash2 = null }; +const Owned = struct { gpa: std.mem.Allocator, e: *engine.Engine, lanes: Lanes, draft: ?*dr.DFlash2 = null, tower: ?*vision.Tower = null }; /// The drafter the Python engine pairs with the 27B once pulled (z-lab/Qwen3.8-27B-DFlash2). const drafter_repo = "models--z-lab--Qwen3.8-27B-DFlash2"; @@ -82,11 +98,20 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [ // the forward's ms by rows: a shared round's tree widths follow the curve try own.lanes.calibrate(io, most_streams); }; + var vision_bytes: usize = 0; + if (o.vision) |v| { + const ops = e.ops(); + own.tower = try vision.Tower.load(gpa, io, .{ .d = ops.k.d, .s = ops.s, .discrete = ops.k.discrete }, dir); + own.lanes.tower = own.tower; + vision_bytes = vision.Tower.scratchBytes(vision.max_patches) + @as(usize, v.image_tokens) * vision.out_hidden * 2 + engine.prompt_rows * 16; + std.log.info("image tower loaded: {d:.2} GiB, {d:.2} GiB kept for one image call", .{ @as(f64, @floatFromInt(own.tower.?.weight_bytes)) / (1 << 30), @as(f64, @floatFromInt(vision_bytes)) / (1 << 30) }); + } return .{ .backend = own.lanes.backend(), .facts = own.lanes.facts(), .rows = st.max_rows, .stream_bytes = st.Seq.bytes(e.w.g, o.context) + if (own.draft != null) dr.Context.bytes() else 0, + .vision_bytes = vision_bytes, .ctx = own, .deinit = release, }; @@ -98,6 +123,8 @@ pub fn explain(_: ?*anyopaque, err: anyerror) ?[]const u8 { error.PromptTooLong => "the prompt and its reply exceed this server's context window: shorten it or lower max_tokens", error.OutOfDeviceMemory => "the GPU had no memory left for this request's caches: retry once another request ends", error.QwenCheckpointHasNoDraftHead => "this checkpoint has no draft head on CUDA yet: serve with --no-drafts", + error.VisionDisabled => "image input needs this server started with --vision", + error.BadImagePlaceholders => "the image placeholders do not match the prepared images", else => null, }; } @@ -106,6 +133,7 @@ fn release(p: *anyopaque) void { const own: *Owned = @ptrCast(@alignCast(p)); own.lanes.deinit(); if (own.draft) |d| d.deinit(); + if (own.tower) |t| t.deinit(); own.e.deinit(); own.gpa.destroy(own); } diff --git a/zig/src/families/qwen27/cuda/round.zig b/zig/src/families/qwen27/cuda/round.zig index 2b2d4e2697..2354420e90 100644 --- a/zig/src/families/qwen27/cuda/round.zig +++ b/zig/src/families/qwen27/cuda/round.zig @@ -55,14 +55,27 @@ pub fn parentsOf(p: Part, buf: []i32) []const i32 { } /// attention.plan_host: rows, (first, rows, keys, slots) per stream, items, then window parents (global rows). -fn attnPlan(gpa: std.mem.Allocator, out: *std.ArrayList(i32), parts: []const Part, group: usize) !usize { +/// Where attnPlan put its segments, in ints from the plan's first: each starts on a 16-byte boundary, so the +/// attention kernels' plan pointers are the aligned ones Triton's captured variants were compiled for. +const AttnAt = struct { n_items: usize, streams: usize, items: usize, parents: usize }; + +fn pad16(out: *std.ArrayList(i32), gpa: std.mem.Allocator, from: usize) !usize { + while ((out.items.len - from) % 4 != 0) try out.append(gpa, 0); + return out.items.len - from; +} + +fn attnPlan(gpa: std.mem.Allocator, out: *std.ArrayList(i32), parts: []const Part, group: usize) !AttnAt { + const from = out.items.len; + var at: AttnAt = undefined; for (parts, 0..) |p, s| for (p.tokens) |_| try out.append(gpa, @intCast(s)); + at.streams = try pad16(out, gpa, from); var start: usize = 0; for (parts) |p| { const w = p.tokens.len; try out.appendSlice(gpa, &.{ @intCast(start), @intCast(w), @intCast(p.seq.pos), @intCast(tri.slots(p.seq.pos, w)) }); start += w; } + at.items = try pad16(out, gpa, from); var n_items: usize = 0; for (parts, 0..) |p, s| { const w = p.tokens.len; @@ -77,13 +90,15 @@ fn attnPlan(gpa: std.mem.Allocator, out: *std.ArrayList(i32), parts: []const Par } } } + at.parents = try pad16(out, gpa, from); start = 0; var buf: [tree.max_nodes]i32 = undefined; for (parts) |p| { for (parentsOf(p, &buf)) |x| try out.append(gpa, if (x < 0) -1 else @intCast(start + @as(usize, @intCast(x)))); start += p.tokens.len; } - return n_items; + at.n_items = n_items; + return at; } /// Each stream's key and value caches as bf16 element offsets from the scratch's origin. @@ -103,7 +118,7 @@ fn plans(gpa: std.mem.Allocator, parts: []const Part, starts: []const usize, mul const parents = parentsOf(p, &buf); const w = parents.len; try tree.depths(parents, dep[0..w]); - for (dep[0..w]) |d| try h.pos.append(gpa, @intCast(p.seq.pos + d)); + for (dep[0..w]) |d| try h.pos.append(gpa, @as(i32, @intCast(p.seq.pos + d)) + p.seq.rope_delta); const s = try tree.schedule(gpa, parents); defer gpa.free(s.entries); for (s.entries, 0..) |x, j| try h.entries.append(gpa, if (j % 3 == 0) x + @as(i32, @intCast(starts[k])) else x); @@ -161,7 +176,7 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, try ops.upload(s.plan, std.mem.sliceAsBytes(host.items)); try ops.s.synchronize(); host.clearRetainingCapacity(); - const n_items = try attnPlan(gpa, &host, parts, group); + const ap = try attnPlan(gpa, &host, parts, group); try ops.upload(s.attn_plan, std.mem.sliceAsBytes(host.items)); try ops.s.synchronize(); for (0..g.layers) |i| try offs.appendSlice(gpa, if (c.linear(i)) &.{ 0, 0 } else &.{ offsetOf(s, p.seq.keys[i]), offsetOf(s, p.seq.values[i]) }); @@ -175,8 +190,8 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, r.starts = s.plan + 3 * W * 4; r.tables = 0; r.offs = s.offs; - r.attn = .{ .rows = s.attn_plan, .streams = s.attn_plan + W * 4, .items = s.attn_plan + (W + 4) * 4, .n_items = n_items, .paths = s.paths, .depths = s.depths, .width = W }; - try t.paths(s.attn_plan + (W + 4 + 3 * n_items) * 4, s.paths, s.depths, W); + r.attn = .{ .rows = s.attn_plan, .streams = s.attn_plan + ap.streams * 4, .items = s.attn_plan + ap.items * 4, .n_items = ap.n_items, .paths = s.paths, .depths = s.depths, .width = W }; + try t.paths(s.attn_plan + ap.parents * 4, s.paths, s.depths, W); return r; } // multi_tree_forward: positions, stream ids, plan entries, starts, ids, windows, attention's plan in one copy @@ -186,8 +201,9 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, for (r.starts_host[0 .. S + 1]) |x| try host.append(gpa, @intCast(x)); for (parts) |p| for (p.tokens) |tok| try host.append(gpa, @intCast(tok)); try host.appendSlice(gpa, h.windows.items); + while (host.items.len % 4 != 0) try host.append(gpa, 0); // the attention plan from a 16-byte boundary const at_attn = host.items.len; - const n_items = try attnPlan(gpa, &host, parts, group); + const ap = try attnPlan(gpa, &host, parts, group); try ops.upload(s.round, std.mem.sliceAsBytes(host.items)); try ops.s.synchronize(); const base = s.round; @@ -198,7 +214,7 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, r.ids = base + (5 * W + S + 1) * 4; r.windows = base + (6 * W + S + 1) * 4; const ab = base + at_attn * 4; - r.attn = .{ .rows = ab, .streams = ab + W * 4, .items = ab + (W + 4 * S) * 4, .n_items = n_items, .paths = s.paths, .depths = s.depths, .width = W }; + r.attn = .{ .rows = ab, .streams = ab + ap.streams * 4, .items = ab + ap.items * 4, .n_items = ap.n_items, .paths = s.paths, .depths = s.depths, .width = W }; // the state pointer table and cache offsets, one copy each; every stream's conv rows stacked by layer var ptrs: std.ArrayList(u64) = .empty; defer ptrs.deinit(gpa); @@ -225,6 +241,6 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, try ops.s.synchronize(); r.tables = s.tables; r.offs = s.offs; - try t.paths(ab + (W + 4 * S + 3 * n_items) * 4, s.paths, s.depths, W); + try t.paths(ab + ap.parents * 4, s.paths, s.depths, W); return r; } diff --git a/zig/src/families/qwen27/cuda/state.zig b/zig/src/families/qwen27/cuda/state.zig index 4cbe06c07e..29d276651b 100644 --- a/zig/src/families/qwen27/cuda/state.zig +++ b/zig/src/families/qwen27/cuda/state.zig @@ -5,6 +5,7 @@ const cuda = @import("cuda"); const c = @import("shape.zig"); const kern = @import("kernels.zig"); const tri = @import("triton.zig"); +const vision = @import("vision.zig"); const torch_ops = kern.torch_ops; /// The most rows a stream's verify window takes (the multi-stream GDN tree kernel's limit). @@ -51,6 +52,8 @@ pub const Seq = struct { head_xs: u64, capacity: usize, pos: usize = 0, + rope_delta: i32 = 0, // an image prompt's rotary shift past its rows (decode positions are pos + rope_delta) + media: ?*const vision.Media = null, // during an image prompt's pass: its rows and positions gpa: std.mem.Allocator, /// Device bytes a stream of `capacity` positions takes. @@ -110,6 +113,8 @@ pub const Seq = struct { try ops.fill32(s.conv[i], 0, g.convBytes() / 4); }; s.pos = 0; + s.rope_delta = 0; + s.media = null; } }; @@ -263,7 +268,7 @@ pub const Scratch = struct { s.prompt_values = try A.take(R * g.kvInner() * bf); s.windows = try A.take(R * c.conv_taps * 4); s.plan = try A.take(R * 3 * 4); - s.attn_plan = try A.take((W + 4 + 3 * items + W) * 4); + s.attn_plan = try A.take((W + 4 + 3 * items + W + 12) * 4); // + each segment's 16-byte padding s.paths = try A.take(W * tri.max_nodes * 4); s.depths = try A.take(W * 4); s.offs = try A.take(g.layers * max_streams * 2 * 8); @@ -276,7 +281,7 @@ pub const Scratch = struct { s.taps = try A.take(R * 5 * g.hidden * bf); s.replay_table = try A.take(g.layers * (4 + max_streams) * 8); s.replay_rows = try A.take(max_streams * (W + 1) * 4); - s.round = try A.take((6 * W + max_streams + 1 + W * c.conv_taps + W + 4 * max_streams + 3 * items * max_streams + W) * 4); + s.round = try A.take((6 * W + max_streams + 1 + W * c.conv_taps + 4 + W + 4 * max_streams + 3 * items * max_streams + W + 12) * 4); s.tables = try A.take(g.layers * max_streams * 8); s.conv_cat = try A.take(g.layers * max_streams * (c.conv_taps - 1) * g.convDim() * bf); for (0..g.layers) |i| { diff --git a/zig/src/families/qwen27/cuda/vision.zig b/zig/src/families/qwen27/cuda/vision.zig index 1305d9b3d6..951cff63d0 100644 --- a/zig/src/families/qwen27/cuda/vision.zig +++ b/zig/src/families/qwen27/cuda/vision.zig @@ -26,6 +26,12 @@ pub const patch_width = channels * temporal * patch * patch; // 1536 pub const grid_side = 48; // sqrt(num_position_embeddings 2304) pub const merged = hidden * merge * merge; // 4608 pub const eps: f32 = 1e-6; +/// The language model's rope_parameters.mrope_section (t, h, w frequencies, interleaved). +pub const mrope_section = [3]u32{ 11, 11, 10 }; +/// Tokens the template writes around an image, and the image placeholder it expands. +pub const vision_start: u32 = 248053; +pub const vision_end: u32 = 248054; +pub const image_token: u32 = 248056; /// One tower call's patches: a 4,096-token image's (0.6.6's MAX_PATCHES). pub const max_patches = 16384; const workspace_bytes = 32 << 20; @@ -56,6 +62,7 @@ const Fns = struct { bias_residual: cuda.Function, bias_cast: cuda.Function, splice: cuda.Function, + attn_prep: cuda.Function, }; /// The image tower's weights on the GPU and its kernels; `encode` turns patches into language-width rows. @@ -98,6 +105,7 @@ pub const Tower = struct { .bias_residual = try t.mod.function("tf_qv_bias_residual"), .bias_cast = try t.mod.function("tf_qv_bias_cast"), .splice = try t.mod.function("tf_qv_splice"), + .attn_prep = try t.mod.function("tf_qv_attn_prep_mrope"), }; errdefer t.freeBuffers(); var src = try source.Source.init(gpa, .{ .d = ops.d, .s = ops.s, .discrete = ops.discrete }); @@ -297,6 +305,11 @@ pub const Tower = struct { try t.go(ops, t.f.splice, rows, 1, 256, .{ x, features, src, int(out_hidden) }); } + /// glue.attn_prep with three position rows (pos [3][rows], device int32): normed, M-RoPE-rotated q and k. + pub fn attnPrep(t: *Tower, ops: Gpu, qg: u64, key: u64, qn: u64, kn: u64, pos: u64, inv: u64, qout: u64, kout: u64, rows: usize, a: struct { heads: usize, kv_heads: usize, dim: usize, half: usize }, eps_: f32) !void { + try t.go(ops, t.f.attn_prep, rows, a.heads + a.kv_heads, @intCast(a.dim), .{ qg, key, qn, kn, pos, inv, qout, kout, eps_, int(a.heads), int(a.kv_heads), int(a.dim), int(a.half), int(rows), int(mrope_section[1]), int(mrope_section[2]) }); + } + fn go(t: *Tower, ops: Gpu, f: cuda.Function, gx: usize, gy: usize, block: u32, args: anytype) !void { _ = t; var a: cuda.Args = .{}; @@ -395,3 +408,124 @@ test "bilinear taps match align_corners resampling at the ends and middle" { try std.testing.expectEqual(@as(i32, 47), b.i[0]); try std.testing.expectEqual(@as(f32, 1), b.w[0]); } + +/// An image prompt's rows for its prompt pass (0.6.6's EncodedVision): the tower's rows on the GPU, each prompt row's +/// feature row (or -1), the three rotary positions of every prompt row and the decode offset after the prompt. +pub const Media = struct { + gpa: std.mem.Allocator, + tower: *Tower, + features: cuda.DeviceBuffer, // bf16 [image rows, out_hidden] + src: []i32, // [prompt]: feature row or -1 + pos: []i32, // [3][prompt]: t, h, w + delta: i32, + dev: cuda.DeviceBuffer, // a chunk's src and [3][rows] positions + + /// Encodes `images` (Image.at: the first of its tokens' rows) and lays out the prompt's positions as + /// transformers' get_rope_index does: text counts up on all three axes; an image sits at the next position with + /// its merged rows and columns on the h and w axes; text after it resumes past its larger side. + pub fn init(gpa: std.mem.Allocator, tower: *Tower, ops: Gpu, prompt: []const u32, images: anytype, chunk_rows: usize) !*Media { + const n = prompt.len; + const m = try gpa.create(Media); + errdefer gpa.destroy(m); + var rows: usize = 0; + for (images) |img| rows += img.tokens; + m.* = .{ .gpa = gpa, .tower = tower, .features = try cuda.DeviceBuffer.alloc(ops.d, rows * out_hidden * 2), .src = &.{}, .pos = &.{}, .delta = 0, .dev = undefined }; + errdefer m.features.free(); + m.dev = try cuda.DeviceBuffer.alloc(ops.d, chunk_rows * 4 * 4); + errdefer m.dev.free(); + m.src = try gpa.alloc(i32, n); + errdefer gpa.free(m.src); + m.pos = try gpa.alloc(i32, 3 * n); + errdefer gpa.free(m.pos); + @memset(m.src, -1); + // the tower: runs of whole images within one call's patches, in prompt order + var done: usize = 0; // feature rows written + var i: usize = 0; + while (i < images.len) { + var j = i; + var patches: usize = 0; + while (j < images.len and (j == i or patches + images[j].gh * images[j].gw <= max_patches)) : (j += 1) patches += images[j].gh * images[j].gw; + const px = try gpa.alloc(f32, patches * patch_width); + defer gpa.free(px); + const grids = try gpa.alloc(Grid, j - i); + defer gpa.free(grids); + var at: usize = 0; + for (images[i..j], grids) |img, *g| { + if (img.pixels.len != @as(usize, img.gh) * img.gw * patch_width or img.tokens * merge * merge != img.gh * img.gw) return error.BadImagePatches; + @memcpy(px[at..][0..img.pixels.len], img.pixels); + at += img.pixels.len; + g.* = .{ .h = img.gh, .w = img.gw }; + } + try tower.encode(ops, px, grids, m.features.ptr + done * out_hidden * 2); + for (images[i..j]) |img| done += img.tokens; + i = j; + } + m.delta = try layout(prompt, images, m.src, m.pos); + return m; + } + + pub fn deinit(m: *Media) void { + m.features.free(); + m.dev.free(); + m.gpa.free(m.src); + m.gpa.free(m.pos); + m.gpa.destroy(m); + } + + /// Whether rows [p0, p0 + w) hold an image row. + pub fn hasImage(m: *const Media, p0: usize, w: usize) bool { + for (m.src[p0..][0..w]) |v| if (v >= 0) return true; + return false; + } +}; + +/// The prompt's three rotary positions ([3][n]) and each placeholder row's feature row (src, -1 for text), as +/// transformers' get_rope_index lays them out; returns the decode offset (rope_delta). +pub fn layout(prompt: []const u32, images: anytype, src: []i32, pos: []i32) !i32 { + const n = prompt.len; + @memset(src, -1); + var next: i32 = 0; + var cursor: usize = 0; + var row: i32 = 0; + for (images) |img| { + const begin: usize = img.at; + const h: usize = img.gh / merge; + const w: usize = img.gw / merge; + const end = begin + h * w; + if (begin < cursor or begin == 0 or end >= n or prompt[begin - 1] != vision_start or prompt[end] != vision_end) return error.BadImagePlaceholders; + for (prompt[begin..end]) |t| if (t != image_token) return error.BadImagePlaceholders; + for (cursor..begin) |r| { + const v = next + @as(i32, @intCast(r - cursor)); + for (0..3) |a| pos[a * n + r] = v; + } + const base = next + @as(i32, @intCast(begin - cursor)); + for (0..h * w) |k| { + const r = begin + k; + pos[r] = base; + pos[n + r] = base + @as(i32, @intCast(k / w)); + pos[2 * n + r] = base + @as(i32, @intCast(k % w)); + src[r] = row; + row += 1; + } + next = base + @as(i32, @intCast(@max(h, w))); + cursor = end; + } + for (cursor..n) |r| { + const v = next + @as(i32, @intCast(r - cursor)); + for (0..3) |a| pos[a * n + r] = v; + } + return next - @as(i32, @intCast(cursor)); +} + +test "media positions: text counts up, an image sits at one position with its grid, text resumes past its side" { + const I = struct { gh: u32, gw: u32, at: u32, tokens: u32 }; + const prompt = [_]u32{ 7, vision_start, image_token, image_token, image_token, image_token, image_token, image_token, vision_end, 9 }; + var src: [10]i32 = undefined; + var pos: [30]i32 = undefined; + const delta = try layout(&prompt, &[_]I{.{ .gh = 4, .gw = 6, .at = 2, .tokens = 6 }}, &src, &pos); + try std.testing.expectEqual(@as(i32, -3), delta); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 2, 2, 2, 2, 2, 5, 6 }, pos[0..10]); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 2, 2, 3, 3, 3, 5, 6 }, pos[10..20]); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 2, 3, 4, 5, 6 }, pos[20..30]); + try std.testing.expectEqualSlices(i32, &.{ -1, -1, 0, 1, 2, 3, 4, 5, -1, -1 }, &src); +} diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 792bbb6632..ba1c01688a 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -428,6 +428,13 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O var family_options: F.Options = .{ .context = @intCast(window), .drafts = o.drafts, .segments = segments }; if (@hasField(F.Options, "slide")) family_options.slide = o.slide; if (@hasField(F.Options, "drafter")) family_options.drafter = o.drafter; + if (o.vision) |v| { + if (!@hasField(F.Options, "vision")) { + problem.* = try std.fmt.allocPrint(a, "--vision: the native CUDA engine reads images for Qwen3.8-27B only, not {s} checkpoints", .{o.model_type}); + return null; + } + family_options.vision = .{ .image_tokens = v.image_tokens, .max_images = v.max_images }; + } const loaded = F.open(gpa, io, &g.ctx, o.dir, kernels, family_options) catch |e| { problem.* = try std.fmt.allocPrint(a, "the native CUDA engine cannot load {s} with {s} glue kernels ({s})", .{ o.dir, kernels orelse "its own", @errorName(e) }); return null; @@ -435,7 +442,8 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O defer if (!opened) loaded.deinit(loaded.ctx); const model = cuda.usage(false).device - held0; const after = try pool(a, io, &g.ctx, problem) orelse return null; - const room = after.room(model) -| loaded.learn_bytes; + const vision_bytes: usize = if (@hasField(@TypeOf(loaded), "vision_bytes")) loaded.vision_bytes else 0; + const room = after.room(model) -| loaded.learn_bytes -| vision_bytes; const free = loaded.free_streams; const streams = budget.admit(room, loaded.stream_bytes, free, o.lanes, o.lanes_fixed) catch |e| { problem.* = switch (e) { @@ -450,7 +458,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O // prompt-cache snapshots are Nemotron's (cuda_reuse.zig); a family without them serves without a cache const cache = if (@hasDecl(F, "snap_bytes")) budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)) else 0; h.startup = try std.fmt.allocPrint(gpa, "CUDA sm_{d} device {d} ({s}{s}): model {d:.2} GiB{s}; {d} stream{s} at once, {d:.2} GiB each at a {d}-token window, of {d:.1} GiB left after a {d:.1} GiB reserve; prompts in {d}-row chunks{s}{s}; {s}", .{ - capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", + capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else if (vision_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for the image tower's calls", .{toGib(vision_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache", .{toGib(cache)}) else ", no prompt cache", if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", }); errdefer gpa.free(h.startup); @@ -485,7 +493,20 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O gpa.free(h.dir); }; // the family cuts its own prompt grid from position 0, as `tensorfold run` does: prefill_step 0 - try h.serve(io, loaded.facts, loaded.rows, .{ .lanes = streams, .context_window = @intCast(window), .startup = h.startup, .prompt_cache = h.store != null, .logprobs = loaded.backend.vtable.first_row != null, .structures = loaded.backend.vtable.first_masked != null }, .{ .ctx = loaded.ctx, .text = F.explain }); + const vision: ?api.Vision = if (@hasDecl(F, "prepareImage")) (if (o.vision) |v| .{ + .ctx = h, + .prepare = struct { + fn f(_: *anyopaque, al: Allocator, bytes: []const u8, max_tokens: u32) anyerror!api.PreparedImage { + const p = try F.prepareImage(al, bytes, max_tokens); + return .{ .pixels = p.pixels, .gh = p.gh, .gw = p.gw, .tokens = p.tokens, .hash = p.hash }; + } + }.f, + .image_token = F.image_token, + .markers = &F.image_markers, + .image_tokens = v.image_tokens, + .max_images = v.max_images, + } else null) else null; + try h.serve(io, loaded.facts, loaded.rows, .{ .lanes = streams, .context_window = @intCast(window), .startup = h.startup, .prompt_cache = h.store != null, .logprobs = loaded.backend.vtable.first_row != null, .structures = loaded.backend.vtable.first_masked != null, .vision = vision }, .{ .ctx = loaded.ctx, .text = F.explain }); opened = true; return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } From 5d8bf48592d41a6496cd907589237f7830455bb9 Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:24:49 +0000 Subject: [PATCH 09/19] qwen27 CUDA: --kv-cache fp8, and attention partials sized for the admitted streams --kv-cache fp8 keeps the 27B's attention caches as e4m3 rows with one fp32 scale a (position, key head) row (amax / 448), half the bytes of bf16: 256K tokens with the image tower in one stream on a 32 GB card. bf16 stays the default and its bits are unchanged (0.6.6 exactness: serial 4/4, windows 54/54, drafted 4/4, shared 0 replies differ). - tf_kv_quant (qwen_ops.cu) quantizes commits' and prompts' rows into the caches - pattn8_kernel: prefill attention staging e4m3 rows widened to bf16, a step ahead in registers (within 5% of bf16 at 100K) - _shared8/_tail8 (tests/cuda/qwen27/attention_kv8.py, captured like the other glue): the decode kernels over e4m3; _tail8 quantizes the window's own rows as their commit will, so a key's bits never depend on windows, drafts or batching (drafted equals serial, concurrent equals solo); attention.py's _merge folds the partials unchanged - attention partials leave the scratch arena: sized after admission for the streams admitted (Engine.fit, native fit hook) and charged to each stream, not for 16 streams at full context (1.4 GiB back at 122,880, 2.9 GiB at 262,144) - tf-qwen27-cuda kvcompare: fp8 against bf16 teacher-forced, top-1 agreement and KL --- zig/kernels/cuda/prefill_attention.cu | 99 +++++++- zig/kernels/cuda/qwen_ops.cu | 40 +++ zig/src/cli/qwen27_cuda.zig | 91 ++++++- zig/src/core/engine_api.zig | 2 + zig/src/families/qwen27/cuda/engine.zig | 12 +- zig/src/families/qwen27/cuda/forward.zig | 58 ++++- zig/src/families/qwen27/cuda/kernels.zig | 30 +++ zig/src/families/qwen27/cuda/native.zig | 13 +- zig/src/families/qwen27/cuda/round.zig | 30 ++- zig/src/families/qwen27/cuda/state.zig | 74 ++++-- zig/src/families/qwen27/cuda/triton.zig | 20 ++ zig/src/native/cuda.zig | 15 +- zig/src/native/metal.zig | 4 + zig/src/server/cli.zig | 5 +- zig/tests/cuda/qwen27/attention_kv8.py | 296 +++++++++++++++++++++++ 15 files changed, 742 insertions(+), 47 deletions(-) create mode 100644 zig/tests/cuda/qwen27/attention_kv8.py diff --git a/zig/kernels/cuda/prefill_attention.cu b/zig/kernels/cuda/prefill_attention.cu index cd9a012fc3..fe241822e3 100644 --- a/zig/kernels/cuda/prefill_attention.cu +++ b/zig/kernels/cuda/prefill_attention.cu @@ -1,5 +1,6 @@ // Device code of src/tensorfold/cuda/kernels/prefill_attention.cu (lines 1-250, comments and ATen includes dropped), checked by zig/tests/cuda/copies.py. #include +#include #include #include @@ -56,11 +57,13 @@ __device__ __forceinline__ float rowsum(const float (&c)[8]) { __fadd_rn(__fadd_rn(w[1], w[5]), __fadd_rn(w[3], w[7]))); } -template -__global__ void __launch_bounds__(32 * WARPS, 1) -pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kc, - const __nv_bfloat16* __restrict__ vc, __nv_bfloat16* __restrict__ out, int p0, int W, int H, int HK, - int G, float scale) { +// F8: the cache is e4m3 rows (kc, vc as bytes) with an fp32 scale a (position, key head) row (ks, vs); staging +// widens each row to bf16 times its scale, so the tiles and their dots are the bf16 kernel's. +template +__device__ __forceinline__ void pattn_body(const __nv_bfloat16* __restrict__ q, const void* __restrict__ kc, + const void* __restrict__ vc, const float* __restrict__ ks, + const float* __restrict__ vs, __nv_bfloat16* __restrict__ out, int p0, + int W, int H, int HK, int G, float scale) { constexpr int C = D / 8, K16 = D / 16, NB = D / 8, RB = WARPS / HPC, THREADS = 32 * WARPS; constexpr bool QREG = D <= 128; // the queries' fragments stay in registers, else in shared memory extern __shared__ uint4 smem[]; @@ -71,14 +74,61 @@ pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restric const int row0 = blockIdx.x * 16 * RB, last = min(row0 + 16 * RB, W) - 1; const int head = hk * G + hg * HPC + warp % HPC, wrow = row0 + (warp / HPC) * 16; const int tiles = (p0 + last) / BN + 1, items = 4 * tiles, limit = p0 + W; + // F8: an item's e4m3 words and scales go to registers a step ahead (fetch), then widen into its slot (put), so + // the loads land while the slot before computes + constexpr int PER = (HALF * C + THREADS - 1) / THREADS; + uint2 pre[F8 ? PER : 1]; + float pre_s[F8 ? PER : 1]; + auto fetch = [&](int item) { + if constexpr (F8) { + const uint8_t* src = static_cast(item % 4 < 2 ? kc : vc); + const float* sc = item % 4 < 2 ? ks : vs; + const int key0 = (item / 4) * BN + (item % 2) * HALF; +#pragma unroll + for (int j = 0; j < PER; ++j) { + const int i = threadIdx.x + j * THREADS, key = key0 + i / C; + pre[j] = make_uint2(0u, 0u); + pre_s[j] = 0.0f; + if (i < HALF * C && key < limit) { + const int64_t r = (int64_t)key * HK + hk; + pre[j] = __ldg(reinterpret_cast(src + r * D + (i % C) * 8)); + pre_s[j] = __ldg(sc + r); + } + } + } + }; + auto put = [&](int item) { + if constexpr (F8) { + uint4* dst = slots + (item % NS) * HALF * C; +#pragma unroll + for (int j = 0; j < PER; ++j) { + const int i = threadIdx.x + j * THREADS; + if (i >= HALF * C) continue; + uint4 w; + const __nv_fp8x2_storage_t* e = reinterpret_cast(&pre[j]); + uint32_t* o = reinterpret_cast(&w); +#pragma unroll + for (int k = 0; k < 4; ++k) { + const float2 x = __half22float2(__half2(__nv_cvt_fp8x2_to_halfraw2(e[k], __NV_E4M3))); + o[k] = pack(__fmul_rn(x.x, pre_s[j]), __fmul_rn(x.y, pre_s[j])); + } + dst[sw(i / C, i % C)] = w; + } + } + }; auto stage = [&](int item) { - const __nv_bfloat16* src = item % 4 < 2 ? kc : vc; const int key0 = (item / 4) * BN + (item % 2) * HALF; uint4* dst = slots + (item % NS) * HALF * C; - for (int i = threadIdx.x; i < HALF * C; i += THREADS) { - const int key = key0 + i / C; - const bool in = key < limit; // past the cache's filled keys: zeros, never read - cp16z(dst + sw(i / C, i % C), src + ((int64_t)(in ? key : 0) * HK + hk) * D + (i % C) * 8, in); + if constexpr (F8) { + fetch(item); + put(item); + } else { + const __nv_bfloat16* src = static_cast(item % 4 < 2 ? kc : vc); + for (int i = threadIdx.x; i < HALF * C; i += THREADS) { + const int key = key0 + i / C; + const bool in = key < limit; // past the cache's filled keys: zeros, never read + cp16z(dst + sw(i / C, i % C), src + ((int64_t)(in ? key : 0) * HK + hk) * D + (i % C) * 8, in); + } } }; uint32_t qa[QREG ? K16 : 1][4]; @@ -106,6 +156,9 @@ pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restric if (i < items) stage(i); commit(); } + if constexpr (F8) { + if (NS < items) fetch(NS); + } float o[NB][4]; #pragma unroll for (int i = 0; i < NB; ++i) o[i][0] = o[i][1] = o[i][2] = o[i][3] = 0.0f; @@ -120,7 +173,12 @@ pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restric }; auto done = [&](int it) { // the slot is free: stage the item NS ahead into it __syncthreads(); - if (it + NS < items) stage(it + NS); + if constexpr (F8) { + if (it + NS < items) { + put(it + NS); + if (it + NS + 1 < items) fetch(it + NS + 1); + } + } else if (it + NS < items) stage(it + NS); commit(); }; auto scores = [&](float (&s)[4][4], const uint4* sl) { @@ -238,6 +296,22 @@ pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restric } } +template +__global__ void __launch_bounds__(32 * WARPS, 1) +pattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kc, + const __nv_bfloat16* __restrict__ vc, __nv_bfloat16* __restrict__ out, int p0, int W, int H, int HK, + int G, float scale) { + pattn_body(q, kc, vc, nullptr, nullptr, out, p0, W, H, HK, G, scale); +} + +template +__global__ void __launch_bounds__(32 * WARPS, 1) +pattn8_kernel(const __nv_bfloat16* __restrict__ q, const uint8_t* __restrict__ kc, const uint8_t* __restrict__ vc, + const float* __restrict__ ks, const float* __restrict__ vs, __nv_bfloat16* __restrict__ out, int p0, + int W, int H, int HK, int G, float scale) { + pattn_body(q, kc, vc, ks, vs, out, p0, W, H, HK, G, scale); +} + } // namespace tf_prefill_attention // Head dim 128, eight warps, eight query heads a block, eight staging slots (Nemotron's 16 heads a KV head). @@ -246,3 +320,6 @@ template __global__ void tf_prefill_attention::pattn_kernel<128, 8, 8, 8>(const // Head dim 256, two query heads a block, two staging slots beside the staged queries (Qwen3.8's 6 heads a KV head). template __global__ void tf_prefill_attention::pattn_kernel<256, 8, 2, 2>(const __nv_bfloat16*, const __nv_bfloat16*, const __nv_bfloat16*, __nv_bfloat16*, int, int, int, int, int, float); +// The same over an fp8 cache (--kv-cache fp8). +template __global__ void tf_prefill_attention::pattn8_kernel<256, 8, 2, 2>(const __nv_bfloat16*, const uint8_t*, + const uint8_t*, const float*, const float*, __nv_bfloat16*, int, int, int, int, int, float); diff --git a/zig/kernels/cuda/qwen_ops.cu b/zig/kernels/cuda/qwen_ops.cu index 778cc1f16e..34b025326d 100644 --- a/zig/kernels/cuda/qwen_ops.cu +++ b/zig/kernels/cuda/qwen_ops.cu @@ -1,5 +1,6 @@ // The 27B's own small kernels: batched state copies, the drafter's taps and its fp32 RMSNorm. #include +#include #include struct TfCopy { @@ -17,6 +18,45 @@ extern "C" __global__ void __launch_bounds__(256) tf_copy_batch(const TfCopy* __ dst[i] = src[i]; } +struct TfQuant { + unsigned long long src, dst, scale; // bf16 rows of 256, their e4m3 rows, one fp32 scale a row + unsigned long long rows; +}; + +// --kv-cache fp8: each 256-wide (position, key head) row as e4m3 of x * (448 / amax), its scale s = amax / 448 (1 for +// a row of zeros). A warp a row, eight rows a block, items on y; attention_kv8.py's quantize and _q8 (the window rows +// _tail8 quantizes itself) give the same bits. +extern "C" __global__ void __launch_bounds__(256) tf_kv_quant(const TfQuant* __restrict__ items) { + const TfQuant it = items[blockIdx.y]; + const unsigned long long row = static_cast(blockIdx.x) * 8 + threadIdx.x / 32; + if (row >= it.rows) return; + const int lane = threadIdx.x % 32; + const uint4 w = reinterpret_cast(it.src)[row * 32 + lane]; + const __nv_bfloat162* h = reinterpret_cast(&w); + float x[8]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const float2 f = __bfloat1622float2(h[i]); + x[2 * i] = f.x; + x[2 * i + 1] = f.y; + } + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < 8; ++i) amax = fmaxf(amax, fabsf(x[i])); +#pragma unroll + for (int o = 16; o > 0; o /= 2) amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, o)); + const float s = amax > 0.0f ? __fdiv_rn(amax, 448.0f) : 1.0f; + const float inv = amax > 0.0f ? __fdiv_rn(448.0f, amax) : 1.0f; + uint2 out; + __nv_fp8x2_storage_t* q = reinterpret_cast<__nv_fp8x2_storage_t*>(&out); +#pragma unroll + for (int i = 0; i < 4; ++i) + q[i] = __nv_cvt_float2_to_fp8x2(make_float2(__fmul_rn(x[2 * i], inv), __fmul_rn(x[2 * i + 1], inv)), + __NV_SATFINITE, __NV_E4M3); + reinterpret_cast(it.dst)[row * 32 + lane] = out; + if (lane == 0) reinterpret_cast(it.scale)[row] = s; +} + // forward.tree_forward's taps: (x.float() + pending.float()).to(bfloat16), row r into columns [col, col + D) of OUT. extern "C" __global__ void __launch_bounds__(256) tf_tap(const __nv_bfloat16* __restrict__ x, const __nv_bfloat16* __restrict__ pending, diff --git a/zig/src/cli/qwen27_cuda.zig b/zig/src/cli/qwen27_cuda.zig index d1d0e879ad..2cb255d077 100644 --- a/zig/src/cli/qwen27_cuda.zig +++ b/zig/src/cli/qwen27_cuda.zig @@ -16,6 +16,8 @@ const usage = \\ Greedy shared rounds of one row a stream over every prompt, timed. \\ tf-qwen27-cuda multi MODEL --kernels DIR --drafter DIR --prompts multi.json \\ capture_multi.py's drafted shared rounds: each round's windows and every reply against the capture's. + \\ tf-qwen27-cuda kvcompare MODEL --kernels DIR --prompts prompts.json [--max-tokens N] + \\ --kv-cache fp8 against bf16, teacher-forced on bf16's greedy reply: top-1 agreement and KL a token. \\ tf-qwen27-cuda vision MODEL --fixtures DIR \\ The image tower on qwen_vision_fixtures.py's patches: each image's rows against 0.6.6's (cosine, max error). \\ @@ -32,6 +34,7 @@ const Options = struct { dump: ?[]const u8 = null, // windows: a case's per-norm rows into this folder cases: ?[]const u8 = null, // windows: only these cases, comma separated drafter: ?[]const u8 = null, // draft: the DFlash2 checkpoint + kv8: bool = false, // --kv-cache fp8: e4m3 attention caches }; pub fn main(init: std.process.Init) !u8 { @@ -42,9 +45,10 @@ pub fn main(init: std.process.Init) !u8 { const rounds = args.len >= 2 and std.mem.eql(u8, args[1], "rounds"); const drafting = args.len >= 2 and std.mem.eql(u8, args[1], "draft"); const multi = args.len >= 2 and std.mem.eql(u8, args[1], "multi"); + const kvcompare = args.len >= 2 and std.mem.eql(u8, args[1], "kvcompare"); if (args.len >= 5 and std.mem.eql(u8, args[1], "vision") and std.mem.eql(u8, args[3], "--fixtures")) return checkVision(gpa, arena, init.io, args[2], args[4]); - if (args.len < 3 or !(std.mem.eql(u8, args[1], "run") or windows or rounds or drafting or multi)) { + if (args.len < 3 or !(std.mem.eql(u8, args[1], "run") or windows or rounds or drafting or multi or kvcompare)) { std.debug.print("{s}", .{usage}); return 2; } @@ -53,7 +57,7 @@ pub fn main(init: std.process.Init) !u8 { while (i + 1 < args.len) : (i += 2) { const k = args[i]; const v = args[i + 1]; - if (std.mem.eql(u8, k, "--kernels")) o.kernels = v else if (std.mem.eql(u8, k, "--prompts")) o.prompts = v else if (std.mem.eql(u8, k, "--expect")) o.expect = v else if (std.mem.eql(u8, k, "--max-tokens")) o.max_tokens = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--context")) o.context = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--device")) o.device = try std.fmt.parseInt(c_int, v, 10) else if (std.mem.eql(u8, k, "--dump")) o.dump = v else if (std.mem.eql(u8, k, "--cases")) o.cases = v else if (std.mem.eql(u8, k, "--drafter")) o.drafter = v else { + if (std.mem.eql(u8, k, "--kernels")) o.kernels = v else if (std.mem.eql(u8, k, "--prompts")) o.prompts = v else if (std.mem.eql(u8, k, "--expect")) o.expect = v else if (std.mem.eql(u8, k, "--max-tokens")) o.max_tokens = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--context")) o.context = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--device")) o.device = try std.fmt.parseInt(c_int, v, 10) else if (std.mem.eql(u8, k, "--dump")) o.dump = v else if (std.mem.eql(u8, k, "--cases")) o.cases = v else if (std.mem.eql(u8, k, "--drafter")) o.drafter = v else if (std.mem.eql(u8, k, "--kv-cache")) o.kv8 = std.mem.eql(u8, v, "fp8") else { std.debug.print("unknown option {s}\n{s}", .{ k, usage }); return 2; } @@ -70,13 +74,15 @@ pub fn main(init: std.process.Init) !u8 { defer ctx.deinit(); const clock = std.Io.Clock.awake; var t0 = clock.now(init.io).toNanoseconds(); - const e = try qwen.Engine.init(gpa, init.io, &ctx, o.model, o.kernels, .{ .context = o.context }); + const e = try qwen.Engine.init(gpa, init.io, &ctx, o.model, o.kernels, .{ .context = o.context, .kv8 = o.kv8 }); defer e.deinit(); + try e.fit(16); // the shared rounds' partials: up to 16 streams of the context (windows' are 4096) std.debug.print("loaded in {d:.1}s, weights {d:.2} GiB\n", .{ @as(f64, @floatFromInt(clock.now(init.io).toNanoseconds() - t0)) / 1e9, @as(f64, @floatFromInt(e.w.bytes())) / (1 << 30) }); if (windows) return checkWindows(gpa, arena, init.io, e, prompts, o.dump, o.cases); if (rounds) return benchRounds(gpa, arena, init.io, e, prompts, o.max_tokens); if (drafting) return runDrafted(gpa, arena, init.io, e, o, prompts, expect); if (multi) return runMulti(gpa, arena, init.io, e, o, prompts); + if (kvcompare) return compareKv(arena, e, prompts, o.max_tokens); var seq = try e.sequence(o.context); defer seq.deinit(); var mismatches: usize = 0; @@ -105,6 +111,85 @@ pub fn main(init: std.process.Init) !u8 { return if (mismatches == 0) 0 else 1; } +/// The scratch's first logits row as fp64 log-probabilities. +fn logProbs(f: *qwen.Forward, bits: []u16, out: []f64) !void { + try f.ops.download(std.mem.sliceAsBytes(bits), f.s.logits); + try f.ops.s.synchronize(); + var most: f64 = -std.math.inf(f64); + for (bits, out) |b, *x| { + x.* = @floatCast(@as(f32, @bitCast(@as(u32, b) << 16))); + most = @max(most, x.*); + } + var total: f64 = 0; + for (out) |x| total += @exp(x - most); + const log_total = most + @log(total); + for (out) |*x| x.* -= log_total; +} + +fn argmaxOf(x: []const f64) u32 { + var best: usize = 0; + for (x, 0..) |v, i| if (v > x[best]) { + best = i; + }; + return @intCast(best); +} + +/// kvcompare: a bf16 stream and an fp8 one on each prompt, both fed bf16's greedy tokens; each step's top-1 +/// agreement, KL(bf16 || fp8) and how much less log-probability fp8 gives bf16's pick. +fn compareKv(arena: std.mem.Allocator, e: *qwen.Engine, prompts: std.json.Value, steps: usize) !u8 { + const vocab = qwen.config.vocab; + const bits = try arena.alloc(u16, vocab); + const pa = try arena.alloc(f64, vocab); + const pb = try arena.alloc(f64, vocab); + var f = e.forward(); + var it = prompts.object.iterator(); + var all_agree: usize = 0; + var all_n: usize = 0; + while (it.next()) |entry| { + const ids = try tokenList(arena, entry.value_ptr.*); + const cap = ids.len + steps + 16; + e.kv8 = false; + var a = try e.sequence(cap); + defer a.deinit(); + e.kv8 = true; + var b = try e.sequence(cap); + defer b.deinit(); + e.kv8 = false; + var agree: usize = 0; + var kl_sum: f64 = 0; + var kl_most: f64 = 0; + var lp_sum: f64 = 0; + var tok: u32 = 0; + for (0..steps) |step| { + for ([_]*qwen.state.Seq{ &a, &b }, [_][]f64{ pa, pb }) |seq, out| { + if (step == 0) { + try e.prefill(seq, ids); + try f.head(seq); + } else { + const part = [_]qwen.Part{.{ .seq = seq, .tokens = &.{tok} }}; + try f.round(&part); + try f.commit(&part, &.{&.{0}}); + } + try logProbs(&f, bits, out); + } + const top_a = argmaxOf(pa); + if (top_a == argmaxOf(pb)) agree += 1; + var kl: f64 = 0; + for (pa, pb) |x, y| kl += @exp(x) * (x - y); + kl_sum += kl; + kl_most = @max(kl_most, kl); + lp_sum += pa[top_a] - pb[top_a]; + tok = top_a; + } + const n: f64 = @floatFromInt(steps); + std.debug.print("{s}: prompt {d}, {d} steps: top-1 agree {d}/{d}, KL mean {d:.5} max {d:.5} nats, pick logprob gap {d:.5}\n", .{ entry.key_ptr.*, ids.len, steps, agree, steps, kl_sum / n, kl_most, lp_sum / n }); + all_agree += agree; + all_n += steps; + } + std.debug.print("overall top-1 agree {d}/{d}\n", .{ all_agree, all_n }); + return 0; +} + fn readJson(a: std.mem.Allocator, io: std.Io, path: []const u8) !std.json.Value { const bytes = try std.Io.Dir.cwd().readFileAlloc(io, path, a, .limited(1 << 30)); return (try std.json.parseFromSliceLeaky(std.json.Value, a, bytes, .{})); diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index cbacca9162..973cb8c2d7 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -255,6 +255,8 @@ pub const Open = struct { policy: []const u8 = "", /// --vision: load the checkpoint's image tower (null: text only; an engine without one refuses). vision: ?VisionOpen = null, + /// --kv-cache fp8: attention caches as e4m3 with a scale a row (the CUDA Qwen3.8-27B engine only). + kv8: bool = false, }; /// An opened engine; ``close`` stops its thread and frees its backend. diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig index daff994c44..de2c8ddd5c 100644 --- a/zig/src/families/qwen27/cuda/engine.zig +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -20,7 +20,7 @@ pub fn chunkBounds(start: usize, end: usize, size: usize, j: usize) [2]usize { return .{ start + (end - start) * j / n, start + (end - start) * (j + 1) / n }; } -pub const Options = struct { context: usize }; +pub const Options = struct { context: usize, kv8: bool = false }; pub const Engine = struct { gpa: std.mem.Allocator, @@ -30,6 +30,7 @@ pub const Engine = struct { w: wts.Weights, scratch: st.Scratch, context: usize, + kv8: bool, // --kv-cache fp8: every stream's attention caches e4m3 with a scale a row pub fn init(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: []const u8, o: Options) !*Engine { const cfg = try c.Config.read(gpa, io, dir); @@ -39,6 +40,7 @@ pub const Engine = struct { e.gpa = gpa; e.ctx = ctx; e.context = o.context; + e.kv8 = o.kv8; e.s = try cuda.Stream.init(ctx.d, true); errdefer e.s.deinit(); e.k = try kern.Kernels.load(gpa, io, ctx, kernels, g.linear_v_heads); @@ -69,7 +71,13 @@ pub const Engine = struct { /// A stream's state for `capacity` positions (its caches, recurrences and conv windows), empty. pub fn sequence(e: *Engine, capacity: usize) !st.Seq { - return st.Seq.init(e.gpa, e.ops(), e.w.g, capacity); + return st.Seq.init(e.gpa, e.ops(), e.w.g, capacity, e.kv8); + } + + /// Attention's partials for `streams` streams of the engine's context in one round. + pub fn fit(e: *Engine, streams: usize) !void { + try e.s.synchronize(); + try e.scratch.fitPartials(e.ctx.d, e.w.g, streams * st.Scratch.streamPartRows(e.context)); } /// prefill_state from an empty state: the prompt in even chunks of at most `prompt_rows`, its last row normed. diff --git a/zig/src/families/qwen27/cuda/forward.zig b/zig/src/families/qwen27/cuda/forward.zig index 3777e8f6aa..e188647925 100644 --- a/zig/src/families/qwen27/cuda/forward.zig +++ b/zig/src/families/qwen27/cuda/forward.zig @@ -121,7 +121,8 @@ pub const Forward = struct { try f.ops.dense(s.y, s.xs, a.k, s.key, W); try f.ops.dense(s.y, s.xs, a.v, rec.v, W); try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, r.pos, w.inv_freq, s.q_rot, rec.k, W, f.attnShape(), c.eps); - try f.t.attention(s.q_rot, rec.k, rec.v, s.origin, r.offsets(i, att), r.attn, .{ .o = s.part_o, .m = s.part_m, .l = s.part_l }, s.attn, f.attnShape()); + const part: tri.Tri.Partials = .{ .o = s.part_o, .m = s.part_m, .l = s.part_l }; + if (r.kv8) try f.t.attention8(s.q_rot, rec.k, rec.v, s.origin, r.offsets(i, att), r.attn, part, s.attn, f.attnShape()) else try f.t.attention(s.q_rot, rec.k, rec.v, s.origin, r.offsets(i, att), r.attn, part, s.attn, f.attnShape()); try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); try f.ops.dense(s.gated, s.gated_xs, a.o, s.r, W); att += 1; @@ -183,6 +184,9 @@ pub const Forward = struct { defer save.deinit(f.gpa); var write: std.ArrayList(kern.Copy) = .empty; defer write.deinit(f.gpa); + var quant: std.ArrayList(kern.Quant) = .empty; // kv8: the accepted keys and values, quantized into the caches + defer quant.deinit(f.gpa); + const hk = g_.kv_heads; base = 0; for (parts, paths, 0..) |p, path, k| { const n = path.len; @@ -199,7 +203,13 @@ pub const Forward = struct { lin += 1; } else { for (path, 0..) |acc, j| { - const at = (p.seq.pos + j) * kv; + const pos = p.seq.pos + j; + if (p.seq.kv8) { + try quant.append(f.gpa, .{ .src = s.attn_records[i].k + (base + acc) * kv, .dst = p.seq.keys[i] + pos * kv / bf, .scale = p.seq.kscale[i] + pos * hk * 4, .rows = hk }); + try quant.append(f.gpa, .{ .src = s.attn_records[i].v + (base + acc) * kv, .dst = p.seq.values[i] + pos * kv / bf, .scale = p.seq.vscale[i] + pos * hk * 4, .rows = hk }); + continue; + } + const at = pos * kv; try write.append(f.gpa, .{ .dst = p.seq.keys[i] + at, .src = s.attn_records[i].k + (base + acc) * kv, .bytes = kv }); try write.append(f.gpa, .{ .dst = p.seq.values[i] + at, .src = s.attn_records[i].v + (base + acc) * kv, .bytes = kv }); } @@ -213,6 +223,9 @@ pub const Forward = struct { try f.ops.upload(second, std.mem.sliceAsBytes(write.items)); try f.ops.copies(s.copy_items, save.items.len); try f.ops.copies(second, write.items.len); + if (quant.items.len > st.copy_items) return error.InvalidQwenCommit; + try f.ops.upload(s.quant_items, std.mem.sliceAsBytes(quant.items)); + try f.ops.quant(s.quant_items, quant.items.len, hk); try f.ops.s.synchronize(); // the tables' host copies go when this returns } @@ -279,13 +292,25 @@ pub const Forward = struct { .attention => |a| { try f.ops.prefillDense(s.y, a.q, s.qg, W); try f.ops.prefillDense(s.y, a.k, s.key, W); - try f.ops.prefillDense(s.y, a.v, seq.values[i] + p0 * kv, W); + // kv8: the chunk's rotated keys and values go through the prompt buffers, quantized into the caches + const kdst = if (seq.kv8) s.prompt_keys else seq.keys[i] + p0 * kv; + const vdst = if (seq.kv8) s.prompt_values else seq.values[i] + p0 * kv; + try f.ops.prefillDense(s.y, a.v, vdst, W); if (media) |m| { const sh = f.attnShape(); - try m.tower.attnPrep(f.gpu(), s.qg, s.key, a.q_norm, a.k_norm, m.dev.ptr + W * 4, w.inv_freq, s.q_rot, seq.keys[i] + p0 * kv, W, .{ .heads = sh.heads, .kv_heads = sh.kv_heads, .dim = sh.dim, .half = sh.half }, c.eps); - } else try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, seq.keys[i] + p0 * kv, W, f.attnShape(), c.eps); + try m.tower.attnPrep(f.gpu(), s.qg, s.key, a.q_norm, a.k_norm, m.dev.ptr + W * 4, w.inv_freq, s.q_rot, kdst, W, .{ .heads = sh.heads, .kv_heads = sh.kv_heads, .dim = sh.dim, .half = sh.half }, c.eps); + } else try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, kdst, W, f.attnShape(), c.eps); const scale: f32 = @floatCast(std.math.pow(f64, c.head_dim, -0.5)); - try f.ops.prefillAttention(s.q_rot, seq.keys[i], seq.values[i], s.attn, p0, W, g_.query_heads, g_.kv_heads, scale); + if (seq.kv8) { + const hk = g_.kv_heads; + const q = [_]kern.Quant{ + .{ .src = s.prompt_keys, .dst = seq.keys[i] + p0 * kv / bf, .scale = seq.kscale[i] + p0 * hk * 4, .rows = W * hk }, + .{ .src = s.prompt_values, .dst = seq.values[i] + p0 * kv / bf, .scale = seq.vscale[i] + p0 * hk * 4, .rows = W * hk }, + }; + try f.ops.upload(s.quant_items, std.mem.sliceAsBytes(&q)); + try f.ops.quant(s.quant_items, q.len, W * hk); + try f.ops.prefillAttention8(s.q_rot, seq.keys[i], seq.values[i], seq.kscale[i], seq.vscale[i], s.attn, p0, W, g_.query_heads, g_.kv_heads, scale); + } else try f.ops.prefillAttention(s.q_rot, seq.keys[i], seq.values[i], s.attn, p0, W, g_.query_heads, g_.kv_heads, scale); try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); try f.ops.prefillDense(s.gated, a.o, s.r, W); }, @@ -355,6 +380,10 @@ pub const Forward = struct { }; const staged = items.items.len; var at_layer: [256]usize = undefined; // each layer's first write, then the end + var quant: std.ArrayList(kern.Quant) = .empty; // kv8: each attention layer's prompt rows to quantize + defer quant.deinit(f.gpa); + var quant_at: [256]usize = @splat(0); + var quant_rows: usize = 0; lin_n = 0; for (0..g_.layers) |i| { at_layer[i] = items.items.len; @@ -373,15 +402,26 @@ pub const Forward = struct { } lin_n += 1; } else { + quant_at[i] = quant.items.len; for (pieces, 0..) |p, k| { const o = starts[k]; const n = p.ids.len; + if (p.seq.kv8) { + const hk = g_.kv_heads; + try quant.append(f.gpa, .{ .src = s.prompt_keys + o * kv, .dst = p.seq.keys[i] + p.seq.pos * kv / bf, .scale = p.seq.kscale[i] + p.seq.pos * hk * 4, .rows = n * hk }); + try quant.append(f.gpa, .{ .src = s.prompt_values + o * kv, .dst = p.seq.values[i] + p.seq.pos * kv / bf, .scale = p.seq.vscale[i] + p.seq.pos * hk * 4, .rows = n * hk }); + quant_rows = @max(quant_rows, n * hk); + continue; + } try items.append(f.gpa, .{ .dst = p.seq.keys[i] + p.seq.pos * kv, .src = s.prompt_keys + o * kv, .bytes = n * kv }); try items.append(f.gpa, .{ .dst = p.seq.values[i] + p.seq.pos * kv, .src = s.prompt_values + o * kv, .bytes = n * kv }); } } } + quant_at[g_.layers] = quant.items.len; at_layer[g_.layers] = items.items.len; + if (quant.items.len > st.copy_items) return error.InvalidQwenWindows; + try f.ops.upload(s.quant_items, std.mem.sliceAsBytes(quant.items)); if (items.items.len > 2 * st.copy_items) return error.InvalidQwenWindows; try f.ops.upload(s.copy_items, std.mem.sliceAsBytes(items.items)); try f.ops.copies(s.copy_items, staged); @@ -416,9 +456,13 @@ pub const Forward = struct { try f.ops.prefillDense(s.y, a.v, s.prompt_values, W); try f.t.attnPrep(s.qg, s.key, a.q_norm, a.k_norm, s.pos, w.inv_freq, s.q_rot, s.prompt_keys, W, f.attnShape(), c.eps); try f.ops.copies(s.copy_items + at_layer[i] * item, at_layer[i + 1] - at_layer[i]); + try f.ops.quant(s.quant_items + quant_at[i] * @sizeOf(kern.Quant), quant_at[i + 1] - quant_at[i], quant_rows); const scale: f32 = @floatCast(std.math.pow(f64, c.head_dim, -0.5)); const qrow = g_.query_heads * c.head_dim * bf; - for (pieces, 0..) |p, k| try f.ops.prefillAttention(s.q_rot + starts[k] * qrow, p.seq.keys[i], p.seq.values[i], s.attn + starts[k] * qrow, p.seq.pos, p.ids.len, g_.query_heads, g_.kv_heads, scale); + for (pieces, 0..) |p, k| if (p.seq.kv8) + try f.ops.prefillAttention8(s.q_rot + starts[k] * qrow, p.seq.keys[i], p.seq.values[i], p.seq.kscale[i], p.seq.vscale[i], s.attn + starts[k] * qrow, p.seq.pos, p.ids.len, g_.query_heads, g_.kv_heads, scale) + else + try f.ops.prefillAttention(s.q_rot + starts[k] * qrow, p.seq.keys[i], p.seq.values[i], s.attn + starts[k] * qrow, p.seq.pos, p.ids.len, g_.query_heads, g_.kv_heads, scale); try f.t.gateMul(s.attn, s.qg, s.gated, s.gated_xs, W, g_.query_heads, c.head_dim); try f.ops.prefillDense(s.gated, a.o, s.r, W); }, diff --git a/zig/src/families/qwen27/cuda/kernels.zig b/zig/src/families/qwen27/cuda/kernels.zig index 4b486922f8..964eb4536b 100644 --- a/zig/src/families/qwen27/cuda/kernels.zig +++ b/zig/src/families/qwen27/cuda/kernels.zig @@ -15,6 +15,7 @@ const sym = struct { const gemv = "_ZN12tf_lane_gemv11gemv_kernelILi64ELi64ELi8EEEvPK13__nv_bfloat16PKfNS_4PartEiii"; const prefill_mm = "_ZN14tf_qmm_prefill14prefill_kernelILi64ELi128ELi128ELi2ELi2ELi2ELb0EEEvPK13__nv_bfloat16PKjS3_S3_Pviiiiii"; const pattn = "_ZN20tf_prefill_attention12pattn_kernelILi256ELi8ELi2ELi2EEEvPK13__nv_bfloat16S3_S3_PS1_iiiiif"; + const pattn8 = "_ZN20tf_prefill_attention13pattn8_kernelILi256ELi8ELi2ELi2EEEvPK13__nv_bfloat16PKhS5_PKfS7_PS1_iiiiif"; const chain128 = "_ZN14tf_gdn_prefill12chain_kernelI13__nv_bfloat16Li128ELi32EEEvPKT_S4_PKS1_PKfS8_S8_PfPS1_iii"; const chain64 = "_ZN14tf_gdn_prefill12chain_kernelI13__nv_bfloat16Li64ELi32EEEvPKT_S4_PKS1_PKfS8_S8_PfPS1_iii"; const argmax = "tf_argmax_rows_i32_kernel"; @@ -44,6 +45,10 @@ pub const pattn_smem: u32 = 98304; // two 32-key slots of 256 dims beside eight /// One item of a batched copy: 16-byte aligned pointers, a multiple of 16 bytes. pub const Copy = extern struct { dst: u64, src: u64, bytes: u64 }; +/// One item of qwen_ops.cu's kv_quant (--kv-cache fp8): `rows` bf16 rows of 256 from src, as e4m3 rows at dst and +/// one fp32 scale a row at scale. +pub const Quant = extern struct { src: u64, dst: u64, scale: u64, rows: u64 }; + /// A tiled 4-bit projection: packed words, (kg, npad) scales and biases, n outputs from k inputs. pub const QLinear = struct { w: u64, s: u64, b: u64, n: usize, k: usize, npad: usize }; @@ -56,6 +61,8 @@ pub const Kernels = struct { gemv: cuda.Function, prefill_mm: cuda.Function, pattn: cuda.Function, + pattn8: cuda.Function, + kv_quant: cuda.Function, trees: [cuda.kernels.gdn_variants.len]cuda.Function, // tree_kernel by gdn_variants: [0] the chain one chip: [3]c_int, // major, minor, SMs: what dispatch_tree reads replay: cuda.Function, @@ -95,6 +102,7 @@ pub const Kernels = struct { k.prefill_mm = try k.mods[1].function(sym.prefill_mm); k.gemv = try k.mods[2].function(sym.gemv); k.pattn = try k.mods[3].function(sym.pattn); + k.pattn8 = try k.mods[3].function(sym.pattn8); for (&k.trees, cuda.kernels.gdn_variants) |*f, v| f.* = try k.mods[4].function(v.symbol); k.replay = try k.mods[4].function(cuda.kernels.gdn_symbols.replay_bf16); k.pack_dense = try k.mods[6].function(cuda.qlinear.symbols.pack_words); @@ -102,6 +110,7 @@ pub const Kernels = struct { k.argmax = try k.mods[7].function(sym.argmax); k.torch = try torch_ops.Functions.resolve(k.mods[7..13]); k.copy_batch = try k.mods[13].function("tf_copy_batch"); + k.kv_quant = try k.mods[13].function("tf_kv_quant"); k.tap = try k.mods[13].function("tf_tap"); k.rms_norm = try k.mods[13].function("tf_rms_norm"); k.inv_freq = try k.mods[13].function("tf_inv_freq"); @@ -118,6 +127,7 @@ pub const Kernels = struct { try k.gemv.allowDynamicShared(group_smem); try k.prefill_mm.allowDynamicShared(prefill_mm_smem); try k.pattn.allowDynamicShared(pattn_smem); + try k.pattn8.allowDynamicShared(pattn_smem); k.gemv_blocks = @max(1, try k.gemv.occupancy(128, group_smem)) * sms; const major = try ctx.attribute(.compute_capability_major); const minor = try ctx.attribute(.compute_capability_minor); @@ -309,6 +319,26 @@ pub const Ops = struct { try o.go(o.k.pattn, .{ (rows + block_rows - 1) / block_rows, kv_heads * (g / hpc), 1 }, 256, pattn_smem, &a); } + /// prefill_attention over an fp8 cache: e4m3 keys and values with a scale a (position, key head) row. + pub fn prefillAttention8(o: Ops, q: u64, kc: u64, vc: u64, ks: u64, vs: u64, out: u64, p0: usize, rows: usize, heads: usize, kv_heads: usize, scale: f32) !void { + const g = heads / kv_heads; + const hpc = 2; + var a: cuda.Args = .{}; + for ([_]u64{ q, kc, vc, ks, vs, out }) |v| a.add(v); + for ([_]usize{ p0, rows, heads, kv_heads, g }) |v| a.add(int(v)); + a.add(scale); + const block_rows = 16 * (8 / hpc); + try o.go(o.k.pattn8, .{ (rows + block_rows - 1) / block_rows, kv_heads * (g / hpc), 1 }, 256, pattn_smem, &a); + } + + /// qwen_ops.cu's tf_kv_quant: `count` items from the device table `items`, none over `most` rows. + pub fn quant(o: Ops, items: u64, count: usize, most: usize) !void { + if (count == 0) return; + var a: cuda.Args = .{}; + a.add(items); + try o.go(o.k.kv_quant, .{ (most + 7) / 8, count, 1 }, 256, 0, &a); + } + /// torch.argmax(rows of bf16 logits) as int32: the first maximum, a NaN first. pub fn argmax(o: Ops, logits: u64, vocab: usize, ld: usize, out: u64, rows: usize) !void { var a: cuda.Args = .{}; diff --git a/zig/src/families/qwen27/cuda/native.zig b/zig/src/families/qwen27/cuda/native.zig index 8b8fdd1ea0..32833fe00c 100644 --- a/zig/src/families/qwen27/cuda/native.zig +++ b/zig/src/families/qwen27/cuda/native.zig @@ -17,7 +17,7 @@ pub const default_context: i64 = 32768; pub const max_segments: u32 = 1; pub const prompt_rows: u32 = engine.prompt_rows; -pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, drafter: ?[]const u8 = null, vision: ?Vision = null }; +pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, drafter: ?[]const u8 = null, vision: ?Vision = null, kv8: bool = false }; /// --vision's limits: the visual tokens a request's images share, and its images. pub const Vision = struct { image_tokens: u32, max_images: u32 }; @@ -83,7 +83,7 @@ const most_streams = 32; pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: ?[]const u8, o: Options) !Loaded { // the 27B's glue runs the Python engine's Triton kernels, captured on this chip (no own glue kernels yet) - const e = try engine.Engine.init(gpa, io, ctx, dir, kernels orelse return error.QwenNeedsCapturedKernels, .{ .context = o.context }); + const e = try engine.Engine.init(gpa, io, ctx, dir, kernels orelse return error.QwenNeedsCapturedKernels, .{ .context = o.context, .kv8 = o.kv8 }); errdefer e.deinit(); const own = try gpa.create(Owned); errdefer gpa.destroy(own); @@ -110,13 +110,20 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [ .backend = own.lanes.backend(), .facts = own.lanes.facts(), .rows = st.max_rows, - .stream_bytes = st.Seq.bytes(e.w.g, o.context) + if (own.draft != null) dr.Context.bytes() else 0, + // each stream's caches, its drafter context, and its share of a round's attention partials (`fit`) + .stream_bytes = st.Seq.bytes(e.w.g, o.context, o.kv8) + st.Scratch.partialBytes(e.w.g, st.Scratch.streamPartRows(o.context)) + if (own.draft != null) dr.Context.bytes() else 0, .vision_bytes = vision_bytes, .ctx = own, .deinit = release, }; } +/// After admission: attention's partials sized for the `streams` the budget admitted. +pub fn fit(p: *anyopaque, streams: u32) !void { + const own: *Owned = @ptrCast(@alignCast(p)); + try own.e.fit(@min(streams, st.max_streams)); +} + /// A request this engine refuses, in words; null: none of its own. pub fn explain(_: ?*anyopaque, err: anyerror) ?[]const u8 { return switch (err) { diff --git a/zig/src/families/qwen27/cuda/round.zig b/zig/src/families/qwen27/cuda/round.zig index 2354420e90..7fd0b2f770 100644 --- a/zig/src/families/qwen27/cuda/round.zig +++ b/zig/src/families/qwen27/cuda/round.zig @@ -27,7 +27,8 @@ pub const Round = struct { most: usize, // the most rows a stream's window holds tables: u64, // multi: (DeltaNet layers, streams) state pointers, a layer's row at its DeltaNet index attn: tri.Tri.Plan, - offs: u64, // (attention layers, streams, 2) int64 cache offsets; single: indexed by layer + offs: u64, // (attention layers, streams, 2) int64 cache offsets (4 under kv8: scales too); single: by layer + kv8: bool, // the streams' caches are e4m3 with scales (attention8) starts_host: [st.max_streams + 1]usize, /// The conv rows gdn_pre reads for layer `i` (its DeltaNet index `j`): the stream's own, or the stacked copy. @@ -38,7 +39,8 @@ pub const Round = struct { /// Layer `i`'s attention cache offsets: tree_forward's (one stream, by layer) or the packed (layer, streams) rows. pub fn offsets(r: *const Round, i: usize, a: usize) u64 { - return if (r.multi) r.offs + a * r.streams * 16 else r.offs + i * 16; + const each: usize = if (r.kv8) 32 else 16; + return if (r.multi) r.offs + a * r.streams * each else r.offs + i * each; } /// The DeltaNet state table of DeltaNet layer `j` (multi only). @@ -106,6 +108,18 @@ fn offsetOf(s: *const st.Scratch, ptr: u64) i64 { return @divExact(@as(i64, @bitCast(ptr -% s.origin)), bf); } +/// Layer `i`'s cache offsets for a stream: bf16 elements of keys and values, or under kv8 e4m3 bytes of keys and +/// values and fp32 elements of their scales. +fn cacheOffsets(s: *const st.Scratch, seq: *const st.Seq, i: usize, out: *std.ArrayList(i64), gpa: std.mem.Allocator) !void { + if (!seq.kv8) return out.appendSlice(gpa, &.{ offsetOf(s, seq.keys[i]), offsetOf(s, seq.values[i]) }); + const d = struct { + fn at(sc: *const st.Scratch, ptr: u64, size: i64) i64 { + return @divExact(@as(i64, @bitCast(ptr -% sc.origin)), size); + } + }; + try out.appendSlice(gpa, &.{ d.at(s, seq.keys[i], 1), d.at(s, seq.values[i], 1), d.at(s, seq.kscale[i], 4), d.at(s, seq.vscale[i], 4) }); +} + /// Every stream's host plans: positions (pos + depth), the GDN schedule (nodes offset by its start), conv windows. const Host = struct { pos: std.ArrayList(i32) = .empty, entries: std.ArrayList(i32) = .empty, windows: std.ArrayList(i32) = .empty, slots: usize = 0, most: usize = 0 }; @@ -144,6 +158,14 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, r.starts_host[k + 1] = W; } if (W > st.round_rows) return error.WindowTooWide; + // attention's partials: the most slots any stream takes, for every row of the round + var most_slots: usize = 0; + for (parts) |p| { + most_slots = @max(most_slots, tri.slots(p.seq.pos, p.tokens.len)); + if (p.seq.kv8 != parts[0].seq.kv8) return error.InvalidQwenWindows; + } + if (most_slots * W > s.part_rows) return error.WindowTooWide; + r.kv8 = parts[0].seq.kv8; r.rows = W; r.streams = parts.len; r.multi = parts.len > 1; @@ -179,7 +201,7 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, const ap = try attnPlan(gpa, &host, parts, group); try ops.upload(s.attn_plan, std.mem.sliceAsBytes(host.items)); try ops.s.synchronize(); - for (0..g.layers) |i| try offs.appendSlice(gpa, if (c.linear(i)) &.{ 0, 0 } else &.{ offsetOf(s, p.seq.keys[i]), offsetOf(s, p.seq.values[i]) }); + for (0..g.layers) |i| if (c.linear(i)) try offs.appendNTimes(gpa, 0, if (r.kv8) 4 else 2) else try cacheOffsets(s, p.seq, i, &offs, gpa); try ops.upload(s.offs, std.mem.sliceAsBytes(offs.items)); try ops.s.synchronize(); r.ids = s.ids; @@ -224,7 +246,7 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, var j: usize = 0; for (0..g.layers) |i| { if (!c.linear(i)) { - for (parts) |p| try offs.appendSlice(gpa, &.{ offsetOf(s, p.seq.keys[i]), offsetOf(s, p.seq.values[i]) }); + for (parts) |p| try cacheOffsets(s, p.seq, i, &offs, gpa); continue; } const cat = s.conv_cat + j * st.max_streams * keep * g.convDim() * bf; diff --git a/zig/src/families/qwen27/cuda/state.zig b/zig/src/families/qwen27/cuda/state.zig index 29d276651b..e85f68a5b5 100644 --- a/zig/src/families/qwen27/cuda/state.zig +++ b/zig/src/families/qwen27/cuda/state.zig @@ -46,8 +46,11 @@ pub const Seq = struct { arena: Arena, rec: []u64, // by layer: (hv, dv, dk) fp32, 0 for attention layers conv: []u64, // by layer: (3, conv dim) bf16 - keys: []u64, // by layer: (capacity, kv heads, head dim) bf16 + keys: []u64, // by layer: (capacity, kv heads, head dim) bf16, or e4m3 under kv8 values: []u64, + kscale: []u64, // kv8: by layer, (capacity, kv heads) fp32, each key row's scale (0 for DeltaNet layers) + vscale: []u64, + kv8: bool, // --kv-cache fp8: keys and values as e4m3 rows with a scale each head_in: u64, // the last prompt row normed, (hidden) bf16, and its group sums: the first draw's input head_xs: u64, capacity: usize, @@ -57,16 +60,16 @@ pub const Seq = struct { gpa: std.mem.Allocator, /// Device bytes a stream of `capacity` positions takes. - pub fn bytes(g: c.Geometry, capacity: usize) usize { + pub fn bytes(g: c.Geometry, capacity: usize, kv8: bool) usize { var n: usize = std.mem.alignForward(usize, g.hidden * 2, 256) + std.mem.alignForward(usize, g.hidden / 64 * 4, 256); const a = std.mem.alignForward; - for (0..g.layers) |i| n += if (c.linear(i)) a(usize, g.deltaBytes(), 256) + a(usize, g.convBytes(), 256) else 2 * a(usize, capacity * g.kvInner() * 2, 256); + for (0..g.layers) |i| n += if (c.linear(i)) a(usize, g.deltaBytes(), 256) + a(usize, g.convBytes(), 256) else if (kv8) 2 * (a(usize, capacity * g.kvInner(), 256) + a(usize, capacity * g.kv_heads * 4, 256)) else 2 * a(usize, capacity * g.kvInner() * 2, 256); return n; } /// A fresh stream: zero recurrences and conv windows, caches for `capacity` positions. - pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize) !Seq { - var s: Seq = .{ .gpa = gpa, .arena = try Arena.init(ops.k.d, bytes(g, capacity)), .rec = &.{}, .conv = &.{}, .keys = &.{}, .values = &.{}, .head_in = 0, .head_xs = 0, .capacity = capacity }; + pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize, kv8: bool) !Seq { + var s: Seq = .{ .gpa = gpa, .arena = try Arena.init(ops.k.d, bytes(g, capacity, kv8)), .rec = &.{}, .conv = &.{}, .keys = &.{}, .values = &.{}, .kscale = &.{}, .vscale = &.{}, .kv8 = kv8, .head_in = 0, .head_xs = 0, .capacity = capacity }; errdefer s.arena.deinit(); s.rec = try gpa.alloc(u64, g.layers); errdefer gpa.free(s.rec); @@ -76,7 +79,11 @@ pub const Seq = struct { errdefer gpa.free(s.keys); s.values = try gpa.alloc(u64, g.layers); errdefer gpa.free(s.values); - const bf = 2; + s.kscale = try gpa.alloc(u64, g.layers); + errdefer gpa.free(s.kscale); + s.vscale = try gpa.alloc(u64, g.layers); + errdefer gpa.free(s.vscale); + const bf: usize = if (kv8) 1 else 2; s.head_in = try s.arena.take(g.hidden * 2); s.head_xs = try s.arena.take(g.hidden / 64 * 4); for (0..g.layers) |i| { @@ -84,6 +91,8 @@ pub const Seq = struct { s.conv[i] = 0; s.keys[i] = 0; s.values[i] = 0; + s.kscale[i] = 0; + s.vscale[i] = 0; const A = &s.arena; if (c.linear(i)) { s.rec[i] = try A.take(g.deltaBytes()); @@ -91,6 +100,10 @@ pub const Seq = struct { } else { s.keys[i] = try A.take(capacity * g.kvInner() * bf); s.values[i] = try A.take(capacity * g.kvInner() * bf); + if (kv8) { + s.kscale[i] = try A.take(capacity * g.kv_heads * 4); + s.vscale[i] = try A.take(capacity * g.kv_heads * 4); + } } } try s.reset(ops, g); @@ -103,6 +116,8 @@ pub const Seq = struct { s.gpa.free(s.conv); s.gpa.free(s.keys); s.gpa.free(s.values); + s.gpa.free(s.kscale); + s.gpa.free(s.vscale); s.* = undefined; } @@ -179,10 +194,12 @@ pub const Scratch = struct { paths: u64, depths: u64, offs: u64, - part_o: u64, + part_o: u64, // attention's partials, (slots, rows, heads, dim) fp32 and (slots, rows, heads) m and l, in `parts` part_m: u64, part_l: u64, - part_chunks: usize, + part_rows: usize, // the slot-rows `parts` holds: a round's most slots times its rows fit under this + parts: ?cuda.DeviceBuffer = null, + quant_items: u64, // --kv-cache fp8: a commit's or a prompt's kv_quant table (kern.Quant) origin: u64, // the bf16 origin cache offsets count from conv_tmp: u64, conv_old: u64, // a round's streams' conv rows that survive its commit, by stream and DeltaNet layer @@ -210,19 +227,49 @@ pub const Scratch = struct { s.arena = try Arena.init(d, s.arena.used); errdefer s.arena.deinit(); try s.layout(g, prompt_rows, capacity); + s.parts = null; + try s.fitPartials(d, g, initial_part_rows); return s; } + /// Partials for calibration's short windows until `fit` sizes them for the admitted streams. + pub const initial_part_rows = 4 * round_rows; + + /// Device bytes partials of `slot_rows` take. + pub fn partialBytes(g: c.Geometry, slot_rows: usize) usize { + const a = std.mem.alignForward; + return a(usize, slot_rows * g.query_heads * c.head_dim * 4, 256) + 2 * a(usize, slot_rows * g.query_heads * 4, 256); + } + + /// The slot-rows a stream of `capacity` positions may add to a round: its most slots (attention.slots of any + /// keys and window it can hold) times its widest window. + pub fn streamPartRows(capacity: usize) usize { + return ((capacity + max_rows + tri.chunk - 1) / tri.chunk + 1) * max_rows; + } + + /// Partials of `slot_rows` (at least the initial), replacing any held. + pub fn fitPartials(s: *Scratch, d: *const cuda.Driver, g: c.Geometry, slot_rows: usize) !void { + const rows = @max(slot_rows, initial_part_rows); + if (s.parts) |*b| b.free(); + s.parts = null; + var b = try cuda.DeviceBuffer.alloc(d, partialBytes(g, rows)); + errdefer b.free(); + var A: Arena = .{ .buf = b }; + s.part_o = try A.take(rows * g.query_heads * c.head_dim * 4); + s.part_m = try A.take(rows * g.query_heads * 4); + s.part_l = try A.take(rows * g.query_heads * 4); + s.part_rows = rows; + s.parts = b; + } + fn layout(s: *Scratch, g: c.Geometry, prompt_rows: usize, capacity: usize) !void { const R = @max(prompt_rows, round_rows); const W = round_rows; const bf = 2; - const chunks = (capacity + W + tri.chunk - 1) / tri.chunk; // attention.plan_host: a query tile's items per committed chunk and folded group const codes = capacity / tri.chunk + capacity / (tri.chunk * tri.group_chunks); const items = codes * ((W * (g.query_heads / g.kv_heads) + 15) / 16); s.rows = R; - s.part_chunks = chunks; s.arena.used = 0; const A = &s.arena; s.origin = try A.take(128); @@ -271,13 +318,11 @@ pub const Scratch = struct { s.attn_plan = try A.take((W + 4 + 3 * items + W + 12) * 4); // + each segment's 16-byte padding s.paths = try A.take(W * tri.max_nodes * 4); s.depths = try A.take(W * 4); - s.offs = try A.take(g.layers * max_streams * 2 * 8); - s.part_o = try A.take(chunks * W * g.query_heads * c.head_dim * 4); - s.part_m = try A.take(chunks * W * g.query_heads * 4); - s.part_l = try A.take(chunks * W * g.query_heads * 4); + s.offs = try A.take(g.layers * max_streams * 4 * 8); s.conv_tmp = try A.take((c.conv_taps - 1 + R) * g.convDim() * bf); s.conv_old = try A.take(max_streams * g.layers * (c.conv_taps - 1) * g.convDim() * bf); s.copy_items = try A.take(2 * copy_items * @sizeOf(kern.Copy)); + s.quant_items = try A.take(copy_items * @sizeOf(kern.Quant)); s.taps = try A.take(R * 5 * g.hidden * bf); s.replay_table = try A.take(g.layers * (4 + max_streams) * 8); s.replay_rows = try A.take(max_streams * (W + 1) * 4); @@ -301,6 +346,7 @@ pub const Scratch = struct { } pub fn deinit(s: *Scratch) void { + if (s.parts) |*b| b.free(); s.arena.deinit(); s.gpa.free(s.records); s.gpa.free(s.attn_records); diff --git a/zig/src/families/qwen27/cuda/triton.zig b/zig/src/families/qwen27/cuda/triton.zig index 5206ff382d..2ed4e7e748 100644 --- a/zig/src/families/qwen27/cuda/triton.zig +++ b/zig/src/families/qwen27/cuda/triton.zig @@ -118,6 +118,26 @@ pub const Tri = struct { p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), p("OUT", "*bf16", out), p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), int("W", w), }, &.{ ci("H", a.heads), ci("D", a.dim), ci("G", g), ci("DS", merge_columns), ci("CH", chunk), ci("GR", group_chunks) }); } + + /// attention_kv8.py's attention8: attention over e4m3 caches, `offs` holding each stream's four offsets (key + /// and value bytes, their scales' fp32 elements) from `base`; the window's own keys and values stay bf16. + pub fn attention8(t: Tri, q: u64, kn: u64, vn: u64, base: u64, offs: u64, plan: Plan, part: Partials, out: u64, a: Attn) !void { + const g = a.heads / a.kv_heads; + const scale: f32 = @floatCast(std.math.pow(f64, @floatFromInt(a.dim), -0.5)); + const w = plan.width; + if (plan.n_items > 0) try t.run("_shared8", .{ u(plan.n_items * a.kv_heads), 1, 1 }, &.{ + p("Q", "*bf16", q), p("KC", "*fp8e4nv", base), p("VC", "*fp8e4nv", base), p("KS", "*fp32", base), p("VS", "*fp32", base), p("OFF", "*i64", offs), + p("STREAM", "*i32", plan.streams), p("ITEMS", "*i32", plan.items), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), + }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), aot.cf("SCALE", scale), ci("GR", group_chunks) }); + const tails = 1 + (max_nodes + chunk - 1) / chunk; + try t.run("_tail8", .{ u(w), u(a.kv_heads), u(tails) }, &.{ + p("Q", "*bf16", q), p("KN", "*bf16", kn), p("VN", "*bf16", vn), p("KC", "*fp8e4nv", base), p("VC", "*fp8e4nv", base), p("KS", "*fp32", base), p("VS", "*fp32", base), p("OFF", "*i64", offs), + p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), p("PATHS", "*i32", plan.paths), p("DEPTHS", "*i32", plan.depths), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), + }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), ci("MAXD", max_nodes), aot.cf("SCALE", scale), ci("GR", group_chunks) }); + try t.run("_merge", .{ u(w), u(a.kv_heads), u(a.dim / merge_columns) }, &.{ + p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), p("OUT", "*bf16", out), p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), int("W", w), + }, &.{ ci("H", a.heads), ci("D", a.dim), ci("G", g), ci("DS", merge_columns), ci("CH", chunk), ci("GR", group_chunks) }); + } }; /// attention.groups: whole committed 2048-key groups fold in their own programs only when they fill the GPU. diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index ba1c01688a..03cb15e17b 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -435,6 +435,13 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O } family_options.vision = .{ .image_tokens = v.image_tokens, .max_images = v.max_images }; } + if (o.kv8) { + if (!@hasField(F.Options, "kv8")) { + problem.* = try std.fmt.allocPrint(a, "--kv-cache fp8: the native CUDA engine keeps fp8 caches for Qwen3.8-27B only, not {s} checkpoints", .{o.model_type}); + return null; + } + family_options.kv8 = true; + } const loaded = F.open(gpa, io, &g.ctx, o.dir, kernels, family_options) catch |e| { problem.* = try std.fmt.allocPrint(a, "the native CUDA engine cannot load {s} with {s} glue kernels ({s})", .{ o.dir, kernels orelse "its own", @errorName(e) }); return null; @@ -452,13 +459,17 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O }; return null; }; + if (@hasDecl(F, "fit")) F.fit(loaded.ctx, streams) catch |e| { + problem.* = try std.fmt.allocPrint(a, "the native CUDA engine cannot size its attention scratch for {d} stream{s} ({s})", .{ streams, if (streams == 1) "" else "s", @errorName(e) }); + return null; + }; const h = try gpa.create(Host); errdefer gpa.destroy(h); h.* = .{ .gpa = gpa, .gpu = g, .family = loaded.ctx, .release = loaded.deinit, .inner = loaded.backend, .vtable = undefined, .cfg = undefined, .clock = undefined, .core = undefined, .host = undefined, .lone = loaded.lone }; // prompt-cache snapshots are Nemotron's (cuda_reuse.zig); a family without them serves without a cache const cache = if (@hasDecl(F, "snap_bytes")) budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)) else 0; - h.startup = try std.fmt.allocPrint(gpa, "CUDA sm_{d} device {d} ({s}{s}): model {d:.2} GiB{s}; {d} stream{s} at once, {d:.2} GiB each at a {d}-token window, of {d:.1} GiB left after a {d:.1} GiB reserve; prompts in {d}-row chunks{s}{s}; {s}", .{ - capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else if (vision_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for the image tower's calls", .{toGib(vision_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", + h.startup = try std.fmt.allocPrint(gpa, "CUDA sm_{d} device {d} ({s}{s}): model {d:.2} GiB{s}; {d} stream{s} at once, {d:.2} GiB each at a {d}-token window{s}, of {d:.1} GiB left after a {d:.1} GiB reserve; prompts in {d}-row chunks{s}{s}; {s}", .{ + capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else if (vision_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for the image tower's calls", .{toGib(vision_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, if (o.kv8) " (fp8 caches)" else "", toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache", .{toGib(cache)}) else ", no prompt cache", if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", }); errdefer gpa.free(h.startup); diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index a8e0283e6c..282d6055d5 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -100,6 +100,10 @@ const Host = struct { /// The engine for `o.dir`, or null with `problem` set when no Metal engine reads the checkpoint. pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]const u8) !?api.Opened { const dense = std.mem.eql(u8, o.model_type, "qwen3_5") and qwen27Engine(a, io, o.dir); + if (o.kv8) { + problem.* = "--kv-cache fp8: the CUDA engine keeps fp8 caches; the Metal engines keep bf16"; + return null; + } if (o.drafter != null and !dense) { problem.* = "--drafter is supported for Qwen3.8-27B-class checkpoints (qwen3_5 with its own output head)"; return null; diff --git a/zig/src/server/cli.zig b/zig/src/server/cli.zig index 02f7578161..ad3d9ffbdb 100644 --- a/zig/src/server/cli.zig +++ b/zig/src/server/cli.zig @@ -50,6 +50,7 @@ pub const flags = [_]Flag{ .{ .name = "--metrics-open", .kind = .store_true, .native = true }, .{ .name = "--dashboard", .kind = .store_true, .native = true }, .{ .name = "--vision", .kind = .store_true, .native = true }, + .{ .name = "--kv-cache", .choices = &.{ "bf16", "fp8" }, .native = true }, .{ .name = "--vision-urls", .kind = .store_true, .native = true }, .{ .name = "--vision-offload", .kind = .store_true }, .{ .name = "--vision-max-images", .native = true }, @@ -167,6 +168,7 @@ pub const Args = struct { master: []const u8 = "", master_port: u16 = 29551, policy: []const u8 = "", // every --policy joined by commas + kv_cache: []const u8 = "bf16", // fp8: e4m3 attention caches, a scale a row (about twice the context) vision: bool = false, // image input (GLM-5.3-Flash): data-URL image_url parts in user messages vision_urls: bool = false, // with --vision, public HTTPS image URLs too (media_fetch.zig) vision_max_images: u32 = 4, @@ -277,7 +279,7 @@ fn apply(a: Allocator, out: *Args, name: []const u8, value: ?[]const u8, u: *Usa const n = try int(u, a, name, v); if (n < 0) return fail(u, a, "argument --compact-keep: expected a token count from 0: '{s}'", .{v}); out.compact_keep = @intCast(n); - } else if (is(name, "--compact-memory")) out.compact_memory = v else if (is(name, "--slide")) out.slide = true else if (is(name, "--slide-graph")) out.slide_graph = v else if (is(name, "--parallel")) out.parallel = v else if (is(name, "--backend")) out.backend = v else if (is(name, "--vision")) out.vision = true else if (is(name, "--vision-urls")) out.vision_urls = true else if (is(name, "--vision-max-images")) { + } else if (is(name, "--compact-memory")) out.compact_memory = v else if (is(name, "--slide")) out.slide = true else if (is(name, "--slide-graph")) out.slide_graph = v else if (is(name, "--parallel")) out.parallel = v else if (is(name, "--backend")) out.backend = v else if (is(name, "--vision")) out.vision = true else if (is(name, "--kv-cache")) out.kv_cache = v else if (is(name, "--vision-urls")) out.vision_urls = true else if (is(name, "--vision-max-images")) { const n = try int(u, a, name, v); if (n < 1 or n > 64) return fail(u, a, "argument --vision-max-images: expected 1 to 64 images: '{s}'", .{v}); out.vision_max_images = @intCast(n); @@ -377,6 +379,7 @@ pub fn request(a: Allocator, dir: []const u8, model_type: []const u8, args: Args .master = args.master, .master_port = args.master_port, .policy = args.policy, + .kv8 = std.mem.eql(u8, args.kv_cache, "fp8"), .vision = if (args.vision) .{ .image_tokens = args.vision_image_tokens, .max_images = args.vision_max_images } else null, }; } diff --git a/zig/tests/cuda/qwen27/attention_kv8.py b/zig/tests/cuda/qwen27/attention_kv8.py new file mode 100644 index 0000000000..386873d247 --- /dev/null +++ b/zig/tests/cuda/qwen27/attention_kv8.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""attention.attention over an fp8 (e4m3) key and value cache, for the native engine's --kv-cache fp8. + +The cache holds each (position, key head) row of 256 as e4m3 times one fp32 scale (the row's absolute max over 448). +_shared8 and _tail8 are attention.py's _shared and _tail with those loads: e4m3 widens to bf16 exactly, the key's +scale multiplies its scores and the value's scale its probabilities, so the dots stay bf16 tensor-core dots. The +window's own rows (KN, VN) are quantized in the kernel exactly as their commit will quantize them, so each key's +contribution is the same committed or not. Partials keep attention.py's layout, so its _merge folds them unchanged. + +``--out DIR`` checks both against attention.attention over the dequantized cache and records their launches for +triton_aot_manifest.py (one specialization each for W of 1, of a multiple of 16, and of anything else). +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import torch +import triton +import triton.language as tl + +FP8_MAX = 448.0 + + +@triton.jit +def _tile8(q, k, v, ks, vs, m, l, o, valid, scale: tl.constexpr): + scores = tl.dot(q, tl.trans(k)).to(tl.float32) * ks[None, :] * scale + scores = tl.where(valid[None, :], scores, float("-inf")) + tile_m = tl.max(scores, 1) + active = tile_m != float("-inf") + next_m = tl.where(active, tl.maximum(m, tile_m), m) + alpha = tl.where(active, tl.where(m == float("-inf"), 0.0, tl.exp(m - next_m)), 1.0) + p = tl.where(valid[None, :] & active[:, None], tl.exp(scores - next_m[:, None]), 0.0) + o = o * alpha[:, None] + tl.dot((p * vs[None, :]).to(tl.bfloat16), v) + l = l * alpha + tl.sum(p, 1) + return next_m, l, o + + +@triton.jit +def _fold8(m, l, o, cm, cl, co): + active = cl > 0.0 + next_m = tl.where(active, tl.maximum(m, cm), m) + a = tl.where(active, tl.where(m == float("-inf"), 0.0, tl.exp(m - next_m)), 1.0) + b = tl.where(active, tl.exp(cm - next_m), 0.0) + return next_m, l * a + cl * b, o * a[:, None] + co * b[:, None] + + +@triton.jit +def _q8(x): + """Rows of bf16 as the cache keeps them: e4m3 of x * (448 / amax) widened back to bf16, and s = amax / 448 + (kv_quant's rule: two IEEE divisions a row, one IEEE product an element).""" + + x = x.to(tl.float32) + amax = tl.max(tl.abs(x), 1) + s = tl.where(amax > 0, tl.math.div_rn(amax, 448.0), 1.0) + inv = tl.where(amax > 0, tl.math.div_rn(448.0, amax), 1.0) + return (x * inv[:, None]).to(tl.float8e4nv).to(tl.bfloat16), s + + +@triton.jit +def _shared8(Q, KC, VC, KS, VS, OFF, STREAM, ITEMS, PO, PM, PL, W, H: tl.constexpr, HK: tl.constexpr, + D: tl.constexpr, G: tl.constexpr, CH: tl.constexpr, SCALE: tl.constexpr, GR: tl.constexpr): + """_shared over the fp8 cache; OFF holds four offsets a stream: keys and values (bytes), their scales (fp32).""" + + item = tl.program_id(0) // HK + hk = tl.program_id(0) % HK + s = tl.load(ITEMS + item * 3) + first = tl.load(ITEMS + item * 3 + 1) + code = tl.load(ITEMS + item * 3 + 2) + start = tl.load(STREAM + s * 4) + rows = tl.load(STREAM + s * 4 + 1) + p = tl.load(STREAM + s * 4 + 2) + groups = ((p + rows + CH - 1) // CH - tl.load(STREAM + s * 4 + 3)) // (GR - 1) + whole = code >= 0 + chunk = tl.where(whole, code * GR, groups * GR - 1 - code) + n = tl.where(whole, GR, 1) + slot = tl.where(whole, code, chunk - groups * (GR - 1)) + if ((chunk + n) * CH <= p) & (code < groups): + koff = tl.multiple_of(tl.load(OFF + s * 4), 16) + voff = tl.multiple_of(tl.load(OFF + s * 4 + 1), 16) + ksoff = tl.load(OFF + s * 4 + 2) + vsoff = tl.load(OFF + s * 4 + 3) + rr = first + tl.arange(0, 16) + ok = rr < rows * G + node = start + rr // G + head = hk * G + rr % G + d = tl.arange(0, D) + q = tl.load(Q + (node[:, None] * H + head[:, None]) * D + d[None, :], mask=ok[:, None], + other=0).to(tl.bfloat16) + gm = tl.full((16,), float("-inf"), tl.float32) + gl = tl.zeros((16,), tl.float32) + go = tl.zeros((16, D), tl.float32) + m, l, o = gm, gl, go + for c in range(n): + key = (chunk + c) * CH + tl.arange(0, 64) + m = tl.full((16,), float("-inf"), tl.float32) + l = tl.zeros((16,), tl.float32) + o = tl.zeros((16, D), tl.float32) + for t in range(CH // 64): + ki = key + t * 64 + kk = tl.load(KC + koff + (ki[:, None] * HK + hk) * D + d[None, :]).to(tl.bfloat16) + vv = tl.load(VC + voff + (ki[:, None] * HK + hk) * D + d[None, :]).to(tl.bfloat16) + ks = tl.load(KS + ksoff + ki * HK + hk) + vs = tl.load(VS + vsoff + ki * HK + hk) + m, l, o = _tile8(q, kk, vv, ks, vs, m, l, o, ki < p, SCALE) + gm, gl, go = _fold8(gm, gl, go, m, l, o) + if not whole: + gm, gl, go = m, l, o + base = (slot * W + node) * H + head + tl.store(PO + base[:, None] * D + d[None, :], go, mask=ok[:, None]) + tl.store(PM + base, gm, mask=ok) + tl.store(PL + base, gl, mask=ok) + + +@triton.jit +def _tail8(Q, KN, VN, KC, VC, KS, VS, OFF, STREAM, ROWS, PATHS, DEPTHS, PO, PM, PL, W, H: tl.constexpr, + HK: tl.constexpr, D: tl.constexpr, G: tl.constexpr, CH: tl.constexpr, MAXD: tl.constexpr, + SCALE: tl.constexpr, GR: tl.constexpr): + """_tail over the fp8 cache; the row's own path quantized as its commit will, so a key's bits never depend on + whether it is committed or in the window (nor on drafts, or how streams share a round).""" + + node = tl.program_id(0) + hk = tl.program_id(1) + s = tl.load(ROWS + node) + p = tl.load(STREAM + s * 4 + 2) + nch = (p + tl.load(STREAM + s * 4 + 1) + CH - 1) // CH + groups = (nch - tl.load(STREAM + s * 4 + 3)) // (GR - 1) + chunk = p // CH + tl.program_id(2) + if chunk < nch: + koff = tl.multiple_of(tl.load(OFF + s * 4), 16) + voff = tl.multiple_of(tl.load(OFF + s * 4 + 1), 16) + ksoff = tl.load(OFF + s * 4 + 2) + vsoff = tl.load(OFF + s * 4 + 3) + gg = tl.arange(0, 16) + d = tl.arange(0, D) + q = tl.load(Q + (node * H + hk * G + gg[:, None]) * D + d[None, :], mask=gg[:, None] < G, other=0).to(tl.bfloat16) + depth = tl.load(DEPTHS + node) + m = tl.full((16,), float("-inf"), tl.float32) + l = tl.zeros((16,), tl.float32) + o = tl.zeros((16, D), tl.float32) + key = chunk * CH + tl.arange(0, 64) + for t in range(CH // 64): + logical = key + t * 64 + committed = logical < p + path_slot = logical - p + on_path = (path_slot >= 0) & (path_slot < depth) + path_node = tl.load(PATHS + node * MAXD + path_slot, mask=on_path, other=0) + kc = tl.load(KC + koff + (logical[:, None] * HK + hk) * D + d[None, :], mask=committed[:, None], other=0.0) + vc = tl.load(VC + voff + (logical[:, None] * HK + hk) * D + d[None, :], mask=committed[:, None], other=0.0) + ks = tl.load(KS + ksoff + logical * HK + hk, mask=committed, other=1.0) + vs = tl.load(VS + vsoff + logical * HK + hk, mask=committed, other=1.0) + kn = tl.zeros((64, D), tl.bfloat16) + vn = tl.zeros((64, D), tl.bfloat16) + kns = tl.full((64,), 1.0, tl.float32) + vns = tl.full((64,), 1.0, tl.float32) + if tl.max(on_path.to(tl.int32), 0) > 0: # a tile of committed keys only skips the quantizing + kn, kns = _q8(tl.load(KN + (path_node[:, None] * HK + hk) * D + d[None, :], mask=on_path[:, None], other=0)) + vn, vns = _q8(tl.load(VN + (path_node[:, None] * HK + hk) * D + d[None, :], mask=on_path[:, None], other=0)) + kk = tl.where(committed[:, None], kc.to(tl.bfloat16), kn) + vv = tl.where(committed[:, None], vc.to(tl.bfloat16), vn) + ks = tl.where(committed, ks, kns) + vs = tl.where(committed, vs, vns) + m, l, o = _tile8(q, kk, vv, ks, vs, m, l, o, committed | on_path, SCALE) + base = ((chunk - groups * (GR - 1)) * W + node) * H + hk * G + gg + tl.store(PO + base[:, None] * D + d[None, :], o, mask=gg[:, None] < G) + tl.store(PM + base, m, mask=gg < G) + tl.store(PL + base, l, mask=gg < G) + + +def quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """(n, HK, D) bf16 rows as e4m3 and one fp32 scale a row: the native kv_quant kernel's rule.""" + + amax = x.float().abs().amax(-1) + # tensor by tensor: IEEE division, as kv_quant's __fdiv_rn (torch divides by a Python float as a reciprocal) + full = torch.full_like(amax, FP8_MAX) + scale = torch.where(amax > 0, amax / full, torch.ones_like(amax)) + inv = torch.where(amax > 0, full / amax, torch.ones_like(amax)) + q = (x.float() * inv[..., None]).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q, scale + + +def attention8(q, kn, vn, origin, kc8, vc8, ks, vs, p: "A.Plan", scale: float): + """attention.attention with the fp8 cache of one stream (offsets from ``origin``, as the engine passes them).""" + + import tensorfold.cuda.kernels.attention as A + + w, h, d = q.shape + hk = kn.shape[1] + g = h // hk + o8 = origin.view(torch.float8_e4m3fn) + of = origin.view(torch.float32) + b = origin.data_ptr() + offs = torch.tensor([[kc8.data_ptr() - b, vc8.data_ptr() - b, (ks.data_ptr() - b) // 4, (vs.data_ptr() - b) // 4]], + dtype=torch.int64, device=q.device) + po = torch.empty((p.chunks, w, h, d), dtype=torch.float32, device=q.device) + pm = torch.empty((p.chunks, w, h), dtype=torch.float32, device=q.device) + pl = torch.empty_like(pm) + if p.items.shape[0]: + _shared8[(p.items.shape[0] * hk,)](q, o8, o8, of, of, offs, p.streams, p.items, po, pm, pl, w, H=h, HK=hk, D=d, + G=g, CH=A.CHUNK, SCALE=scale, GR=A.GROUP, num_warps=4, num_stages=1) + tails = 1 + -(-A.MAX_NODES // A.CHUNK) + _tail8[(w, hk, tails)](q, kn, vn, o8, o8, of, of, offs, p.streams, p.rows, p.paths, p.depths, po, pm, pl, w, H=h, + HK=hk, D=d, G=g, CH=A.CHUNK, MAXD=A.MAX_NODES, SCALE=scale, GR=A.GROUP, num_warps=4, + num_stages=1) + out = torch.empty_like(q) + A._merge[(w, hk, d // A.MERGE_COLUMNS)](po, pm, pl, out, p.streams, p.rows, w, H=h, D=d, G=g, DS=A.MERGE_COLUMNS, + CH=A.CHUNK, GR=A.GROUP, num_warps=4) + return out + + +def check(p_len: int, parents: list[int], dev) -> float: + """attention8 against attention.attention over the dequantized cache: the worst row's cosine.""" + + import tensorfold.cuda.kernels.attention as A + + torch.manual_seed(p_len * 131 + len(parents)) + h, hk, d = 24, 4, 256 + w = len(parents) + cap = p_len + 64 + k = torch.randn((cap, hk, d), device=dev).mul(3).to(torch.bfloat16) + v = torch.randn((cap, hk, d), device=dev).to(torch.bfloat16) + k8, ks = quantize(k) + v8, vs = quantize(v) + kd = (k8.float() * ks[..., None]).to(torch.bfloat16) + vd = (v8.float() * vs[..., None]).to(torch.bfloat16) + q = torch.randn((w, h, d), device=dev).to(torch.bfloat16) + kn = torch.randn((w, hk, d), device=dev).mul(3).to(torch.bfloat16) + vn = torch.randn((w, hk, d), device=dev).to(torch.bfloat16) + plan = A.plan([parents], [p_len], h // hk, dev) + scale = d ** -0.5 + kq, kqs = quantize(kn) + vq, vqs = quantize(vn) + knd = (kq.float() * kqs[..., None]).to(torch.bfloat16) + vnd = (vq.float() * vqs[..., None]).to(torch.bfloat16) + ref = A.attention(q, knd, vnd, torch.tensor([A.offsets([(kd, vd)], dev)], dtype=torch.int64, device=dev), plan, + scale=scale) + got = attention8(q, kn, vn, A.base(dev).view(torch.uint8), k8, v8, ks, vs, plan, scale) + cos = torch.nn.functional.cosine_similarity(ref.float().view(w * h, d), got.float().view(w * h, d), dim=1) + if parents == list(range(-1, w - 1)) and w > 1: + # a chain's first row committed instead: the other rows' outputs must keep their bits + k8b = torch.cat([k8[:p_len], kq[:1], k8[p_len + 1:]]) + ksb = torch.cat([ks[:p_len], kqs[:1], ks[p_len + 1:]]) + v8b = torch.cat([v8[:p_len], vq[:1], v8[p_len + 1:]]) + vsb = torch.cat([vs[:p_len], vqs[:1], vs[p_len + 1:]]) + plan_b = A.plan([list(range(-1, w - 2))], [p_len + 1], h // hk, dev) + got_b = attention8(q[1:].contiguous(), kn[1:].contiguous(), vn[1:].contiguous(), A.base(dev).view(torch.uint8), + k8b, v8b, ksb, vsb, plan_b, scale) + same = torch.equal(got[1:], got_b) + print(f" committed vs in-window keys: {'bit-identical' if same else 'DIFFER'}", flush=True) + if not same: + return 0.0 + return float(cos.min()) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--out", required=True) + ap.add_argument("--tools", required=True, help="folder of triton_aot_manifest.py") + a = ap.parse_args() + sys.path.insert(0, a.tools) + import triton_aot_manifest as aot + + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + rec = aot.Recorder().install() + rec.detail = True + dev = torch.device("cuda:0") + worst = 1.0 + # W of 1 (decode), 16 (a divisible verify window), 7 and 33 (neither); committed keys under and over a group + cases = [(0, [-1]), (700, [-1]), (5000, [-1]), (9000, list(range(-1, 15))), (3000, [-1, 0, 1, 1, 3, 4, 2]), + (20000, [-1] + list(range(32))), (300, list(range(-1, 15)))] + for p_len, parents in cases: + c = check(p_len, parents, dev) + worst = min(worst, c) + print(f"p={p_len} w={len(parents)}: worst row cosine {c:.6f}", flush=True) + rec.dump(out / "launches.json") + from triton.runtime.jit import JITFunction + import gc + + spec = {} + for fn in gc.get_objects(): + if isinstance(fn, JITFunction): + spec[f"{fn.fn.__module__}.{fn.fn.__qualname__}"] = { + "params": [x.name for x in fn.params], + "do_not_specialize": [x.name for x in fn.params if x.do_not_specialize], + "no_align": [x.name for x in fn.params if x.do_not_specialize_on_alignment]} + (out / "jit.json").write_text(json.dumps(spec, indent=1, sort_keys=True) + "\n") + print(f"worst {worst:.6f}") + return 0 if worst > 0.995 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 06cc8115601c9599598d6309d0124264a7bfec0c Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:42:06 +0000 Subject: [PATCH 10/19] qwen27 CUDA: a prompt cache, so a chat's next turn resumes its kept state The 27B serves the shared prompt cache (core/prompt_cache.zig) as Nemotron does, its copies in pinned host memory: a long window's caches leave the device no room for them. - reuse.zig: a kept state is a stream's DeltaNet recurrences and conv windows whole and its attention rows (and fp8 scales) through the mark, about 275 MiB at 4,096 tokens under fp8 - with a cache, prompts cut the zero-anchored 4,096-row grid (engine.Chunks), so a pass resumed or kept at a grid point computes what a fresh pass of the same prompt does; without one (--prompt-cache-gib 0, and tf-qwen27-cuda) the even splits stay, bit-exact against 0.6.6 - prefillFrom (engine, lone): a restored stream's pass from its grid point, stopping at the lane host's marks; a resumed drafting stream's context holds only the rows it read (drafts change no token) - image prompts neither resume nor keep (their rows' positions are the tower's) - native/cuda.zig: each family's snapshot vtable, and a host-memory budget (8 GiB or --prompt-cache-gib, at most half of MemAvailable) for a family whose copies live there - zig fmt over native/cuda.zig and qwen27/cuda/triton.zig (the fmt check) The qwen suite's soak check: 60/60 replies equal to the same request alone and 60/60 turns resumed (0/60 resumed before), in 117 s against 244 s; drafts, concurrent and behaviour unchanged. --- zig/src/families/qwen27/cuda.zig | 2 + zig/src/families/qwen27/cuda/engine.zig | 70 +++++++++++-- zig/src/families/qwen27/cuda/lanes.zig | 36 ++++++- zig/src/families/qwen27/cuda/lone.zig | 23 +++-- zig/src/families/qwen27/cuda/native.zig | 20 +++- zig/src/families/qwen27/cuda/reuse.zig | 126 ++++++++++++++++++++++++ zig/src/families/qwen27/cuda/triton.zig | 6 +- zig/src/native/cuda.zig | 34 ++++--- 8 files changed, 283 insertions(+), 34 deletions(-) create mode 100644 zig/src/families/qwen27/cuda/reuse.zig diff --git a/zig/src/families/qwen27/cuda.zig b/zig/src/families/qwen27/cuda.zig index 650f1748ef..3e10f18896 100644 --- a/zig/src/families/qwen27/cuda.zig +++ b/zig/src/families/qwen27/cuda.zig @@ -16,6 +16,7 @@ pub const lone = @import("cuda/lone.zig"); pub const drafts = @import("cuda/drafts.zig"); pub const vision = @import("cuda/vision.zig"); pub const image = @import("cuda/image.zig"); +pub const reuse = @import("cuda/reuse.zig"); test { _ = config; @@ -34,4 +35,5 @@ test { _ = native; _ = vision; _ = image; + _ = reuse; } diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig index de2c8ddd5c..af3c6594b4 100644 --- a/zig/src/families/qwen27/cuda/engine.zig +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -20,6 +20,45 @@ pub fn chunkBounds(start: usize, end: usize, size: usize, j: usize) [2]usize { return .{ start + (end - start) * j / n, start + (end - start) * (j + 1) / n }; } +/// A prompt's chunks from `from`: even splits of the whole prompt (0.6.6's PrefillPlan), or under a prompt cache the +/// zero-anchored grid, so a pass resumed or kept at a grid point cuts where a fresh pass of the same prompt does. +pub const Chunks = struct { + len: usize, + grid: bool, + at: usize, + j: usize = 0, + + pub fn init(len: usize, from: usize, grid: bool) Chunks { + return .{ .len = len, .grid = grid, .at = from }; + } + + pub fn next(x: *Chunks) ?[2]usize { + if (x.at >= x.len) return null; + const b: [2]usize = if (x.grid) .{ x.at, @min(x.len, x.at - x.at % prompt_rows + prompt_rows) } else chunkBounds(0, x.len, prompt_rows, x.j); + x.at = b[1]; + x.j += 1; + return b; + } +}; + +/// A state the cache may resume: a grid point inside the prompt. +pub fn resumable(from: usize, len: usize) bool { + return from > 0 and from < len and from % prompt_rows == 0; +} + +/// Where a prompt pass stops to let the cache copy its state (ascending grid points), and who copies it. +pub const Keep = struct { + at: []const u32, + ctx: *anyopaque, + call: *const fn (ctx: *anyopaque, at: u32) void, + + /// Call the keep for a chunk that ended at `end`. + pub fn after(k: ?Keep, end: usize) void { + const x = k orelse return; + for (x.at) |m| if (m == end) x.call(x.ctx, m); + } +}; + pub const Options = struct { context: usize, kv8: bool = false }; pub const Engine = struct { @@ -31,6 +70,7 @@ pub const Engine = struct { scratch: st.Scratch, context: usize, kv8: bool, // --kv-cache fp8: every stream's attention caches e4m3 with a scale a row + grid: bool = false, // a prompt cache: prompts in zero-anchored chunks (Chunks), not even splits pub fn init(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: []const u8, o: Options) !*Engine { const cfg = try c.Config.read(gpa, io, dir); @@ -87,21 +127,25 @@ pub const Engine = struct { /// prefill with an image prompt's rows and positions (null: text); its decode offset stays on the stream. pub fn prefillMedia(e: *Engine, seq: *st.Seq, prompt: []const u32, media: ?*const vision.Media) !void { + return e.prefillFrom(seq, prompt, 0, media, null); + } + + /// The prompt from `from` (0, or a grid point the stream's state was restored at), keeping states at `keep`'s marks. + pub fn prefillFrom(e: *Engine, seq: *st.Seq, prompt: []const u32, from: usize, media: ?*const vision.Media, keep: ?Keep) !void { if (prompt.len == 0) return error.EmptyPrompt; if (prompt.len >= seq.capacity) return error.PromptTooLong; - try seq.reset(e.ops(), e.w.g); + if (from == 0) try seq.reset(e.ops(), e.w.g) else if (from != seq.pos or !resumable(from, prompt.len)) return error.BadReuse; seq.media = media; defer seq.media = null; var f = e.forward(); - const n = (prompt.len + prompt_rows - 1) / prompt_rows; - for (0..n) |j| { - const b = chunkBounds(0, prompt.len, prompt_rows, j); - try f.chunk(prompt[b[0]..b[1]], seq, j == n - 1); + var it = Chunks.init(prompt.len, from, e.grid); + while (it.next()) |b| { + try f.chunk(prompt[b[0]..b[1]], seq, b[1] == prompt.len); + Keep.after(keep, b[1]); } if (media) |m| seq.rope_delta = m.delta; } - /// decode.serial_decode, greedy: one-row windows from `pending` until `count` tokens or an end token. pub fn decode(e: *Engine, seq: *st.Seq, pending: u32, count: usize, out: *std.ArrayList(u32)) !void { var f = e.forward(); try out.append(e.gpa, pending); @@ -128,6 +172,20 @@ pub const Engine = struct { } }; +test "a cached prompt cuts the zero-anchored grid, resumed or not" { + var it = Chunks.init(9000, 0, true); + try std.testing.expectEqual([2]usize{ 0, 4096 }, it.next().?); + try std.testing.expectEqual([2]usize{ 4096, 8192 }, it.next().?); + try std.testing.expectEqual([2]usize{ 8192, 9000 }, it.next().?); + try std.testing.expectEqual(@as(?[2]usize, null), it.next()); + var resumed = Chunks.init(9000, 4096, true); + try std.testing.expectEqual([2]usize{ 4096, 8192 }, resumed.next().?); + var even = Chunks.init(5078, 0, false); + try std.testing.expectEqual([2]usize{ 0, 2539 }, even.next().?); + try std.testing.expectEqual([2]usize{ 2539, 5078 }, even.next().?); + try std.testing.expect(resumable(4096, 5000) and !resumable(4096, 4096) and !resumable(3000, 5000) and !resumable(0, 5000)); +} + test "prefill.chunks splits evenly" { try std.testing.expectEqual([2]usize{ 0, 2539 }, chunkBounds(0, 5078, 4096, 0)); try std.testing.expectEqual([2]usize{ 2539, 5078 }, chunkBounds(0, 5078, 4096, 1)); diff --git a/zig/src/families/qwen27/cuda/lanes.zig b/zig/src/families/qwen27/cuda/lanes.zig index d88d43baf6..adae5d4913 100644 --- a/zig/src/families/qwen27/cuda/lanes.zig +++ b/zig/src/families/qwen27/cuda/lanes.zig @@ -7,7 +7,9 @@ const st = @import("state.zig"); const Engine = @import("engine.zig").Engine; const Part = @import("forward.zig").Part; const Forward = @import("forward.zig").Forward; -const prompt_rows = @import("engine.zig").prompt_rows; +const eng = @import("engine.zig"); +const prompt_rows = eng.prompt_rows; +const reuse = @import("reuse.zig"); const dl = @import("draft_load.zig"); const dr = @import("draft.zig"); const drafts = @import("drafts.zig"); @@ -228,7 +230,8 @@ pub const Cuda = struct { try x.prefillOpened(s, seq); } - /// prefill_state into an opened stream: the prompt in chunks, a drafting stream's last window of taps absorbed. + /// prefill_state into an opened stream: the prompt in chunks, a drafting stream's last window of taps absorbed. A + /// kept state the lane host found is restored first, and the pass stops at its marks for the cache to copy. fn prefillOpened(x: *Cuda, s: *lanes.Stream, seq: *st.Seq) !void { // an image prompt: the tower's rows and the prompt's three positions for its pass (EncodedVision) const media: ?*vision.Media = if (s.images.len == 0) null else blk: { @@ -237,11 +240,36 @@ pub const Cuda = struct { break :blk try vision.Media.init(x.gpa, tower, gpu, s.prompt(), s.images, prompt_rows); }; defer if (media) |m| m.deinit(); + const from = x.restoreKept(s, seq, media != null); + // an image prompt keeps nothing: its rows' positions are the tower's, not the tokens' + const marks: ?eng.Keep = if (s.reuse.marks.len == 0 or media != null) null else .{ .at = s.reuse.marks, .ctx = s, .call = keptAt }; if (x.drafter) |d| if (s.drafts) { const ds = try x.draftingOf(s); - return lone.prefillMedia(x.e, seq, &ds.ctx, d, s.prompt(), media); + return lone.prefillFrom(x.e, seq, &ds.ctx, d, s.prompt(), from, media, marks); }; - try x.e.prefillMedia(seq, s.prompt(), media); + try x.e.prefillFrom(seq, s.prompt(), from, media, marks); + } + + /// The stream's kept state restored on its sequence: where its prompt pass starts (0: none, off the grid, failed). + fn restoreKept(x: *Cuda, s: *lanes.Stream, seq: *st.Seq, image: bool) usize { + s.cached = 0; + const saved = s.reuse.saved orelse return 0; + if (image or !x.e.grid or !eng.resumable(s.reuse.at, s.prompt().len)) { + s.reuse_failed = true; + return 0; + } + reuse.restoreSeq(x.e, seq, saved) catch { + s.reuse_failed = true; + return 0; + }; + s.cached = s.reuse.at; + return s.reuse.at; + } + + /// The prompt pass stands at a mark: the lane host copies the stream's state there. + fn keptAt(ctx: *anyopaque, at: u32) void { + const s: *lanes.Stream = @ptrCast(@alignCast(ctx)); + if (s.reuse.hook) |k| k.at(k.ptr, s, at); } /// prefill_batch: prompts in shared passes of at most a chunk's rows (a longer one alone, in its chunks). diff --git a/zig/src/families/qwen27/cuda/lone.zig b/zig/src/families/qwen27/cuda/lone.zig index 6aa91e3fe3..bb384e9ce4 100644 --- a/zig/src/families/qwen27/cuda/lone.zig +++ b/zig/src/families/qwen27/cuda/lone.zig @@ -8,7 +8,8 @@ const tree = @import("tree.zig"); const kern = @import("kernels.zig"); const dr = @import("draft.zig"); const dl = @import("draft_load.zig"); -const Engine = @import("engine.zig").Engine; +const eng = @import("engine.zig"); +const Engine = eng.Engine; const Forward = @import("forward.zig").Forward; const Part = @import("forward.zig").Part; const Lanes = @import("lanes.zig").Cuda; @@ -77,7 +78,14 @@ pub fn prefill(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, promp /// prefill with an image prompt's rows and positions (null: text); its decode offset stays on the stream. pub fn prefillMedia(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, prompt: []const u32, media: ?*const @import("vision.zig").Media) !void { - try seq.reset(e.ops(), e.w.g); + return prefillFrom(e, seq, ctx, d, prompt, 0, media, null); +} + +/// prefillMedia from `from` (0, or the grid point a kept state was restored at), keeping states at `keep`'s marks. A +/// resumed pass's context holds only the rows it read: drafts change no token, so a reply is the same either way. +pub fn prefillFrom(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, prompt: []const u32, from: usize, media: ?*const @import("vision.zig").Media, keep: ?eng.Keep) !void { + if (prompt.len >= seq.capacity) return error.PromptTooLong; + if (from == 0) try seq.reset(e.ops(), e.w.g) else if (from != seq.pos or !eng.resumable(from, prompt.len)) return error.BadReuse; seq.media = media; defer seq.media = null; defer if (media) |m| { @@ -85,15 +93,14 @@ pub fn prefillMedia(e: *Engine, seq: *st.Seq, ctx: *dr.Context, d: *dr.DFlash2, }; ctx.len = 0; ctx.end = 0; - const tap_from = prompt.len -| dl.window; + const tap_from = @max(from, prompt.len -| dl.window); ctx.skip(tap_from); var f = e.forward(); f.taps = e.scratch.taps; - const rows = @import("engine.zig").prompt_rows; - const chunks = (prompt.len + rows - 1) / rows; - for (0..chunks) |j| { - const b = @import("engine.zig").chunkBounds(0, prompt.len, rows, j); - try f.chunk(prompt[b[0]..b[1]], seq, j == chunks - 1); + var it = eng.Chunks.init(prompt.len, from, e.grid); + while (it.next()) |b| { + try f.chunk(prompt[b[0]..b[1]], seq, b[1] == prompt.len); + eng.Keep.after(keep, b[1]); if (b[1] <= tap_from) continue; const skip = tap_from -| b[0]; try d.absorb(ctx, e.scratch.taps + skip * taps_width * bf, b[1] - b[0] - skip); diff --git a/zig/src/families/qwen27/cuda/native.zig b/zig/src/families/qwen27/cuda/native.zig index 32833fe00c..0add28111d 100644 --- a/zig/src/families/qwen27/cuda/native.zig +++ b/zig/src/families/qwen27/cuda/native.zig @@ -10,6 +10,7 @@ const Lanes = @import("lanes.zig").Cuda; const dr = @import("draft.zig"); const vision = @import("vision.zig"); const image = @import("image.zig"); +const reuse = @import("reuse.zig"); pub const model_type = "qwen3_5"; pub const formats: []const []const u8 = &.{"mlx-q4g64"}; @@ -17,6 +18,20 @@ pub const default_context: i64 = 32768; pub const max_segments: u32 = 1; pub const prompt_rows: u32 = engine.prompt_rows; +/// Prompt-cache copies (reuse.zig), kept in pinned host memory; a state ends where its prompt pass stands. +pub const snap_bytes = reuse.bytes; +pub const snap_save = reuse.save; +pub const snap_restore = reuse.restore; +pub const snap_drop = reuse.drop; +pub const snap_host = true; +pub const snap_lookahead: u32 = 0; + +/// The server keeps prompt states: prompts cut the zero-anchored grid, where kept states resume. +pub fn useCache(p: *anyopaque) void { + const own: *Owned = @ptrCast(@alignCast(p)); + own.e.grid = true; +} + pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, drafter: ?[]const u8 = null, vision: ?Vision = null, kv8: bool = false }; /// --vision's limits: the visual tokens a request's images share, and its images. @@ -50,9 +65,10 @@ pub const Loaded = struct { ctx: *anyopaque, deinit: *const fn (*anyopaque) void, lone: ?LoneRun = null, + target: *reuse.Target, // what the prompt cache copies (reuse.zig) }; -const Owned = struct { gpa: std.mem.Allocator, e: *engine.Engine, lanes: Lanes, draft: ?*dr.DFlash2 = null, tower: ?*vision.Tower = null }; +const Owned = struct { gpa: std.mem.Allocator, e: *engine.Engine, lanes: Lanes, target: reuse.Target = undefined, draft: ?*dr.DFlash2 = null, tower: ?*vision.Tower = null }; /// The drafter the Python engine pairs with the 27B once pulled (z-lab/Qwen3.8-27B-DFlash2). const drafter_repo = "models--z-lab--Qwen3.8-27B-DFlash2"; @@ -89,6 +105,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [ errdefer gpa.destroy(own); own.* = .{ .gpa = gpa, .e = e, .lanes = try Lanes.init(gpa, e, most_streams) }; errdefer own.lanes.deinit(); + own.target = .{ .e = e, .seqs = &own.lanes.seqs }; if (o.drafts) if (try drafterDir(gpa, io, o.drafter)) |draft_dir| { defer gpa.free(draft_dir); const f = e.forward(); @@ -115,6 +132,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [ .vision_bytes = vision_bytes, .ctx = own, .deinit = release, + .target = &own.target, }; } diff --git a/zig/src/families/qwen27/cuda/reuse.zig b/zig/src/families/qwen27/cuda/reuse.zig new file mode 100644 index 0000000000..750569c852 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reuse.zig @@ -0,0 +1,126 @@ +//! Qwen3.8-27B CUDA prompt states for the shared cache: a stream's DeltaNet recurrences, conv windows and its attention +//! rows so far, copied to pinned host memory (a long window's caches leave the device no room for copies). + +const std = @import("std"); +const cuda = @import("cuda"); +const lanes = @import("lanes"); +const c = @import("shape.zig"); +const st = @import("state.zig"); +const Engine = @import("engine.zig").Engine; + +/// One kept state: the copy and the position it holds. +pub const Copy = struct { + host: cuda.HostBuffer, + at: usize, +}; + +/// The engine and the streams' states a save copies. The native host keeps one of these for the loaded model. +pub const Target = struct { + e: *Engine, + seqs: *std.AutoHashMapUnmanaged(*lanes.Stream, *st.Seq), +}; + +/// One piece of a stream's state: its device address and length. +const Piece = struct { dev: u64, len: usize }; + +/// The pieces a state at `at` holds, in their order in a copy: each DeltaNet layer's recurrence and conv window, each +/// attention layer's first `at` key and value rows (and their scales under kv8). `seq` null: lengths only. +fn pieces(g: c.Geometry, kv8: bool, seq: ?*const st.Seq, at: usize, out: []Piece) []Piece { + var n: usize = 0; + const row: usize = g.kvInner() * @as(usize, if (kv8) 1 else 2); + for (0..g.layers) |i| { + if (c.linear(i)) { + out[n] = .{ .dev = if (seq) |s| s.rec[i] else 0, .len = g.deltaBytes() }; + out[n + 1] = .{ .dev = if (seq) |s| s.conv[i] else 0, .len = g.convBytes() }; + n += 2; + continue; + } + out[n] = .{ .dev = if (seq) |s| s.keys[i] else 0, .len = at * row }; + out[n + 1] = .{ .dev = if (seq) |s| s.values[i] else 0, .len = at * row }; + n += 2; + if (kv8) { + out[n] = .{ .dev = if (seq) |s| s.kscale[i] else 0, .len = at * g.kv_heads * 4 }; + out[n + 1] = .{ .dev = if (seq) |s| s.vscale[i] else 0, .len = at * g.kv_heads * 4 }; + n += 2; + } + } + return out[0..n]; +} + +const max_pieces = 4 * 128; + +fn total(g: c.Geometry, kv8: bool, at: usize) u64 { + var buf: [max_pieces]Piece = undefined; + var n: u64 = 0; + for (pieces(g, kv8, null, at, &buf)) |p| n += p.len; + return n; +} + +/// Host bytes a save at `at` takes. +pub fn bytes(ptr: *anyopaque, at: u32) u64 { + const t: *Target = @ptrCast(@alignCast(ptr)); + return total(t.e.w.g, t.e.kv8, at); +} + +/// Copy the stream's state after `at` prompt tokens. Its pass stands there. +pub fn save(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque { + const t: *Target = @ptrCast(@alignCast(ptr)); + const s: *lanes.Stream = @ptrCast(@alignCast(owner orelse return error.NoStream)); + const seq = t.seqs.get(s) orelse return error.UnknownStream; + if (seq.pos != at or seq.rope_delta != 0) return error.NotAtMark; + const e = t.e; + const copy = try e.gpa.create(Copy); + errdefer e.gpa.destroy(copy); + copy.* = .{ .host = try cuda.HostBuffer.alloc(e.ctx.d, total(e.w.g, e.kv8, at)), .at = at }; + errdefer copy.host.free(); + var buf: [max_pieces]Piece = undefined; + var off: usize = 0; + for (pieces(e.w.g, e.kv8, seq, at, &buf)) |p| { + try e.ops().download(copy.host.bytes[off..][0..p.len], p.dev); + off += p.len; + } + try e.s.synchronize(); + return copy; +} + +/// Make `seq` the kept state's: the next prompt chunk starts at its position. +pub fn restoreSeq(e: *Engine, seq: *st.Seq, saved: *anyopaque) !void { + const copy: *Copy = @ptrCast(@alignCast(saved)); + if (copy.at >= seq.capacity) return error.BadReuse; + var buf: [max_pieces]Piece = undefined; + var off: usize = 0; + for (pieces(e.w.g, e.kv8, seq, copy.at, &buf)) |p| { + try e.ops().upload(p.dev, copy.host.bytes[off..][0..p.len]); + off += p.len; + } + try e.s.synchronize(); // the host copy may be dropped once this returns + seq.pos = copy.at; + seq.rope_delta = 0; + seq.media = null; +} + +/// The store's restore: the owner's state made the kept one's. +pub fn restore(ptr: *anyopaque, owner: ?*anyopaque, saved: *anyopaque) anyerror!void { + const t: *Target = @ptrCast(@alignCast(ptr)); + const s: *lanes.Stream = @ptrCast(@alignCast(owner orelse return error.NoStream)); + return restoreSeq(t.e, t.seqs.get(s) orelse return error.UnknownStream, saved); +} + +/// Free a kept state's host copy. +pub fn drop(ptr: *anyopaque, saved: *anyopaque) void { + const t: *Target = @ptrCast(@alignCast(ptr)); + const copy: *Copy = @ptrCast(@alignCast(saved)); + copy.host.free(); + t.e.gpa.destroy(copy); +} + +test "a kept state holds the recurrences whole and the attention rows it read" { + const g = c.the_27b; + var buf: [max_pieces]Piece = undefined; + try std.testing.expectEqual(@as(usize, 48 * 2 + 16 * 2), pieces(g, false, null, 1, &buf).len); + try std.testing.expectEqual(@as(usize, 48 * 2 + 16 * 4), pieces(g, true, null, 1, &buf).len); + const fixed = total(g, false, 0); + try std.testing.expectEqual(fixed, 48 * (g.deltaBytes() + g.convBytes())); + try std.testing.expectEqual(fixed + 16 * 2 * 1000 * g.kvInner() * 2, total(g, false, 1000)); + try std.testing.expectEqual(fixed + 16 * 2 * 1000 * (g.kvInner() + g.kv_heads * 4), total(g, true, 1000)); +} diff --git a/zig/src/families/qwen27/cuda/triton.zig b/zig/src/families/qwen27/cuda/triton.zig index 2ed4e7e748..fcab03c8ef 100644 --- a/zig/src/families/qwen27/cuda/triton.zig +++ b/zig/src/families/qwen27/cuda/triton.zig @@ -126,13 +126,13 @@ pub const Tri = struct { const scale: f32 = @floatCast(std.math.pow(f64, @floatFromInt(a.dim), -0.5)); const w = plan.width; if (plan.n_items > 0) try t.run("_shared8", .{ u(plan.n_items * a.kv_heads), 1, 1 }, &.{ - p("Q", "*bf16", q), p("KC", "*fp8e4nv", base), p("VC", "*fp8e4nv", base), p("KS", "*fp32", base), p("VS", "*fp32", base), p("OFF", "*i64", offs), + p("Q", "*bf16", q), p("KC", "*fp8e4nv", base), p("VC", "*fp8e4nv", base), p("KS", "*fp32", base), p("VS", "*fp32", base), p("OFF", "*i64", offs), p("STREAM", "*i32", plan.streams), p("ITEMS", "*i32", plan.items), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), aot.cf("SCALE", scale), ci("GR", group_chunks) }); const tails = 1 + (max_nodes + chunk - 1) / chunk; try t.run("_tail8", .{ u(w), u(a.kv_heads), u(tails) }, &.{ - p("Q", "*bf16", q), p("KN", "*bf16", kn), p("VN", "*bf16", vn), p("KC", "*fp8e4nv", base), p("VC", "*fp8e4nv", base), p("KS", "*fp32", base), p("VS", "*fp32", base), p("OFF", "*i64", offs), - p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), p("PATHS", "*i32", plan.paths), p("DEPTHS", "*i32", plan.depths), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), + p("Q", "*bf16", q), p("KN", "*bf16", kn), p("VN", "*bf16", vn), p("KC", "*fp8e4nv", base), p("VC", "*fp8e4nv", base), p("KS", "*fp32", base), p("VS", "*fp32", base), p("OFF", "*i64", offs), + p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), p("PATHS", "*i32", plan.paths), p("DEPTHS", "*i32", plan.depths), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), ci("MAXD", max_nodes), aot.cf("SCALE", scale), ci("GR", group_chunks) }); try t.run("_merge", .{ u(w), u(a.kv_heads), u(a.dim / merge_columns) }, &.{ p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), p("OUT", "*bf16", out), p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), int("W", w), diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 03cb15e17b..d089a10e8b 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -13,13 +13,12 @@ const Pool = budget.Pool; /// CUDA families provide metadata, open their lane backend and explain their refusals. const registry = .{ nemotron.native, qwen27.native }; -/// Nemotron's prompt-cache copies. A file-level value so the store's pointer stays valid for the process. -const nemotron_snaps: api.prompt_cache.Snapshots.VTable = .{ - .bytes = nemotron.native.snap_bytes, - .save = nemotron.native.snap_save, - .restore = nemotron.native.snap_restore, - .drop = nemotron.native.snap_drop, -}; +/// A family's prompt-cache copies. A comptime value per family, so the store's pointer stays valid for the process. +fn Snaps(comptime F: type) type { + return struct { + const vtable: api.prompt_cache.Snapshots.VTable = .{ .bytes = F.snap_bytes, .save = F.snap_save, .restore = F.snap_restore, .drop = F.snap_drop }; + }; +} pub const backends: []const []const u8 = &.{"cuda"}; pub const families: []const api.Family = blk: { @@ -466,19 +465,22 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O const h = try gpa.create(Host); errdefer gpa.destroy(h); h.* = .{ .gpa = gpa, .gpu = g, .family = loaded.ctx, .release = loaded.deinit, .inner = loaded.backend, .vtable = undefined, .cfg = undefined, .clock = undefined, .core = undefined, .host = undefined, .lone = loaded.lone }; - // prompt-cache snapshots are Nemotron's (cuda_reuse.zig); a family without them serves without a cache - const cache = if (@hasDecl(F, "snap_bytes")) budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)) else 0; + // a family with prompt-cache snapshots keeps them in device room (Nemotron) or in pinned host memory (the 27B, whose + // long window leaves the device none); a family without them serves without a cache + const cache = if (!@hasDecl(F, "snap_bytes")) 0 else if (@hasDecl(F, "snap_host")) hostCacheBytes(a, io, o.prompt_cache_gib) else budget.cacheBytes(o.prompt_cache_gib, room, loaded.stream_bytes * (streams -| free)); h.startup = try std.fmt.allocPrint(gpa, "CUDA sm_{d} device {d} ({s}{s}): model {d:.2} GiB{s}; {d} stream{s} at once, {d:.2} GiB each at a {d}-token window{s}, of {d:.1} GiB left after a {d:.1} GiB reserve; prompts in {d}-row chunks{s}{s}; {s}", .{ - capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else if (vision_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for the image tower's calls", .{toGib(vision_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, if (o.kv8) " (fp8 caches)" else "", toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", - if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache", .{toGib(cache)}) else ", no prompt cache", if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", + capability, device, name, if (after.unified) ", memory shared with the host" else "", toGib(model), if (loaded.learn_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for --slide's trainer", .{toGib(loaded.learn_bytes)}) else if (vision_bytes > 0) try std.fmt.allocPrint(a, " and {d:.2} GiB kept for the image tower's calls", .{toGib(vision_bytes)}) else "", streams, if (streams == 1) "" else "s", toGib(loaded.stream_bytes), window, if (o.kv8) " (fp8 caches)" else "", toGib(room), toGib(after.reserve), F.prompt_rows, if (segments > 1) try std.fmt.allocPrint(a, ", {d} staggered segments a call", .{segments}) else "", + if (cache > 0) try std.fmt.allocPrint(a, ", a {d:.1} GiB prompt cache{s}", .{ toGib(cache), if (@hasDecl(F, "snap_host")) " in host memory" else "" }) else ", no prompt cache", if (kernels) |dir| try std.fmt.allocPrint(a, "glue kernels captured at {s}", .{dir}) else "own glue kernels", }); errdefer gpa.free(h.startup); if (@hasDecl(F, "snap_bytes") and cache > 0) { const pc = api.prompt_cache; const store = try gpa.create(pc.Store); // states kept and resumed on the family's grid, where a cache-off pass cuts; the head reads one token on - store.* = pc.Store.init(gpa, .{ .ptr = loaded.target, .vtable = &nemotron_snaps }, .{ .lookahead = 1, .planned = true, .grid = F.prompt_rows }, cache); + // Nemotron's head reads one token on (lookahead 1); the 27B's states end where its prompt pass stands + store.* = pc.Store.init(gpa, .{ .ptr = loaded.target, .vtable = &Snaps(F).vtable }, .{ .lookahead = if (@hasDecl(F, "snap_lookahead")) F.snap_lookahead else 1, .planned = true, .grid = F.prompt_rows }, cache); h.store = store; + if (@hasDecl(F, "useCache")) F.useCache(loaded.ctx); } errdefer if (h.store) |store| { store.deinit(); @@ -522,6 +524,14 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; } +/// A host-memory prompt cache: --prompt-cache-gib, else 8 GiB, at most half the host's available memory. +fn hostCacheBytes(a: Allocator, io: std.Io, asked: ?f64) u64 { + const want: u64 = if (asked) |g| (if (g > 0) std.math.lossyCast(u64, g * gib) else 0) else 8 << 30; + const text = procText(a, io, "/proc/meminfo") orelse return 0; + const m = budget.meminfo(text) orelse return 0; + return @min(want, m.available / 2); +} + fn toGib(bytes: u64) f64 { return @as(f64, @floatFromInt(bytes)) / gib; } From d49a906ffc168d707f62be598d329bf8dd8f5e2d Mon Sep 17 00:00:00 2001 From: BobClawblaw <288991054+BobClawblaw@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:08:50 +0000 Subject: [PATCH 11/19] qwen27 CUDA: no tooling that imports the Python engine in the tree attention_kv8.py (which records the fp8 attention launches) and qwen_vision_fixtures.py (which records the vision fixtures) import tensorfold, so they move out of the tree; their recordings stay fixture files, made once outside it. --- tools/zig/qwen_vision_fixtures.py | 103 --------- zig/src/cli/qwen27_cuda.zig | 5 +- zig/src/families/qwen27/cuda/triton.zig | 5 +- zig/tests/cuda/qwen27/attention_kv8.py | 296 ------------------------ 4 files changed, 6 insertions(+), 403 deletions(-) delete mode 100644 tools/zig/qwen_vision_fixtures.py delete mode 100644 zig/tests/cuda/qwen27/attention_kv8.py diff --git a/tools/zig/qwen_vision_fixtures.py b/tools/zig/qwen_vision_fixtures.py deleted file mode 100644 index 5f989a42ef..0000000000 --- a/tools/zig/qwen_vision_fixtures.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -"""Fixtures for the native Qwen vision path, from TensorFold 0.6.6's --vision frontend and tower on CUDA. - -For a fixed image set: each image's file, its decoded RGB (images.py), its patches and grid (QwenImageProcessor's -processor at 0.6.6's per-image cap), and the tower's merged rows (QwenCudaVision, bf16, flash attention). The Zig -checks compare their own decode, patches and rows against these: ``tf-qwen27-cuda vision MODEL --fixtures DIR``. -""" - -from __future__ import annotations - -import argparse -import io -import json -from pathlib import Path - -import numpy as np - - -def images(): - """(name, encoded bytes) for the set: PNG layouts, sizes from tiny to large, an EXIF-rotated JPEG.""" - from PIL import Image, ImageDraw - - out = [] - shapes = Image.new("RGB", (517, 389), "white") - d = ImageDraw.Draw(shapes) - d.ellipse((30, 40, 230, 240), fill=(220, 20, 20)) - d.rectangle((280, 60, 490, 260), fill=(20, 60, 220)) - d.text((60, 300), "ORANGE 7351", fill="black") - rng = np.random.default_rng(7) - y, x = np.mgrid[0:768, 0:1024] - photo = np.stack([(x / 4) % 256, (y / 3) % 256, ((x + y) / 7) % 256], -1) + rng.normal(0, 12, (768, 1024, 3)) - photo = Image.fromarray(np.clip(photo, 0, 255).astype(np.uint8)) - large = photo.resize((2000, 1500)) - tiny = shapes.resize((20, 15)) - logo = Image.new("RGBA", (96, 96), (0, 0, 0, 0)) - ImageDraw.Draw(logo).ellipse((8, 8, 88, 88), fill=(30, 160, 90, 200)) - palette = shapes.convert("P", palette=Image.Palette.ADAPTIVE, colors=16) - palette.info["transparency"] = 0 - gray = photo.convert("L") - - def png(img, **kw): - b = io.BytesIO() - img.save(b, "PNG", **kw) - return b.getvalue() - - def jpeg(img, exif=None): - b = io.BytesIO() - img.save(b, "JPEG", quality=90, **({"exif": exif} if exif else {})) - return b.getvalue() - - exif = Image.Exif() - exif[0x0112] = 6 # rotate 90 on display - out += [("shapes.png", png(shapes)), ("photo.png", png(photo)), ("large.png", png(large)), - ("tiny.png", png(tiny)), ("logo.png", png(logo)), ("palette.png", png(palette)), ("gray.png", png(gray)), - ("photo.jpg", jpeg(photo)), ("rotated.jpg", jpeg(shapes, exif.tobytes()))] - return out - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--model", required=True) - ap.add_argument("--out", required=True) - a = ap.parse_args() - import torch - from torch.nn.attention import SDPBackend, sdpa_kernel - from tensorfold.vision.images import _decode, DEFAULT_LIMITS - from tensorfold.vision.qwen_cuda import QwenCudaVision, TOKENS_PER_IMAGE - - out = Path(a.out) - out.mkdir(parents=True, exist_ok=True) - vision = QwenCudaVision(a.model, torch.device("cuda:0")) - proc = vision.frontend.processor - factor = vision.config["patch_size"] * vision.config["spatial_merge_size"] - cap = min(int(getattr(proc, "max_pixels", TOKENS_PER_IMAGE * factor**2)), TOKENS_PER_IMAGE * factor**2) - floor = min(int(getattr(proc, "min_pixels", factor**2)), cap) - entries = [] - for name, data in images(): - stem = name.replace(".", "_") - (out / name).write_bytes(data) - decoded = _decode(data, "auto", DEFAULT_LIMITS, DEFAULT_LIMITS.max_pixels) - pil = decoded.to_pil() - rgb = np.asarray(pil, dtype=np.uint8) - (out / f"{stem}.rgb").write_bytes(rgb.tobytes()) - result = proc(images=[pil], max_pixels=cap, min_pixels=floor) - pixels = np.asarray(result["pixel_values"], dtype=np.float32) - t, h, w = (int(v) for v in np.asarray(result["image_grid_thw"])[0]) - (out / f"{stem}.f32").write_bytes(pixels.tobytes()) - with torch.inference_mode(), sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]): - feats = vision.tower(torch.tensor(pixels, dtype=torch.bfloat16, device="cuda:0"), - grid_thw=torch.tensor([[t, h, w]], device="cuda:0"), return_dict=True).pooler_output - feats = feats.to(torch.bfloat16).contiguous().cpu() - (out / f"{stem}.bf16").write_bytes(feats.view(torch.int16).numpy().tobytes()) - entries.append({"name": name, "file": name, "rgb": f"{stem}.rgb", "size": [rgb.shape[1], rgb.shape[0]], - "grid": [h, w], "pixels": f"{stem}.f32", "features": f"{stem}.bf16", - "tokens": h * w // 4}) - print(f"{name}: {rgb.shape[1]}x{rgb.shape[0]} -> grid {h}x{w}, {h * w // 4} tokens", flush=True) - (out / "manifest.json").write_text(json.dumps({"images": entries, "cap_pixels": cap, "min_pixels": floor}, - indent=1) + "\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/zig/src/cli/qwen27_cuda.zig b/zig/src/cli/qwen27_cuda.zig index 2cb255d077..84c8bda959 100644 --- a/zig/src/cli/qwen27_cuda.zig +++ b/zig/src/cli/qwen27_cuda.zig @@ -19,7 +19,7 @@ const usage = \\ tf-qwen27-cuda kvcompare MODEL --kernels DIR --prompts prompts.json [--max-tokens N] \\ --kv-cache fp8 against bf16, teacher-forced on bf16's greedy reply: top-1 agreement and KL a token. \\ tf-qwen27-cuda vision MODEL --fixtures DIR - \\ The image tower on qwen_vision_fixtures.py's patches: each image's rows against 0.6.6's (cosine, max error). + \\ The image tower on recorded 0.6.6 fixtures' patches: each image's rows against 0.6.6's (cosine, max error). \\ ; @@ -547,7 +547,8 @@ fn runMulti(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.E return if (bad == 0 and round == want_rounds.len) 0 else 1; } -/// The tower against qwen_vision_fixtures.py's manifest: per image, the features' row cosines and largest error. +/// The tower against a manifest recorded from 0.6.6 outside the tree (qwen_vision_fixtures.py): per image, the +/// features' row cosines and largest error. fn checkVision(gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, model: []const u8, dir: []const u8) !u8 { const V = qwen.vision; const manifest = try readJson(arena, io, try std.fs.path.join(arena, &.{ dir, "manifest.json" })); diff --git a/zig/src/families/qwen27/cuda/triton.zig b/zig/src/families/qwen27/cuda/triton.zig index fcab03c8ef..bc737a39ec 100644 --- a/zig/src/families/qwen27/cuda/triton.zig +++ b/zig/src/families/qwen27/cuda/triton.zig @@ -119,8 +119,9 @@ pub const Tri = struct { }, &.{ ci("H", a.heads), ci("D", a.dim), ci("G", g), ci("DS", merge_columns), ci("CH", chunk), ci("GR", group_chunks) }); } - /// attention_kv8.py's attention8: attention over e4m3 caches, `offs` holding each stream's four offsets (key - /// and value bytes, their scales' fp32 elements) from `base`; the window's own keys and values stay bf16. + /// attention8 (attention_kv8.py, captured outside the tree like the rest of the set): attention over e4m3 + /// caches, `offs` holding each stream's four offsets (key and value bytes, their scales' fp32 elements) from + /// `base`; the window's own keys and values stay bf16. pub fn attention8(t: Tri, q: u64, kn: u64, vn: u64, base: u64, offs: u64, plan: Plan, part: Partials, out: u64, a: Attn) !void { const g = a.heads / a.kv_heads; const scale: f32 = @floatCast(std.math.pow(f64, @floatFromInt(a.dim), -0.5)); diff --git a/zig/tests/cuda/qwen27/attention_kv8.py b/zig/tests/cuda/qwen27/attention_kv8.py deleted file mode 100644 index 386873d247..0000000000 --- a/zig/tests/cuda/qwen27/attention_kv8.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python3 -"""attention.attention over an fp8 (e4m3) key and value cache, for the native engine's --kv-cache fp8. - -The cache holds each (position, key head) row of 256 as e4m3 times one fp32 scale (the row's absolute max over 448). -_shared8 and _tail8 are attention.py's _shared and _tail with those loads: e4m3 widens to bf16 exactly, the key's -scale multiplies its scores and the value's scale its probabilities, so the dots stay bf16 tensor-core dots. The -window's own rows (KN, VN) are quantized in the kernel exactly as their commit will quantize them, so each key's -contribution is the same committed or not. Partials keep attention.py's layout, so its _merge folds them unchanged. - -``--out DIR`` checks both against attention.attention over the dequantized cache and records their launches for -triton_aot_manifest.py (one specialization each for W of 1, of a multiple of 16, and of anything else). -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -import torch -import triton -import triton.language as tl - -FP8_MAX = 448.0 - - -@triton.jit -def _tile8(q, k, v, ks, vs, m, l, o, valid, scale: tl.constexpr): - scores = tl.dot(q, tl.trans(k)).to(tl.float32) * ks[None, :] * scale - scores = tl.where(valid[None, :], scores, float("-inf")) - tile_m = tl.max(scores, 1) - active = tile_m != float("-inf") - next_m = tl.where(active, tl.maximum(m, tile_m), m) - alpha = tl.where(active, tl.where(m == float("-inf"), 0.0, tl.exp(m - next_m)), 1.0) - p = tl.where(valid[None, :] & active[:, None], tl.exp(scores - next_m[:, None]), 0.0) - o = o * alpha[:, None] + tl.dot((p * vs[None, :]).to(tl.bfloat16), v) - l = l * alpha + tl.sum(p, 1) - return next_m, l, o - - -@triton.jit -def _fold8(m, l, o, cm, cl, co): - active = cl > 0.0 - next_m = tl.where(active, tl.maximum(m, cm), m) - a = tl.where(active, tl.where(m == float("-inf"), 0.0, tl.exp(m - next_m)), 1.0) - b = tl.where(active, tl.exp(cm - next_m), 0.0) - return next_m, l * a + cl * b, o * a[:, None] + co * b[:, None] - - -@triton.jit -def _q8(x): - """Rows of bf16 as the cache keeps them: e4m3 of x * (448 / amax) widened back to bf16, and s = amax / 448 - (kv_quant's rule: two IEEE divisions a row, one IEEE product an element).""" - - x = x.to(tl.float32) - amax = tl.max(tl.abs(x), 1) - s = tl.where(amax > 0, tl.math.div_rn(amax, 448.0), 1.0) - inv = tl.where(amax > 0, tl.math.div_rn(448.0, amax), 1.0) - return (x * inv[:, None]).to(tl.float8e4nv).to(tl.bfloat16), s - - -@triton.jit -def _shared8(Q, KC, VC, KS, VS, OFF, STREAM, ITEMS, PO, PM, PL, W, H: tl.constexpr, HK: tl.constexpr, - D: tl.constexpr, G: tl.constexpr, CH: tl.constexpr, SCALE: tl.constexpr, GR: tl.constexpr): - """_shared over the fp8 cache; OFF holds four offsets a stream: keys and values (bytes), their scales (fp32).""" - - item = tl.program_id(0) // HK - hk = tl.program_id(0) % HK - s = tl.load(ITEMS + item * 3) - first = tl.load(ITEMS + item * 3 + 1) - code = tl.load(ITEMS + item * 3 + 2) - start = tl.load(STREAM + s * 4) - rows = tl.load(STREAM + s * 4 + 1) - p = tl.load(STREAM + s * 4 + 2) - groups = ((p + rows + CH - 1) // CH - tl.load(STREAM + s * 4 + 3)) // (GR - 1) - whole = code >= 0 - chunk = tl.where(whole, code * GR, groups * GR - 1 - code) - n = tl.where(whole, GR, 1) - slot = tl.where(whole, code, chunk - groups * (GR - 1)) - if ((chunk + n) * CH <= p) & (code < groups): - koff = tl.multiple_of(tl.load(OFF + s * 4), 16) - voff = tl.multiple_of(tl.load(OFF + s * 4 + 1), 16) - ksoff = tl.load(OFF + s * 4 + 2) - vsoff = tl.load(OFF + s * 4 + 3) - rr = first + tl.arange(0, 16) - ok = rr < rows * G - node = start + rr // G - head = hk * G + rr % G - d = tl.arange(0, D) - q = tl.load(Q + (node[:, None] * H + head[:, None]) * D + d[None, :], mask=ok[:, None], - other=0).to(tl.bfloat16) - gm = tl.full((16,), float("-inf"), tl.float32) - gl = tl.zeros((16,), tl.float32) - go = tl.zeros((16, D), tl.float32) - m, l, o = gm, gl, go - for c in range(n): - key = (chunk + c) * CH + tl.arange(0, 64) - m = tl.full((16,), float("-inf"), tl.float32) - l = tl.zeros((16,), tl.float32) - o = tl.zeros((16, D), tl.float32) - for t in range(CH // 64): - ki = key + t * 64 - kk = tl.load(KC + koff + (ki[:, None] * HK + hk) * D + d[None, :]).to(tl.bfloat16) - vv = tl.load(VC + voff + (ki[:, None] * HK + hk) * D + d[None, :]).to(tl.bfloat16) - ks = tl.load(KS + ksoff + ki * HK + hk) - vs = tl.load(VS + vsoff + ki * HK + hk) - m, l, o = _tile8(q, kk, vv, ks, vs, m, l, o, ki < p, SCALE) - gm, gl, go = _fold8(gm, gl, go, m, l, o) - if not whole: - gm, gl, go = m, l, o - base = (slot * W + node) * H + head - tl.store(PO + base[:, None] * D + d[None, :], go, mask=ok[:, None]) - tl.store(PM + base, gm, mask=ok) - tl.store(PL + base, gl, mask=ok) - - -@triton.jit -def _tail8(Q, KN, VN, KC, VC, KS, VS, OFF, STREAM, ROWS, PATHS, DEPTHS, PO, PM, PL, W, H: tl.constexpr, - HK: tl.constexpr, D: tl.constexpr, G: tl.constexpr, CH: tl.constexpr, MAXD: tl.constexpr, - SCALE: tl.constexpr, GR: tl.constexpr): - """_tail over the fp8 cache; the row's own path quantized as its commit will, so a key's bits never depend on - whether it is committed or in the window (nor on drafts, or how streams share a round).""" - - node = tl.program_id(0) - hk = tl.program_id(1) - s = tl.load(ROWS + node) - p = tl.load(STREAM + s * 4 + 2) - nch = (p + tl.load(STREAM + s * 4 + 1) + CH - 1) // CH - groups = (nch - tl.load(STREAM + s * 4 + 3)) // (GR - 1) - chunk = p // CH + tl.program_id(2) - if chunk < nch: - koff = tl.multiple_of(tl.load(OFF + s * 4), 16) - voff = tl.multiple_of(tl.load(OFF + s * 4 + 1), 16) - ksoff = tl.load(OFF + s * 4 + 2) - vsoff = tl.load(OFF + s * 4 + 3) - gg = tl.arange(0, 16) - d = tl.arange(0, D) - q = tl.load(Q + (node * H + hk * G + gg[:, None]) * D + d[None, :], mask=gg[:, None] < G, other=0).to(tl.bfloat16) - depth = tl.load(DEPTHS + node) - m = tl.full((16,), float("-inf"), tl.float32) - l = tl.zeros((16,), tl.float32) - o = tl.zeros((16, D), tl.float32) - key = chunk * CH + tl.arange(0, 64) - for t in range(CH // 64): - logical = key + t * 64 - committed = logical < p - path_slot = logical - p - on_path = (path_slot >= 0) & (path_slot < depth) - path_node = tl.load(PATHS + node * MAXD + path_slot, mask=on_path, other=0) - kc = tl.load(KC + koff + (logical[:, None] * HK + hk) * D + d[None, :], mask=committed[:, None], other=0.0) - vc = tl.load(VC + voff + (logical[:, None] * HK + hk) * D + d[None, :], mask=committed[:, None], other=0.0) - ks = tl.load(KS + ksoff + logical * HK + hk, mask=committed, other=1.0) - vs = tl.load(VS + vsoff + logical * HK + hk, mask=committed, other=1.0) - kn = tl.zeros((64, D), tl.bfloat16) - vn = tl.zeros((64, D), tl.bfloat16) - kns = tl.full((64,), 1.0, tl.float32) - vns = tl.full((64,), 1.0, tl.float32) - if tl.max(on_path.to(tl.int32), 0) > 0: # a tile of committed keys only skips the quantizing - kn, kns = _q8(tl.load(KN + (path_node[:, None] * HK + hk) * D + d[None, :], mask=on_path[:, None], other=0)) - vn, vns = _q8(tl.load(VN + (path_node[:, None] * HK + hk) * D + d[None, :], mask=on_path[:, None], other=0)) - kk = tl.where(committed[:, None], kc.to(tl.bfloat16), kn) - vv = tl.where(committed[:, None], vc.to(tl.bfloat16), vn) - ks = tl.where(committed, ks, kns) - vs = tl.where(committed, vs, vns) - m, l, o = _tile8(q, kk, vv, ks, vs, m, l, o, committed | on_path, SCALE) - base = ((chunk - groups * (GR - 1)) * W + node) * H + hk * G + gg - tl.store(PO + base[:, None] * D + d[None, :], o, mask=gg[:, None] < G) - tl.store(PM + base, m, mask=gg < G) - tl.store(PL + base, l, mask=gg < G) - - -def quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """(n, HK, D) bf16 rows as e4m3 and one fp32 scale a row: the native kv_quant kernel's rule.""" - - amax = x.float().abs().amax(-1) - # tensor by tensor: IEEE division, as kv_quant's __fdiv_rn (torch divides by a Python float as a reciprocal) - full = torch.full_like(amax, FP8_MAX) - scale = torch.where(amax > 0, amax / full, torch.ones_like(amax)) - inv = torch.where(amax > 0, full / amax, torch.ones_like(amax)) - q = (x.float() * inv[..., None]).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) - return q, scale - - -def attention8(q, kn, vn, origin, kc8, vc8, ks, vs, p: "A.Plan", scale: float): - """attention.attention with the fp8 cache of one stream (offsets from ``origin``, as the engine passes them).""" - - import tensorfold.cuda.kernels.attention as A - - w, h, d = q.shape - hk = kn.shape[1] - g = h // hk - o8 = origin.view(torch.float8_e4m3fn) - of = origin.view(torch.float32) - b = origin.data_ptr() - offs = torch.tensor([[kc8.data_ptr() - b, vc8.data_ptr() - b, (ks.data_ptr() - b) // 4, (vs.data_ptr() - b) // 4]], - dtype=torch.int64, device=q.device) - po = torch.empty((p.chunks, w, h, d), dtype=torch.float32, device=q.device) - pm = torch.empty((p.chunks, w, h), dtype=torch.float32, device=q.device) - pl = torch.empty_like(pm) - if p.items.shape[0]: - _shared8[(p.items.shape[0] * hk,)](q, o8, o8, of, of, offs, p.streams, p.items, po, pm, pl, w, H=h, HK=hk, D=d, - G=g, CH=A.CHUNK, SCALE=scale, GR=A.GROUP, num_warps=4, num_stages=1) - tails = 1 + -(-A.MAX_NODES // A.CHUNK) - _tail8[(w, hk, tails)](q, kn, vn, o8, o8, of, of, offs, p.streams, p.rows, p.paths, p.depths, po, pm, pl, w, H=h, - HK=hk, D=d, G=g, CH=A.CHUNK, MAXD=A.MAX_NODES, SCALE=scale, GR=A.GROUP, num_warps=4, - num_stages=1) - out = torch.empty_like(q) - A._merge[(w, hk, d // A.MERGE_COLUMNS)](po, pm, pl, out, p.streams, p.rows, w, H=h, D=d, G=g, DS=A.MERGE_COLUMNS, - CH=A.CHUNK, GR=A.GROUP, num_warps=4) - return out - - -def check(p_len: int, parents: list[int], dev) -> float: - """attention8 against attention.attention over the dequantized cache: the worst row's cosine.""" - - import tensorfold.cuda.kernels.attention as A - - torch.manual_seed(p_len * 131 + len(parents)) - h, hk, d = 24, 4, 256 - w = len(parents) - cap = p_len + 64 - k = torch.randn((cap, hk, d), device=dev).mul(3).to(torch.bfloat16) - v = torch.randn((cap, hk, d), device=dev).to(torch.bfloat16) - k8, ks = quantize(k) - v8, vs = quantize(v) - kd = (k8.float() * ks[..., None]).to(torch.bfloat16) - vd = (v8.float() * vs[..., None]).to(torch.bfloat16) - q = torch.randn((w, h, d), device=dev).to(torch.bfloat16) - kn = torch.randn((w, hk, d), device=dev).mul(3).to(torch.bfloat16) - vn = torch.randn((w, hk, d), device=dev).to(torch.bfloat16) - plan = A.plan([parents], [p_len], h // hk, dev) - scale = d ** -0.5 - kq, kqs = quantize(kn) - vq, vqs = quantize(vn) - knd = (kq.float() * kqs[..., None]).to(torch.bfloat16) - vnd = (vq.float() * vqs[..., None]).to(torch.bfloat16) - ref = A.attention(q, knd, vnd, torch.tensor([A.offsets([(kd, vd)], dev)], dtype=torch.int64, device=dev), plan, - scale=scale) - got = attention8(q, kn, vn, A.base(dev).view(torch.uint8), k8, v8, ks, vs, plan, scale) - cos = torch.nn.functional.cosine_similarity(ref.float().view(w * h, d), got.float().view(w * h, d), dim=1) - if parents == list(range(-1, w - 1)) and w > 1: - # a chain's first row committed instead: the other rows' outputs must keep their bits - k8b = torch.cat([k8[:p_len], kq[:1], k8[p_len + 1:]]) - ksb = torch.cat([ks[:p_len], kqs[:1], ks[p_len + 1:]]) - v8b = torch.cat([v8[:p_len], vq[:1], v8[p_len + 1:]]) - vsb = torch.cat([vs[:p_len], vqs[:1], vs[p_len + 1:]]) - plan_b = A.plan([list(range(-1, w - 2))], [p_len + 1], h // hk, dev) - got_b = attention8(q[1:].contiguous(), kn[1:].contiguous(), vn[1:].contiguous(), A.base(dev).view(torch.uint8), - k8b, v8b, ksb, vsb, plan_b, scale) - same = torch.equal(got[1:], got_b) - print(f" committed vs in-window keys: {'bit-identical' if same else 'DIFFER'}", flush=True) - if not same: - return 0.0 - return float(cos.min()) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--out", required=True) - ap.add_argument("--tools", required=True, help="folder of triton_aot_manifest.py") - a = ap.parse_args() - sys.path.insert(0, a.tools) - import triton_aot_manifest as aot - - out = Path(a.out) - out.mkdir(parents=True, exist_ok=True) - rec = aot.Recorder().install() - rec.detail = True - dev = torch.device("cuda:0") - worst = 1.0 - # W of 1 (decode), 16 (a divisible verify window), 7 and 33 (neither); committed keys under and over a group - cases = [(0, [-1]), (700, [-1]), (5000, [-1]), (9000, list(range(-1, 15))), (3000, [-1, 0, 1, 1, 3, 4, 2]), - (20000, [-1] + list(range(32))), (300, list(range(-1, 15)))] - for p_len, parents in cases: - c = check(p_len, parents, dev) - worst = min(worst, c) - print(f"p={p_len} w={len(parents)}: worst row cosine {c:.6f}", flush=True) - rec.dump(out / "launches.json") - from triton.runtime.jit import JITFunction - import gc - - spec = {} - for fn in gc.get_objects(): - if isinstance(fn, JITFunction): - spec[f"{fn.fn.__module__}.{fn.fn.__qualname__}"] = { - "params": [x.name for x in fn.params], - "do_not_specialize": [x.name for x in fn.params if x.do_not_specialize], - "no_align": [x.name for x in fn.params if x.do_not_specialize_on_alignment]} - (out / "jit.json").write_text(json.dumps(spec, indent=1, sort_keys=True) + "\n") - print(f"worst {worst:.6f}") - return 0 if worst > 0.995 else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) From 5e08e130bfb32e944390c2b9dfa3359a8859ea76 Mon Sep 17 00:00:00 2001 From: Jonathan Regan-Beasley <197533392+jregan-beasley-smc@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:40:39 +1100 Subject: [PATCH 12/19] perf(qwen27 cuda): reduce repeated-prompt TTFT Port early first-token delivery and opt-in exact-prompt state reuse to PR726, preserving its host-prefix cache and admission flow. Keep the experimental caches restricted to supported BF16 text requests. --- zig/src/core/engine_api.zig | 1 + zig/src/core/lane_host.zig | 12 +- zig/src/core/lane_host_first_test.zig | 119 +++++ zig/src/core/lanes/backend.zig | 3 + zig/src/core/lanes/engine.zig | 2 + zig/src/core/lanes/lanes.zig | 1 + zig/src/core/lanes/open.zig | 17 +- zig/src/core/lanes/open_first.zig | 64 +++ zig/src/core/lanes/open_first_test.zig | 163 ++++++ zig/src/families/qwen27/cuda/draft_pool.zig | 147 ++++++ .../families/qwen27/cuda/draft_pool_test.zig | 105 ++++ zig/src/families/qwen27/cuda/drafts.zig | 15 + zig/src/families/qwen27/cuda/engine.zig | 6 + .../qwen27/cuda/exact_cache_policy.zig | 58 +++ .../families/qwen27/cuda/exact_cache_test.zig | 491 ++++++++++++++++++ .../qwen27/cuda/exact_draft_reuse.zig | 173 ++++++ .../families/qwen27/cuda/exact_eligible.zig | 40 ++ .../qwen27/cuda/exact_first_logits.zig | 31 ++ zig/src/families/qwen27/cuda/exact_init.zig | 30 ++ .../qwen27/cuda/exact_logits_policy.zig | 75 +++ .../qwen27/cuda/exact_prompt_cache.zig | 339 ++++++++++++ .../families/qwen27/cuda/exact_seq_reuse.zig | 189 +++++++ zig/src/families/qwen27/cuda/lanes.zig | 142 ++--- zig/src/families/qwen27/cuda/reset_skip.zig | 58 +++ .../qwen27/cuda/reset_skip_factory_test.zig | 179 +++++++ .../families/qwen27/cuda/reset_skip_test.zig | 105 ++++ .../families/qwen27/cuda/restore_proof.zig | 76 +++ zig/src/families/qwen27/cuda/seq_pool.zig | 120 +++++ .../families/qwen27/cuda/seq_pool_test.zig | 172 ++++++ zig/src/families/qwen27/cuda/state.zig | 15 +- zig/src/native/cuda.zig | 2 +- 31 files changed, 2880 insertions(+), 70 deletions(-) create mode 100644 zig/src/core/lane_host_first_test.zig create mode 100644 zig/src/core/lanes/open_first.zig create mode 100644 zig/src/core/lanes/open_first_test.zig create mode 100644 zig/src/families/qwen27/cuda/draft_pool.zig create mode 100644 zig/src/families/qwen27/cuda/draft_pool_test.zig create mode 100644 zig/src/families/qwen27/cuda/exact_cache_policy.zig create mode 100644 zig/src/families/qwen27/cuda/exact_cache_test.zig create mode 100644 zig/src/families/qwen27/cuda/exact_draft_reuse.zig create mode 100644 zig/src/families/qwen27/cuda/exact_eligible.zig create mode 100644 zig/src/families/qwen27/cuda/exact_first_logits.zig create mode 100644 zig/src/families/qwen27/cuda/exact_init.zig create mode 100644 zig/src/families/qwen27/cuda/exact_logits_policy.zig create mode 100644 zig/src/families/qwen27/cuda/exact_prompt_cache.zig create mode 100644 zig/src/families/qwen27/cuda/exact_seq_reuse.zig create mode 100644 zig/src/families/qwen27/cuda/reset_skip.zig create mode 100644 zig/src/families/qwen27/cuda/reset_skip_factory_test.zig create mode 100644 zig/src/families/qwen27/cuda/reset_skip_test.zig create mode 100644 zig/src/families/qwen27/cuda/restore_proof.zig create mode 100644 zig/src/families/qwen27/cuda/seq_pool.zig create mode 100644 zig/src/families/qwen27/cuda/seq_pool_test.zig diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index af6289bd68..b7216318c3 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -458,6 +458,7 @@ test { _ = @import("lane_host_reuse_test.zig"); _ = @import("lane_host_warm_test.zig"); _ = @import("lane_host_fill_test.zig"); + _ = @import("lane_host_first_test.zig"); _ = @import("prompt_cache.zig"); _ = @import("prompt_radix.zig"); _ = @import("prompt_imprint.zig"); diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index aabe76ef41..ae4b948779 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -345,7 +345,7 @@ pub const LaneHost = struct { h.filling.append(h.gpa, job) catch return h.drop(job, "out of memory"); return true; } - h.core.addStream(&job.stream) catch |e| return switch (e) { + h.core.addStreamCommitted(&job.stream, .{ .ptr = job, .call = firstCommitted }) catch |e| return switch (e) { error.Cancelled => h.cancel(job), error.LaneMemory => h.requeue(job), else => h.drop(job, h.words(e)), @@ -362,7 +362,7 @@ pub const LaneHost = struct { const job = h.filling.items[at]; const first = !job.fill_began; job.fill_began = true; - const done = h.core.fillStream(&job.stream, first) catch |e| { + const done = h.core.fillStreamCommitted(&job.stream, first, .{ .ptr = job, .call = firstCommitted }) catch |e| { _ = switch (e) { error.Cancelled => h.cancel(job), error.LaneMemory => blk: { @@ -385,7 +385,14 @@ pub const LaneHost = struct { return c.constraint(h.gpa, h.io, @fromBackingInt(@intCast(@backingInt(st.kind))), st.text, st.after); } + fn firstCommitted(ptr: *anyopaque) void { + const job: *Job = @ptrCast(@alignCast(ptr)); + job.host.prefilled(job, job.began); + job.host.send(job); + } + pub fn prefilled(h: *LaneHost, job: *Job, began: i96) void { + if (job.prefilled != null) return; const done = std.Io.Clock.awake.now(h.io).toNanoseconds(); h.lock(); if (done > began) h.prefill_rate = @as(f64, @floatFromInt(job.request.prompt.len - job.stream.cached)) / (@as(f64, @floatFromInt(done - began)) / 1e9); @@ -428,6 +435,7 @@ pub const LaneHost = struct { /// No memory for another lane yet: the job waits at the queue's head, and admission holds until a job ends. fn requeue(h: *LaneHost, job: *Job) bool { + if (job.stream.pending != null or job.delivered != 0) return h.drop(job, "memory failure after first-token commit"); if (job.stream.grammar) |g| g.deinit(h.gpa); // the next admission compiles its own job.stream.deinit(h.gpa); job.proposer.deinit(); diff --git a/zig/src/core/lane_host_first_test.zig b/zig/src/core/lane_host_first_test.zig new file mode 100644 index 0000000000..04b71df7b5 --- /dev/null +++ b/zig/src/core/lane_host_first_test.zig @@ -0,0 +1,119 @@ +//! Actual lane-host delivery keeps terminal jobs alive through the callback and never requeues an emitted first token. +const std = @import("std"); +const lanes = @import("lanes"); +const api = @import("engine_api.zig"); +const LaneHost = @import("lane_host.zig").LaneHost; +const a = std.testing.allocator; + +const Box = struct { + mutex: std.Io.Mutex = .init, + tokens: usize = 0, + logprobs: usize = 0, + prefilled: usize = 0, + finishes: usize = 0, + reason: ?api.Reason = null, + fn event(ptr: *anyopaque, _: api.Id, ev: *const api.Event) void { + const b: *Box = @ptrCast(@alignCast(ptr)); + b.mutex.lockUncancelable(std.testing.io); + defer b.mutex.unlock(std.testing.io); + switch (ev.*) { + .tokens => |t| b.tokens += t.len, + .logprobs => |r| b.logprobs += r.len, + .prefilled => b.prefilled += 1, + .finished => |f| { + b.reason = f.reason; + b.finishes += 1; + }, + else => {}, + } + } + fn wait(b: *Box) !void { + for (0..1000) |_| { + b.mutex.lockUncancelable(std.testing.io); + const done = b.reason != null; + b.mutex.unlock(std.testing.io); + if (done) return; + try std.Io.sleep(std.testing.io, .fromMilliseconds(1), .awake); + } + return error.TestTimedOut; + } +}; + +const Target = struct { + fake: lanes.fake.Fake = .{ .gpa = a }, + table: lanes.backend.Backend.VTable = undefined, + drafts: usize = 0, + releases: usize = 0, + lane_memory: bool, + observed: *Box, + delivered_before_draft: bool = false, + fn backend(t: *Target) lanes.backend.Backend { + t.table = t.fake.backend().vtable.*; + t.table.draft = draft; + t.table.release = release; + return .{ .ptr = &t.fake, .vtable = &t.table, .first_before_draft = true }; + } + fn self(ptr: *anyopaque) *Target { + const fake: *lanes.fake.Fake = @ptrCast(@alignCast(ptr)); + return @fieldParentPtr("fake", fake); + } + fn draft(ptr: *anyopaque, _: []const lanes.backend.DraftRequest) !void { + const t = self(ptr); + t.drafts += 1; + t.observed.mutex.lockUncancelable(std.testing.io); + t.delivered_before_draft = t.observed.tokens == 1 and t.observed.logprobs == 1; + t.observed.mutex.unlock(std.testing.io); + return if (t.lane_memory) error.LaneMemory else error.InitialDraftFailed; + } + fn release(ptr: *anyopaque, s: *lanes.Stream) void { + const t = self(ptr); + t.releases += 1; + t.fake.backend().release(s); + } +}; + +fn check(lane_memory: bool) !void { + var one: Box = .{}; + var other: Box = .{}; + var t: Target = .{ .lane_memory = lane_memory, .observed = &other }; + defer t.fake.deinit(); + var cfg = try lanes.Config.init(a, .{ .exact_width = 8, .hidden_rows = true, .mtp = true, .speculate = true, .drafts = 7 }, 8, 7); + defer cfg.deinit(a); + var clock: lanes.fake.FixedClock = .{}; + var core = lanes.Engine.init(a, &cfg, t.backend(), clock.clock()); + defer core.deinit(); + var h = LaneHost.init(a, std.testing.io, &core, .{ .lanes = 2 }); + const e = h.engine(); + const terminal: api.Request = .{ .prompt = &.{ 3, 1, 4 }, .max_tokens = 1, .logprobs = 2 }; + const failing: api.Request = .{ .prompt = &.{ 3, 1, 4 }, .max_tokens = 8, .logprobs = 2 }; + try e.submit(1, &terminal, .{ .ctx = &one, .event = Box.event }); + try e.submit(2, &failing, .{ .ctx = &other, .event = Box.event }); + try h.start(); + var stopped = false; + defer if (!stopped) h.stop(); + try one.wait(); + try other.wait(); + h.stop(); + stopped = true; + try std.testing.expectEqual(api.Reason.length, one.reason.?); + try std.testing.expectEqual(api.Reason.failed, other.reason.?); + try std.testing.expectEqual(@as(usize, 1), one.tokens); + try std.testing.expectEqual(@as(usize, 1), other.tokens); + try std.testing.expectEqual(@as(usize, 1), one.logprobs); + try std.testing.expectEqual(@as(usize, 1), other.logprobs); + try std.testing.expectEqual(@as(usize, 1), one.prefilled); + try std.testing.expectEqual(@as(usize, 1), other.prefilled); + try std.testing.expectEqual(@as(usize, 1), one.finishes); + try std.testing.expectEqual(@as(usize, 1), other.finishes); + try std.testing.expectEqual(@as(usize, 1), t.drafts); + try std.testing.expectEqual(@as(usize, 2), t.releases); + try std.testing.expect(t.delivered_before_draft); +} + +test "first token and logprob sent once; terminal job survives another job's draft error" { + try check(false); +} + +test "LaneMemory after the committed handoff fails instead of resetting and requeueing the reply" { + try check(true); +} diff --git a/zig/src/core/lanes/backend.zig b/zig/src/core/lanes/backend.zig index 7d2eb1fb08..8bfef2869b 100644 --- a/zig/src/core/lanes/backend.zig +++ b/zig/src/core/lanes/backend.zig @@ -59,9 +59,12 @@ pub const DraftRequest = struct { ranks: bool = false, // also keep each chained level's best tokens (alternatives(), for grafts) }; +pub const Committed = struct { ptr: *anyopaque, call: *const fn (*anyopaque) void }; + pub const Backend = struct { ptr: *anyopaque, vtable: *const VTable, + first_before_draft: bool = false, pub const VTable = struct { /// Prefill the stream's prompt into its caches (and its draft head's). diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index 0efdfc0067..69cde5f3dc 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -78,9 +78,11 @@ pub const Engine = struct { } pub const addStream = open.add; + pub const addStreamCommitted = open.addCommitted; pub const opened = open.opened; pub const fills = open.fills; pub const fillStream = open.fillStream; + pub const fillStreamCommitted = open.fillStreamCommitted; /// Take over a stream another driver decoded so far: its cache settled, `pending` and `cache_len` set, no drafts held. pub fn adopt(e: *Engine, s: *Stream) !void { diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index 5d50a16ef3..e961a2aed3 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -38,6 +38,7 @@ pub const LogprobRow = logprob.Row; test { std.testing.refAllDecls(@This()); _ = @import("engine_test.zig"); + _ = @import("open_first_test.zig"); _ = @import("gpu_full_test.zig"); _ = @import("drafted_test.zig"); _ = @import("pricing_test.zig"); diff --git a/zig/src/core/lanes/open.zig b/zig/src/core/lanes/open.zig index 7cddf9a064..fcb4a52196 100644 --- a/zig/src/core/lanes/open.zig +++ b/zig/src/core/lanes/open.zig @@ -14,17 +14,26 @@ const int = trail.int; /// Prefill a stream, draw its first token and ask for its first drafts; it takes part from the next round. pub fn add(e: *Engine, s: *Stream) !void { + return addCommitted(e, s, null); +} + +pub fn addCommitted(e: *Engine, s: *Stream, hook: ?be.Committed) !void { _ = e.arena.reset(.retain_capacity); try trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }); e.backend.prefill(s) catch |err| { if (err == error.Cancelled) e.backend.release(s); // the host finishes a cancelled stream without the core return err; }; - try opened(e, s); + try openedCommitted(e, s, hook); } /// After the prompt pass: the first token drawn and committed, the first drafts asked; rounds from the next. pub fn opened(e: *Engine, s: *Stream) !void { + return openedCommitted(e, s, null); +} + +fn openedCommitted(e: *Engine, s: *Stream, hook: ?be.Committed) !void { + if (e.backend.first_before_draft) return @import("open_first.zig").opened(e, s, hook); if (s.isCancelled()) { // cancelled in its last chunk: no first token e.backend.release(s); return error.Cancelled; @@ -77,6 +86,10 @@ pub fn fills(e: *const Engine) bool { /// One chunk of a stream's prompt pass; true once the stream is opened and joins the next round. pub fn fillStream(e: *Engine, s: *Stream, first: bool) !bool { + return fillStreamCommitted(e, s, first, null); +} + +pub fn fillStreamCommitted(e: *Engine, s: *Stream, first: bool, hook: ?be.Committed) !bool { _ = e.arena.reset(.retain_capacity); if (first) try trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }); const chunk = e.backend.vtable.prefill_step orelse return error.NoPrefillSteps; @@ -85,6 +98,6 @@ pub fn fillStream(e: *Engine, s: *Stream, first: bool) !bool { return err; }; if (!done) return false; - try opened(e, s); + try openedCommitted(e, s, hook); return true; } diff --git a/zig/src/core/lanes/open_first.zig b/zig/src/core/lanes/open_first.zig new file mode 100644 index 0000000000..8638695786 --- /dev/null +++ b/zig/src/core/lanes/open_first.zig @@ -0,0 +1,64 @@ +//! An opt-in single-stream admission commits and hands off its first token before preparing initial drafts. +const Engine = @import("engine.zig").Engine; +const be = @import("backend.zig"); +const Stream = @import("stream.zig").Stream; +const ev = @import("events.zig"); +const win = @import("windows.zig"); +const trail = @import("trail.zig"); +const LogRow = @import("logprob.zig").Row; +const f = ev.f; +const str = trail.str; +const int = trail.int; + +pub fn opened(e: *Engine, s: *Stream, hook: ?be.Committed) !void { + if (s.isCancelled()) { + e.backend.release(s); + return error.Cancelled; + } + try e.live.ensureUnusedCapacity(e.gpa, 1); + s.context.shrinkRetainingCapacity(s.prompt_len); + s.rows.clearRetainingCapacity(); + s.pending = null; + s.cache_len = s.prompt_len; + const position: u64 = s.prompt_len; + const drawn = try e.drawFirst(s, position); + var feed: be.Feed = .{ .handle = drawn }; + if (try e.forcedNext(s)) |t| feed = .{ .value = t }; + const depth: ?u32 = if (e.cfg.family_mtp and s.drafts) @intCast(try e.rule.depth(win.who(s))) else null; + const value = try e.readFeed(feed); + if (e.log != null) { + const actual = if (feed == .handle) value else try e.backend.read(drawn); + try trail.event(e, &.{ f("ev", str("first")), f("stream", str(s.id)), f("position", int(position)), f("drawn", int(actual)), f("token", int(value)) }); + } + var row: [1]LogRow = undefined; + if (s.logprobs != null) row[0] = (try e.backend.firstRow(s)).forToken(value); + _ = try s.commit(e.gpa, &.{value}, if (s.logprobs != null) &row else &.{}); + s.pending = value; + try trail.resolve(e); + if (s.finished) { + try trail.finish(e, s); + e.release(s); + if (hook) |h| h.call(h.ptr); + return; + } + if (hook) |h| h.call(h.ptr); + if (s.isCancelled()) { + e.release(s); + return error.Cancelled; + } + if (depth) |d| { + e.backend.draft(&.{.{ .stream = s, .follow = &.{}, .first = .{ .value = value }, .rows = null, .start = s.prompt_len, .position = position + 1, .depth = d }}) catch |err| { + if (err == error.Cancelled) e.release(s); + return err; + }; + s.dropHeld(e.gpa); + s.next = .{ .count = d }; + if (e.backend.vtable.tree) |tree| if (try tree(e.backend.ptr, s, e.gpa)) |held| { + s.next = held; + }; + try trail.event(e, &.{ f("ev", str("draft")), f("stream", str(s.id)), f("depth", int(d)), f("position", int(position + 1)), f("follow", .{ .u32s = &.{value} }), f("rows", .null) }); + } else if (e.cfg.pipelined and s.logprobs == null and s.grammar == null) { + try e.queueNext(s, feed); + } + e.live.appendAssumeCapacity(s); +} diff --git a/zig/src/core/lanes/open_first_test.zig b/zig/src/core/lanes/open_first_test.zig new file mode 100644 index 0000000000..274994dd10 --- /dev/null +++ b/zig/src/core/lanes/open_first_test.zig @@ -0,0 +1,163 @@ +//! Single-stream opt-in admission preserves committed tokens and default ordering across terminal and failure paths. +const std = @import("std"); +const Engine = @import("engine.zig").Engine; +const Config = @import("config.zig").Config; +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const fake = @import("fake.zig"); +const a = std.testing.allocator; +const ids = [_]u32{ 3, 1, 4 }; + +const Probe = struct { + target: fake.Fake = .{ .gpa = a }, + table: be.Backend.VTable = undefined, + hooks: usize = 0, + drafts: usize = 0, + releases: usize = 0, + committed_in_draft: bool = false, + hook_in_draft: bool = false, + fail_draft: bool = false, + cancel: bool = false, + selected: u32 = 0, + + fn backend(p: *Probe, early: bool, stepped: bool) be.Backend { + p.table = (if (stepped) p.target.stepped() else p.target.backend()).vtable.*; + p.table.draft = draft; + p.table.release = release; + return .{ .ptr = &p.target, .vtable = &p.table, .first_before_draft = early }; + } + fn self(ptr: *anyopaque) *Probe { + const t: *fake.Fake = @ptrCast(@alignCast(ptr)); + return @fieldParentPtr("target", t); + } + fn draft(ptr: *anyopaque, req: []const be.DraftRequest) !void { + const p = self(ptr); + p.drafts += 1; + p.committed_in_draft = req[0].stream.emitted().len == 1; + p.hook_in_draft = p.hooks == 1; + if (req[0].first) |f| p.selected = switch (f) { + .value => |v| v, + .handle => |h| try p.target.backend().read(h), + }; + if (p.fail_draft) return error.InitialDraftFailed; + try p.target.backend().draft(req); + } + fn release(ptr: *anyopaque, s: *sm.Stream) void { + const p = self(ptr); + p.releases += 1; + p.target.backend().release(s); + } + fn hook(ptr: *anyopaque) void { + const p: *Probe = @ptrCast(@alignCast(ptr)); + p.hooks += 1; + } + fn cancelled(ptr: *anyopaque) bool { + const p: *Probe = @ptrCast(@alignCast(ptr)); + return p.cancel and p.hooks > 0; + } +}; + +fn config() !Config { + return Config.init(a, .{ .exact_width = 8, .hidden_rows = true, .mtp = true, .speculate = true, .drafts = 7 }, 8, 7); +} + +fn runOrder(early: bool) !void { + var p: Probe = .{}; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(early, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "order", .prompt = &ids, .max_new = 8 }); + defer s.deinit(a); + try e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook }); + try std.testing.expectEqual(early, p.committed_in_draft); + try std.testing.expectEqual(early, p.hook_in_draft); + try std.testing.expectEqual(@as(usize, @intFromBool(early)), p.hooks); + try std.testing.expectEqual(@as(usize, 1), s.emitted().len); + e.discard(&s); + try std.testing.expectEqual(@as(usize, 1), p.releases); +} + +test "opt-in commits and hands off before draft; default retains old ordering" { + try runOrder(true); + try runOrder(false); +} + +test "max-one and EOS terminate successfully without preparing any drafts" { + for ([_]bool{ false, true }) |eos| { + var p: Probe = .{ .fail_draft = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + const token = fake.next(&ids, null, ids.len); + var s = try sm.Stream.init(a, .{ .id = "terminal", .prompt = &ids, .max_new = if (eos) 8 else 1, .eos = if (eos) &.{token} else &.{} }); + defer s.deinit(a); + try e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook }); + try std.testing.expect(s.finished); + try std.testing.expectEqual(@as(usize, 0), p.drafts); + try std.testing.expectEqual(@as(usize, 1), p.hooks); + try std.testing.expectEqual(@as(usize, 1), p.releases); + try std.testing.expectEqual(@as(usize, 0), e.activeCount()); + } +} + +test "forced first and its logprob are committed exactly once before a later draft failure" { + var p: Probe = .{ .fail_draft = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "forced", .prompt = &ids, .max_new = 8, .logprobs = 2 }); + defer s.deinit(a); + try s.force.append(a, 42); + try std.testing.expectError(error.InitialDraftFailed, e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expectEqualSlices(u32, &.{42}, s.emitted()); + try std.testing.expectEqual(@as(u32, 42), p.selected); + try std.testing.expectEqual(@as(usize, 1), s.rows.items.len); + try std.testing.expectEqual(@as(u32, 42), s.rows.items[0].token); + try std.testing.expectEqual(@as(usize, 1), p.hooks); + e.discard(&s); + try std.testing.expectEqual(@as(usize, 1), p.releases); +} + +test "cancellation at handoff releases once and skips initial drafts" { + var p: Probe = .{ .cancel = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "cancel", .prompt = &ids, .max_new = 8, .cancel_check = .{ .ptr = &p, .check = Probe.cancelled } }); + defer s.deinit(a); + try std.testing.expectError(error.Cancelled, e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expectEqual(@as(usize, 1), p.hooks); + try std.testing.expectEqual(@as(usize, 1), p.releases); + try std.testing.expectEqual(@as(usize, 0), p.drafts); +} + +test "stepped prefill uses the same committed handoff only after its final chunk" { + var p: Probe = .{}; + p.target.prefill_chunks = 2; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, true), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "step", .prompt = &ids, .max_new = 8 }); + defer s.deinit(a); + try std.testing.expect(!try e.fillStreamCommitted(&s, true, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expectEqual(@as(usize, 0), p.hooks); + while (!try e.fillStreamCommitted(&s, false, .{ .ptr = &p, .call = Probe.hook })) {} + try std.testing.expectEqual(@as(usize, 1), p.hooks); + try std.testing.expect(p.committed_in_draft and p.hook_in_draft); + e.discard(&s); +} diff --git a/zig/src/families/qwen27/cuda/draft_pool.zig b/zig/src/families/qwen27/cuda/draft_pool.zig new file mode 100644 index 0000000000..7b2da57e8d --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_pool.zig @@ -0,0 +1,147 @@ +//! Detached fixed-class draft Context ownership; no CUDA allocation, copy or fence occurs in this policy. +const std = @import("std"); +pub const max_slots: usize = 16; +pub const max_capacity: usize = 512; +pub const max_bytes: usize = 1536 * 1024 * 1024; + +pub const Stats = struct { + enabled: bool = false, + failed: bool = false, + driver_creations: u64 = 0, + reuse_hits: u64 = 0, + evictions: u64 = 0, + discarded: u64 = 0, + initialized: u64 = 0, + retained_slots: usize = 0, + retained_bytes: usize = 0, + quarantined_slots: usize = 0, + quarantined_bytes: usize = 0, + fence_failures: u64 = 0, +}; + +pub fn enabled(raw: ?[]const u8, cache_enabled: bool) !bool { + const text = raw orelse return false; + if (std.mem.eql(u8, text, "0")) return false; + if (!std.mem.eql(u8, text, "1")) return error.InvalidDraftReuseFlag; + if (!cache_enabled) return error.DraftReuseRequiresExactCache; + return true; +} + +pub fn configured(cache_enabled: bool) !bool { + const env = std.c.getenv("TF_PR608_EXACT_CACHE_DRAFT_REUSE"); + return enabled(if (env) |v| std.mem.span(v) else null, cache_enabled); +} + +pub fn eligible(drafted: bool, drafter: bool, tokens: usize, capacity: usize, fresh: bool, cancelled: bool) bool { + return drafted and drafter and tokens > 0 and tokens <= max_capacity and capacity <= max_capacity and + tokens < capacity and fresh and !cancelled; +} + +/// A completed request authorizes storage recycling only when every independent success gate holds. +pub fn completed(initialized: bool, terminal: bool, uncancelled: bool, settled: bool, cache_ok: bool, cache_failed: bool) bool { + return initialized and terminal and uncancelled and settled and cache_ok and !cache_failed; +} + +pub const Proof = struct { + stream: *const anyopaque, + seq: *const anyopaque, + wrapper: *const anyopaque, + arena: u64, + tokens: usize, + capacity: usize, +}; + +pub const Proofs = struct { + items: [max_slots]?Proof = @splat(null), + pub fn mark(p: *Proofs, proof: Proof) !void { + if (proof.arena == 0 or proof.tokens == 0 or proof.tokens >= proof.capacity or proof.capacity > max_capacity) + return error.InvalidDraftInitialization; + for (p.items) |slot| if (slot) |old| { + if (old.stream == proof.stream) return error.DuplicateDraftInitialization; + }; + for (&p.items) |*slot| if (slot.* == null) { + slot.* = proof; + return; + }; + return error.DraftInitializationLimit; + } + pub fn remove(p: *Proofs, stream: *const anyopaque) ?Proof { + for (&p.items) |*slot| if (slot.*) |proof| { + if (proof.stream != stream) continue; + slot.* = null; + return proof; + }; + return null; + } +}; + +/// Idle entries were fenced; quarantined entries must not be freed or taken until a new successful fence. +pub fn Pool(comptime T: type) type { + return struct { + const Self = @This(); + entries: [max_slots]*T = undefined, + quarantine: [max_slots]*T = undefined, + limit: usize, + item_bytes: usize, + stats: Stats, + + pub fn init(on: bool, slots: usize, item_bytes: usize) Self { + return .{ .limit = @min(slots, max_slots), .item_bytes = item_bytes, .stats = .{ .enabled = on } }; + } + pub fn active(p: *const Self) bool { + return p.stats.enabled and !p.stats.failed; + } + pub fn hasRoom(p: *const Self, live: usize) bool { + return live < p.limit and p.stats.retained_slots + p.stats.quarantined_slots < p.limit - live; + } + fn duplicate(p: *const Self, ptr: *T) bool { + for (p.entries[0..p.stats.retained_slots]) |old| if (old == ptr) return true; + for (p.quarantine[0..p.stats.quarantined_slots]) |old| if (old == ptr) return true; + return false; + } + pub fn take(p: *Self) ?*T { + if (!p.active() or p.stats.retained_slots == 0) return null; + p.stats.retained_slots -= 1; + p.stats.retained_bytes -= p.item_bytes; + p.stats.reuse_hits += 1; + return p.entries[p.stats.retained_slots]; + } + pub fn evict(p: *Self) ?*T { + if (p.stats.retained_slots == 0) return null; + p.stats.retained_slots -= 1; + p.stats.retained_bytes -= p.item_bytes; + p.stats.evictions += 1; + return p.entries[p.stats.retained_slots]; + } + pub fn retain(p: *Self, ptr: *T, live: usize) bool { + if (!p.active() or !p.hasRoom(live) or p.item_bytes > max_bytes or + p.stats.retained_bytes > max_bytes - p.item_bytes) return false; + if (p.duplicate(ptr)) @panic("duplicate draft pool owner"); + p.entries[p.stats.retained_slots] = ptr; + p.stats.retained_slots += 1; + p.stats.retained_bytes += p.item_bytes; + return true; + } + pub fn fail(p: *Self) void { + if (p.stats.enabled) p.stats.failed = true; + } + pub fn park(p: *Self, ptr: *T, live: usize) void { + p.fail(); + if (!p.hasRoom(live) or p.duplicate(ptr)) @panic("draft quarantine ownership limit"); + p.quarantine[p.stats.quarantined_slots] = ptr; + p.stats.quarantined_slots += 1; + p.stats.quarantined_bytes += p.item_bytes; + p.stats.fence_failures += 1; + } + pub fn reap(p: *Self, fenced: bool) ?*T { + if (!fenced or p.stats.quarantined_slots == 0) return null; + p.stats.quarantined_slots -= 1; + p.stats.quarantined_bytes -= p.item_bytes; + return p.quarantine[p.stats.quarantined_slots]; + } + }; +} + +test { + _ = @import("draft_pool_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/draft_pool_test.zig b/zig/src/families/qwen27/cuda/draft_pool_test.zig new file mode 100644 index 0000000000..e1fc92575d --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_pool_test.zig @@ -0,0 +1,105 @@ +//! Host-only Context storage ownership and initialization provenance regressions. +const std = @import("std"); +const p = @import("draft_pool.zig"); +const expect = std.testing.expect; +const equal = std.testing.expectEqual; + +test "strict flag is default off and requires exact cache" { + try expect(!try p.enabled(null, false)); + try expect(!try p.enabled("0", false)); + try expect(try p.enabled("1", true)); + try std.testing.expectError(error.DraftReuseRequiresExactCache, p.enabled("1", false)); + try std.testing.expectError(error.InvalidDraftReuseFlag, p.enabled("yes", true)); +} + +test "fresh small drafted requests include cross-key and capacity changes" { + try expect(p.eligible(true, true, 14, 286, true, false)); + try expect(p.eligible(true, true, 31, 303, true, false)); + try expect(p.eligible(true, true, 1, 512, true, false)); + try expect(!p.eligible(false, true, 14, 286, true, false)); + try expect(!p.eligible(true, false, 14, 286, true, false)); + try expect(!p.eligible(true, true, 31, 513, true, false)); + try expect(!p.eligible(true, true, 0, 286, true, false)); + try expect(!p.eligible(true, true, 31, 31, true, false)); + try expect(!p.eligible(true, true, 31, 303, false, false)); + try expect(!p.eligible(true, true, 31, 303, true, true)); +} + +test "detached storage transfers once and total ownership is bounded" { + var owners = [_]u8{ 1, 2, 3 }; + var pool = p.Pool(u8).init(true, 2, 80); + try expect(pool.retain(&owners[0], 1)); + try expect(!pool.retain(&owners[1], 1)); + try expect(!pool.hasRoom(1)); + try expect(pool.take().? == &owners[0]); + try expect(pool.take() == null); + try equal(@as(u64, 1), pool.stats.reuse_hits); + try expect(pool.retain(&owners[0], 0)); + try expect(pool.retain(&owners[1], 0)); + try expect(!pool.retain(&owners[2], 0)); + try equal(@as(usize, 160), pool.stats.retained_bytes); + try expect(pool.evict().? == &owners[1]); + try expect(pool.evict().? == &owners[0]); + try equal(@as(usize, 0), pool.stats.retained_bytes); + try equal(@as(u64, 2), pool.stats.evictions); +} + +test "retained bytes and backend configured ownership cap are enforced" { + var owners = [_]u8{ 1, 2, 3 }; + var pool = p.Pool(u8).init(true, 32, p.max_bytes / 2); + try equal(@as(usize, 16), pool.limit); + try expect(pool.retain(&owners[0], 0)); + try expect(pool.retain(&owners[1], 0)); + try expect(!pool.retain(&owners[2], 0)); + try equal(p.max_bytes, pool.stats.retained_bytes); + var disabled = p.Pool(u8).init(false, 16, 1); + try expect(!disabled.retain(&owners[0], 0)); + var zero = p.Pool(u8).init(true, 0, 1); + try expect(!zero.hasRoom(0)); +} + +test "failed fence quarantines without reuse or free until successful fence" { + var owners = [_]u8{ 1, 2 }; + var pool = p.Pool(u8).init(true, 2, 80); + pool.park(&owners[0], 1); + try expect(pool.stats.failed); + try expect(!pool.hasRoom(1)); + try expect(pool.take() == null); + try expect(pool.reap(false) == null); + try equal(@as(usize, 1), pool.stats.quarantined_slots); + try equal(@as(usize, 80), pool.stats.quarantined_bytes); + try expect(pool.reap(true).? == &owners[0]); + try expect(pool.reap(true) == null); + try equal(@as(usize, 0), pool.stats.quarantined_bytes); + try expect(!pool.retain(&owners[1], 0)); + try equal(@as(u64, 1), pool.stats.fence_failures); +} + +test "initialization proof binds independent request target wrapper and arena" { + var owners = [_]u8{ 1, 2, 3, 4 }; + const proof: p.Proof = .{ .stream = &owners[0], .seq = &owners[1], .wrapper = &owners[2], .arena = 4096, .tokens = 31, .capacity = 303 }; + var proofs: p.Proofs = .{}; + try proofs.mark(proof); + try std.testing.expectError(error.DuplicateDraftInitialization, proofs.mark(proof)); + try expect(proofs.remove(&owners[3]) == null); + const taken = proofs.remove(&owners[0]).?; + try expect(taken.seq == @as(*const anyopaque, &owners[1]) and taken.wrapper == @as(*const anyopaque, &owners[2])); + try equal(@as(u64, 4096), taken.arena); + try expect(proofs.remove(&owners[0]) == null); + var invalid = proof; + invalid.capacity = 513; + try std.testing.expectError(error.InvalidDraftInitialization, proofs.mark(invalid)); + invalid = proof; + invalid.arena = 0; + try std.testing.expectError(error.InvalidDraftInitialization, proofs.mark(invalid)); +} + +test "retention requires every independent clean completion gate" { + try expect(p.completed(true, true, true, true, true, false)); + try expect(!p.completed(false, true, true, true, true, false)); + try expect(!p.completed(true, false, true, true, true, false)); + try expect(!p.completed(true, true, false, true, true, false)); + try expect(!p.completed(true, true, true, false, true, false)); + try expect(!p.completed(true, true, true, true, false, false)); + try expect(!p.completed(true, true, true, true, true, true)); +} diff --git a/zig/src/families/qwen27/cuda/drafts.zig b/zig/src/families/qwen27/cuda/drafts.zig index 02e3659524..99303e9832 100644 --- a/zig/src/families/qwen27/cuda/drafts.zig +++ b/zig/src/families/qwen27/cuda/drafts.zig @@ -31,12 +31,27 @@ pub const Stream = struct { pub fn deinit(x: *Stream) void { x.ctx.deinit(); + x.deinitHost(); + } + + fn deinitHost(x: *Stream) void { x.copies.deinit(); x.tokens.deinit(x.gpa); x.parents.deinit(x.gpa); x.scores.deinit(x.gpa); } + /// Reconstruct every request-specific host field around exclusively owned, already-fenced storage. + pub fn resetStorage(x: *Stream) void { + const gpa = x.gpa; + var ctx = x.ctx; + x.deinitHost(); + ctx.cur = 0; + ctx.len = 0; + ctx.end = 0; + x.* = init(gpa, ctx); + } + fn clear(x: *Stream) void { x.kind = .none; x.tokens.clearRetainingCapacity(); diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig index af3c6594b4..0b2d2a5b81 100644 --- a/zig/src/families/qwen27/cuda/engine.zig +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -114,6 +114,12 @@ pub const Engine = struct { return st.Seq.init(e.gpa, e.ops(), e.w.g, capacity, e.kv8); } + /// Private BF16 admission only; no target may read this destination before restore or reset. + pub fn sequenceForRestore(e: *Engine, capacity: usize) !st.Seq { + if (e.kv8) return error.ExactCacheRequiresBf16; + return st.Seq.initRestoreDestination(e.gpa, e.ops(), e.w.g, capacity); + } + /// Attention's partials for `streams` streams of the engine's context in one round. pub fn fit(e: *Engine, streams: usize) !void { try e.s.synchronize(); diff --git a/zig/src/families/qwen27/cuda/exact_cache_policy.zig b/zig/src/families/qwen27/cuda/exact_cache_policy.zig new file mode 100644 index 0000000000..3aa664485b --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_cache_policy.zig @@ -0,0 +1,58 @@ +//! Private full-prompt cache eligibility and bounded ownership decisions. +const std = @import("std"); +pub const max_tokens = 512; +pub const max_bytes = 256 * 1024 * 1024; + +pub fn enabled(value: ?[]const u8) !bool { + const text = value orelse return false; + if (std.mem.eql(u8, text, "0")) return false; + if (std.mem.eql(u8, text, "1")) return true; + return error.InvalidExactPromptCacheFlag; +} + +pub fn eligible(tokens: usize, chunks: usize, fresh: bool, cancelled: bool) bool { + return tokens > 0 and tokens <= max_tokens and chunks <= max_tokens and fresh and !cancelled; +} + +pub fn clean(finished: bool, normal_reason: bool, cancelled: bool, settled: bool) bool { + return finished and normal_reason and !cancelled and settled; +} + +pub const Decision = enum { refuse, busy, retain, evict }; +pub fn admission(active_bytes: usize, pending_bytes: usize, requested: usize) Decision { + if (requested == 0 or requested > max_bytes) return .refuse; + if (pending_bytes != 0) return .busy; + return if (active_bytes > max_bytes - requested) .evict else .retain; +} + +test "flag is strict and disabled by default" { + try std.testing.expect(!try enabled(null)); + try std.testing.expect(!try enabled("0")); + try std.testing.expect(try enabled("1")); + try std.testing.expectError(error.InvalidExactPromptCacheFlag, enabled("true")); +} + +test "only bounded fresh uncancelled prompts can enter" { + try std.testing.expect(eligible(1, 0, true, false)); + try std.testing.expect(eligible(512, 512, true, false)); + try std.testing.expect(!eligible(0, 0, true, false)); + try std.testing.expect(!eligible(513, 0, true, false)); + try std.testing.expect(!eligible(14, 0, false, false)); + try std.testing.expect(!eligible(14, 0, true, true)); +} + +test "clean completion never includes uncertain settle or cancellation" { + try std.testing.expect(clean(true, true, false, true)); + try std.testing.expect(!clean(true, true, false, false)); + try std.testing.expect(!clean(true, true, true, true)); + try std.testing.expect(!clean(true, false, false, true)); + try std.testing.expect(!clean(false, true, false, true)); +} + +test "one pending owner and total byte bound include replacement with room" { + try std.testing.expectEqual(.busy, admission(0, 1, 100)); + try std.testing.expectEqual(.refuse, admission(0, 0, max_bytes + 1)); + try std.testing.expectEqual(.retain, admission(100, 0, 100)); + try std.testing.expectEqual(.evict, admission(197995008, 0, 197995008)); + try std.testing.expectEqual(.retain, admission(0, 0, 197995008)); +} diff --git a/zig/src/families/qwen27/cuda/exact_cache_test.zig b/zig/src/families/qwen27/cuda/exact_cache_test.zig new file mode 100644 index 0000000000..db3166b367 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_cache_test.zig @@ -0,0 +1,491 @@ +//! Host-only CUDA-driver mocks exercise snapshot ownership and failure paths. +const std = @import("std"); +const cuda = @import("cuda"); +const lanes = @import("lanes"); +const cache_mod = @import("exact_prompt_cache.zig"); +const st = @import("state.zig"); +const shape = @import("shape.zig"); +const kern = @import("kernels.zig"); +const dr = @import("draft.zig"); +const g = shape.the_27b; +const a = std.testing.allocator; + +const Mock = struct { + var live: usize = 0; + var freed: usize = 0; + const Copy = struct { dst: u64, src: u64, bytes: usize }; + var copy_rows: [512]Copy = undefined; + var copies: usize = 0; + var fences: usize = 0; + var fail_copy = false; + var fail_fence = false; + var next: u64 = 0x100000000; + fn alloc(ptr: *cuda.abi.DevicePtr, _: usize) callconv(.c) cuda.abi.Result { + ptr.* = next; + next += 0x100000000; + live += 1; + return cuda.abi.success; + } + fn free(_: cuda.abi.DevicePtr) callconv(.c) cuda.abi.Result { + live -= 1; + freed += 1; + return cuda.abi.success; + } + fn copy(dst: cuda.abi.DevicePtr, src: cuda.abi.DevicePtr, bytes: usize, _: cuda.abi.Stream) callconv(.c) cuda.abi.Result { + if (fail_copy) return cuda.abi.error_not_ready; + std.debug.assert(copies < copy_rows.len); + copy_rows[copies] = .{ .dst = dst, .src = src, .bytes = bytes }; + copies += 1; + return cuda.abi.success; + } + fn fence(_: cuda.abi.Stream) callconv(.c) cuda.abi.Result { + if (fail_fence) return cuda.abi.error_not_ready; + fences += 1; + return cuda.abi.success; + } + fn reset() void { + live = 0; + freed = 0; + copies = 0; + fences = 0; + fail_copy = false; + fail_fence = false; + } +}; + +const Fixture = struct { + driver: cuda.Driver = undefined, + kernels: kern.Kernels = undefined, + rec: [64]u64 = @splat(0x1000), + conv: [64]u64 = @splat(0x2000), + keys: [64]u64 = @splat(0x3000), + values: [64]u64 = @splat(0x4000), + fn ops(f: *Fixture) kern.Ops { + f.driver.api.cuMemAlloc_v2 = Mock.alloc; + f.driver.api.cuMemFree_v2 = Mock.free; + f.driver.api.cuMemcpyDtoDAsync_v2 = Mock.copy; + f.driver.api.cuStreamSynchronize = Mock.fence; + f.kernels.d = &f.driver; + return .{ .k = &f.kernels, .s = .{ .d = &f.driver, .handle = null } }; + } + fn seq(f: *Fixture, n: usize) st.Seq { + return .{ .gpa = a, .arena = .{ .buf = null }, .rec = &f.rec, .conv = &f.conv, .keys = &f.keys, .values = &f.values, .kscale = &.{}, .vscale = &.{}, .kv8 = false, .head_in = 0x5000, .head_xs = 0x6000, .pos = n, .capacity = n + 256 }; + } +}; + +fn stream(ids: []const u32, drafted: bool) !lanes.Stream { + return lanes.Stream.init(a, .{ .id = "cache-test", .prompt = ids, .max_new = 256, .drafts = drafted }); +} +fn finish(s: *lanes.Stream) void { + s.finished = true; + s.reason = .length; +} + +test "actual snapshot owner replacement and hit copy never alias a live destination" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + finish(&seed); + try cache.release(&seed, true, ops); + const saved = cache.active.?; + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + var dest = f.seq(0); + try std.testing.expect(try cache.lookup(&hit, &dest, null, ops, g)); + try std.testing.expectEqual(@as(u32, 3), hit.cached); + try std.testing.expectEqual(@as(usize, 3), dest.pos); + try std.testing.expect(saved.head_in != dest.head_in); + var miss = try stream(&.{ 4, 5, 6 }, false); + defer miss.deinit(a); + try cache.capture(&miss, &seq, null, ops, g); + try std.testing.expectEqual(@as(u64, 1), cache.stats.evictions); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + try std.testing.expect(Mock.fences >= 2); + finish(&miss); + try cache.release(&miss, true, ops); + try std.testing.expectEqual(@as(u64, 2), cache.stats.promotions); +} + +test "pending is never hit and error or cancellation cannot promote" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var s = try stream(&.{ 1, 2 }, false); + defer s.deinit(a); + var seq = f.seq(2); + try cache.capture(&s, &seq, null, ops, g); + var other = try stream(&.{ 1, 2 }, false); + defer other.deinit(a); + try std.testing.expect(!try cache.lookup(&other, &seq, null, ops, g)); + try cache.capture(&other, &seq, null, ops, g); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + s.finished = true; + s.reason = .cancelled; + try cache.release(&s, true, ops); + try std.testing.expect(cache.active == null and cache.pending == null); + s.finished = false; + s.reason = .none; + try cache.capture(&s, &seq, null, ops, g); + finish(&s); + try cache.release(&other, false, ops); + try std.testing.expect(cache.failed and cache.active == null and Mock.live == 0); + try cache.release(&s, true, ops); + try std.testing.expectEqual(@as(u64, 0), cache.stats.promotions); +} + +test "copy failure purges and failed fence defers freeing in-flight source" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var s = try stream(&.{1}, false); + defer s.deinit(a); + var seq = f.seq(1); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, cache.capture(&s, &seq, null, ops, g)); + Mock.fail_copy = false; + Mock.fail_fence = true; + try std.testing.expectError(error.NotReady, cache.abandon(&s, ops)); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + Mock.fail_fence = false; + try cache.abandon(&s, ops); + try std.testing.expectEqual(@as(usize, 0), Mock.live); +} + +test "allocation refusal and disabled path perform no CUDA copy or fence" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var s = try stream(&.{1}, false); + defer s.deinit(a); + var seq = f.seq(1); + var off = cache_mod.Cache.init(a, false); + defer off.deinit(ops) catch unreachable; + try off.capture(&s, &seq, null, ops, g); + try std.testing.expect(!try off.lookup(&s, &seq, null, ops, g)); + var failing = std.testing.FailingAllocator.init(a, .{ .fail_index = 0 }); + var refused = cache_mod.Cache.init(failing.allocator(), true); + defer refused.deinit(ops) catch unreachable; + try refused.capture(&s, &seq, null, ops, g); + try std.testing.expectEqual(@as(u64, 1), refused.stats.refusals); + try std.testing.expect(Mock.copies == 0 and Mock.fences == 0 and Mock.live == 0); +} + +test "draft active-bank compact copies preserve metadata and bound maximum snapshot" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var s = try stream(&.{ 1, 2 }, true); + defer s.deinit(a); + var seq = f.seq(2); + var ctx: dr.Context = .{ .arena = .{ .buf = null }, .keys = @splat(@splat(0x10000)), .values = @splat(@splat(0x20000)), .cur = 1, .len = 2, .end = 2 }; + try cache.capture(&s, &seq, &ctx, ops, g); + // absorbMany and _block_attention pack each head at len*128 BF16 values, not the allocation window. + const head_bytes = 2 * 128 * 2; + const target_copies = 2 + 48 * 2 + 16 * 2; + const saved = cache.pending.?; + for (0..5) |layer| for (0..8) |head| { + const pair = Mock.copy_rows[target_copies + layer * 16 + head * 2 ..][0..2]; + try std.testing.expectEqual(0x10000 + head * head_bytes, pair[0].src); + try std.testing.expectEqual(0x20000 + head * head_bytes, pair[1].src); + try std.testing.expectEqual(saved.draft_keys[layer] + head * head_bytes, pair[0].dst); + try std.testing.expectEqual(saved.draft_values[layer] + head * head_bytes, pair[1].dst); + try std.testing.expectEqual(head_bytes, pair[0].bytes); + try std.testing.expectEqual(head_bytes, pair[1].bytes); + }; + const capture_copies = Mock.copies; + finish(&s); + try cache.release(&s, true, ops); + var hit = try stream(&.{ 1, 2 }, true); + defer hit.deinit(a); + ctx.cur = 0; + ctx.len = 0; + ctx.end = 0; + try std.testing.expect(try cache.lookup(&hit, &seq, &ctx, ops, g)); + try std.testing.expect(ctx.cur == 1 and ctx.len == 2 and ctx.end == 2); + for (0..5) |layer| for (0..8) |head| { + const pair = Mock.copy_rows[capture_copies + target_copies + layer * 16 + head * 2 ..][0..2]; + try std.testing.expectEqual(0x10000 + head * head_bytes, pair[0].dst); + try std.testing.expectEqual(0x20000 + head * head_bytes, pair[1].dst); + try std.testing.expectEqual(saved.draft_keys[layer] + head * head_bytes, pair[0].src); + try std.testing.expectEqual(saved.draft_values[layer] + head * head_bytes, pair[1].src); + }; + try std.testing.expectEqual(@as(usize, 197995008), cache_mod.Snapshot.bytes(g, 512, true)); + try std.testing.expect(cache_mod.Snapshot.bytes(g, 512, true) <= cache_mod.policy.max_bytes); +} + +test "acquisition hint shares exact eligibility but never copies or reports a hit" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + const before = cache.stats; + const copies = Mock.copies; + const fences = Mock.fences; + const active = cache.active; + try std.testing.expect(cache.canRestore(&hit, false)); + try std.testing.expect(!cache.canRestore(&hit, true)); + hit.cached = 1; + try std.testing.expect(!cache.canRestore(&hit, false)); + hit.cached = 0; + cache.failed = true; + try std.testing.expect(!cache.canRestore(&hit, false)); + cache.failed = false; + var miss = try stream(&.{ 1, 2, 4 }, false); + defer miss.deinit(a); + try std.testing.expect(!cache.canRestore(&miss, false)); + try std.testing.expect(std.meta.eql(before, cache.stats)); + try std.testing.expectEqual(copies, Mock.copies); + try std.testing.expectEqual(fences, Mock.fences); + try std.testing.expect(cache.active == active and hit.cached == 0); +} + +test "enabled first logits add one bounded row and absent marker prevents promotion" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + cache.logits_enabled = true; + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + const snap = cache.pending.?; + try std.testing.expect(!snap.logits_valid and snap.first_logits != 0); + try std.testing.expectEqual(cache_mod.Snapshot.bytes(g, 3, false) + 496640, snap.device_bytes); + try std.testing.expectEqual(@as(usize, 198491648), cache_mod.Snapshot.bytesWithLogits(g, 512, true, true)); + try std.testing.expect(cache_mod.Snapshot.bytesWithLogits(g, 512, true, true) < cache_mod.policy.max_bytes); + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expect(cache.active == null and cache.pending == null and Mock.live == 0); + try std.testing.expectEqual(@as(u64, 1), cache.stats.purges); +} + +test "actual raw capture and proven current-active copy preserve direction and eviction fallback" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + cache.logits_enabled = true; + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + const saved_logits = cache.pending.?.first_logits; + const before = Mock.copies; + try std.testing.expect(try cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + try std.testing.expectEqual(before + 1, Mock.copies); + try std.testing.expectEqual(saved_logits, Mock.copy_rows[before].dst); + try std.testing.expectEqual(@as(u64, 0x7770), Mock.copy_rows[before].src); + try std.testing.expectEqual(@as(usize, 496640), Mock.copy_rows[before].bytes); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + var dest = f.seq(0); + try std.testing.expect(try cache.lookup(&hit, &dest, null, ops, g)); + const copied = Mock.copies; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, false)); + try std.testing.expectEqual(copied, Mock.copies); + dest.pos = 4; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + dest.pos = 3; + try std.testing.expect(try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + try std.testing.expectEqual(saved_logits, Mock.copy_rows[copied].src); + try std.testing.expectEqual(@as(u64, 0x8880), Mock.copy_rows[copied].dst); + try std.testing.expectEqual(@as(usize, 496640), Mock.copy_rows[copied].bytes); + try std.testing.expect(cache.active.?.logits_valid); + try std.testing.expectEqual(@as(u64, 1), cache.stats.first_logits_hits); + Mock.copies = 0; + var miss = try stream(&.{ 4, 5, 6 }, false); + defer miss.deinit(a); + try cache.capture(&miss, &seq, null, ops, g); + try std.testing.expect(cache.active == null and cache.pending != null); + const after_eviction = Mock.copies; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + try std.testing.expectEqual(after_eviction, Mock.copies); + try std.testing.expect(Mock.fences > 0); +} + +test "raw capture error clears validity and hit copy error disables future use" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + cache.logits_enabled = true; + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{1}, false); + defer seed.deinit(a); + var seq = f.seq(1); + try cache.capture(&seed, &seq, null, ops, g); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + try std.testing.expect(cache.failed and !cache.pending.?.logits_valid); + Mock.fail_copy = false; + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expect(cache.active == null and cache.pending == null); + cache.failed = false; + seed.finished = false; + seed.reason = .none; + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expect(try cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{1}, false); + defer hit.deinit(a); + var dest = f.seq(0); + try std.testing.expect(try cache.lookup(&hit, &dest, null, ops, g)); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + try std.testing.expect(cache.failed); + Mock.fail_copy = false; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); +} + +test "logits-off capture and first-logit methods add no row copy or fence" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{1}, false); + defer seed.deinit(a); + var seq = f.seq(1); + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expectEqual(cache_mod.Snapshot.bytes(g, 1, false), cache.pending.?.device_bytes); + try std.testing.expectEqual(@as(u64, 0), cache.pending.?.first_logits); + const copied = Mock.copies; + const fenced = Mock.fences; + try std.testing.expect(!try cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + try std.testing.expect(!try cache.restoreFirstLogits(&seed, &seq, ops, 0x8880, true)); + try std.testing.expectEqual(copied, Mock.copies); + try std.testing.expectEqual(fenced, Mock.fences); + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expect(cache.active != null); +} + +test "unsupported Seq formats decline active lookup and new capture before any device operation" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2 }, false); + defer seed.deinit(a); + var seq = f.seq(2); + try cache.capture(&seed, &seq, null, ops, g); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2 }, false); + defer hit.deinit(a); + var miss = try stream(&.{ 3, 4 }, false); + defer miss.deinit(a); + const raw = try cache_mod.Snapshot.init(a, ops, g, &hit); + defer raw.deinit(); + var media: @import("vision.zig").Media = undefined; + const before = .{ Mock.live, Mock.copies, Mock.fences }; + for (0..3) |kind| { + seq.kv8 = kind == 0; + seq.media = if (kind == 1) &media else null; + seq.rope_delta = if (kind == 2) 7 else 0; + try cache.capture(&miss, &seq, null, ops, g); + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(@as(u32, 0), hit.cached); + try std.testing.expectError(error.UnsupportedExactCacheState, raw.save(ops, g, &seq, null)); + try std.testing.expectError(error.UnsupportedExactCacheState, raw.restore(ops, g, &seq, null)); + try std.testing.expectEqual(before[0], Mock.live); + try std.testing.expectEqual(before[1], Mock.copies); + try std.testing.expectEqual(before[2], Mock.fences); + } + seq.kv8 = false; + seq.media = null; + seq.rope_delta = 0; + hit.images = &.{.{ .pixels = &.{}, .gh = 1, .gw = 1, .at = 0, .tokens = 1 }}; + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(before[1], Mock.copies); + hit.images = &.{}; + try std.testing.expect(try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(@as(u32, 2), hit.cached); +} + +test "partial host-prefix metadata cannot become full-N exact restoration proof" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + const before = Mock.copies; + hit.reuse.at = 2; + try std.testing.expect(!cache.canRestore(&hit, false)); + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + hit.reuse.at = 0; + hit.reuse.marks = &.{2}; + try std.testing.expect(!cache.canRestore(&hit, false)); + hit.reuse.marks = &.{}; + hit.cached = 2; + try std.testing.expect(!cache.canRestore(&hit, false)); + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(before, Mock.copies); + try std.testing.expectEqual(@as(u32, 2), hit.cached); + hit.cached = 0; + try std.testing.expect(cache.canRestore(&hit, false)); +} + +test "normal chat assistant-header marker permits full-N snapshot promotion and repeat hit" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + const ids: [31]u32 = @splat(1); + var seed = try lanes.Stream.init(a, .{ .id = "chat-cold", .prompt = &ids, .max_new = 256, .drafts = false, .decide = 29 }); + defer seed.deinit(a); + var seq = f.seq(ids.len); + try std.testing.expect(!cache.canRestore(&seed, false)); + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expectEqual(@as(u64, 1), cache.stats.captures); + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expectEqual(@as(u64, 1), cache.stats.promotions); + var repeat = try lanes.Stream.init(a, .{ .id = "chat-repeat", .prompt = &ids, .max_new = 256, .drafts = false, .decide = 29 }); + defer repeat.deinit(a); + var dest = f.seq(0); + try std.testing.expect(cache.canRestore(&repeat, false)); + try std.testing.expect(try cache.lookup(&repeat, &dest, null, ops, g)); + try std.testing.expectEqual(@as(u32, 31), repeat.cached); + try std.testing.expectEqual(@as(usize, 31), dest.pos); + try std.testing.expectEqual(@as(u64, 1), cache.stats.hits); +} diff --git a/zig/src/families/qwen27/cuda/exact_draft_reuse.zig b/zig/src/families/qwen27/cuda/exact_draft_reuse.zig new file mode 100644 index 0000000000..94a27b05d2 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_draft_reuse.zig @@ -0,0 +1,173 @@ +//! Private cold-capable Context storage recycling; uncertain in-flight ownership is quarantined, never reused. +const std = @import("std"); +const Cuda = @import("lanes.zig").Cuda; +const lanes = @import("lanes"); +const drafts = @import("drafts.zig"); +const dr = @import("draft.zig"); +const seq_reuse = @import("exact_seq_reuse.zig"); +pub const policy = @import("draft_pool.zig"); +pub const Storage = policy.Pool(drafts.Stream); + +fn destroy(x: *Cuda, ds: *drafts.Stream) void { + ds.deinit(); + x.gpa.destroy(ds); +} + +fn clearIdle(x: *Cuda) void { + while (x.draft_pool.evict()) |ds| destroy(x, ds); +} + +/// Only already-fenced idle storage can be freed without another fence. +pub fn fail(x: *Cuda) void { + if (!x.draft_pool.stats.enabled) return; + x.draft_pool.fail(); + clearIdle(x); +} + +fn fresh(s: *lanes.Stream) bool { + return s.chunks.len == 0 and !s.reuse_failed and s.reuse.saved == null and s.reuse.at == 0 and + s.reuse.marks.len == 0 and s.reuse.hook == null; +} + +fn eligible(x: *Cuda, s: *lanes.Stream, capacity: usize) bool { + return @import("exact_eligible.zig").text(x.e.kv8, s) and policy.eligible(s.drafts, x.drafter != null, s.prompt().len, capacity, fresh(s), s.isCancelled()); +} + +fn reap(x: *Cuda) !void { + if (x.draft_pool.stats.quarantined_slots == 0) return; + try x.e.s.synchronize(); + while (x.draft_pool.reap(true)) |ds| destroy(x, ds); +} + +/// Misses within the fixed small class may take storage; unsupported admissions drain before target allocation. +pub fn beforeOpen(x: *Cuda, s: *lanes.Stream, capacity: usize) !void { + if (!x.draft_pool.stats.enabled) return; + try reap(x); + if (x.seqs.count() >= x.draft_pool.limit) return error.DraftPoolOwnershipLimit; + if (!x.draft_pool.active() or !eligible(x, s, capacity)) clearIdle(x); +} + +pub fn obtain(x: *Cuda, s: *lanes.Stream) !*drafts.Stream { + errdefer seq_reuse.fail(x); + if (x.drafting.get(s)) |ds| return ds; + var ds: *drafts.Stream = undefined; + if (x.draft_pool.stats.enabled) { + try reap(x); + const seq = x.seqs.get(s) orelse return error.UnknownQwenStream; + if (x.draft_pool.active() and eligible(x, s, seq.capacity)) { + if (x.draft_pool.take()) |saved| { + ds = saved; + errdefer destroy(x, ds); + ds.resetStorage(); + try x.drafting.put(x.gpa, s, ds); + return ds; + } + } else clearIdle(x); + if (!x.draft_pool.hasRoom(x.drafting.count())) return error.DraftPoolOwnershipLimit; + } + ds = try x.gpa.create(drafts.Stream); + errdefer x.gpa.destroy(ds); + const ctx = dr.Context.init(x.e.ctx.d) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + if (x.draft_pool.stats.enabled) x.draft_pool.stats.driver_creations += 1; + ds.* = drafts.Stream.init(x.gpa, ctx); + errdefer ds.deinit(); + try x.drafting.put(x.gpa, s, ds); + return ds; +} + +/// The tag authorizes recycling empty storage later, not reuse of prior request state now. +pub fn initialized(x: *Cuda, s: *lanes.Stream) !void { + if (!x.draft_pool.active()) return; + errdefer fail(x); + const seq = x.seqs.get(s) orelse return error.UnknownQwenStream; + if (!eligible(x, s, seq.capacity)) return; + const ds = x.drafting.get(s) orelse return error.UnknownDraftStream; + const buf = ds.ctx.arena.buf orelse return error.InvalidDraftInitialization; + if (seq.pos != s.prompt_len or ds.ctx.len != s.prompt_len or ds.ctx.end != s.prompt_len) + return error.InvalidDraftInitialization; + try x.draft_initialized.mark(.{ .stream = s, .seq = seq, .wrapper = ds, .arena = buf.ptr, .tokens = s.prompt_len, .capacity = seq.capacity }); + x.draft_pool.stats.initialized += 1; +} + +fn detach(x: *Cuda, s: *lanes.Stream) ?*drafts.Stream { + if (x.drafting.fetchRemove(s)) |entry| return entry.value; + return null; +} + +/// Error ownership never enters the reusable pool, even when a later fence succeeds. +pub fn drop(x: *Cuda, s: *lanes.Stream) void { + _ = x.draft_initialized.remove(s); + const ds = detach(x, s) orelse return; + if (!x.draft_pool.stats.enabled) return destroy(x, ds); + fail(x); + x.e.s.synchronize() catch |err| { + std.log.err("quarantining failed draft Context: {s}", .{@errorName(err)}); + x.draft_pool.park(ds, x.drafting.count()); + return; + }; + x.draft_pool.stats.discarded += 1; + destroy(x, ds); +} + +/// Target release has already happened; pointer comparison does not dereference that former Seq owner. +pub fn release(x: *Cuda, s: *lanes.Stream, seq: ?*const anyopaque, settled: bool, cache_ok: bool) void { + const proof = x.draft_initialized.remove(s); + const ds = detach(x, s) orelse return; + if (!x.draft_pool.stats.enabled) return destroy(x, ds); + const normal = s.finished and (s.reason == .stop or s.reason == .length); + const valid = if (proof) |p| p.seq == seq and p.wrapper == @as(*const anyopaque, ds) and + p.arena == (if (ds.ctx.arena.buf) |buf| buf.ptr else 0) and p.tokens == s.prompt_len and + eligible(x, s, p.capacity) else false; + const clean = policy.completed(valid, normal, !s.isCancelled(), settled, cache_ok, x.exact_cache.failed); + if (!settled or !cache_ok or x.exact_cache.failed) fail(x); + x.e.s.synchronize() catch |err| { + std.log.err("quarantining released draft Context: {s}", .{@errorName(err)}); + fail(x); + x.draft_pool.park(ds, x.drafting.count()); + return; + }; + if (clean and !s.isCancelled() and x.draft_pool.active()) { + ds.resetStorage(); + if (x.draft_pool.retain(ds, x.drafting.count())) return; + } + x.draft_pool.stats.discarded += 1; + destroy(x, ds); +} + +fn shutdownFence(stream: anytype) !void { + stream.synchronize() catch |first_err| { + std.log.warn("draft teardown fence, retrying once: {s}", .{@errorName(first_err)}); + try stream.synchronize(); + }; +} + +/// One bounded retry precedes a fatal caller exit; no uncertain Context is freed on either failed fence. +pub fn shutdown(x: *Cuda) !void { + if (!x.draft_pool.stats.enabled) return; + if (x.draft_pool.stats.quarantined_slots > 0 or x.drafting.count() > 0) { + try shutdownFence(&x.e.s); + } + while (x.draft_pool.reap(true)) |ds| destroy(x, ds); + clearIdle(x); + std.debug.print("PR608_EXACT_CACHE_DRAFT_POOL {f}\n", .{std.json.fmt(x.draft_pool.stats, .{})}); +} + +test "shutdown fence has one bounded retry and propagates persistent failure" { + const Mock = struct { + calls: usize = 0, + failures: usize, + pub fn synchronize(x: *@This()) !void { + x.calls += 1; + if (x.calls <= x.failures) return error.MockFenceFailed; + } + }; + var good: Mock = .{ .failures = 0 }; + try shutdownFence(&good); + try std.testing.expectEqual(@as(usize, 1), good.calls); + var recovered: Mock = .{ .failures = 1 }; + try shutdownFence(&recovered); + try std.testing.expectEqual(@as(usize, 2), recovered.calls); + var failed: Mock = .{ .failures = 3 }; + try std.testing.expectError(error.MockFenceFailed, shutdownFence(&failed)); + try std.testing.expectEqual(@as(usize, 2), failed.calls); +} diff --git a/zig/src/families/qwen27/cuda/exact_eligible.zig b/zig/src/families/qwen27/cuda/exact_eligible.zig new file mode 100644 index 0000000000..7f9703b016 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_eligible.zig @@ -0,0 +1,40 @@ +//! The exact-cache helpers support only the original BF16 fresh text arithmetic and state layout. +const lanes = @import("lanes"); + +pub fn text(kv8: bool, s: *const lanes.Stream) bool { + // The server's assistant-header decide marker is inert in Qwen's nonlearning CUDA backend. + return !kv8 and s.images.len == 0 and s.prompt_logprobs == null and s.decode_spans.len == 0 and + !s.untaught and !s.reuse_failed; +} + +test "FP8, images and changed prompt arithmetic decline every exact-cache treatment" { + const std = @import("std"); + var s = try lanes.Stream.init(std.testing.allocator, .{ .id = "eligible", .prompt = &.{1}, .max_new = 4 }); + defer s.deinit(std.testing.allocator); + try std.testing.expect(text(false, &s)); + try std.testing.expect(!text(true, &s)); + s.images = &.{.{ .pixels = &.{}, .gh = 1, .gw = 1, .at = 0, .tokens = 1 }}; + try std.testing.expect(!text(false, &s)); + s.images = &.{}; + s.prompt_logprobs = 1; + try std.testing.expect(!text(false, &s)); + s.prompt_logprobs = null; + s.decode_spans = &.{.{ 0, 1 }}; + try std.testing.expect(!text(false, &s)); + s.decode_spans = &.{}; + s.untaught = true; + try std.testing.expect(!text(false, &s)); + s.untaught = false; + s.reuse_failed = true; + try std.testing.expect(!text(false, &s)); +} + +test "ordinary chat assistant-header metadata does not change Qwen text eligibility" { + const std = @import("std"); + const ids: [31]u32 = @splat(1); + var s = try lanes.Stream.init(std.testing.allocator, .{ .id = "chat-marker", .prompt = &ids, .max_new = 256, .decide = 29 }); + defer s.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(?u32, 29), s.decide); + try std.testing.expect(text(false, &s)); + try std.testing.expect(!text(true, &s)); +} diff --git a/zig/src/families/qwen27/cuda/exact_first_logits.zig b/zig/src/families/qwen27/cuda/exact_first_logits.zig new file mode 100644 index 0000000000..5b709a7bfa --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_first_logits.zig @@ -0,0 +1,31 @@ +//! First-head substitution only; the core forced-token selection and every sampling operation remain unchanged. +const std = @import("std"); +const lanes = @import("lanes"); +const Cuda = @import("lanes.zig").Cuda; +const reuse = @import("exact_seq_reuse.zig"); + +pub fn first(p: *anyopaque, s: *lanes.Stream, position: u64) !u64 { + const x: *Cuda = @ptrCast(@alignCast(p)); + errdefer { + reuse.fail(x); + if (x.exact_cache.logits_enabled) { + _ = x.restored.remove(s); + x.exact_cache.failed = true; + x.exact_cache.abandon(s, x.e.ops()) catch |err| std.log.err("first-logits purge: {s}", .{@errorName(err)}); + } + } + const seq = try x.settled(s); + if (position != seq.pos) return error.InvalidQwenPosition; + var f = x.e.forward(); + if (x.exact_cache.logits_enabled) { + const proven = x.seqs.get(s) == seq and x.restored.matches(s, seq, s.prompt().len); + if (!try x.exact_cache.restoreFirstLogits(s, seq, f.ops, f.s.logits, proven)) { + x.exact_cache.stats.first_logits_fallbacks += 1; + try f.head(seq); + _ = try x.exact_cache.captureFirstLogits(s, seq, f.ops, f.s.logits); + } + } else try f.head(seq); + var out: [1]u32 = undefined; + try x.drawRows(&.{s.sampling}, &.{position}, &out); + return out[0]; +} diff --git a/zig/src/families/qwen27/cuda/exact_init.zig b/zig/src/families/qwen27/cuda/exact_init.zig new file mode 100644 index 0000000000..b4657aec4c --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_init.zig @@ -0,0 +1,30 @@ +//! The default-off exact-cache owners initialized beside the unchanged Qwen CUDA host buffers. +const std = @import("std"); +const Cuda = @import("lanes.zig").Cuda; +const Engine = @import("engine.zig").Engine; +const c = @import("shape.zig"); +const st = @import("state.zig"); +const exact = @import("exact_prompt_cache.zig"); +const exact_reuse = @import("exact_seq_reuse.zig"); +const draft_reuse = @import("exact_draft_reuse.zig"); +const dr = @import("draft.zig"); + +pub fn init(gpa: std.mem.Allocator, e: *Engine, streams: u32) !Cuda { + const env = std.c.getenv("TF_PR608_EXACT_PROMPT_CACHE"); + const on = try exact.policy.enabled(if (env) |v| std.mem.span(v) else null); + var cache = exact.Cache.init(gpa, on); + cache.logits_enabled = try exact.logits_policy.configured(on); + return .{ + .gpa = gpa, + .e = e, + .streams = streams, + .exact_cache = cache, + .reset_skip = .{ .enabled = try exact_reuse.reset_skip.configured(on) }, + .seq_pool = exact_reuse.Storage.init(try exact_reuse.policy.configured(on), streams), + .draft_pool = draft_reuse.Storage.init(try draft_reuse.policy.configured(on), streams, dr.Context.bytes()), + .logits = try gpa.alloc(u16, c.vocab), + .cand_values = try gpa.alloc(f32, st.round_rows * st.max_candidates), + .cand_ids = try gpa.alloc(i64, st.round_rows * st.max_candidates), + .values = try gpa.alloc(f64, st.max_candidates), + }; +} diff --git a/zig/src/families/qwen27/cuda/exact_logits_policy.zig b/zig/src/families/qwen27/cuda/exact_logits_policy.zig new file mode 100644 index 0000000000..f4dc1f6d3f --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_logits_policy.zig @@ -0,0 +1,75 @@ +//! Bounded raw first-logit cache policy; decisions never cache sampling or a selected token. +const std = @import("std"); +pub const row_bytes: usize = 248320 * 2; // shape.zig's fixed BF16 vocabulary row, already 256-byte aligned + +pub fn enabled(text: ?[]const u8, cache_enabled: bool) !bool { + const value = text orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (!std.mem.eql(u8, value, "1")) return error.InvalidExactCacheLogitsFlag; + if (!cache_enabled) return error.FirstLogitsRequireExactCache; + return true; +} + +pub fn configured(cache_enabled: bool) !bool { + const env = std.c.getenv("TF_PR608_EXACT_CACHE_LOGITS"); + return enabled(if (env) |v| std.mem.span(v) else null, cache_enabled); +} + +pub fn canUse(healthy: bool, proven: bool, key_matches: bool, valid: bool, position_matches: bool, cancelled: bool) bool { + return healthy and proven and key_matches and valid and position_matches and !cancelled; +} + +pub fn promotable(on: bool, valid: bool) bool { + return !on or valid; +} + +/// Only successful submission grants validity; the existing promotion fence supplies completion. +pub fn capture(valid: *bool, copier: anytype, dst: u64, src: u64) !void { + valid.* = false; + try copier.copy(dst, src, row_bytes); + valid.* = true; +} + +test "raw logits are off by default and strictly require the full-prompt cache" { + try std.testing.expect(!try enabled(null, false)); + try std.testing.expect(!try enabled("0", false)); + try std.testing.expect(try enabled("1", true)); + try std.testing.expectError(error.FirstLogitsRequireExactCache, enabled("1", false)); + try std.testing.expectError(error.InvalidExactCacheLogitsFlag, enabled("true", true)); + try std.testing.expectError(error.InvalidExactCacheLogitsFlag, enabled("", true)); +} + +test "current active key and actual stream restore proof are both required" { + try std.testing.expect(canUse(true, true, true, true, true, false)); + try std.testing.expect(!canUse(false, true, true, true, true, false)); + try std.testing.expect(!canUse(true, false, true, true, true, false)); + try std.testing.expect(!canUse(true, true, false, true, true, false)); + try std.testing.expect(!canUse(true, true, true, false, true, false)); + try std.testing.expect(!canUse(true, true, true, true, false, false)); + try std.testing.expect(!canUse(true, true, true, true, true, true)); + try std.testing.expect(!promotable(true, false)); + try std.testing.expect(promotable(true, true)); + try std.testing.expect(promotable(false, false)); +} + +test "copy submission failure cannot create a valid raw row or promotion" { + const Copier = struct { + fail: bool, + calls: usize = 0, + fn copy(self: *@This(), dst: u64, src: u64, n: usize) !void { + self.calls += 1; + try std.testing.expectEqual(@as(u64, 256), dst); + try std.testing.expectEqual(@as(u64, 512), src); + try std.testing.expectEqual(@as(usize, 496640), n); + if (self.fail) return error.InjectedCopyFailure; + } + }; + var valid = true; + var copier: Copier = .{ .fail = true }; + try std.testing.expectError(error.InjectedCopyFailure, capture(&valid, &copier, 256, 512)); + try std.testing.expect(!valid and !promotable(true, valid)); + copier.fail = false; + try capture(&valid, &copier, 256, 512); + try std.testing.expect(valid and promotable(true, valid)); + try std.testing.expectEqual(@as(usize, 2), copier.calls); +} diff --git a/zig/src/families/qwen27/cuda/exact_prompt_cache.zig b/zig/src/families/qwen27/cuda/exact_prompt_cache.zig new file mode 100644 index 0000000000..41f5662879 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_prompt_cache.zig @@ -0,0 +1,339 @@ +//! Default-off full-prompt snapshots into independent normal Qwen allocations. +const std = @import("std"); +const lanes = @import("lanes"); +const st = @import("state.zig"); +const c = @import("shape.zig"); +const dr = @import("draft.zig"); +const dl = @import("draft_load.zig"); +const kern = @import("kernels.zig"); +pub const policy = @import("exact_cache_policy.zig"); +pub const logits_policy = @import("exact_logits_policy.zig"); +const roundedBytes = struct { + fn bytes(n: usize) usize { + return std.mem.alignForward(usize, n, 256); + } +}.bytes; + +pub const Snapshot = struct { + gpa: std.mem.Allocator, + arena: st.Arena, + prompt: []u32, + chunks: []u32, + drafted: bool, + pos: usize, + cur: u1 = 0, + len: usize = 0, + end: usize = 0, + rec: [64]u64 = @splat(0), + conv: [64]u64 = @splat(0), + keys: [64]u64 = @splat(0), + values: [64]u64 = @splat(0), + head_in: u64, + head_xs: u64, + draft_keys: [dl.layers]u64 = @splat(0), + draft_values: [dl.layers]u64 = @splat(0), + device_bytes: usize, + first_logits: u64 = 0, + logits_valid: bool = false, + + pub fn bytes(g: c.Geometry, n: usize, drafted: bool) usize { + return bytesWithLogits(g, n, drafted, false); + } + + pub fn bytesWithLogits(g: c.Geometry, n: usize, drafted: bool, logits: bool) usize { + return st.Seq.bytes(g, n, false) + (if (drafted) 2 * dl.layers * roundedBytes(dl.kv_heads * n * dl.head_dim * 2) else 0) + + (if (logits) logits_policy.row_bytes else 0); + } + + pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, s: *lanes.Stream) !*Snapshot { + return initWithLogits(gpa, ops, g, s, false); + } + + pub fn initWithLogits(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, s: *lanes.Stream, logits: bool) !*Snapshot { + if (g.layers != 64) return error.UnsupportedExactCacheGeometry; + const n = s.prompt().len; + const total = bytesWithLogits(g, n, s.drafts, logits); + if (total > policy.max_bytes) return error.ExactCacheBudget; + const out = try gpa.create(Snapshot); + errdefer gpa.destroy(out); + const prompt = try gpa.dupe(u32, s.prompt()); + errdefer gpa.free(prompt); + const chunks = try gpa.dupe(u32, s.chunks); + errdefer gpa.free(chunks); + out.* = .{ .gpa = gpa, .arena = try st.Arena.init(ops.k.d, total), .prompt = prompt, .chunks = chunks, .drafted = s.drafts, .pos = n, .head_in = 0, .head_xs = 0, .device_bytes = total }; + errdefer out.arena.deinit(); + out.head_in = try out.arena.take(g.hidden * 2); + out.head_xs = try out.arena.take(g.hidden / 64 * 4); + for (0..g.layers) |i| { + if (c.linear(i)) { + out.rec[i] = try out.arena.take(g.deltaBytes()); + out.conv[i] = try out.arena.take(g.convBytes()); + } else { + out.keys[i] = try out.arena.take(n * g.kvInner() * 2); + out.values[i] = try out.arena.take(n * g.kvInner() * 2); + } + } + if (s.drafts) for (0..dl.layers) |i| { + out.draft_keys[i] = try out.arena.take(dl.kv_heads * n * dl.head_dim * 2); + out.draft_values[i] = try out.arena.take(dl.kv_heads * n * dl.head_dim * 2); + }; + if (logits) out.first_logits = try out.arena.take(logits_policy.row_bytes); + return out; + } + + fn target(snap: *const Snapshot, ops: kern.Ops, g: c.Geometry, seq: *st.Seq, restoring: bool) !void { + if (seq.capacity < snap.pos) return error.ExactCacheCapacity; + const pair = struct { + fn run(o: kern.Ops, saved: u64, live: u64, n: usize, restored: bool) !void { + try o.copy(if (restored) live else saved, if (restored) saved else live, n); + } + }.run; + try pair(ops, snap.head_in, seq.head_in, g.hidden * 2, restoring); + try pair(ops, snap.head_xs, seq.head_xs, g.hidden / 64 * 4, restoring); + for (0..g.layers) |i| if (c.linear(i)) { + try pair(ops, snap.rec[i], seq.rec[i], g.deltaBytes(), restoring); + try pair(ops, snap.conv[i], seq.conv[i], g.convBytes(), restoring); + } else { + try pair(ops, snap.keys[i], seq.keys[i], snap.pos * g.kvInner() * 2, restoring); + try pair(ops, snap.values[i], seq.values[i], snap.pos * g.kvInner() * 2, restoring); + }; + } + + fn context(snap: *const Snapshot, ops: kern.Ops, ctx: *dr.Context, restoring: bool) !void { + const n = snap.len * dl.head_dim * 2; + const live_stride = n; + for (0..dl.layers) |i| for (0..dl.kv_heads) |h| { + const k = ctx.keys[i][snap.cur] + h * live_stride; + const v = ctx.values[i][snap.cur] + h * live_stride; + const sk = snap.draft_keys[i] + h * n; + const sv = snap.draft_values[i] + h * n; + try ops.copy(if (restoring) k else sk, if (restoring) sk else k, n); + try ops.copy(if (restoring) v else sv, if (restoring) sv else v, n); + }; + } + + pub fn save(snap: *Snapshot, ops: kern.Ops, g: c.Geometry, seq: *st.Seq, ctx: ?*dr.Context) !void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0) return error.UnsupportedExactCacheState; + if (seq.pos != snap.pos or (snap.drafted != (ctx != null))) return error.InvalidExactCacheState; + if (ctx) |x| { + if (x.len != snap.pos or x.end != snap.pos) return error.InvalidExactCacheContext; + snap.cur = x.cur; + snap.len = x.len; + snap.end = x.end; + } + try snap.target(ops, g, seq, false); + if (ctx) |x| try snap.context(ops, x, false); + } + + pub fn restore(snap: *const Snapshot, ops: kern.Ops, g: c.Geometry, seq: *st.Seq, ctx: ?*dr.Context) !void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0) return error.UnsupportedExactCacheState; + if (snap.drafted != (ctx != null)) return error.InvalidExactCacheState; + try snap.target(ops, g, seq, true); + if (ctx) |x| { + try snap.context(ops, x, true); + x.cur = snap.cur; + x.len = snap.len; + x.end = snap.end; + } + seq.pos = snap.pos; + } + + pub fn matches(snap: *const Snapshot, s: *lanes.Stream) bool { + return snap.drafted == s.drafts and std.mem.eql(u32, snap.prompt, s.prompt()) and std.mem.eql(u32, snap.chunks, s.chunks); + } + + pub fn deinit(snap: *Snapshot) void { + snap.arena.deinit(); + snap.gpa.free(snap.prompt); + snap.gpa.free(snap.chunks); + snap.gpa.destroy(snap); + } +}; + +pub const Stats = struct { + lookups: u64 = 0, + hits: u64 = 0, + misses: u64 = 0, + captures: u64 = 0, + promotions: u64 = 0, + purges: u64 = 0, + evictions: u64 = 0, + refusals: u64 = 0, + active_bytes: usize = 0, + pending_bytes: usize = 0, + first_logits_captures: u64 = 0, + first_logits_hits: u64 = 0, + first_logits_fallbacks: u64 = 0, +}; + +pub const Cache = struct { + gpa: std.mem.Allocator, + enabled: bool, + logits_enabled: bool = false, + failed: bool = false, + active: ?*Snapshot = null, + pending: ?*Snapshot = null, + owner: ?*lanes.Stream = null, + stats: Stats = .{}, + + pub fn init(gpa: std.mem.Allocator, on: bool) Cache { + return .{ .gpa = gpa, .enabled = on }; + } + + pub fn eligible(cache: *const Cache, s: *lanes.Stream) bool { + const fresh = s.reuse.saved == null and s.reuse.at == 0 and s.reuse.marks.len == 0 and s.cached == 0; + return cache.enabled and !cache.failed and @import("exact_eligible.zig").text(false, s) and policy.eligible(s.prompt().len, s.chunks.len, fresh, s.isCancelled()); + } + + /// Pure acquisition hint; successful lookup, not this hint, proves restoration. + pub fn canRestore(cache: *const Cache, s: *lanes.Stream, drafted: bool) bool { + if (!cache.eligible(s) or s.drafts != drafted) return false; + const snap = cache.active orelse return false; + return snap.matches(s); + } + + fn fence(cache: *Cache, ops: kern.Ops) !void { + ops.s.synchronize() catch |err| { + cache.failed = true; + return err; + }; + } + + fn freeActive(cache: *Cache, ops: kern.Ops) !void { + if (cache.active) |snap| { + try cache.fence(ops); + snap.deinit(); + cache.active = null; + cache.stats.active_bytes = 0; + } + } + + fn freePending(cache: *Cache, ops: kern.Ops) !void { + if (cache.pending) |snap| { + try cache.fence(ops); + snap.deinit(); + cache.pending = null; + cache.owner = null; + cache.stats.pending_bytes = 0; + } + } + + pub fn lookup(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry) !bool { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0 or !cache.eligible(s) or s.drafts != (ctx != null)) return false; + cache.stats.lookups += 1; + if (cache.active) |snap| if (snap.matches(s)) { + snap.restore(ops, g, seq, ctx) catch |err| { + cache.failed = true; + return err; + }; + s.cached = @intCast(snap.pos); + cache.stats.hits += 1; + return true; + }; + cache.stats.misses += 1; + return false; + } + + pub fn capture(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry) !void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0 or !cache.eligible(s) or s.drafts != (ctx != null) or (cache.active != null and cache.active.?.matches(s))) return; + const size = Snapshot.bytesWithLogits(g, s.prompt().len, s.drafts, cache.logits_enabled); + switch (policy.admission(cache.stats.active_bytes, cache.stats.pending_bytes, size)) { + .refuse, .busy => { + cache.stats.refusals += 1; + return; + }, + .evict => { + try cache.freeActive(ops); + cache.stats.evictions += 1; + }, + .retain => {}, + } + const snap = Snapshot.initWithLogits(cache.gpa, ops, g, s, cache.logits_enabled) catch { + cache.stats.refusals += 1; + return; + }; + cache.pending = snap; + cache.owner = s; + cache.stats.pending_bytes = snap.device_bytes; + snap.save(ops, g, seq, ctx) catch |err| { + cache.failed = true; + return err; + }; + cache.stats.captures += 1; + } + + /// Copy only from the current matching immutable entry; eviction means an ordinary head fallback. + pub fn restoreFirstLogits(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ops: kern.Ops, raw: u64, proven: bool) !bool { + if (!cache.logits_enabled or seq.kv8 or !@import("exact_eligible.zig").text(false, s)) return false; + const snap = cache.active orelse return false; + if (!logits_policy.canUse(cache.enabled and !cache.failed, proven, snap.matches(s), snap.logits_valid, seq.pos == snap.pos and s.prompt().len == snap.pos and s.cached == snap.pos, s.isCancelled())) return false; + if (snap.first_logits == 0) return error.InvalidFirstLogitsStorage; + ops.copy(raw, snap.first_logits, logits_policy.row_bytes) catch |err| { + cache.failed = true; + return err; + }; + cache.stats.first_logits_hits += 1; + return true; + } + + /// Preserve the cold owner's raw row before any draw; the normal promotion fence completes this copy. + pub fn captureFirstLogits(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ops: kern.Ops, raw: u64) !bool { + if (seq.kv8 or !@import("exact_eligible.zig").text(false, s) or !cache.logits_enabled or cache.failed or cache.owner != s) return false; + const snap = cache.pending orelse return false; + if (snap.logits_valid) return false; + if (!snap.matches(s) or seq.pos != snap.pos or snap.first_logits == 0) return error.InvalidFirstLogitsCapture; + logits_policy.capture(&snap.logits_valid, ops, snap.first_logits, raw) catch |err| { + cache.failed = true; + snap.logits_valid = false; + return err; + }; + cache.stats.first_logits_captures += 1; + return true; + } + + pub fn release(cache: *Cache, s: *lanes.Stream, settled: bool, ops: kern.Ops) !void { + if (!settled) { + cache.failed = true; + if (cache.pending != null) { + try cache.freePending(ops); + cache.stats.purges += 1; + } + return; + } + if (cache.owner != s) return; + const normal = s.reason == .stop or s.reason == .length; + const logits_ready = if (cache.pending) |snap| logits_policy.promotable(cache.logits_enabled, snap.logits_valid) else false; + if (cache.failed or !logits_ready or !policy.clean(s.finished, normal, s.isCancelled(), settled)) { + try cache.freePending(ops); + cache.stats.purges += 1; + return; + } + try cache.fence(ops); + try cache.freeActive(ops); + cache.active = cache.pending; + cache.pending = null; + cache.owner = null; + cache.stats.active_bytes = cache.stats.pending_bytes; + cache.stats.pending_bytes = 0; + cache.stats.promotions += 1; + } + + pub fn abandon(cache: *Cache, s: *lanes.Stream, ops: kern.Ops) !void { + if (cache.owner == s) { + try cache.freePending(ops); + cache.stats.purges += 1; + } + } + + pub fn deinit(cache: *Cache, ops: kern.Ops) !void { + if (cache.enabled) std.debug.print("PR608_EXACT_CACHE {{\"hits\":{d},\"misses\":{d},\"captures\":{d},\"promotions\":{d},\"purges\":{d},\"evictions\":{d},\"refusals\":{d},\"active_bytes\":{d},\"pending_bytes\":{d},\"failed\":{}}}\n", .{ cache.stats.hits, cache.stats.misses, cache.stats.captures, cache.stats.promotions, cache.stats.purges, cache.stats.evictions, cache.stats.refusals, cache.stats.active_bytes, cache.stats.pending_bytes, cache.failed }); + if (cache.logits_enabled) std.debug.print("PR608_EXACT_CACHE_LOGITS {f}\n", .{std.json.fmt(.{ + .captures = cache.stats.first_logits_captures, + .hits = cache.stats.first_logits_hits, + .fallbacks = cache.stats.first_logits_fallbacks, + .failed = cache.failed, + }, .{})}); + try cache.freePending(ops); + try cache.freeActive(ops); + } +}; diff --git a/zig/src/families/qwen27/cuda/exact_seq_reuse.zig b/zig/src/families/qwen27/cuda/exact_seq_reuse.zig new file mode 100644 index 0000000000..a101769248 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_seq_reuse.zig @@ -0,0 +1,189 @@ +//! Private target-only allocation reuse after proven full-N cache restore; reset and draft lifecycle are unchanged. +const std = @import("std"); +const st = @import("state.zig"); +const dr = @import("draft.zig"); +const lanes = @import("lanes"); +const Cuda = @import("lanes.zig").Cuda; +const draft_reuse = @import("exact_draft_reuse.zig"); +pub const policy = @import("seq_pool.zig"); +pub const Proofs = @import("restore_proof.zig").Proofs; +pub const Storage = policy.Pool(st.Seq); +pub const reset_skip = @import("reset_skip.zig"); +pub const Opened = struct { seq: *st.Seq, deferred_reset: bool = false, pooled: bool = false }; + +fn destroy(x: *Cuda, seq: *st.Seq) void { + seq.deinit(); + x.gpa.destroy(seq); +} + +pub fn clear(x: *Cuda) void { + while (x.seq_pool.evict()) |seq| destroy(x, seq); +} + +pub fn fail(x: *Cuda) void { + draft_reuse.fail(x); + if (!x.seq_pool.stats.enabled) return; + x.seq_pool.fail(); + clear(x); +} + +pub fn shutdown(x: *Cuda) void { + if (x.reset_skip.enabled) std.debug.print("PR608_EXACT_CACHE_RESET_SKIP {f}\n", .{std.json.fmt(x.reset_skip, .{})}); + if (x.seq_pool.stats.enabled) std.debug.print("PR608_EXACT_CACHE_SEQ_POOL {f}\n", .{std.json.fmt(x.seq_pool.stats, .{})}); + clear(x); +} + +/// A new stream's state, sized for its prompt and reply. +pub fn open(x: *Cuda, s: *lanes.Stream) !*st.Seq { + errdefer fail(x); + try x.settle(); + const ids = s.prompt(); + if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; + const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); + if (ids.len >= capacity) return error.PromptTooLong; + try draft_reuse.beforeOpen(x, s, capacity); + return obtain(x, s, capacity); +} + +/// A peek permits acquisition only; a separate successful lookup records restore provenance. +pub fn obtain(x: *Cuda, s: *lanes.Stream, capacity: usize) !*st.Seq { + return (try obtainState(x, s, capacity, false)).seq; +} + +fn obtainState(x: *Cuda, s: *lanes.Stream, capacity: usize, defer_reset: bool) !Opened { + errdefer fail(x); + if (x.seq_pool.active()) { + if (x.seqs.count() >= x.seq_pool.limit) return error.SeqReuseOwnershipLimit; + if (@import("exact_eligible.zig").text(x.e.kv8, s) and s.reuse.hook == null and + x.exact_cache.canRestore(s, x.drafter != null and s.drafts) and capacity <= x.e.context) + { + if (x.seq_pool.take(capacity)) |seq| { + errdefer destroy(x, seq); + if (seq.capacity != capacity) return error.SeqReuseCapacityMismatch; + if (defer_reset) { + seq.pos = 0; + seq.rope_delta = 0; + seq.media = null; + } else try seq.reset(x.e.ops(), x.e.w.g); + try x.seqs.put(x.gpa, s, seq); + if (defer_reset) x.reset_skip.deferred_pool += 1; + return .{ .seq = seq, .deferred_reset = defer_reset, .pooled = true }; + } + } + // Every fresh creation drains retained ownership, including a prompt-key miss with room. + x.seq_pool.beforeCreate(x, destroy); + if (!x.seq_pool.hasRoom(x.seqs.count())) return error.SeqReuseOwnershipLimit; + } + const seq = try x.gpa.create(st.Seq); + errdefer x.gpa.destroy(seq); + x.seq_pool.stats.create_attempts += 1; + seq.* = (if (defer_reset) x.e.sequenceForRestore(capacity) else x.e.sequence(capacity)) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + x.seq_pool.stats.driver_creations += 1; + errdefer seq.deinit(); + try x.seqs.put(x.gpa, s, seq); + if (defer_reset) x.reset_skip.deferred_fresh += 1; + return .{ .seq = seq, .deferred_reset = defer_reset }; +} + +/// Private admission only; public Cuda.open always initializes the target. +pub fn openPrefill(x: *Cuda, s: *lanes.Stream) !Opened { + if (!x.reset_skip.enabled) return .{ .seq = try x.open(s) }; + errdefer fail(x); + try x.settle(); + const ids = s.prompt(); + if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; + const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); + if (ids.len >= capacity) return error.PromptTooLong; + try draft_reuse.beforeOpen(x, s, capacity); + const hint = capacity <= reset_skip.max_capacity and @import("exact_eligible.zig").text(x.e.kv8, s) and s.reuse.hook == null and + x.exact_cache.canRestore(s, x.drafter != null and s.drafts); + return obtainState(x, s, capacity, hint); +} + +const Resolve = struct { + x: *Cuda, + stream: *lanes.Stream, + seq: *st.Seq, + ctx: ?*dr.Context, + pub fn cancelled(a: Resolve) bool { + return a.stream.isCancelled(); + } + pub fn lookup(a: Resolve) !bool { + return @import("exact_seq_reuse.zig").lookup(a.x, a.stream, a.seq, a.ctx); + } + pub fn reset(a: Resolve) !void { + try a.seq.reset(a.x.e.ops(), a.x.e.w.g); + } +}; + +/// No target call may precede resolution; restore errors abort, never fall back from a partial copy. +pub fn lookupPrefill(x: *Cuda, s: *lanes.Stream, opened: *Opened, ctx: ?*dr.Context) !bool { + errdefer fail(x); + if (!opened.deferred_reset) return lookup(x, s, opened.seq, ctx); + const action: Resolve = .{ .x = x, .stream = s, .seq = opened.seq, .ctx = ctx }; + return reset_skip.resolve(action, &opened.deferred_reset, opened.pooled, &x.reset_skip, reset_skip.resetWork(x.e.w.g)); +} + +pub fn lookup(x: *Cuda, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context) !bool { + errdefer fail(x); + if (!@import("exact_eligible.zig").text(x.e.kv8, s)) return false; + const hit = try x.exact_cache.lookup(s, seq, ctx, x.e.ops(), x.e.w.g); + if (hit and (x.seq_pool.active() or x.exact_cache.logits_enabled)) { + if (x.seqs.get(s) != seq or seq.pos != s.prompt().len or s.cached != s.prompt().len) + return error.InvalidSeqRestoreProof; + try x.restored.mark(s, seq, s.prompt().len); + } + if (hit) try draft_reuse.initialized(x, s); + return hit; +} + +fn retainOrDestroy(x: *Cuda, s: *lanes.Stream, seq: *st.Seq, restored: bool, settled: bool, cache_ok: bool) void { + const bytes = st.Seq.bytes(x.e.w.g, seq.capacity, seq.kv8); + const safe = @import("exact_eligible.zig").text(x.e.kv8, s) and !seq.kv8 and cache_ok and !x.exact_cache.failed and seq.capacity <= x.e.context and + seq.pos >= s.prompt_len and seq.pos <= seq.capacity; + if (safe and policy.completed(s, settled, restored, cache_ok, x.exact_cache.failed) and x.seq_pool.canRetain(x.seqs.count(), seq.capacity, bytes)) { + x.e.s.synchronize() catch |err| { + std.log.err("synchronizing before exact-cache Seq retention: {s}", .{@errorName(err)}); + fail(x); + x.seq_pool.stats.discarded += 1; + destroy(x, seq); + return; + }; + if (policy.completed(s, settled, restored, cache_ok, x.exact_cache.failed) and !x.exact_cache.failed and + x.seq_pool.retain(seq, x.seqs.count(), seq.capacity, bytes)) return; + } + x.seq_pool.stats.discarded += 1; + destroy(x, seq); +} + +/// Release retains only detached, synchronized ownership after actual settle and cache-release success. +pub fn release(ptr: *anyopaque, s: *lanes.Stream) void { + const x: *Cuda = @ptrCast(@alignCast(ptr)); + const settled_ok = blk: { + x.settle() catch |err| { + std.log.err("committing the last round before a release: {s}", .{@errorName(err)}); + break :blk false; + }; + break :blk true; + }; + const cache_ok = blk: { + x.exact_cache.release(s, settled_ok, x.e.ops()) catch |err| { + std.log.err("exact cache release: {s}", .{@errorName(err)}); + fail(x); + break :blk false; + }; + break :blk true; + }; + const draft_seq: ?*const anyopaque = if (x.seqs.get(s)) |seq| seq else null; + const proof = x.restored.remove(s); + if (x.seqs.fetchRemove(s)) |entry| { + const restored = if (proof) |p| p.seq == @as(*const anyopaque, entry.value) and p.tokens == s.prompt_len else false; + retainOrDestroy(x, s, entry.value, restored, settled_ok, cache_ok); + } + draft_reuse.release(x, s, draft_seq, settled_ok, cache_ok); +} + +test { + _ = policy; + _ = @import("restore_proof.zig"); +} diff --git a/zig/src/families/qwen27/cuda/lanes.zig b/zig/src/families/qwen27/cuda/lanes.zig index adae5d4913..2008211bb6 100644 --- a/zig/src/families/qwen27/cuda/lanes.zig +++ b/zig/src/families/qwen27/cuda/lanes.zig @@ -10,6 +10,9 @@ const Forward = @import("forward.zig").Forward; const eng = @import("engine.zig"); const prompt_rows = eng.prompt_rows; const reuse = @import("reuse.zig"); +const exact = @import("exact_prompt_cache.zig"); +const exact_reuse = @import("exact_seq_reuse.zig"); +const draft_reuse = @import("exact_draft_reuse.zig"); const dl = @import("draft_load.zig"); const dr = @import("draft.zig"); const drafts = @import("drafts.zig"); @@ -48,20 +51,27 @@ pub const Cuda = struct { drafting: std.AutoHashMapUnmanaged(*lanes.Stream, *drafts.Stream) = .empty, costs: [calibration.len + 1]lanes.config.Cost = undefined, // the forward's ms by a round's rows (calibrate) cost_count: usize = 0, + exact_cache: exact.Cache, + seq_pool: exact_reuse.Storage, + restored: exact_reuse.Proofs = .{}, + draft_pool: draft_reuse.Storage, + draft_initialized: draft_reuse.policy.Proofs = .{}, + reset_skip: exact_reuse.reset_skip.Stats, + + pub const openPrefill = exact_reuse.openPrefill; + pub const lookupPrefill = exact_reuse.lookupPrefill; pub fn init(gpa: std.mem.Allocator, e: *Engine, streams: u32) !Cuda { - return .{ - .gpa = gpa, - .e = e, - .streams = streams, - .logits = try gpa.alloc(u16, c.vocab), - .cand_values = try gpa.alloc(f32, st.round_rows * st.max_candidates), - .cand_ids = try gpa.alloc(i64, st.round_rows * st.max_candidates), - .values = try gpa.alloc(f64, st.max_candidates), - }; + return @import("exact_init.zig").init(gpa, e, streams); } pub fn deinit(x: *Cuda) void { + draft_reuse.shutdown(x) catch |err| { + std.log.err("fatal unresolved draft Context fence; terminating before device frees: {s}", .{@errorName(err)}); + std.process.exit(1); + }; + exact_reuse.shutdown(x); + x.exact_cache.deinit(x.e.ops()) catch |err| std.log.err("exact cache shutdown: {s}", .{@errorName(err)}); var it = x.seqs.valueIterator(); while (it.next()) |seq| { seq.*.deinit(); @@ -134,19 +144,12 @@ pub const Cuda = struct { } pub fn backend(x: *Cuda) be.Backend { - return .{ .ptr = x, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany } }; + return .{ .ptr = x, .first_before_draft = true, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany } }; } /// The stream's drafting (its DFlash2 context and copy index), made on first use. pub fn draftingOf(x: *Cuda, s: *lanes.Stream) !*drafts.Stream { - if (x.drafting.get(s)) |ds| return ds; - const ds = try x.gpa.create(drafts.Stream); - errdefer x.gpa.destroy(ds); - const ctx = dr.Context.init(x.e.ctx.d) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; - ds.* = drafts.Stream.init(x.gpa, ctx); - errdefer ds.deinit(); - try x.drafting.put(x.gpa, s, ds); - return ds; + return draft_reuse.obtain(x, s); } fn cast(p: *anyopaque) *Cuda { @@ -158,7 +161,7 @@ pub const Cuda = struct { } /// Commits the last verified round: every stream its kept rows (its whole window unless keep cut it). - fn settle(x: *Cuda) !void { + pub fn settle(x: *Cuda) !void { const p = &x.pending; if (p.n == 0) return; const n = p.n; @@ -168,7 +171,7 @@ pub const Cuda = struct { } /// The stream's state, the last round committed first. - fn settled(x: *Cuda, s: *lanes.Stream) !*st.Seq { + pub fn settled(x: *Cuda, s: *lanes.Stream) !*st.Seq { try x.settle(); return x.seqOf(s); } @@ -208,31 +211,24 @@ pub const Cuda = struct { } } - /// A new stream's state, sized for its prompt and reply. - pub fn open(x: *Cuda, s: *lanes.Stream) !*st.Seq { - try x.settle(); - const ids = s.prompt(); - if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; - const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); - if (ids.len >= capacity) return error.PromptTooLong; - const seq = try x.gpa.create(st.Seq); - errdefer x.gpa.destroy(seq); - seq.* = x.e.sequence(capacity) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; - errdefer seq.deinit(); - try x.seqs.put(x.gpa, s, seq); - return seq; - } + pub const open = exact_reuse.open; fn prefill(p: *anyopaque, s: *lanes.Stream) !void { const x = cast(p); - const seq = try x.open(s); + var opened = try x.openPrefill(s); errdefer x.dropSeq(s); - try x.prefillOpened(s, seq); + try x.prefillOpened(s, &opened); } /// prefill_state into an opened stream: the prompt in chunks, a drafting stream's last window of taps absorbed. A /// kept state the lane host found is restored first, and the pass stops at its marks for the cache to copy. - fn prefillOpened(x: *Cuda, s: *lanes.Stream, seq: *st.Seq) !void { + fn prefillOpened(x: *Cuda, s: *lanes.Stream, opened: *exact_reuse.Opened) !void { + errdefer exact_reuse.fail(x); + const seq = opened.seq; + if (x.exact_cache.eligible(s) and !x.e.kv8) { + const ctx = if (x.drafter != null and s.drafts) &(try x.draftingOf(s)).ctx else null; + if (try exact_reuse.lookupPrefill(x, s, opened, ctx)) return; + } // an image prompt: the tower's rows and the prompt's three positions for its pass (EncodedVision) const media: ?*vision.Media = if (s.images.len == 0) null else blk: { const tower = x.tower orelse return error.VisionDisabled; @@ -245,9 +241,12 @@ pub const Cuda = struct { const marks: ?eng.Keep = if (s.reuse.marks.len == 0 or media != null) null else .{ .at = s.reuse.marks, .ctx = s, .call = keptAt }; if (x.drafter) |d| if (s.drafts) { const ds = try x.draftingOf(s); - return lone.prefillFrom(x.e, seq, &ds.ctx, d, s.prompt(), from, media, marks); + try lone.prefillFrom(x.e, seq, &ds.ctx, d, s.prompt(), from, media, marks); + try x.exact_cache.capture(s, seq, &ds.ctx, x.e.ops(), x.e.w.g); + return draft_reuse.initialized(x, s); }; try x.e.prefillFrom(seq, s.prompt(), from, media, marks); + try x.exact_cache.capture(s, seq, null, x.e.ops(), x.e.w.g); } /// The stream's kept state restored on its sequence: where its prompt pass starts (0: none, off the grid, failed). @@ -280,13 +279,27 @@ pub const Cuda = struct { var n: usize = 0; var rows: usize = 0; for (streams, errs, 0..) |s, *err, i| { - const seq = x.open(s) catch |e| { + var opened = x.openPrefill(s) catch |e| { err.* = e; continue; }; + const seq = opened.seq; const ids = s.prompt(); + if (x.exact_cache.eligible(s) and !x.e.kv8) { + const ctx = if (x.drafter != null and s.drafts) &(x.draftingOf(s) catch |e| { + x.dropSeq(s); + err.* = e; + continue; + }).ctx else null; + const hit = exact_reuse.lookupPrefill(x, s, &opened, ctx) catch |e| { + x.dropSeq(s); + err.* = e; + continue; + }; + if (hit) continue; + } if (ids.len > prompt_rows) { - x.prefillOpened(s, seq) catch |e| { + x.prefillOpened(s, &opened) catch |e| { x.dropSeq(s); err.* = e; }; @@ -315,7 +328,13 @@ pub const Cuda = struct { fn passPieces(x: *Cuda, streams: []const *lanes.Stream, pieces: []const Forward.Piece, which: []const usize) !void { var f = x.e.forward(); - const d = x.drafter orelse return f.chunks(pieces); + const d = x.drafter orelse { + try f.chunks(pieces); + if (x.exact_cache.enabled and !x.e.kv8) for (pieces, which) |pc, i| { + try x.exact_cache.capture(streams[i], pc.seq, null, x.e.ops(), x.e.w.g); + }; + return; + }; f.taps = x.e.scratch.taps; try f.chunks(pieces); // each drafting stream's last window of taps, consecutive whole prompts absorbed together @@ -348,25 +367,26 @@ pub const Cuda = struct { g_rows += taken; } if (g > 0) try d.absorbMany(group[0..g], x.e.scratch.taps + g_from * row); + for (pieces, which) |pc, i| { + const s = streams[i]; + const ctx = if (s.drafts) &(try x.draftingOf(s)).ctx else null; + if (!x.e.kv8) try x.exact_cache.capture(s, pc.seq, ctx, x.e.ops(), x.e.w.g); + if (ctx != null) try draft_reuse.initialized(x, s); + } } fn dropSeq(x: *Cuda, s: *lanes.Stream) void { + if (x.draft_pool.stats.enabled) draft_reuse.drop(x, s); + _ = x.restored.remove(s); + exact_reuse.fail(x); + x.exact_cache.abandon(s, x.e.ops()) catch |err| std.log.err("exact cache abandon: {s}", .{@errorName(err)}); if (x.seqs.fetchRemove(s)) |entry| { entry.value.deinit(); x.gpa.destroy(entry.value); } } - fn first(p: *anyopaque, s: *lanes.Stream, position: u64) !u64 { - const x = cast(p); - const seq = try x.settled(s); - if (position != seq.pos) return error.InvalidQwenPosition; - var f = x.e.forward(); - try f.head(seq); - var out: [1]u32 = undefined; - try x.drawRows(&.{s.sampling}, &.{position}, &out); - return out[0]; - } + const first = @import("exact_first_logits.zig").first; fn queue(p: *anyopaque, s: *lanes.Stream, feed: be.Feed, position: u64) !u64 { const x = cast(p); @@ -500,7 +520,7 @@ pub const Cuda = struct { .value => |v| v, .handle => |h| try read(p, h), }; - try firsts[na].appendSlice(x.gpa, context); + try firsts[na].appendSlice(x.gpa, r.stream.prompt()); try firsts[na].append(x.gpa, token); context = firsts[na].items; } @@ -531,18 +551,7 @@ pub const Cuda = struct { return true; } - fn release(p: *anyopaque, s: *lanes.Stream) void { - const x = cast(p); - x.settle() catch |err| std.log.err("committing the last round before a release: {s}", .{@errorName(err)}); - if (x.seqs.fetchRemove(s)) |entry| { - entry.value.deinit(); - x.gpa.destroy(entry.value); - } - if (x.drafting.fetchRemove(s)) |entry| { - entry.value.deinit(); - x.gpa.destroy(entry.value); - } - } + const release = exact_reuse.release; }; fn value(word: u16) f64 { @@ -584,3 +593,8 @@ pub fn draw(gpa: std.mem.Allocator, logits: []const u16, settings: ?lanes.Sampli } return @intCast(try lanes.sampling.choose(gpa, values, ids, position, sampling)); } + +test { + _ = @import("exact_cache_test.zig"); + _ = @import("exact_eligible.zig"); +} diff --git a/zig/src/families/qwen27/cuda/reset_skip.zig b/zig/src/families/qwen27/cuda/reset_skip.zig new file mode 100644 index 0000000000..fc625e7103 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reset_skip.zig @@ -0,0 +1,58 @@ +//! Private prefill reset transaction: only actual full-state restoration resolves uninitialized target state. +const std = @import("std"); +pub const max_capacity: usize = 512; +pub const Stats = struct { + enabled: bool = false, + deferred_fresh: u64 = 0, + deferred_pool: u64 = 0, + restored_fresh: u64 = 0, + restored_pool: u64 = 0, + avoided_fill_calls: u64 = 0, + avoided_fill_bytes: u64 = 0, + fallback_resets: u64 = 0, + failed_deferred_transactions: u64 = 0, +}; + +pub fn compatible(raw: ?[]const u8, cache_enabled: bool) !bool { + const value = raw orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (!std.mem.eql(u8, value, "1")) return error.InvalidResetSkip; + if (!cache_enabled) return error.ResetSkipNeedsExactCache; + return true; +} + +pub fn configured(cache_enabled: bool) !bool { + const raw = std.c.getenv("TF_PR608_EXACT_CACHE_SKIP_RESET"); + return compatible(if (raw) |value| std.mem.span(value) else null, cache_enabled); +} + +/// Counts the actual reset domains, excluding attention KV, heads and alignment padding. +pub fn resetWork(g: anytype) struct { calls: u64, bytes: u64 } { + var linear_layers: u64 = 0; + for (0..g.layers) |i| if (i % 4 != 3) { + linear_layers += 1; + }; + return .{ .calls = linear_layers * 2, .bytes = linear_layers * @as(u64, @intCast(g.deltaBytes() + g.convBytes())) }; +} + +/// A hint never resolves deferred state: actual hit or original reset must happen before target work. +pub fn resolve(action: anytype, deferred: *bool, pooled: bool, stats: *Stats, work: anytype) !bool { + if (!deferred.*) return action.lookup(); + errdefer stats.failed_deferred_transactions += 1; + if (action.cancelled()) return error.Cancelled; + const hit = try action.lookup(); + if (hit) { + if (pooled) stats.restored_pool += 1 else stats.restored_fresh += 1; + stats.avoided_fill_calls += work.calls; + stats.avoided_fill_bytes += work.bytes; + } else { + try action.reset(); + stats.fallback_resets += 1; + } + deferred.* = false; + return hit; +} + +test { + _ = @import("reset_skip_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/reset_skip_factory_test.zig b/zig/src/families/qwen27/cuda/reset_skip_factory_test.zig new file mode 100644 index 0000000000..31aaa4fe15 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reset_skip_factory_test.zig @@ -0,0 +1,179 @@ +//! Real Seq factories and Ops.fill32 wired to synthetic CUDA driver calls; no device or model is used. +const std = @import("std"); +const cuda = @import("cuda"); +const st = @import("state.zig"); +const shape = @import("shape.zig"); +const kern = @import("kernels.zig"); +const policy = @import("reset_skip.zig"); +const Call = struct { dst: u64, value: c_uint, words: usize }; +var calls: [96]Call = undefined; +var call_count: usize = 0; +var allocations: usize = 0; +var frees: usize = 0; +var fail_alloc: bool = false; +var fail_fill: bool = false; + +fn allocate(ptr: *u64, bytes: usize) callconv(.c) c_int { + if (fail_alloc) return cuda.abi.error_not_ready; + if (bytes == 0) return 1; + allocations += 1; + ptr.* = 0x100000000; + return 0; +} +fn freeDevice(ptr: u64) callconv(.c) c_int { + if (ptr == 0x100000000) frees += 1; + return 0; +} +fn fill(dst: u64, value: c_uint, words: usize, stream: cuda.abi.Stream) callconv(.c) c_int { + _ = stream; + if (fail_fill) return cuda.abi.error_not_ready; + if (call_count >= calls.len) return 1; + calls[call_count] = .{ .dst = dst, .value = value, .words = words }; + call_count += 1; + return 0; +} +fn errorText(code: c_int, out: *?[*:0]const u8) callconv(.c) c_int { + _ = code; + out.* = "synthetic reset failure"; + return 0; +} +fn setup(driver: *cuda.Driver, kernels: *kern.Kernels) kern.Ops { + call_count = 0; + allocations = 0; + frees = 0; + fail_alloc = false; + fail_fill = false; + driver.* = undefined; + driver.api.cuMemAlloc_v2 = allocate; + driver.api.cuMemFree_v2 = freeDevice; + driver.api.cuMemsetD32Async = fill; + driver.api.cuGetErrorName = errorText; + driver.api.cuGetErrorString = errorText; + kernels.* = undefined; + kernels.d = driver; + return .{ .k = kernels, .s = .{ .d = driver, .handle = null } }; +} +fn checkReset(seq: *const st.Seq) !void { + const g = shape.the_27b; + var index: usize = 0; + var bytes: u64 = 0; + for (0..g.layers) |layer| if (shape.linear(layer)) { + try std.testing.expectEqual(seq.rec[layer], calls[index].dst); + try std.testing.expectEqual(g.deltaBytes() / 4, calls[index].words); + try std.testing.expectEqual(seq.conv[layer], calls[index + 1].dst); + try std.testing.expectEqual(g.convBytes() / 4, calls[index + 1].words); + try std.testing.expectEqual(@as(c_uint, 0), calls[index].value | calls[index + 1].value); + bytes += (calls[index].words + calls[index + 1].words) * 4; + index += 2; + }; + try std.testing.expectEqual(@as(usize, 96), index); + try std.testing.expectEqual(index, call_count); + try std.testing.expectEqual(@as(u64, 153944064), bytes); + const actual = policy.resetWork(g); + try std.testing.expectEqual(@as(u64, @intCast(index)), actual.calls); + try std.testing.expectEqual(bytes, actual.bytes); +} + +test "public Seq factory calls all original driver fills and owns one arena" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + var seq = try st.Seq.init(std.testing.allocator, ops, shape.the_27b, 303, false); + try checkReset(&seq); + try std.testing.expectEqual(@as(usize, 303), seq.capacity); + try std.testing.expectEqual(@as(usize, 0), seq.pos); + seq.deinit(); + try std.testing.expectEqual(@as(usize, 1), allocations); + try std.testing.expectEqual(allocations, frees); +} + +test "restore destination preserves capacity and layout while omitting only initial fills" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + var seq = try st.Seq.initRestoreDestination(std.testing.allocator, ops, shape.the_27b, 303); + defer seq.deinit(); + try std.testing.expectEqual(@as(usize, 0), call_count); + try std.testing.expectEqual(@as(usize, 303), seq.capacity); + try std.testing.expectEqual(st.Seq.bytes(shape.the_27b, 303, false), seq.arena.buf.?.len); + var layout: st.Arena = .{ .buf = null }; + _ = try layout.take(shape.the_27b.hidden * 2); + _ = try layout.take(shape.the_27b.hidden / 64 * 4); + try std.testing.expectEqual(@as(u64, 0x100000000), seq.head_in); + try std.testing.expectEqual(@as(u64, 0x100000000 + 10240), seq.head_xs); + for (0..shape.the_27b.layers) |layer| { + if (shape.linear(layer)) { + const rec_at = std.mem.alignForward(usize, layout.used, 256); + _ = try layout.take(shape.the_27b.deltaBytes()); + const conv_at = std.mem.alignForward(usize, layout.used, 256); + _ = try layout.take(shape.the_27b.convBytes()); + try std.testing.expectEqual(0x100000000 + @as(u64, @intCast(rec_at)), seq.rec[layer]); + try std.testing.expectEqual(0x100000000 + @as(u64, @intCast(conv_at)), seq.conv[layer]); + } else { + _ = try layout.take(303 * shape.the_27b.kvInner() * 2); + _ = try layout.take(303 * shape.the_27b.kvInner() * 2); + } + } + try seq.reset(ops, shape.the_27b); + try checkReset(&seq); +} + +test "restore factory allocation failure owns no device buffer" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + fail_alloc = true; + try std.testing.expectError(error.NotReady, st.Seq.initRestoreDestination(std.testing.allocator, ops, shape.the_27b, 303)); + try std.testing.expectEqual(@as(usize, 0), allocations + frees + call_count); +} + +test "public factory reset error frees the allocated arena and host arrays" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + fail_fill = true; + try std.testing.expectError(error.NotReady, st.Seq.init(std.testing.allocator, ops, shape.the_27b, 303, false)); + try std.testing.expectEqual(@as(usize, 1), allocations); + try std.testing.expectEqual(allocations, frees); +} + +test "public Engine sequence stays initialized and private restore factory omits fills" { + var d: cuda.Driver = undefined; + var engine: @import("engine.zig").Engine = undefined; + const ops = setup(&d, &engine.k); + engine.gpa = std.testing.allocator; + engine.s = ops.s; + engine.w.g = shape.the_27b; + engine.kv8 = false; + var safe = try engine.sequence(303); + try checkReset(&safe); + safe.deinit(); + call_count = 0; + var deferred = try engine.sequenceForRestore(303); + try std.testing.expectEqual(@as(usize, 0), call_count); + try std.testing.expectEqual(@as(usize, 303), deferred.capacity); + deferred.deinit(); + try std.testing.expectEqual(@as(usize, 2), allocations); + try std.testing.expectEqual(allocations, frees); +} + +test "FP8 public factory keeps scales and fills; private BF16 factory rejects FP8 before allocation" { + var d: cuda.Driver = undefined; + var engine: @import("engine.zig").Engine = undefined; + const ops = setup(&d, &engine.k); + engine.gpa = std.testing.allocator; + engine.s = ops.s; + engine.w.g = shape.the_27b; + engine.kv8 = true; + try std.testing.expectError(error.ExactCacheRequiresBf16, engine.sequenceForRestore(303)); + try std.testing.expectEqual(@as(usize, 0), allocations + call_count); + var safe = try engine.sequence(303); + defer safe.deinit(); + try checkReset(&safe); + try std.testing.expect(safe.kv8); + try std.testing.expectEqual(st.Seq.bytes(shape.the_27b, 303, true), safe.arena.buf.?.len); + for (0..shape.the_27b.layers) |layer| { + if (shape.linear(layer)) continue; + try std.testing.expect(safe.kscale[layer] != 0 and safe.vscale[layer] != 0); + } +} diff --git a/zig/src/families/qwen27/cuda/reset_skip_test.zig b/zig/src/families/qwen27/cuda/reset_skip_test.zig new file mode 100644 index 0000000000..732303ae87 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reset_skip_test.zig @@ -0,0 +1,105 @@ +//! Host transaction checks prove deferred target state never reaches work after a miss or failed restore. +const std = @import("std"); +const policy = @import("reset_skip.zig"); +const work = .{ .calls = @as(u64, 96), .bytes = @as(u64, 153944064) }; +const Mock = struct { + hit: bool = true, + cancel: bool = false, + copy_error: bool = false, + reset_error: bool = false, + copies: usize = 0, + resets: usize = 0, + initialized: bool = false, + pub fn cancelled(a: *Mock) bool { + return a.cancel; + } + pub fn lookup(a: *Mock) !bool { + a.copies += 1; + if (a.copy_error) return error.PartialCopy; + if (a.hit) a.initialized = true; + return a.hit; + } + pub fn reset(a: *Mock) !void { + a.resets += 1; + if (a.reset_error) return error.ResetFailed; + a.initialized = true; + } + fn target(a: *Mock) !void { + if (!a.initialized) return error.UninitializedTarget; + } +}; + +test "strict opt-in requires exact cache and rejects malformed flags" { + try std.testing.expect(!try policy.compatible(null, true)); + try std.testing.expect(!try policy.compatible("0", false)); + try std.testing.expect(try policy.compatible("1", true)); + try std.testing.expectError(error.ResetSkipNeedsExactCache, policy.compatible("1", false)); + try std.testing.expectError(error.InvalidResetSkip, policy.compatible("true", true)); +} + +test "actual hit resolves fresh and retained state without reset" { + for ([_]bool{ false, true }) |pooled| { + var m: Mock = .{}; + var deferred = true; + var stats: policy.Stats = .{ .enabled = true }; + try std.testing.expect(try policy.resolve(&m, &deferred, pooled, &stats, work)); + try m.target(); + try std.testing.expect(!deferred); + try std.testing.expectEqual(@as(usize, 0), m.resets); + try std.testing.expectEqual(@as(u64, 96), stats.avoided_fill_calls); + try std.testing.expectEqual(work.bytes, stats.avoided_fill_bytes); + try std.testing.expectEqual(@as(u64, 1), stats.restored_fresh + stats.restored_pool); + } +} + +test "positive hint then actual miss resets before target work" { + var m: Mock = .{ .hit = false }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.UninitializedTarget, m.target()); + try std.testing.expect(!try policy.resolve(&m, &deferred, false, &stats, work)); + try m.target(); + try std.testing.expectEqual(@as(usize, 1), m.copies); + try std.testing.expectEqual(@as(usize, 1), m.resets); + try std.testing.expectEqual(@as(u64, 1), stats.fallback_resets); + try std.testing.expectEqual(@as(u64, 0), stats.avoided_fill_calls); +} + +test "partial copy aborts without reset fallback or resolved admission" { + var m: Mock = .{ .copy_error = true }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.PartialCopy, policy.resolve(&m, &deferred, false, &stats, work)); + try std.testing.expect(deferred); + try std.testing.expectEqual(@as(usize, 0), m.resets); + try std.testing.expectEqual(@as(u64, 1), stats.failed_deferred_transactions); +} + +test "cancelled deferred destination aborts before copying or resetting" { + var m: Mock = .{ .cancel = true }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.Cancelled, policy.resolve(&m, &deferred, true, &stats, work)); + try std.testing.expect(deferred); + try std.testing.expectEqual(@as(usize, 0), m.copies + m.resets); +} + +test "reset failure on actual miss remains unresolved and propagates" { + var m: Mock = .{ .hit = false, .reset_error = true }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.ResetFailed, policy.resolve(&m, &deferred, false, &stats, work)); + try std.testing.expect(deferred); + try std.testing.expectEqual(@as(u64, 0), stats.fallback_resets + stats.avoided_fill_calls); +} + +test "ordinary initialized path retains original lookup with no extra reset or cancel check" { + var m: Mock = .{ .hit = false, .cancel = true, .initialized = true }; + var deferred = false; + var stats: policy.Stats = .{}; + try std.testing.expect(!try policy.resolve(&m, &deferred, false, &stats, work)); + try m.target(); + try std.testing.expectEqual(@as(usize, 1), m.copies); + try std.testing.expectEqual(@as(usize, 0), m.resets); + try std.testing.expectEqual(@as(u64, 0), stats.failed_deferred_transactions); +} diff --git a/zig/src/families/qwen27/cuda/restore_proof.zig b/zig/src/families/qwen27/cuda/restore_proof.zig new file mode 100644 index 0000000000..455e1ed9b9 --- /dev/null +++ b/zig/src/families/qwen27/cuda/restore_proof.zig @@ -0,0 +1,76 @@ +//! Bounded provenance binds a successful full-N restore to one stream and its exclusively owned Seq. +const std = @import("std"); +pub const Proof = struct { stream: *const anyopaque, seq: *const anyopaque, tokens: usize }; +pub const Proofs = struct { + items: [16]?Proof = @splat(null), + + pub fn mark(p: *Proofs, stream: *const anyopaque, seq: *const anyopaque, tokens: usize) !void { + if (tokens == 0 or tokens > 512) return error.InvalidSeqRestoreProof; + for (&p.items) |*slot| if (slot.*) |existing| { + if (existing.stream == stream) return error.DuplicateSeqRestoreProof; + }; + for (&p.items) |*slot| if (slot.* == null) { + slot.* = .{ .stream = stream, .seq = seq, .tokens = tokens }; + return; + }; + return error.SeqRestoreProofCapacity; + } + + /// A read-only binding check leaves the release-time pool proof intact. + pub fn matches(p: *const Proofs, stream: *const anyopaque, seq: *const anyopaque, tokens: usize) bool { + for (p.items) |slot| if (slot) |proof| { + if (proof.stream == stream) return proof.seq == seq and proof.tokens == tokens; + }; + return false; + } + + pub fn remove(p: *Proofs, stream: *const anyopaque) ?Proof { + for (&p.items) |*slot| if (slot.*) |existing| { + if (existing.stream != stream) continue; + slot.* = null; + return existing; + }; + return null; + } +}; + +test "provenance binds stream and Seq once and is cleared before pointer reuse" { + var p: Proofs = .{}; + var streams: [2]u8 = @splat(0); + var seqs: [2]u8 = @splat(0); + try p.mark(&streams[0], &seqs[0], 14); + try std.testing.expect(p.remove(&streams[1]) == null); + try std.testing.expectError(error.DuplicateSeqRestoreProof, p.mark(&streams[0], &seqs[1], 14)); + const proof = p.remove(&streams[0]).?; + try std.testing.expect(proof.seq == @as(*const anyopaque, &seqs[0]) and proof.seq != @as(*const anyopaque, &seqs[1]) and proof.tokens == 14); + try std.testing.expect(p.remove(&streams[0]) == null); + try p.mark(&streams[0], &seqs[1], 31); + const next = p.remove(&streams[0]).?; + try std.testing.expect(next.seq == @as(*const anyopaque, &seqs[1]) and next.tokens == 31); +} + +test "fixed proof ownership refuses overflow and unsupported restoration lengths" { + var p: Proofs = .{}; + var streams: [17]u8 = @splat(0); + var seqs: [17]u8 = @splat(0); + try std.testing.expectError(error.InvalidSeqRestoreProof, p.mark(&streams[0], &seqs[0], 0)); + try std.testing.expectError(error.InvalidSeqRestoreProof, p.mark(&streams[0], &seqs[0], 513)); + for (0..16) |i| try p.mark(&streams[i], &seqs[i], 14); + try std.testing.expectError(error.SeqRestoreProofCapacity, p.mark(&streams[16], &seqs[16], 14)); + _ = p.remove(&streams[0]); + try p.mark(&streams[16], &seqs[16], 31); +} + +test "read-only first-logit proof rejects another owner, Seq and position without consuming release authority" { + var proofs: Proofs = .{}; + var streams: [2]u8 = @splat(0); + var seqs: [2]u8 = @splat(0); + try proofs.mark(&streams[0], &seqs[0], 14); + try std.testing.expect(proofs.matches(&streams[0], &seqs[0], 14)); + try std.testing.expect(!proofs.matches(&streams[1], &seqs[0], 14)); + try std.testing.expect(!proofs.matches(&streams[0], &seqs[1], 14)); + try std.testing.expect(!proofs.matches(&streams[0], &seqs[0], 31)); + try std.testing.expect(proofs.matches(&streams[0], &seqs[0], 14)); + _ = proofs.remove(&streams[0]); + try std.testing.expect(!proofs.matches(&streams[0], &seqs[0], 14)); +} diff --git a/zig/src/families/qwen27/cuda/seq_pool.zig b/zig/src/families/qwen27/cuda/seq_pool.zig new file mode 100644 index 0000000000..2a20f88ef9 --- /dev/null +++ b/zig/src/families/qwen27/cuda/seq_pool.zig @@ -0,0 +1,120 @@ +//! Fixed ownership storage for the private exact-capacity short Seq experiment; no allocator or device work. +const std = @import("std"); +pub const max_slots = 16; +pub const max_capacity = 512; +pub const max_bytes = 3 * 1024 * 1024 * 1024; + +pub const Stats = struct { + enabled: bool = false, + failed: bool = false, + create_attempts: u64 = 0, + driver_creations: u64 = 0, + reuse_hits: u64 = 0, + evictions: u64 = 0, + discarded: u64 = 0, + retained_bytes: usize = 0, + retained_slots: usize = 0, +}; + +pub fn parse(raw: ?[]const u8) !bool { + const value = raw orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (std.mem.eql(u8, value, "1")) return true; + return error.InvalidSeqReuse; +} + +pub fn configured(cache_enabled: bool) !bool { + const raw = std.c.getenv("TF_PR608_EXACT_CACHE_SEQ_REUSE"); + return compatible(if (raw) |value| std.mem.span(value) else null, cache_enabled); +} + +pub fn compatible(raw: ?[]const u8, cache_enabled: bool) !bool { + const on = try parse(raw); + if (on and !cache_enabled) return error.SeqReuseNeedsExactCache; + return on; +} + +/// Normal completion requires proven full-N restoration, never a cached count alone. +pub fn completed(s: anytype, settled: bool, restored: bool, cache_ok: bool, cache_failed: bool) bool { + return cache_ok and !cache_failed and restored and settled and s.finished and (s.reason == .stop or s.reason == .length) and + !s.isCancelled() and s.cached == s.prompt_len and !s.reuse_failed and s.reuse.saved == null and + s.reuse.at == 0 and s.reuse.marks.len == 0 and s.reuse.hook == null; +} + +/// A taken item is caller-owned immediately; a retained item is pool-owned until take/pop. +pub fn Pool(comptime T: type) type { + return struct { + const Self = @This(); + const Entry = struct { ptr: *T, capacity: usize, bytes: usize }; + entries: [max_slots]Entry = undefined, + limit: usize = 0, + stats: Stats = .{}, + + pub fn init(enabled: bool, slots: usize) Self { + return .{ .limit = @min(slots, max_slots), .stats = .{ .enabled = enabled } }; + } + + pub fn active(p: *const Self) bool { + return p.stats.enabled and !p.stats.failed; + } + + fn remove(p: *Self, index: usize) *T { + const entry = p.entries[index]; + p.stats.retained_slots -= 1; + p.stats.retained_bytes -= entry.bytes; + for (index..p.stats.retained_slots) |i| p.entries[i] = p.entries[i + 1]; + return entry.ptr; + } + + pub fn take(p: *Self, capacity: usize) ?*T { + if (!p.active() or capacity > max_capacity) return null; + for (p.entries[0..p.stats.retained_slots], 0..) |entry, i| { + if (entry.capacity != capacity) continue; + p.stats.reuse_hits += 1; + return p.remove(i); + } + return null; + } + + /// Eviction detaches ownership; the caller frees exactly this item. + pub fn evict(p: *Self) ?*T { + if (p.stats.retained_slots == 0) return null; + p.stats.evictions += 1; + return p.remove(0); + } + + /// An exact-capacity miss drains all retained bytes, even when a count slot remains available. + pub fn beforeCreate(p: *Self, context: anytype, comptime free: anytype) void { + while (p.evict()) |ptr| free(context, ptr); + } + + pub fn hasRoom(p: *const Self, active_count: usize) bool { + return active_count < p.limit and p.stats.retained_slots < p.limit - active_count; + } + + pub fn canRetain(p: *const Self, active_count: usize, capacity: usize, bytes: usize) bool { + return p.active() and capacity <= max_capacity and bytes <= max_bytes and + p.stats.retained_bytes <= max_bytes - bytes and p.hasRoom(active_count); + } + + pub fn retain(p: *Self, ptr: *T, active_count: usize, capacity: usize, bytes: usize) bool { + if (!p.active()) return false; + for (p.entries[0..p.stats.retained_slots]) |entry| + if (entry.ptr == ptr) @panic("duplicate Seq pool ownership"); + if (!p.canRetain(active_count, capacity, bytes)) return false; + p.entries[p.stats.retained_slots] = .{ .ptr = ptr, .capacity = capacity, .bytes = bytes }; + p.stats.retained_slots += 1; + p.stats.retained_bytes += bytes; + return true; + } + + /// Failure disables future take/retain; the caller must evict and free outstanding ownership. + pub fn fail(p: *Self) void { + if (p.stats.enabled) p.stats.failed = true; + } + }; +} + +test { + _ = @import("seq_pool_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/seq_pool_test.zig b/zig/src/families/qwen27/cuda/seq_pool_test.zig new file mode 100644 index 0000000000..9ffe378818 --- /dev/null +++ b/zig/src/families/qwen27/cuda/seq_pool_test.zig @@ -0,0 +1,172 @@ +//! Exact-capacity ownership, footprint and failure checks independent of CUDA or model allocation. +const std = @import("std"); +const policy = @import("seq_pool.zig"); +const Object = struct { alive: bool = true }; +const Pool = policy.Pool(Object); + +fn free(object: *Object) !void { + try std.testing.expect(object.alive); + object.alive = false; +} + +test "default-off and invalid settings cannot silently enable retention" { + try std.testing.expect(!try policy.parse(null)); + try std.testing.expect(!try policy.parse("0")); + try std.testing.expect(try policy.parse("1")); + for ([_][]const u8{ "", "2", "true", "-1" }) |value| + try std.testing.expectError(error.InvalidSeqReuse, policy.parse(value)); + var p = Pool.init(false, 16); + var object: Object = .{}; + try std.testing.expect(!p.retain(&object, 0, 158, 100)); + try std.testing.expect(p.take(158) == null); + try std.testing.expect(object.alive); +} + +test "take transfers exclusive ownership and does not round capacity or change the object" { + var p = Pool.init(true, 2); + var first: Object = .{}; + var second: Object = .{}; + try std.testing.expect(p.retain(&first, 0, 158, 100)); + try std.testing.expect(p.retain(&second, 0, 272, 120)); + try std.testing.expect(p.take(157) == null); + try std.testing.expect(p.take(159) == null); + const taken = p.take(158).?; + try std.testing.expect(taken == &first and taken.alive); + try std.testing.expect(p.take(158) == null); + try std.testing.expectEqual(@as(usize, 1), p.stats.retained_slots); + try std.testing.expectEqual(@as(usize, 120), p.stats.retained_bytes); + try free(taken); + try free(p.evict().?); + try std.testing.expect(p.evict() == null); + try std.testing.expect(!first.alive and !second.alive); +} + +test "capacity miss must evict unused ownership before a fresh active slot fits" { + var p = Pool.init(true, 2); + var retained: Object = .{}; + var active: Object = .{}; + var replacement: Object = .{}; + try std.testing.expect(p.retain(&retained, 1, 158, 100)); + try std.testing.expect(!p.hasRoom(1)); + try std.testing.expect(p.take(272) == null); + try free(p.evict().?); + try std.testing.expect(p.hasRoom(1)); + // The caller can now own active and replacement, never active+retained+replacement. + try std.testing.expect(!p.retain(&active, 2, 158, 100)); + try std.testing.expectEqual(@as(u64, 1), p.stats.evictions); + try free(&active); + try free(&replacement); + try std.testing.expect(!retained.alive); +} + +test "capacity miss drains every retained byte even with room for a fresh large allocation" { + const Ledger = struct { + freed: usize = 0, + fn release(x: *@This(), object: *Object) void { + std.debug.assert(object.alive); + object.alive = false; + x.freed += 1; + } + }; + var p = Pool.init(true, 16); + var objects: [2]Object = @splat(.{}); + var ledger: Ledger = .{}; + try std.testing.expect(p.retain(&objects[0], 0, 158, 100)); + try std.testing.expect(p.retain(&objects[1], 0, 272, 120)); + try std.testing.expect(p.hasRoom(1)); + try std.testing.expect(p.take(16384) == null); + p.beforeCreate(&ledger, Ledger.release); + try std.testing.expectEqual(@as(usize, 2), ledger.freed); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_bytes); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_slots); + for (objects) |object| try std.testing.expect(!object.alive); +} + +test "total ownership byte cap short-capacity cap and configured stream bound are enforced" { + var p = Pool.init(true, 20); + var objects: [16]Object = @splat(.{}); + try std.testing.expectEqual(@as(usize, 16), p.limit); + try std.testing.expect(!p.retain(&objects[0], 0, 513, 1)); + try std.testing.expect(!p.retain(&objects[0], 0, 512, policy.max_bytes + 1)); + try std.testing.expect(p.retain(&objects[0], 15, 512, policy.max_bytes)); + try std.testing.expect(!p.retain(&objects[1], 14, 158, 1)); + try free(p.evict().?); + p = Pool.init(true, 2); + try std.testing.expect(!p.retain(&objects[1], 2, 158, 1)); + try std.testing.expect(p.retain(&objects[1], 1, 158, 1)); + try std.testing.expect(!p.retain(&objects[2], 1, 158, 1)); + try free(p.evict().?); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_bytes); +} + +test "failure disables reuse and shutdown drains each detached object once" { + var p = Pool.init(true, 3); + var objects: [3]Object = @splat(.{}); + for (&objects, 0..) |*object, i| try std.testing.expect(p.retain(object, 0, 158, i + 1)); + p.fail(); + try std.testing.expect(p.stats.failed); + try std.testing.expect(p.take(158) == null); + try std.testing.expect(!p.retain(&objects[0], 0, 158, 1)); + while (p.evict()) |object| try free(object); + for (objects) |object| try std.testing.expect(!object.alive); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_slots); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_bytes); + try std.testing.expect(p.evict() == null); +} + +const Stream = struct { + finished: bool = true, + reason: enum { none, stop, length, cancelled } = .length, + cancelled: bool = false, + cached: u32 = 14, + prompt_len: u32 = 14, + reuse_failed: bool = false, + reuse: struct { + saved: ?*anyopaque = null, + at: u32 = 0, + marks: []const u32 = &.{}, + hook: ?u8 = null, + } = .{}, + pub fn isCancelled(s: *Stream) bool { + return s.cancelled; + } +}; + +test "only normally completed settled uncancelled proven full-N restored streams may transfer to the pool" { + var s: Stream = .{}; + try std.testing.expect(policy.completed(&s, true, true, true, false)); + try std.testing.expect(!policy.completed(&s, false, true, true, false)); + try std.testing.expect(!policy.completed(&s, true, false, true, false)); + try std.testing.expect(!policy.completed(&s, true, true, false, false)); + try std.testing.expect(!policy.completed(&s, true, true, true, true)); + for (0..9) |case| { + s = .{}; + switch (case) { + 0 => s.finished = false, + 1 => s.reason = .cancelled, + 2 => s.reason = .none, + 3 => s.cancelled = true, + 4 => s.cached = 1, + 5 => s.reuse_failed = true, + 6 => s.reuse.at = 1, + 7 => s.reuse.marks = &.{1}, + 8 => s.reuse.hook = 1, + else => unreachable, + } + try std.testing.expect(!policy.completed(&s, true, true, true, false)); + } + s = .{}; + var sentinel: u8 = 0; + s.reuse.saved = &sentinel; + try std.testing.expect(!policy.completed(&s, true, true, true, false)); + s = .{ .reason = .stop }; + try std.testing.expect(policy.completed(&s, true, true, true, false)); +} + +test "independent pool flag requires the exact cache and remains strict" { + try std.testing.expect(!try policy.compatible(null, false)); + try std.testing.expect(!try policy.compatible("0", true)); + try std.testing.expect(try policy.compatible("1", true)); + try std.testing.expectError(error.SeqReuseNeedsExactCache, policy.compatible("1", false)); + try std.testing.expectError(error.InvalidSeqReuse, policy.compatible("yes", true)); +} diff --git a/zig/src/families/qwen27/cuda/state.zig b/zig/src/families/qwen27/cuda/state.zig index e85f68a5b5..e318cb0747 100644 --- a/zig/src/families/qwen27/cuda/state.zig +++ b/zig/src/families/qwen27/cuda/state.zig @@ -69,6 +69,15 @@ pub const Seq = struct { /// A fresh stream: zero recurrences and conv windows, caches for `capacity` positions. pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize, kv8: bool) !Seq { + return initState(gpa, ops, g, capacity, kv8, true); + } + + /// Private BF16 restore destination; actual restore or reset must precede any target work. + pub fn initRestoreDestination(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize) !Seq { + return initState(gpa, ops, g, capacity, false, false); + } + + fn initState(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize, kv8: bool, initialize: bool) !Seq { var s: Seq = .{ .gpa = gpa, .arena = try Arena.init(ops.k.d, bytes(g, capacity, kv8)), .rec = &.{}, .conv = &.{}, .keys = &.{}, .values = &.{}, .kscale = &.{}, .vscale = &.{}, .kv8 = kv8, .head_in = 0, .head_xs = 0, .capacity = capacity }; errdefer s.arena.deinit(); s.rec = try gpa.alloc(u64, g.layers); @@ -106,7 +115,7 @@ pub const Seq = struct { } } } - try s.reset(ops, g); + if (initialize) try s.reset(ops, g); return s; } @@ -353,3 +362,7 @@ pub const Scratch = struct { s.* = undefined; } }; + +test { + _ = @import("reset_skip_factory_test.zig"); +} diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index fdfdf56c43..3b31643c0e 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -332,7 +332,7 @@ const Host = struct { } }.f, }; - return .{ .ptr = h, .vtable = &h.vtable }; + return .{ .ptr = h, .vtable = &h.vtable, .first_before_draft = h.inner.first_before_draft }; } }; From 10e7fe0763f299d8ce84ff8f3557eea5c8e25aa5 Mon Sep 17 00:00:00 2001 From: Jonathan Regan-Beasley <197533392+jregan-beasley-smc@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:48:47 +1100 Subject: [PATCH 13/19] test(qwen27 cuda): record Spark TTFT overlap with PR726 --- validation/pr726-ttft/README.md | 52 + validation/pr726-ttft/bench_concurrent.py | 275 + validation/pr726-ttft/bench_evidence.py | 54 + validation/pr726-ttft/bundle-manifest.json | 25 + validation/pr726-ttft/counters.json | 121 + validation/pr726-ttft/long-manifest.json | 17776 +++++++++++++++++++ validation/pr726-ttft/provenance.json | 401 + validation/pr726-ttft/qualification.json | 34 + validation/pr726-ttft/requests.jsonl | 152 + validation/pr726-ttft/run_overlap.py | 269 + validation/pr726-ttft/summary.json | 177 + validation/pr726-ttft/test_verify.py | 95 + validation/pr726-ttft/verify.py | 199 + 13 files changed, 19630 insertions(+) create mode 100644 validation/pr726-ttft/README.md create mode 100644 validation/pr726-ttft/bench_concurrent.py create mode 100644 validation/pr726-ttft/bench_evidence.py create mode 100644 validation/pr726-ttft/bundle-manifest.json create mode 100644 validation/pr726-ttft/counters.json create mode 100644 validation/pr726-ttft/long-manifest.json create mode 100644 validation/pr726-ttft/provenance.json create mode 100644 validation/pr726-ttft/qualification.json create mode 100644 validation/pr726-ttft/requests.jsonl create mode 100644 validation/pr726-ttft/run_overlap.py create mode 100644 validation/pr726-ttft/summary.json create mode 100644 validation/pr726-ttft/test_verify.py create mode 100644 validation/pr726-ttft/verify.py diff --git a/validation/pr726-ttft/README.md b/validation/pr726-ttft/README.md new file mode 100644 index 0000000000..5efc01c7f8 --- /dev/null +++ b/validation/pr726-ttft/README.md @@ -0,0 +1,52 @@ +## What this changes + +This receipt covers the BF16 text overlap of [PR726](https://github.com/ashhart/TensorFold/pull/726) with the earlier [PR608 TTFT contribution](https://github.com/machinegenieorg/TensorFold/pull/2). The source base is PR726 `d49a906f`; candidate runtime commit is `5e08e130`. The 31 runtime/test files are pinned in `provenance.json`. + +The candidate adds committed first-token delivery before initial drafts, whole-identical-short-prompt snapshots, target and draft allocation reuse, raw first-logit reuse, and guarded omission of initialization fills. All five `TF_PR608_EXACT_*` switches remain default off. The latter four require `TF_PR608_EXACT_PROMPT_CACHE=1`. Exact snapshots admit at most 512 prompt tokens and use a 256 MiB budget; target/draft pools have 3 GiB/1.5 GiB retained budgets, at most 16 owners, and normal target capacity at most 512. Public target factories remain initialized; unsupported FP8/image requests decline the exact paths. + +PR726's host prefix cache remains separate. It can restore a 4096-token long-chat prefix; the short cache restores the whole identical prompt, not a general prefix. Appending conversation turns does not automatically produce a short-cache hit. + +Credit @philip-pentatonic for the Qwen CUDA engine and DFlash work in [PR608](https://github.com/ashhart/TensorFold/pull/608), and @BobClawblaw for PR726's host-prefix, vision and FP8 work. Jonathan Regan-Beasley contributed and tested this integration in a personal capacity. + +## Receipt + +One fresh process per arm, in order **B, D, A, C**: A stock/cache-off; B stock/1 GiB host cache; C candidate/cache-off; D candidate/1 GiB host cache. C/D enable all five switches. This is one sequential correctness panel, not balanced timing. Hardware was DGX Spark GB10, Linux aarch64, with Zig 0.17.0. Inference ran as a native host process. CPU-only builds used the immutable container image pinned in `provenance.json`; both binaries used common AOT/fatbins. Driver/library versions were not independently retained. + +Each arm ran code14 and chat31: first use, one repeat and one n16 burst, all with 256 output tokens. Two genuine long-chat requests used 32 output tokens. Context was 32768, parallel 16, BF16 KV, greedy seed 1234, no thinking, and ignore-EOS. “First use” means a prompt in an already-loaded process; chat follows code. Four live `/tokenize` checks matched each arm's frozen fixture. All 152 literal request bodies, output SHA256 values and token fingerprints matched across arms. The accepted actual-CUDA gate and 439/439 Linux host checks are attested separately. + +Selected **B → D** diagnostics (TTFT milliseconds; client decode tokens/s; full-response or burst wall seconds): + +| Request | TTFT | Decode | Full response / wall | +|---|---:|---:|---:| +| Code first n1 | 140.04 → 140.93 | 69.48 → 65.44 | 3.810 → 4.037 | +| Code repeat n1 | 125.53 → 12.39 | 69.50 → 65.35 | 3.795 → 3.914 | +| Code n16 | 1066.41 → 144.74 | 23.47 → 25.31 | 11.937 → 10.223 | +| Chat first n1 | 131.17 → 132.35 | 49.80 → 49.15 | 5.252 → 5.320 | +| Chat repeat n1 | 127.84 → 13.79 | 49.78 → 49.08 | 5.250 → 5.209 | +| Chat n16 | 1088.51 → 153.76 | 17.54 → 18.76 | 15.628 → 13.749 | +| Long follow-up | 351.15 → 338.28 | See request evidence | 0.944 → 1.025 | + +N16 TTFT/decode are per-stream medians. Slowest TTFT and all four arms are retained in `summary.json`; burst wall runs from earliest send to latest completion. TTFT starts at send and ends at the first nonempty SSE text. Decode is `(tokens−1)/(last−first)`; it is not aggregate throughput. Code repeat loses about 6% decode speed and 119 ms overall; cold code costs 227 ms more and long follow-up costs 81 ms more. Identical output does not imply equal work: code n1 uses 45 rounds in B and 48 in D. + +Offline verification needs only Python 3.11+: + +```sh +python3 -B validation/pr726-ttft/verify.py --source-root . +python3 -B -m unittest discover -s validation/pr726-ttft -p test_verify.py +``` + +To repeat a panel against an already-running pinned server, use the included standard-library client: + +```sh +python3 -B validation/pr726-ttft/run_overlap.py --base "$BASE_URL" \ + --model qwen27-overlap --arm D --suite both \ + --long-manifest validation/pr726-ttft/long-manifest.json --output D-client.json +``` + +Start a fresh server per arm with the recorded context/parallel/precision/settings, `TENSORFOLD_CUDA_KERNELS="$AOT"`, reserve 8 GiB, and `--prompt-cache-gib 0` for A/C or `1` for B/D. Use the stock binary for A/B and candidate for C/D. The client does not start servers or load models. Binary/model/kernel hashes, original receipt hashes and exact request bodies are retained; no weights or operational scripts are included. + +## Limits + +D's host-prefix hook disables pool reuse and reset omission: both pools record zero reuse and zero avoided fills. Exact snapshots and first-logit reuse do show 34 hits. This is not evidence that all mechanisms executed with the host cache on. The device qualifier used private build-only registration; its source is excluded and its accepted gates/hash receipt are retained. + +FP8, images, higher limits, error paths and other models were not qualified by this HTTP panel. The preserved failing gates preceded this final fixture; R3 changes only the genuine chat-history test, not runtime. Timings need repeated balanced ordering before stable performance magnitudes can be claimed. Investigating the remaining speculative-work differences and conservative host-hook exclusions is further work, not an implemented optimization. diff --git a/validation/pr726-ttft/bench_concurrent.py b/validation/pr726-ttft/bench_concurrent.py new file mode 100644 index 0000000000..8e4d60f0ca --- /dev/null +++ b/validation/pr726-ttft/bench_concurrent.py @@ -0,0 +1,275 @@ +"""Decode speed of an OpenAI-compatible server under concurrent requests (bench_openai's prompts and settings). + +Default: four cells (code and chat prompts, sampled and greedy). In each cell N requests of the same prompt start +at once, stream i with seed ``seed + i`` (greedy: all ``seed``), each ``--tokens`` long with ignore_eos. Per stream: decode tok/s = +(tokens - 1) / (last token - first token) and time to first token. Aggregate tok/s = the streams' tokens after +their first / (last token of any stream - first token of any stream). Medians over ``--reps``. + +``--mixed``: request i takes prompt i % 2 and seed seed + i // 2; also reports the steady aggregate, over the time +every stream is live (tokens split by the characters each stream delivered inside that window). +``--alone``: runs every request alone first and checks each concurrent reply's token-id hash (``token_sha``) +against it; ``--serial`` checks the alone runs against ``"draft": false``. ``--no-seed``: sampled requests carry +no seed (mlx_lm's server batches only those), for measuring a standard; no hash checks then. Run a standard with its +prompt cache off (mlx_lm: ``--prompt-cache-size 0``): a cached reply can arrive as one text piece. Tokens come from +the reply's usage. A request that fails to connect or errors counts in ``failed``; one whose text came in fewer than +``MIN_PIECES`` pieces counts in ``unmeasured``; neither enters the rates. ``--stagger-ms`` spaces the starts (a +server with a small listen backlog resets simultaneous connections; those resets are reported, not hidden). +Standard library only. + + python3 tools/bench_concurrent.py $BASE_URL MODEL --levels 1,2,4,8 --alone --output out.json +""" + +import argparse +import hashlib +import json +import statistics +import sys +import threading +import time +import urllib.request + +from bench_evidence import Evidence, token_equal + +if sys.version_info < (3, 11): + sys.exit(f"Python 3.11+ is required (this is {sys.version.split()[0]}); " + "TensorFold's tools use PEP 604 unions. Try: python3.12 tools/bench_concurrent.py ...") + +REQUEST_TIMEOUT = 180 + +MIN_PIECES = 4 # text pieces a reply needs before its span is timed + +PROMPTS = [ + {"name": "code", "kind": "completion", + "prompt": "Write a short Python function that computes the Fibonacci sequence and explain it."}, + {"name": "chat", "kind": "chat", + "prompt": "Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example."}, +] + + +def stream(base: str, model: str, item: dict, tokens: int, temperature: float, seed: int | None, + draft: bool = True, gate: threading.Barrier | None = None) -> dict: + try: + return _stream(base, model, item, tokens, temperature, seed, draft, gate) + except Exception as exc: # noqa: BLE001 - a failed request is reported, not fatal + if gate is not None: + gate.abort() + return {"prompt": item["name"], "seed": seed, "sent": None, "first": None, "last": None, "tokens": None, + "pieces": [], "ttft_s": None, "decode_tps": None, "token_sha": None, + "error": type(exc).__name__, "complete": False, "prompt_tokens": None, + "cached_tokens": None, "cache_state": "unverified", "output_sha256": None} + + +def _stream(base: str, model: str, item: dict, tokens: int, temperature: float, seed: int | None, draft: bool, + gate: threading.Barrier | None) -> dict: + body = {"model": model, "max_tokens": tokens, "temperature": temperature, "stream": True, + "stream_options": {"include_usage": True}, "ignore_eos": True} + if seed is not None: + body["seed"] = seed + if not draft: + body["draft"] = False + if temperature > 0: + body.update(top_k=20, top_p=0.95) + if item["kind"] == "chat": + url = base + "/v1/chat/completions" + body["messages"] = item.get("messages") or [{"role": "user", "content": item["prompt"]}] + body["chat_template_kwargs"] = {"enable_thinking": False} + else: + url = base + "/v1/completions" + body["prompt"] = item["prompt"] + body_bytes = json.dumps(body).encode() + body_sha = hashlib.sha256(body_bytes).hexdigest() + req = urllib.request.Request(url, data=body_bytes, headers={"Content-Type": "application/json"}) + if gate is not None: + gate.wait(timeout=30) + sent = time.perf_counter() + response_id: str | None = None + pieces: list[tuple[float, int]] = [] # (arrival, characters) of every text chunk + usage: dict = {} + runtime: dict = {} + evidence = Evidence() + complete = False + done = None + finish_reason = None + speculative = {} + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp: + for raw in resp: + line = raw.decode().strip() + if line == "data: [DONE]": + done = time.perf_counter() + complete = True + break + if not line.startswith("data:"): + continue + chunk = json.loads(line[5:]) + if "error" in chunk: + raise RuntimeError("server_stream_error") + usage = chunk.get("usage") or usage + runtime = chunk.get("tensorfold") or runtime + speculative = chunk.get("speculative") or speculative + for choice in chunk.get("choices", []): + evidence.add(choice) + reason = choice.get("finish_reason") + if reason is not None: + if finish_reason is not None and finish_reason != reason: + raise ValueError("finish reason changed within a stream") + finish_reason = reason + delta = choice.get("delta") or {} + # reasoning counts: its tokens are in completion_tokens, so the clock starts with them + piece = choice.get("text") or delta.get("content") or delta.get("reasoning_content") or "" + if piece: + pieces.append((time.perf_counter(), len(piece))) + if chunk.get("id") is not None: + if response_id is not None and response_id != chunk["id"]: + raise ValueError("response ID changed within a stream") + response_id = chunk["id"] + observed = evidence.finish(usage, runtime, complete) + n = observed["tokens"] + work_source = speculative if speculative else runtime + work = {k: work_source.get(k) for k in ("rounds", "accepted", "drafted")} + out = {"prompt": item["name"], "seed": seed, "sent": sent, "first": None, "last": None, "tokens": n, + "pieces": pieces, "ttft_s": None, "decode_tps": None, "response_id": response_id, + "case_id": item.get("case_id"), "request_body": body, "request_body_sha256": body_sha, "request_url": url, + "usage": usage, "runtime": runtime, "speculative": speculative, "finish_reason": finish_reason, + "work": work, "work_source": "speculative" if speculative else "tensorfold", + "done": done, "full_response_s": done - sent if done is not None else None, **observed} + if not complete: + out["error"] = "incomplete_stream" + if pieces: + first, last = pieces[0][0], pieces[-1][0] + out.update(first=first, last=last, ttft_s=first - sent if complete else None) + if complete and n is not None and n > 1 and last > first and len(pieces) >= MIN_PIECES: + out["decode_tps"] = (n - 1) / (last - first) + else: + out["unmeasured"] = True + else: + out["unmeasured"] = True + return out + + +def together(base: str, model: str, specs: list[tuple[dict, int | None]], tokens: int, temperature: float, + stagger_ms: float = 0.0) -> list[dict]: + gate = threading.Barrier(len(specs)) + out: list[dict | None] = [None] * len(specs) + + def run(i: int) -> None: + if stagger_ms: + gate.wait(timeout=30) + time.sleep(i * stagger_ms / 1e3) + out[i] = stream(base, model, specs[i][0], tokens, temperature, specs[i][1]) + else: + out[i] = stream(base, model, specs[i][0], tokens, temperature, specs[i][1], gate=gate) + + threads = [threading.Thread(target=run, args=(i,)) for i in range(len(specs))] + for t in threads: + t.start() + for t in threads: + t.join() + return [r for r in out if r is not None] + + +def aggregates(runs: list[dict]) -> dict: + done = [r for r in runs if r["decode_tps"] is not None] + out: dict = {"failed": sum(1 for r in runs if r.get("error")), "unmeasured": sum(1 for r in runs if r.get("unmeasured")), + "aggregate_tps": 0.0} + if not done: + return out + first = min(r["first"] for r in done) + last = max(r["last"] for r in done) + if last > first: + out["aggregate_tps"] = sum(r["tokens"] - 1 for r in done) / (last - first) + lo, hi = max(r["first"] for r in done), min(r["last"] for r in done) # every stream live + if len(done) > 1 and hi > lo: + share = 0.0 + for r in done: + chars = sum(c for _, c in r["pieces"]) or 1 + share += r["tokens"] * sum(c for t, c in r["pieces"] if lo <= t <= hi) / chars + out["steady_tps"] = share / (hi - lo) + return out + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("base") + p.add_argument("model") + p.add_argument("--levels", default="4", help="concurrent requests, a comma list") + p.add_argument("--tokens", type=int, default=256) + p.add_argument("--reps", type=int, default=3) + p.add_argument("--seed", type=int, default=1234) + p.add_argument("--temperatures", default="1.0,0") + p.add_argument("--mixed", action="store_true", help="request i takes prompt i %% 2 (sampled only)") + p.add_argument("--alone", action="store_true", help="check each reply's token_sha against its request alone") + p.add_argument("--serial", action="store_true", help="check the alone runs against \"draft\": false") + p.add_argument("--no-seed", action="store_true", help="sampled requests without a seed (standards; no checks)") + p.add_argument("--stagger-ms", type=float, default=0.0, help="start request i this many ms after request i - 1") + p.add_argument("--label", default="") + p.add_argument("--output") + args = p.parse_args() + if args.no_seed: + args.alone = args.serial = False + levels = [int(x) for x in args.levels.split(",")] + temps = [1.0] if args.mixed else [float(t) for t in args.temperatures.split(",")] + cells = [None] if args.mixed else PROMPTS + report: dict = {"label": args.label, "tokens": args.tokens, "mixed": args.mixed, "cells": []} + for temp in temps: + for item in cells: + specs: list[tuple[dict, int | None]] + if args.mixed or item is None: + specs = [(PROMPTS[i % 2], args.seed + i // 2) for i in range(max(levels))] + else: + specs = [(item, args.seed + i if temp > 0 else args.seed) for i in range(max(levels))] + if args.no_seed: + specs = [(spec[0], None) for spec in specs] + together(args.base, args.model, specs[:1], 16, temp) # warm-up + alone: dict[tuple[str, int | None], dict] = {} + if args.alone or args.serial: + for spec in specs: + key = (spec[0]["name"], spec[1]) + if key not in alone: + alone[key] = stream(args.base, args.model, spec[0], args.tokens, temp, spec[1]) + if args.serial: + for key, ref in alone.items(): + serial = stream(args.base, args.model, next(q for q in PROMPTS if q["name"] == key[0]), + args.tokens, temp, key[1], draft=False) + ref["serial_equal"] = token_equal(serial, ref) + for n in levels: + reps = [] + for _ in range(args.reps): + runs = together(args.base, args.model, specs[:n], args.tokens, temp, args.stagger_ms) + rep = aggregates(runs) + rep["per_stream_tps"] = [round(r["decode_tps"], 1) for r in runs if r["decode_tps"] is not None] + rep["ttft_s"] = [round(r["ttft_s"], 2) if r["ttft_s"] is not None else None for r in runs] + rep["errors"] = [r["error"] for r in runs if r.get("error")] + rep["evidence"] = [{k: r.get(k) for k in ("prompt_tokens", "cached_tokens", "tokens", + "cache_state", "complete", "token_sha", "output_sha256")} for r in runs] + if alone: + # None: the request or its solo run failed (not an exactness result) + rep["equal_alone"] = [token_equal(alone[(r["prompt"], r["seed"])], r) for r in runs] + reps.append(rep) + cell = {"prompt": "mixed" if args.mixed or item is None else item["name"], "temperature": temp, + "streams": n, "failed": sum(r["failed"] for r in reps), + "unmeasured": sum(r["unmeasured"] for r in reps), + "aggregate_tps": round(statistics.median(r["aggregate_tps"] for r in reps), 1), + "per_stream_tps": round(statistics.median([v for r in reps for v in r["per_stream_tps"]] or [0.0]), 1), + "ttft_s_max": max((t for r in reps for t in r["ttft_s"] if t is not None), default=None)} + if all("steady_tps" in r for r in reps): + cell["steady_tps"] = round(statistics.median(r["steady_tps"] for r in reps), 1) + if alone: + checks = [v for r in reps for v in r["equal_alone"]] + cell["alone"] = {"equal": checks.count(True), "unequal": checks.count(False), + "failed": checks.count(None)} + if args.serial: + checks = [v.get("serial_equal") for v in alone.values()] + cell["serial"] = {"equal": checks.count(True), "unequal": checks.count(False), + "failed": checks.count(None)} + errors = sorted({e for r in reps for e in r["errors"]}) + if errors: + cell["errors"] = errors[:3] + print(json.dumps({"label": args.label, **cell}), flush=True) + report["cells"].append({**cell, "reps": reps}) + if args.output: + with open(args.output, "w") as f: + json.dump(report, f, indent=1) + + +if __name__ == "__main__": + main() diff --git a/validation/pr726-ttft/bench_evidence.py b/validation/pr726-ttft/bench_evidence.py new file mode 100644 index 0000000000..a9fe6c257d --- /dev/null +++ b/validation/pr726-ttft/bench_evidence.py @@ -0,0 +1,54 @@ +"""Validated usage and output fingerprints shared by the HTTP benchmark clients.""" +import hashlib +import re + + +def count(value): + """A server-reported nonnegative integer; missing, boolean and coerced numbers stay unknown.""" + return value if type(value) is int and value >= 0 else None + + +class Evidence: + """Hash content and reasoning independently so network chunk boundaries cannot alter the fingerprint.""" + + def __init__(self): + self.content = hashlib.sha256() + self.reasoning = hashlib.sha256() + + def add(self, choice): + delta = choice.get("delta") or {} + content = choice.get("text") or delta.get("content") or "" + reasoning = delta.get("reasoning_content") or "" + if not isinstance(content, str) or not isinstance(reasoning, str): + raise ValueError("invalid_text_chunk") + self.content.update(content.encode("utf-8")) + self.reasoning.update(reasoning.encode("utf-8")) + + def finish(self, usage, runtime, complete): + usage = usage if isinstance(usage, dict) else {} + runtime = runtime if isinstance(runtime, dict) else {} + details = usage.get("prompt_tokens_details") + details = details if isinstance(details, dict) else {} + prompt = count(usage.get("prompt_tokens")) + cached = count(details.get("cached_tokens")) + tokens = count(usage.get("completion_tokens")) + valid = prompt is not None and cached is not None and cached <= prompt + state = "cold" if valid and cached == 0 else "reused" if valid else "unverified" + if not complete: + state = "unverified" + token_sha = runtime.get("token_sha") + if not isinstance(token_sha, str) or not re.fullmatch(r"(?:[0-9a-f]{12}|[0-9a-f]{64})", token_sha): + token_sha = None + digest = hashlib.sha256(self.content.digest() + self.reasoning.digest()).hexdigest() + return {"prompt_tokens": prompt, "cached_tokens": cached, "tokens": tokens, "cache_state": state, + "complete": complete, "token_sha": token_sha if complete else None, + "output_sha256": digest if complete else None} + + +def token_equal(left, right): + """Missing token hashes or failed/incomplete streams are unverified, never an equality pass.""" + if left.get("error") or right.get("error") or not left.get("complete") or not right.get("complete"): + return None + if not left.get("token_sha") or not right.get("token_sha"): + return None + return left["token_sha"] == right["token_sha"] diff --git a/validation/pr726-ttft/bundle-manifest.json b/validation/pr726-ttft/bundle-manifest.json new file mode 100644 index 0000000000..d14a7fff03 --- /dev/null +++ b/validation/pr726-ttft/bundle-manifest.json @@ -0,0 +1,25 @@ +{ + "files_sha256": { + "README.md": "a9976a37af2aa4622460f75c0777adfefdb29a87e40cf4912f14630ff2146c25", + "bench_concurrent.py": "6274692888d28cc38003eeddd91c5fd990b5c2dedb28b9002fc42187dfd5cb80", + "bench_evidence.py": "119c396f24f3c13e181429bc69acff39f34254efd4789c5673a3dcbc54738d93", + "counters.json": "79fb15d79736e4df225d74b0072a09200809630c25094121d902e69deaa16c6d", + "long-manifest.json": "bfbb7cbb63e5d6696c0be700edf820dfdcdc03fb149e76467cfa8fb572f4fb53", + "provenance.json": "290721b93fd9cfa30e7377c5987ec1746eaa8449bed241720f0a55b8650b8819", + "qualification.json": "e8d86be5bf3107e3576232e694af7ecbce870e145d0603e5667c174b032417cd", + "requests.jsonl": "50499c2e13a67685aac84231c7cd62aba108c7e069c9f76d87261f0cbc939ad9", + "run_overlap.py": "9ce3f4b06f2b90ae13ef52259cd495d7f28d50e66a3089f1c4f5bd541e8549ee", + "summary.json": "7fb9857f5b15fd726559be89a459eab362c6bba33bac06d37f9ac97b3672258d", + "test_verify.py": "08cec265255224c266f517e00aec98be025550871880136b23d4463e896c443b", + "verify.py": "a389b3d6cb4138158cbb9323e0ee0b503bb5c9bdf5de0c35ceb8af5be6482edb" + }, + "host_checks": { + "independent_metrics_exact": true, + "portable_mutation_tests_passed": 9, + "python_ast": "3.11", + "runtime_files_rehashed_unchanged": 31 + }, + "request_count": 152, + "sanitized": true, + "schema": "pr726_portable_bundle_v1" +} diff --git a/validation/pr726-ttft/counters.json b/validation/pr726-ttft/counters.json new file mode 100644 index 0000000000..dbdc804d30 --- /dev/null +++ b/validation/pr726-ttft/counters.json @@ -0,0 +1,121 @@ +{ + "arms": { + "C": { + "draft_pool": { + "discarded": 2, + "driver_creations": 18, + "enabled": true, + "evictions": 16, + "failed": false, + "fence_failures": 0, + "initialized": 36, + "quarantined_bytes": 0, + "quarantined_slots": 0, + "retained_bytes": 0, + "retained_slots": 0, + "reuse_hits": 20 + }, + "exact_cache": { + "active_bytes": 157117952, + "captures": 2, + "evictions": 1, + "failed": false, + "hits": 34, + "misses": 2, + "pending_bytes": 0, + "promotions": 2, + "purges": 0, + "refusals": 0 + }, + "logits": { + "captures": 2, + "failed": false, + "fallbacks": 4, + "hits": 34 + }, + "reset_skip": { + "avoided_fill_bytes": 5234098176, + "avoided_fill_calls": 3264, + "deferred_fresh": 32, + "deferred_pool": 2, + "enabled": true, + "failed_deferred_transactions": 0, + "fallback_resets": 0, + "restored_fresh": 32, + "restored_pool": 2 + }, + "seq_pool": { + "create_attempts": 36, + "discarded": 4, + "driver_creations": 36, + "enabled": true, + "evictions": 32, + "failed": false, + "retained_bytes": 0, + "retained_slots": 0, + "reuse_hits": 2 + } + }, + "D": { + "draft_pool": { + "discarded": 38, + "driver_creations": 38, + "enabled": true, + "evictions": 0, + "failed": false, + "fence_failures": 0, + "initialized": 0, + "quarantined_bytes": 0, + "quarantined_slots": 0, + "retained_bytes": 0, + "retained_slots": 0, + "reuse_hits": 0 + }, + "exact_cache": { + "active_bytes": 157117952, + "captures": 2, + "evictions": 1, + "failed": false, + "hits": 34, + "misses": 2, + "pending_bytes": 0, + "promotions": 2, + "purges": 0, + "refusals": 0 + }, + "logits": { + "captures": 2, + "failed": false, + "fallbacks": 4, + "hits": 34 + }, + "reset_skip": { + "avoided_fill_bytes": 0, + "avoided_fill_calls": 0, + "deferred_fresh": 0, + "deferred_pool": 0, + "enabled": true, + "failed_deferred_transactions": 0, + "fallback_resets": 0, + "restored_fresh": 0, + "restored_pool": 0 + }, + "seq_pool": { + "create_attempts": 38, + "discarded": 38, + "driver_creations": 38, + "enabled": true, + "evictions": 0, + "failed": false, + "retained_bytes": 0, + "retained_slots": 0, + "reuse_hits": 0 + } + } + }, + "schema": "pr726_sanitized_owner_counters_v1", + "source_server_log_sha256": { + "C": "ce4e09bdfe3ee8bdad1f27b3289ce398b1d1d12244b03fa90a94aa4f2a2e0fd5", + "D": "3edc97475dc3d5fb4d1c833d19ad72827f85d45a498e5c19a47db4526da5b6d6" + } +} diff --git a/validation/pr726-ttft/long-manifest.json b/validation/pr726-ttft/long-manifest.json new file mode 100644 index 0000000000..c75a30375f --- /dev/null +++ b/validation/pr726-ttft/long-manifest.json @@ -0,0 +1,17776 @@ +{ + "chat_template_sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "common_full_prefix_tokens": 4419, + "followup_full_tokens": 4445, + "followup_history_len": 4438, + "history_len": 4412, + "id_sha256": { + "followup": { + "generation": "f8910192b1eab51b6aeda7b21f20c8a166773d30da8e1aabe5d5d5b1d533402f", + "history": "f7b62f6caad3524eb7c1cf9020d22c5e2155a5930cccaab8f519b62b034e610d" + }, + "initial": { + "generation": "1e72d9495c1182f66f1664b2bf3cd4db8b8e08dbf2a8175b6903fff1c4857e14", + "history": "3c01cc896a0c90055aa87d78aaf9700c92ab1d46f6dab7eea9ef7163e8413c82" + } + }, + "initial_full_tokens": 4419, + "lookahead": 0, + "messages": { + "followup": [ + { + "content": "atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?", + "role": "user" + }, + { + "content": "Acknowledged.", + "role": "assistant" + }, + { + "content": "Summarize the document in one sentence.", + "role": "user" + } + ], + "initial": [ + { + "content": "atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?", + "role": "user" + } + ] + }, + "planned_grid": 4096, + "planned_mark": 4096, + "rendered": { + "followup": { + "generation": [ + 248045, + 846, + 198, + 266, + 14078, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 198, + 22365, + 3299, + 13273, + 1118, + 30, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271, + 87132, + 3438, + 13, + 248046, + 198, + 248045, + 846, + 198, + 8917, + 5437, + 537, + 279, + 2128, + 303, + 799, + 11316, + 13, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271 + ], + "history": [ + 248045, + 846, + 198, + 266, + 14078, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 198, + 22365, + 3299, + 13273, + 1118, + 30, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271, + 87132, + 3438, + 13, + 248046, + 198, + 248045, + 846, + 198, + 8917, + 5437, + 537, + 279, + 2128, + 303, + 799, + 11316, + 13, + 248046, + 198 + ] + }, + "initial": { + "generation": [ + 248045, + 846, + 198, + 266, + 14078, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 198, + 22365, + 3299, + 13273, + 1118, + 30, + 248046, + 198, + 248045, + 74455, + 198, + 248068, + 271, + 248069, + 271 + ], + "history": [ + 248045, + 846, + 198, + 266, + 14078, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 58880, + 49577, + 72730, + 9197, + 198, + 22365, + 3299, + 13273, + 1118, + 30, + 248046, + 198 + ] + } + }, + "schema": "pr726_chat_history_fixture_v1", + "synthetic_text_only": true, + "tokenizer_config_sha256": "792fa3f0cb88b111e54ef3134c873531008c4df471d108da17903426e308aa7b", + "tokenizer_sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", + "vocab_size": 248077 +} diff --git a/validation/pr726-ttft/provenance.json b/validation/pr726-ttft/provenance.json new file mode 100644 index 0000000000..ba6103b53a --- /dev/null +++ b/validation/pr726-ttft/provenance.json @@ -0,0 +1,401 @@ +{ + "all_five_flags_default_off": true, + "aot": { + "aot_sha256": "dce85c91013171fd92c66c9401dfce3998df4f7753a3a3c2e22efd8911b22d37", + "extension_receipt_sha256": "036325e585cf5a41284dd40c033c9800c9aa1ed266ca5421aae9c66d97694c94", + "original_receipt_sha256": "050e43af502ad64e3940ead8b9215dd70b56cc3c43453017f9b777afcc1ba493" + }, + "arm_reports": { + "A": { + "raw_report_sha256": "550b9c70a065eb8e93074dec3e41f273022412d9f90f0a7e564fd30af3f019fb", + "tokenize_preflight": { + "followup": { + "generation": { + "count": 4445, + "sha256": "f8910192b1eab51b6aeda7b21f20c8a166773d30da8e1aabe5d5d5b1d533402f" + }, + "history": { + "count": 4438, + "sha256": "f7b62f6caad3524eb7c1cf9020d22c5e2155a5930cccaab8f519b62b034e610d" + } + }, + "initial": { + "generation": { + "count": 4419, + "sha256": "1e72d9495c1182f66f1664b2bf3cd4db8b8e08dbf2a8175b6903fff1c4857e14" + }, + "history": { + "count": 4412, + "sha256": "3c01cc896a0c90055aa87d78aaf9700c92ab1d46f6dab7eea9ef7163e8413c82" + } + } + } + }, + "B": { + "raw_report_sha256": "979d44a300008d6e4222579085032204ecd51e1076a2ee633d98246c8147b4e4", + "tokenize_preflight": { + "followup": { + "generation": { + "count": 4445, + "sha256": "f8910192b1eab51b6aeda7b21f20c8a166773d30da8e1aabe5d5d5b1d533402f" + }, + "history": { + "count": 4438, + "sha256": "f7b62f6caad3524eb7c1cf9020d22c5e2155a5930cccaab8f519b62b034e610d" + } + }, + "initial": { + "generation": { + "count": 4419, + "sha256": "1e72d9495c1182f66f1664b2bf3cd4db8b8e08dbf2a8175b6903fff1c4857e14" + }, + "history": { + "count": 4412, + "sha256": "3c01cc896a0c90055aa87d78aaf9700c92ab1d46f6dab7eea9ef7163e8413c82" + } + } + } + }, + "C": { + "raw_report_sha256": "1cec63943b3f668f2eb119ca97ba70ab26c2f1a63b7c2f4d9074c5c265c5a053", + "tokenize_preflight": { + "followup": { + "generation": { + "count": 4445, + "sha256": "f8910192b1eab51b6aeda7b21f20c8a166773d30da8e1aabe5d5d5b1d533402f" + }, + "history": { + "count": 4438, + "sha256": "f7b62f6caad3524eb7c1cf9020d22c5e2155a5930cccaab8f519b62b034e610d" + } + }, + "initial": { + "generation": { + "count": 4419, + "sha256": "1e72d9495c1182f66f1664b2bf3cd4db8b8e08dbf2a8175b6903fff1c4857e14" + }, + "history": { + "count": 4412, + "sha256": "3c01cc896a0c90055aa87d78aaf9700c92ab1d46f6dab7eea9ef7163e8413c82" + } + } + } + }, + "D": { + "raw_report_sha256": "1ff41461cfe06857e4f0278e773c3f584b36127109b3e8f6739c730c563e3db5", + "tokenize_preflight": { + "followup": { + "generation": { + "count": 4445, + "sha256": "f8910192b1eab51b6aeda7b21f20c8a166773d30da8e1aabe5d5d5b1d533402f" + }, + "history": { + "count": 4438, + "sha256": "f7b62f6caad3524eb7c1cf9020d22c5e2155a5930cccaab8f519b62b034e610d" + } + }, + "initial": { + "generation": { + "count": 4419, + "sha256": "1e72d9495c1182f66f1664b2bf3cd4db8b8e08dbf2a8175b6903fff1c4857e14" + }, + "history": { + "count": 4412, + "sha256": "3c01cc896a0c90055aa87d78aaf9700c92ab1d46f6dab7eea9ef7163e8413c82" + } + } + } + } + }, + "arms": { + "A": { + "flags": { + "TF_PR608_EXACT_CACHE_DRAFT_REUSE": "0", + "TF_PR608_EXACT_CACHE_LOGITS": "0", + "TF_PR608_EXACT_CACHE_SEQ_REUSE": "0", + "TF_PR608_EXACT_CACHE_SKIP_RESET": "0", + "TF_PR608_EXACT_PROMPT_CACHE": "0" + }, + "host_cache_gib": 0 + }, + "B": { + "flags": { + "TF_PR608_EXACT_CACHE_DRAFT_REUSE": "0", + "TF_PR608_EXACT_CACHE_LOGITS": "0", + "TF_PR608_EXACT_CACHE_SEQ_REUSE": "0", + "TF_PR608_EXACT_CACHE_SKIP_RESET": "0", + "TF_PR608_EXACT_PROMPT_CACHE": "0" + }, + "host_cache_gib": 1 + }, + "C": { + "flags": { + "TF_PR608_EXACT_CACHE_DRAFT_REUSE": "1", + "TF_PR608_EXACT_CACHE_LOGITS": "1", + "TF_PR608_EXACT_CACHE_SEQ_REUSE": "1", + "TF_PR608_EXACT_CACHE_SKIP_RESET": "1", + "TF_PR608_EXACT_PROMPT_CACHE": "1" + }, + "host_cache_gib": 0 + }, + "D": { + "flags": { + "TF_PR608_EXACT_CACHE_DRAFT_REUSE": "1", + "TF_PR608_EXACT_CACHE_LOGITS": "1", + "TF_PR608_EXACT_CACHE_SEQ_REUSE": "1", + "TF_PR608_EXACT_CACHE_SKIP_RESET": "1", + "TF_PR608_EXACT_PROMPT_CACHE": "1" + }, + "host_cache_gib": 1 + } + }, + "base_commit": "d49a906ffc168d707f62be598d329bf8dd8f5e2d", + "binary_sha256": { + "base": "97b1a443773157d4534d6005ab5baae6506dba7f739dad590324efbe3968f313", + "candidate": "d2c6d38c020580c5673a76a355ca1d3720078c18ad6d6f0116ee18c184abb976" + }, + "build_only_qualifier_registration_sha256": { + "zig/build/cuda.zig": "4a50b39f059ebb7ff6dec551e84a904f33d29374903d71fee6f39fecdbc7cf22", + "zig/src/families/qwen27/cuda.zig": "6637ab2105f65bcd3804808a2aaa98a126de8ffbd348f1bd6ecee9c8aa98fd5d", + "zig/src/pr726_overlap_qual.zig": "fc0d4d260fd4e7a6971949b7b8586427cf8aa2f6783fd7654c066f1dc328a126" + }, + "build_receipts_sha256": { + "base": "56be95ef52a15a11ee622c9f34ee2f69a375c6873ce3fe033ff99915bec3bbbe", + "candidate": "927f47565f45642a71be0893386b8bc188d957b5be4a619d444b6bc1170d51b2" + }, + "candidate_commit": "5e08e130bfb32e944390c2b9dfa3359a8859ea76", + "candidate_patch_sha256": "600597764c141275a544e29fedab1f6d1db7c8589f2425860b5e9daeb433ee9c", + "client_source_sha256": { + "bench_concurrent.py": "6274692888d28cc38003eeddd91c5fd990b5c2dedb28b9002fc42187dfd5cb80", + "bench_evidence.py": "119c396f24f3c13e181429bc69acff39f34254efd4789c5673a3dcbc54738d93", + "run_overlap.py": "9ce3f4b06f2b90ae13ef52259cd495d7f28d50e66a3089f1c4f5bd541e8549ee" + }, + "cpu_checks": { + "lean_baseline_findings": 1207, + "lean_findings_added": 0, + "linux_cuda_host_failed": 0, + "linux_cuda_host_passed": 439 + }, + "environment": { + "build_image_sha256": "sha256:15b8de8baabe68e46afc51e88f3ccad5042013bafe8f244ec082ce0bb06d47e3", + "build_image_usage": "CPU-only builds; not the inference runtime", + "cuda_arch": "sm_121", + "driver_library_versions": "Not independently retained in this receipt.", + "hardware": "NVIDIA DGX Spark GB10", + "inference_runtime": "Native binary launched on the Linux host", + "os": "Linux aarch64", + "zig": "0.17.0" + }, + "fatbins_sha256": { + "affine4_pack.fatbin": "f9ecce62413628aa76709055cadbf9f5f88ac049d91de02cb0b06e8a0902aee5", + "exlr8.fatbin": "ee272291e0811d8d1a0be51322422f604f5f711f42b8d273419ece531bff580d", + "exlr8_dense.fatbin": "481fff255df9a6306208f8b4ad7dfb46e9358d3fa039b1f9035e95335e720936", + "experts.fatbin": "bc1d9bba17ce24f4ff1b03641f5dc6956f282ffccd8fcfac2390821278a5c944", + "experts_pack.fatbin": "2b98cda2bd6e8352e95b825ea5a96f0170508f9a75c2a3688eeafc021540fb4a", + "experts_prefill.fatbin": "6c66bf58f890d2be8b010eb6ceec7cbba37e4c3171cd3bb3305f29192f373a12", + "fp8_experts.fatbin": "e392b32b3a601a197003bedce59484e4406033e1eabdc67b0d12b4c3756bf364", + "gdn.fatbin": "9a14a00e676962e6952c69f33a3dcb9f41e84b85963894895c4008ac69c274e8", + "gdn_prefill.fatbin": "a49872b46ff0cdf1d76e0cf8c17eaa77030aa33c040f98ccfbfc02bf6a29d19f", + "lane_gemv.fatbin": "8802dfa394dc9d7b7b6ce0b33e96969d694e4967c2d730f0761207a194503182", + "modelopt_pack.fatbin": "bf2592982c750f3d4ac9dc6aa5039a5f00597ac8e6328f3daec91485f2171a3d", + "nemotron_attention.fatbin": "22ea1e1e8f8dbcde8ce698506c172424758f96e0a273088188a0ca3d50208f6c", + "nemotron_keyed.fatbin": "575cff009a5d9955430364e1a94a622675c8bb4a98b39968264eea2201134735", + "nemotron_mamba.fatbin": "852ccca5959fe47ede55b52ffef4e9dae5e2d6b5f293ca29cbb69fb33cfbcba0", + "nemotron_norms.fatbin": "04a581247d1de5101ea6325091a30b5f5ddb5b21b417ef4ea81fa5719a32630a", + "nemotron_ops.fatbin": "e1fc392da48897719b08d8470ab0ef6bd9ce36a8b12941b829c65918c6ba88c8", + "nemotron_route.fatbin": "95efa984541a851b36efbc8cbaae0b51cee06bdf9da5a9819815e9b0f516b4f6", + "nvfp4_experts.fatbin": "1e0a25830113d8d27fb97d31e31157e83ce892f49364583406d94670c7fae2c3", + "prefill_attention.fatbin": "9b9701a7a1ee3f176add4f33ed9f3e2b3731924e924dbe5ed6413fcff60f7fb4", + "probe.fatbin": "a657823cc25bd0c9d4ace72faee7a60c21998a1c6605eb5e5ac34a76cb28a147", + "prompt16.fatbin": "898295332302643f675220a1868840c825f7e960e395208ed25d7366a1776863", + "qmm_group.fatbin": "650e16799049dbc1106fc7042e31787f96a332add28b6868d05c7d3fdb7c7e7f", + "qmm_prefill.fatbin": "9303bba885ccbc56182be8cd1ffc95b341d17b3c0bcec0a62a543e5b178f94fe", + "qmmf.fatbin": "cedfe8b726c9bf6f013e5cef7ab8995e6727884b984266d83c1b6e3f9001a314", + "qwen_ops.fatbin": "2bf8413839c85fcd52592591a8741add98e5560367ed7132b61dbcca419172d9", + "qwen_vision.fatbin": "11df8b7df0dd5e2a93ed7460266d3e2df7b2c7bde50c194d55e5fca716d163da", + "sample.fatbin": "2ec1c3abc32659641cba9f90846dd73f0b1f1b473d23c7a99f3a6f3b04efd55b", + "scan_rows.fatbin": "752a6ff9fe576def53311a8c6e972b9178632122539ba8ed3bfa398070a1d2aa", + "torch_argmax.fatbin": "131cc45a0e327c07351100aeea86725fa23ce27b3e7532d069286af016f65818", + "torch_indexing.fatbin": "96cc93928384c391c499482cb6d3cdf25146c5ce130e1e8469aee9707177e6b3", + "torch_movement.fatbin": "87580de4781083eaddc10f11d47d73b4f0f57f84777266f22be77be7d5aa143c", + "torch_nemotron_constants.fatbin": "3ff622c0e500e8899c1397f60ac999119db60c9e437cb297f8a40cdb295829b3", + "torch_norm_rotary.fatbin": "96d91e897d32c95b1efebefeb0dc8a6610560a5a99131c4c46145376772446ff", + "torch_pointwise.fatbin": "b784ed09041e6b8e3eed592b502ae7e2373ab2a23b3e246d559056ced183a8fe", + "torch_topk.fatbin": "d6d9820c4b4f7e158ce7c95d12685f5c4160b4bb5bcfa110fa4383a5ee69276a", + "train.fatbin": "191b48d3d42076ba36fd5170ee20d36e45163b85d5a90f5981e28ba6e865e0a5" + }, + "flags_require_exact_cache": [ + "TF_PR608_EXACT_CACHE_SEQ_REUSE", + "TF_PR608_EXACT_CACHE_LOGITS", + "TF_PR608_EXACT_CACHE_DRAFT_REUSE", + "TF_PR608_EXACT_CACHE_SKIP_RESET" + ], + "independent_acceptance_sha256": "35ae2131536df842b19ec1730b5b21160ab908c4a622d7d12a56ad289cfd7cec", + "memory_caps": { + "draft_retained_bytes": 1610612736, + "exact_prompt_tokens_max": 512, + "exact_snapshot_bytes": 268435456, + "pool_capacity_max": 512, + "pool_slots": 16, + "target_retained_bytes": 3221225472 + }, + "model_manifest_sha256": "1a556f5bd87c302a86cec21e437e58cede87e7492492b78bf24df5c2882633e4", + "models": [ + { + "files": [ + { + "name": "chat_template.jinja", + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "size": 8952 + }, + { + "name": "config.json", + "sha256": "14b65a0ee06517060a6bbd979bb1a8ff54e7b304b1a1f01d54344b88b8285e85", + "size": 4932 + }, + { + "name": "generation_config.json", + "sha256": "e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e", + "size": 202 + }, + { + "name": "model-00001-of-00003.safetensors", + "sha256": "6cc1508e96fb5d0865dfd5753a79f4ec60651bf3e2a82844a7e8ae9c60528c0d", + "size": 5343268662 + }, + { + "name": "model-00002-of-00003.safetensors", + "sha256": "83f2a20ca8058f486a3634a27faf99587f4cd3c156a83dee34fb99e6ac178670", + "size": 5354185130 + }, + { + "name": "model-00003-of-00003.safetensors", + "sha256": "31b8c91ef899f79efaaa69e3d2c096f6e2ebeb2ff20e29222abbd9ebc79e560a", + "size": 5357087557 + }, + { + "name": "model.safetensors.index.json", + "sha256": "13b840162b4cb35c66fef7df072f7dbb4717908204364f5e5d9f9655a2758fa8", + "size": 218281 + }, + { + "name": "preprocessor_config.json", + "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", + "size": 390 + }, + { + "name": "processor_config.json", + "sha256": "45fc17c8dd2474af6b493b52483c26c0584b0082d368c480f9fa611e73070040", + "size": 991 + }, + { + "name": "tokenizer.json", + "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", + "size": 19989325 + }, + { + "name": "tokenizer_config.json", + "sha256": "792fa3f0cb88b111e54ef3134c873531008c4df471d108da17903426e308aa7b", + "size": 1165 + }, + { + "name": "video_preprocessor_config.json", + "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13", + "size": 385 + }, + { + "name": "vocab.json", + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + "size": 6722759 + } + ], + "name": "target", + "repo": "Vontra/Qwen3.8-27B-MLX-4bit", + "revision": "70ae7fac63274ff2eac54152031433374cb80f2f" + }, + { + "files": [ + { + "name": "config.json", + "sha256": "873e3556509b0da06e29654ba00d4944888d4b5e8a33afde25f7eb27d321e980", + "size": 1239 + }, + { + "name": "model.safetensors", + "sha256": "67fc76d68dc5a9415511a4f394ef744d67510cd20e93b37cc2cc7d28e4bab65c", + "size": 3848817896 + } + ], + "name": "drafter", + "repo": "z-lab/Qwen3.8-27B-DFlash2", + "revision": "50307d4c4cde6860d4eee73e2547cd786fe8e8a4" + } + ], + "order": [ + "B", + "D", + "A", + "C" + ], + "prior_public_contribution_commit": "10c70be288c9a7f3b6cd995f84e1517d21b07bae", + "qualifier_source_included": false, + "raw_operations_included": false, + "raw_receipts_sha256": { + "A-client.json": "550b9c70a065eb8e93074dec3e41f273022412d9f90f0a7e564fd30af3f019fb", + "B-client.json": "979d44a300008d6e4222579085032204ecd51e1076a2ee633d98246c8147b4e4", + "C-client.json": "1cec63943b3f668f2eb119ca97ba70ab26c2f1a63b7c2f4d9074c5c265c5a053", + "D-client.json": "1ff41461cfe06857e4f0278e773c3f584b36127109b3e8f6739c730c563e3db5", + "after.json": "58eb732ac45d631c1fd695dc20f65585de115c0323ab11970538f0058a4c5e24", + "before.json": "8cb97249256f8df2c76b8b25ae660b9249f9280db47fd8f5d005e513ace0a6a7", + "comparison.json": "5dcd106c39401d23a683b0d61d6fc9b1fa2d8a3b6551638efdeaecb84a5a44ab", + "provenance.json": "c11d0d231e1e0b053a438f2c72d1456534af0ba3017bdfd8716279ccf0a4d856", + "status.json": "0f0fcc5e29d477161b857e3c18f62e7d798c7585bff43e55b22f3a2350e9fe1e" + }, + "restoration_attested": true, + "results_note_sha256": "8e3c416aa27f257b37bec84f93299d21dd840862747977fc304c97b8f868a3dd", + "runtime_source_manifest_sha256": "691b37dbb638bd0249adbaeaae252028d799b0678ce22a9ef824636c468be558", + "runtime_source_sha256": { + "zig/src/core/engine_api.zig": "0b96e2c12bdad63d94f4c5dea96fdebd14cc6706b64839eac6b3c75336b73eab", + "zig/src/core/lane_host.zig": "70159b3fa553a0f7446df79a745145dce9e629675cc28a002385a6be10bd0087", + "zig/src/core/lane_host_first_test.zig": "2aee5f44c1d2658039dd5a92b7ce99262e70026e97a5f7a69898c5b2f68ccd6f", + "zig/src/core/lanes/backend.zig": "dcf2f6a67b4d1e57afa8fbc8f3bac0ff272a7586c9b9ccebdc767bdd36088fa4", + "zig/src/core/lanes/engine.zig": "dfe3933976a20e6c9a61f9b9ec878cb07372b715835e134b5466ee975cd199ca", + "zig/src/core/lanes/lanes.zig": "c941dfaf8b3fcacddf11b50168f018f6d6b33bd700dd18e548b35fc9ae8e4a44", + "zig/src/core/lanes/open.zig": "180ba32613a8963cebbedf2bdccb708895d4d38b54622a163845d9230655131e", + "zig/src/core/lanes/open_first.zig": "b86ca87d2280d1cb3be71f5769b796943643bed69652ab271569ae06a5c9b314", + "zig/src/core/lanes/open_first_test.zig": "b35895f85bedf816e5f1cb8b6c5f2fd1306f5cfff657f7eeaa662d242f5b0ecd", + "zig/src/families/qwen27/cuda/draft_pool.zig": "3ca8a67b16ff5d019998c5242caea5a351dff94ddeaa461027108dcac0b8fff5", + "zig/src/families/qwen27/cuda/draft_pool_test.zig": "2e5792ccd7c8fbef33c50644fa721be9af284cdfcf2bedb78e920330fb28b88c", + "zig/src/families/qwen27/cuda/drafts.zig": "6d6de998f31e9b50cb9831a32f46c66bdae8a6d41e6bf98c55afc992d3dcc345", + "zig/src/families/qwen27/cuda/engine.zig": "0e97ff7eaec52059d23ca918d30b0107acf0a89ad39bf5f1b054464befd1f4ae", + "zig/src/families/qwen27/cuda/exact_cache_policy.zig": "5e550ddcbdd1db0ca50dbc543488b88516809945b8665820a06a0bc3150ca4ab", + "zig/src/families/qwen27/cuda/exact_cache_test.zig": "99d4d6ee8d94eb180c1e8783e54c5fea0f93e49f3b33d20d2a27c2fc5396d860", + "zig/src/families/qwen27/cuda/exact_draft_reuse.zig": "a20d1953fdaad54cd8892bce338b06c8d1afabc4df3ba4e9f00fa46696d43d05", + "zig/src/families/qwen27/cuda/exact_eligible.zig": "0fd9aaf4cb710df8553778c4edcb8d28302955e6d065fd9af7261fd4a2a5f29d", + "zig/src/families/qwen27/cuda/exact_first_logits.zig": "eb99f17648837d5026238ee9270cd30895b39a32c9646597fff5953ef511b103", + "zig/src/families/qwen27/cuda/exact_init.zig": "7f4ce6a85dfb8bed2aa62bf2edec900ca6cb4e658c90ea6aef323adaa59d0212", + "zig/src/families/qwen27/cuda/exact_logits_policy.zig": "c12f1d89bedfeb9192330c9eca0967abf7fe44886c9e736b09be9747f1ed57ca", + "zig/src/families/qwen27/cuda/exact_prompt_cache.zig": "3a39bc71549380628f1e1c237ace83251178de9e2c0724051cefa696366fe98a", + "zig/src/families/qwen27/cuda/exact_seq_reuse.zig": "3c99063cf26064c64e452ca96d980c5a780b84cffe7c7e2fbbbba4d4393f4005", + "zig/src/families/qwen27/cuda/lanes.zig": "5264dced52a22d5d1313e00069d62d361e3d6998eccddbb56b623c1d8352f99d", + "zig/src/families/qwen27/cuda/reset_skip.zig": "715816eddb923dbada206179405e8863fd0d82da66fa6db5050b44c9d88e46e2", + "zig/src/families/qwen27/cuda/reset_skip_factory_test.zig": "db2e5f04e0e12b7600b3b7624e70316c9ed76624cb4ee76d561830c05dc67922", + "zig/src/families/qwen27/cuda/reset_skip_test.zig": "5386b56c2191d840d216d160deac31705f2b4ff5df6fec4f7ef0f7cbe2bbf058", + "zig/src/families/qwen27/cuda/restore_proof.zig": "73357234268ab6c7a5f0fb72bb8fdc980be08743144a5364c56a0c9b8fd2e7bb", + "zig/src/families/qwen27/cuda/seq_pool.zig": "88978229ebd42339041992ea489c4cdf82f7cf64ff99295039d8f75508b6ca95", + "zig/src/families/qwen27/cuda/seq_pool_test.zig": "362733783b4bd6e5f647eee93f6eab44c34bd94c136398c1c8bcfc382a0819ef", + "zig/src/families/qwen27/cuda/state.zig": "7af35dceab6d271eae823f53b4f21c8c697b5967286dff77f3c63596ef8d7766", + "zig/src/native/cuda.zig": "02d8595f56e2e85755ca1a44104b7136d35bb07a6792525f373068cf8998bc60" + }, + "sanitized": true, + "schema": "pr726_sanitized_provenance_v1", + "server_settings": { + "context": 32768, + "kv_cache": "bf16", + "memory_reserve_gib": 8, + "model_alias": "qwen27-overlap", + "parallel": 16, + "temperature": 0, + "thinking": false + } +} diff --git a/validation/pr726-ttft/qualification.json b/validation/pr726-ttft/qualification.json new file mode 100644 index 0000000000..ec37393c67 --- /dev/null +++ b/validation/pr726-ttft/qualification.json @@ -0,0 +1,34 @@ +{ + "cases": { + "chat_drafted": { + "cached": 31, + "full_state_and_first_continuation_equal": true + }, + "code_drafted": { + "cached": 14, + "full_state_and_first_continuation_equal": true + }, + "code_plain": { + "cached": 14, + "full_state_and_first_continuation_equal": true + }, + "grid_code_drafted": { + "cached": 14, + "full_state_and_first_continuation_equal": true + } + }, + "cold_cross_key_reuse_proved": true, + "grid_full_n_proved": true, + "independent_qualification_receipt_sha256": "c6fbfdb2adb56f96a2481eb6d0bee342743403838460252712cc349c80bf367c", + "integrated_live_hits": 16, + "passed": true, + "provenance_sha256": "aa43a89fdd850f8d4d23bf4dbc600fe5ea6fe675358d4fa85c9f810c4425ef4d", + "qualification_command_sha256": "9893d4d9f996eafc7899730724c3d573fa36504c625ab9cc3b9a2110b7716d45", + "qualification_json_sha256": "d6dab2e845ca3c9aad63b4d99d96af60b05512ce75435b2d914eadb6704fe009", + "qualification_log_sha256": "8a5ae14433a7158a8587fe0af10a6caef415c0701d66152b31c9654a80cea946", + "qualifier_binary_sha256": "0488bb2a3dc2764f70ebd491ce4ac5146ef83bec86310c9c4dd4c9ac0a160e6b", + "reset_skip_proved": true, + "schema": "pr726_overlap_r2_independent_qualification_v1", + "scope": "Accepted actual-CUDA BF16 short full-N and integrated state/first/continuation gates; R3 HTTP separately proves long 4096-prefix reuse. Device qualifier source is not included.", + "source_qualifier_sha256": "fc0d4d260fd4e7a6971949b7b8586427cf8aa2f6783fd7654c066f1dc328a126" +} diff --git a/validation/pr726-ttft/requests.jsonl b/validation/pr726-ttft/requests.jsonl new file mode 100644 index 0000000000..8292e7d662 --- /dev/null +++ b/validation/pr726-ttft/requests.jsonl @@ -0,0 +1,152 @@ +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":69.46525076037533,"done":1097809.502546333,"finish_reason":"length","first":1097805.831594345,"full_response_s":3.8124594420660287,"last":1097809.502494503,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097805.831594345,2],[1097805.911534849,7],[1097805.992289764,4],[1097806.073072995,73],[1097806.153268845,51],[1097806.233685273,21],[1097806.314548187,14],[1097806.394695984,20],[1097806.481811276,16],[1097806.566920812,15],[1097806.649106042,23],[1097806.728949112,23],[1097806.810575805,25],[1097806.890759706,36],[1097806.971228932,14],[1097807.052257781,6],[1097807.133031909,18],[1097807.213178945,18],[1097807.294054414,19],[1097807.374298613,8],[1097807.458333334,11],[1097807.543437935,18],[1097807.624749613,15],[1097807.705229788,39],[1097807.787185974,30],[1097807.867486455,14],[1097807.948129845,15],[1097808.028561062,8],[1097808.109651288,6],[1097808.189925172,15],[1097808.271095418,23],[1097808.351394769,22],[1097808.432743685,26],[1097808.520896822,35],[1097808.604090068,31],[1097808.684390188,12],[1097808.765302929,13],[1097808.845784802,38],[1097808.926869882,47],[1097809.006965986,25],[1097809.088520312,11],[1097809.168951506,34],[1097809.249353872,12],[1097809.33044241,36],[1097809.409855816,60],[1097809.502494503,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-da5f22604461ee72bc09549ad5c032f5","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":15,"prefill_raised":[],"prefill_seconds":0.126399588,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.671969402,"time_to_first_token":0.127618002,"token_sha":"2b28aa2108de","tokens_per_second":69.44502311514631},"seed":1234,"sent":1097805.690086891,"speculative":{"acceptance_rate":0.31602373887240354,"accepted":213,"drafted":674,"rounds":45,"tokens_per_round":5.688888888888889},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.1415074539836496,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":213,"drafted":674,"rounds":45},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":69.55600513446439,"done":1097813.298634538,"finish_reason":"length","first":1097809.632474283,"full_response_s":3.7954030090477318,"last":1097813.298584772,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097809.632474283,2],[1097809.713106317,7],[1097809.793928387,4],[1097809.874940046,73],[1097809.955405123,51],[1097810.0358621,21],[1097810.118702576,14],[1097810.198801558,20],[1097810.279654338,16],[1097810.36014946,15],[1097810.441936214,23],[1097810.529472746,23],[1097810.612550653,25],[1097810.692494079,36],[1097810.772881986,14],[1097810.854313311,6],[1097810.93476934,18],[1097811.014973244,18],[1097811.095662014,19],[1097811.176247235,8],[1097811.257100107,11],[1097811.337446297,18],[1097811.416949563,15],[1097811.50361954,39],[1097811.587631969,30],[1097811.667724481,14],[1097811.748123189,15],[1097811.828987111,8],[1097811.909678792,6],[1097811.990498383,15],[1097812.07117729,23],[1097812.15158336,22],[1097812.232532703,26],[1097812.313861169,35],[1097812.394345993,31],[1097812.481727394,12],[1097812.566746165,13],[1097812.647613153,38],[1097812.72806854,47],[1097812.808942107,25],[1097812.890114714,11],[1097812.971330506,34],[1097813.052559551,12],[1097813.132856358,36],[1097813.213283412,60],[1097813.298584772,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-4e11fcfa52fa1bf5c58b9baf38483e18","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":15,"prefill_raised":[],"prefill_seconds":0.127324588,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.667161878,"time_to_first_token":0.128130845,"token_sha":"2b28aa2108de","tokens_per_second":69.5360631691209},"seed":1234,"sent":1097809.503231529,"speculative":{"acceptance_rate":0.3164933135215453,"accepted":213,"drafted":673,"rounds":45,"tokens_per_round":5.688888888888889},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.1292427540756762,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":213,"drafted":673,"rounds":45},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.016808028810452,"done":1097825.332260388,"finish_reason":"length","first":1097814.714651818,"full_response_s":12.0275407389272,"last":1097825.332215971,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.714651818,2],[1097815.470614543,7],[1097815.625486069,4],[1097815.826041031,63],[1097816.060859328,10],[1097816.242881417,31],[1097816.422976376,29],[1097816.614564995,22],[1097816.795778328,14],[1097816.97841744,24],[1097817.160527128,4],[1097817.305837955,13],[1097817.495522231,21],[1097817.677776811,2],[1097817.858160883,18],[1097818.04098456,29],[1097818.276543265,27],[1097818.519322474,24],[1097818.669045664,4],[1097818.816913335,16],[1097819.056456156,10],[1097819.236042591,31],[1097819.384398844,8],[1097819.573310107,11],[1097819.75445549,16],[1097819.937147929,12],[1097820.121016309,39],[1097820.294200891,18],[1097820.482660581,11],[1097820.667189505,16],[1097820.850659728,16],[1097820.998336362,11],[1097821.181710015,6],[1097821.364707258,9],[1097821.553395674,29],[1097821.73645387,22],[1097821.919320018,8],[1097822.065731745,18],[1097822.248333668,18],[1097822.433447892,34],[1097822.623768967,10],[1097822.807351865,14],[1097822.988950904,13],[1097823.176660168,2],[1097823.355944842,36],[1097823.546915119,42],[1097823.732360348,26],[1097823.878633706,17],[1097824.064987373,34],[1097824.301487247,12],[1097824.489078179,36],[1097824.776035745,50],[1097824.959867448,2],[1097825.332215971,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-8e96939142b80ce4f6121129b7bcbcca","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.124474468,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.616020635,"time_to_first_token":1.4037474,"token_sha":"2b28aa2108de","tokens_per_second":24.020299956773776},"seed":1234,"sent":1097813.304719649,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.409932168899104,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.159265125387865,"done":1097825.331284254,"finish_reason":"length","first":1097813.823667534,"full_response_s":12.026401416165754,"last":1097825.331269368,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097813.823667534,2],[1097815.471576325,7],[1097815.624445405,4],[1097815.82743328,63],[1097816.062946701,10],[1097816.241999815,31],[1097816.423301889,29],[1097816.615810126,22],[1097816.797062676,14],[1097816.980555629,24],[1097817.159878489,4],[1097817.306431355,13],[1097817.494068568,21],[1097817.678684918,2],[1097817.859845391,18],[1097818.039596352,29],[1097818.277447033,27],[1097818.517842867,24],[1097818.669275408,4],[1097818.817296773,16],[1097819.056041345,10],[1097819.236558821,31],[1097819.385905018,8],[1097819.573689608,11],[1097819.753228295,16],[1097819.936600712,12],[1097820.121682647,39],[1097820.294686931,18],[1097820.481401904,11],[1097820.670977877,16],[1097820.851990688,16],[1097820.997968964,11],[1097821.181453462,6],[1097821.363029228,9],[1097821.553241755,29],[1097821.736234997,22],[1097821.91968215,8],[1097822.067301823,18],[1097822.251486613,18],[1097822.432330654,34],[1097822.626470465,10],[1097822.809874634,14],[1097822.989732084,13],[1097823.173998036,2],[1097823.354940884,36],[1097823.545013615,42],[1097823.732037961,26],[1097823.878562078,17],[1097824.062280381,34],[1097824.300285628,12],[1097824.48935818,36],[1097824.7783525,50],[1097824.961128272,2],[1097825.331269368,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-05c7f754dc9a818839976af2f4686a03","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127628433,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.506679138,"time_to_first_token":0.516074291,"token_sha":"2b28aa2108de","tokens_per_second":22.161042029744305},"seed":1234,"sent":1097813.304882838,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.5187846960034221,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.17240007961242,"done":1097825.331615651,"finish_reason":"length","first":1097814.326042428,"full_response_s":12.025551459984854,"last":1097825.330513226,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.326042428,2],[1097815.470723084,7],[1097815.624786061,4],[1097815.829163943,63],[1097816.062834494,10],[1097816.242274426,31],[1097816.422548741,29],[1097816.616282245,22],[1097816.79594571,14],[1097816.979004997,24],[1097817.160380859,4],[1097817.307072087,13],[1097817.493625743,21],[1097817.680193613,2],[1097817.859660626,18],[1097818.040483104,29],[1097818.278478965,27],[1097818.519528463,24],[1097818.668069467,4],[1097818.816542109,16],[1097819.058052928,10],[1097819.238665243,31],[1097819.385804193,8],[1097819.572398019,11],[1097819.754019166,16],[1097819.936024587,12],[1097820.123230355,39],[1097820.293892684,18],[1097820.48154337,11],[1097820.669642945,16],[1097820.851220773,16],[1097820.999251696,11],[1097821.181659114,6],[1097821.36446287,9],[1097821.554870661,29],[1097821.73883201,22],[1097821.919656651,8],[1097822.067851183,18],[1097822.248637359,18],[1097822.433521121,34],[1097822.626900128,10],[1097822.807519421,14],[1097822.988827142,13],[1097823.175705526,2],[1097823.357767387,36],[1097823.546843939,42],[1097823.732306454,26],[1097823.879819975,17],[1097824.063576758,34],[1097824.298703568,12],[1097824.488380042,36],[1097824.776806551,50],[1097824.963802957,2],[1097825.330513226,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-3fa5e993f6ec89793926b443c58b2847","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125836743,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.005297982,"time_to_first_token":1.015716818,"token_sha":"2b28aa2108de","tokens_per_second":23.170658388084707},"seed":1234,"sent":1097813.306064191,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.0199782368727028,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":21.43618888797706,"done":1097825.331238156,"finish_reason":"length","first":1097813.435417726,"full_response_s":12.026226189918816,"last":1097825.331189,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097813.435417726,2],[1097815.470224682,7],[1097815.621602796,4],[1097815.826918504,63],[1097816.061174165,10],[1097816.242981219,31],[1097816.423026861,29],[1097816.616462384,22],[1097816.795687138,14],[1097816.97856094,24],[1097817.160464974,4],[1097817.308146456,13],[1097817.493970335,21],[1097817.677648597,2],[1097817.860374379,18],[1097818.039847272,29],[1097818.277031532,27],[1097818.519248907,24],[1097818.668482535,4],[1097818.81602532,16],[1097819.055685374,10],[1097819.23722055,31],[1097819.385699413,8],[1097819.573364674,11],[1097819.753764693,16],[1097819.936976178,12],[1097820.122594861,39],[1097820.29382957,18],[1097820.481353852,11],[1097820.667429027,16],[1097820.85008839,16],[1097820.997862504,11],[1097821.181500873,6],[1097821.364052398,9],[1097821.553923074,29],[1097821.739108347,22],[1097821.920021151,8],[1097822.065142386,18],[1097822.248415493,18],[1097822.433334454,34],[1097822.624947334,10],[1097822.807787498,14],[1097822.989238925,13],[1097823.177183195,2],[1097823.35655805,36],[1097823.545358026,42],[1097823.732943736,26],[1097823.879094404,17],[1097824.062833804,34],[1097824.299418655,12],[1097824.488057929,36],[1097824.776663357,50],[1097824.96263028,2],[1097825.331189,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-07a5cca4e389515b38cb7023d087158f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127669328,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.89489082,"time_to_first_token":0.128308049,"token_sha":"2b28aa2108de","tokens_per_second":21.437775584391616},"seed":1234,"sent":1097813.305011966,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.13040575990453362,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.90661497638556,"done":1097825.332511209,"finish_reason":"length","first":1097815.094229423,"full_response_s":12.02818385604769,"last":1097825.332473369,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097815.094229423,2],[1097815.472163885,7],[1097815.623397298,4],[1097815.828310977,63],[1097816.061264218,10],[1097816.245316536,31],[1097816.422608029,29],[1097816.614903553,22],[1097816.799812823,14],[1097816.980890729,24],[1097817.161988026,4],[1097817.30779823,13],[1097817.495089602,21],[1097817.678430348,2],[1097817.85946416,18],[1097818.039742348,29],[1097818.278265933,27],[1097818.51977098,24],[1097818.66936114,4],[1097818.818386952,16],[1097819.056997356,10],[1097819.238135984,31],[1097819.384535284,8],[1097819.573560867,11],[1097819.754252719,16],[1097819.936131463,12],[1097820.121302971,39],[1097820.295301295,18],[1097820.481186264,11],[1097820.671447958,16],[1097820.851873936,16],[1097820.998697101,11],[1097821.181394366,6],[1097821.364987532,9],[1097821.555720496,29],[1097821.736922446,22],[1097821.91896978,8],[1097822.066984895,18],[1097822.249014585,18],[1097822.43278026,34],[1097822.627802591,10],[1097822.806339357,14],[1097822.988735297,13],[1097823.172292931,2],[1097823.359238925,36],[1097823.547014344,42],[1097823.732592842,26],[1097823.879575253,17],[1097824.063503689,34],[1097824.298254635,12],[1097824.488164308,36],[1097824.775209109,50],[1097824.958868056,2],[1097825.332473369,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-b47b8777868e71ba1235f0b5027b67a8","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125925636,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.237396039,"time_to_first_token":1.780294306,"token_sha":"2b28aa2108de","tokens_per_second":24.908677854071637},"seed":1234,"sent":1097813.304327353,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.7899020700715482,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.654925635886073,"done":1097825.331392841,"finish_reason":"length","first":1097814.07480953,"full_response_s":12.025800403906032,"last":1097825.3306399,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.07480953,2],[1097815.472245583,7],[1097815.622079526,4],[1097815.82807054,63],[1097816.062154208,10],[1097816.243096227,31],[1097816.42315101,29],[1097816.615604776,22],[1097816.800765127,14],[1097816.981106419,24],[1097817.160194669,4],[1097817.306378149,13],[1097817.49560641,21],[1097817.678027172,2],[1097817.859268319,18],[1097818.039787727,29],[1097818.276676088,27],[1097818.517779353,24],[1097818.667740754,4],[1097818.81681984,16],[1097819.056896594,10],[1097819.23841085,31],[1097819.384729957,8],[1097819.572894576,11],[1097819.753827935,16],[1097819.936348641,12],[1097820.123173163,39],[1097820.294769845,18],[1097820.48219996,11],[1097820.669346663,16],[1097820.852410315,16],[1097820.99862997,11],[1097821.181213044,6],[1097821.365548449,9],[1097821.553469929,29],[1097821.737757827,22],[1097821.919195072,8],[1097822.068442079,18],[1097822.250303877,18],[1097822.432107747,34],[1097822.62461792,10],[1097822.807085837,14],[1097822.989438541,13],[1097823.1744135,2],[1097823.358643581,36],[1097823.546795535,42],[1097823.731028161,26],[1097823.878780229,17],[1097824.063427594,34],[1097824.29913619,12],[1097824.488473424,36],[1097824.777825664,50],[1097824.95938844,2],[1097825.3306399,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-44171c18e47fe7c645ea4214f4988d10","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.12562979,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.256643185,"time_to_first_token":0.765194495,"token_sha":"2b28aa2108de","tokens_per_second":22.653289778235074},"seed":1234,"sent":1097813.305592437,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.7692170930095017,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.910229295331607,"done":1097825.330400189,"finish_reason":"length","first":1097814.199964831,"full_response_s":12.023930880939588,"last":1097825.330364126,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.199964831,2],[1097815.470857045,7],[1097815.621258395,4],[1097815.828912429,63],[1097816.062680077,10],[1097816.241185505,31],[1097816.423257902,29],[1097816.61439041,22],[1097816.794874637,14],[1097816.978331036,24],[1097817.161266605,4],[1097817.305675728,13],[1097817.495847903,21],[1097817.678732074,2],[1097817.858869737,18],[1097818.040773481,29],[1097818.276426273,27],[1097818.518681007,24],[1097818.668745892,4],[1097818.818938251,16],[1097819.054742937,10],[1097819.236464782,31],[1097819.38395442,8],[1097819.572249509,11],[1097819.752827906,16],[1097819.937100261,12],[1097820.119943404,39],[1097820.293749714,18],[1097820.482325804,11],[1097820.670260927,16],[1097820.851296452,16],[1097820.998414778,11],[1097821.180712455,6],[1097821.365092983,9],[1097821.554694509,29],[1097821.73604223,22],[1097821.918598653,8],[1097822.067380175,18],[1097822.247837882,18],[1097822.433151163,34],[1097822.625275116,10],[1097822.808952019,14],[1097822.989702072,13],[1097823.172740872,2],[1097823.355429434,36],[1097823.54646092,42],[1097823.729047745,26],[1097823.879358446,17],[1097824.06188547,34],[1097824.300607549,12],[1097824.488705949,36],[1097824.776576074,50],[1097824.962750344,2],[1097825.330364126,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-a11d6b9f97500fb57c17d8daa5c307a8","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125499608,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.130646338,"time_to_first_token":0.889679159,"token_sha":"2b28aa2108de","tokens_per_second":22.90972080654747},"seed":1234,"sent":1097813.306469308,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.8934955229051411,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.73965823464757,"done":1097825.332349512,"finish_reason":"length","first":1097814.590787353,"full_response_s":12.026411727070808,"last":1097825.332306775,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.590787353,2],[1097815.47001128,7],[1097815.623091073,4],[1097815.828678827,63],[1097816.061017459,10],[1097816.245040868,31],[1097816.422503185,29],[1097816.615300552,22],[1097816.800187316,14],[1097816.980477725,24],[1097817.162115439,4],[1097817.308504189,13],[1097817.494989096,21],[1097817.678508781,2],[1097817.861357843,18],[1097818.040876404,29],[1097818.276140602,27],[1097818.51844699,24],[1097818.667953563,4],[1097818.817963449,16],[1097819.055949547,10],[1097819.236618573,31],[1097819.385018796,8],[1097819.572506767,11],[1097819.755240263,16],[1097819.936417933,12],[1097820.121554354,39],[1097820.293612841,18],[1097820.481108248,11],[1097820.668061815,16],[1097820.85232508,16],[1097820.998910468,11],[1097821.181980989,6],[1097821.363926891,9],[1097821.553844514,29],[1097821.736862101,22],[1097821.919134071,8],[1097822.065310694,18],[1097822.251190157,18],[1097822.431989507,34],[1097822.625821225,10],[1097822.810359855,14],[1097822.989282702,13],[1097823.176926467,2],[1097823.355715501,36],[1097823.546359871,42],[1097823.73172217,26],[1097823.880533878,17],[1097824.06488434,34],[1097824.298920903,12],[1097824.48913433,36],[1097824.775892455,50],[1097824.960343222,2],[1097825.332306775,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-c2684159561fb970fe544570c9cc59a4","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.134356202,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.740728876,"time_to_first_token":1.279955591,"token_sha":"2b28aa2108de","tokens_per_second":23.74140553624752},"seed":1234,"sent":1097813.305937785,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.2848495680373162,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":25.526035974601424,"done":1097825.333179428,"finish_reason":"length","first":1097815.342599528,"full_response_s":12.02265917393379,"last":1097825.332399756,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097815.342599528,2],[1097815.472344937,7],[1097815.623763453,4],[1097815.829240983,63],[1097816.062390467,10],[1097816.243938597,31],[1097816.42293868,29],[1097816.61633393,22],[1097816.798154078,14],[1097816.979281609,24],[1097817.1618496,4],[1097817.306845608,13],[1097817.494139125,21],[1097817.677428618,2],[1097817.859538703,18],[1097818.03953471,29],[1097818.277085795,27],[1097818.518836336,24],[1097818.669547537,4],[1097818.817613256,16],[1097819.055468068,10],[1097819.237580459,31],[1097819.384192102,8],[1097819.572971808,11],[1097819.754208316,16],[1097819.936244854,12],[1097820.121921865,39],[1097820.293417145,18],[1097820.481868352,11],[1097820.667306465,16],[1097820.85210275,16],[1097820.999157225,11],[1097821.180285128,6],[1097821.3631184,9],[1097821.554923723,29],[1097821.737598738,22],[1097821.919722358,8],[1097822.066212965,18],[1097822.247919564,18],[1097822.432547702,34],[1097822.627227909,10],[1097822.806616126,14],[1097822.98888651,13],[1097823.173681188,2],[1097823.358458274,36],[1097823.545704902,42],[1097823.729551949,26],[1097823.878933443,17],[1097824.064245331,34],[1097824.301148343,12],[1097824.489492185,36],[1097824.778108162,50],[1097824.960165679,2],[1097825.332399756,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-e33a950040f914050539399c38a29d24","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125927188,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.989229827,"time_to_first_token":2.029129472,"token_sha":"2b28aa2108de","tokens_per_second":25.527493552181337},"seed":1234,"sent":1097813.310520254,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":2.0320792738348246,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.309909640157027,"done":1097825.331718204,"finish_reason":"length","first":1097814.841030445,"full_response_s":12.026207769988105,"last":1097825.330579924,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.841030445,2],[1097815.470119854,7],[1097815.622227348,4],[1097815.827586977,63],[1097816.061974741,10],[1097816.242551326,31],[1097816.42280424,29],[1097816.615379273,22],[1097816.795906046,14],[1097816.979356008,24],[1097817.159701327,4],[1097817.308800313,13],[1097817.494679575,21],[1097817.681153373,2],[1097817.860847967,18],[1097818.040456597,29],[1097818.277385775,27],[1097818.520031392,24],[1097818.667618464,4],[1097818.817528325,16],[1097819.056208614,10],[1097819.236779984,31],[1097819.385117637,8],[1097819.573424539,11],[1097819.754888709,16],[1097819.936905717,12],[1097820.122217442,39],[1097820.295895251,18],[1097820.48146433,11],[1097820.669029685,16],[1097820.850766444,16],[1097820.998488888,11],[1097821.181913298,6],[1097821.364338371,9],[1097821.555490675,29],[1097821.738166282,22],[1097821.91925548,8],[1097822.065849409,18],[1097822.24868373,18],[1097822.433401969,34],[1097822.625535093,10],[1097822.807845201,14],[1097822.989168784,13],[1097823.172995134,2],[1097823.35822045,36],[1097823.545927152,42],[1097823.732245789,26],[1097823.879866442,17],[1097824.062515211,34],[1097824.299332493,12],[1097824.489416955,36],[1097824.778998056,50],[1097824.961242029,2],[1097825.330579924,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-6188f4bdf98ee719d32baecabaf15cae","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126194342,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.490108708,"time_to_first_token":1.529408746,"token_sha":"2b28aa2108de","tokens_per_second":24.30861367580787},"seed":1234,"sent":1097813.305510434,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.5355200108606368,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.448870294468826,"done":1097825.33047377,"finish_reason":"length","first":1097814.455720994,"full_response_s":12.022407139884308,"last":1097825.330445311,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.455720994,2],[1097815.471775529,7],[1097815.625138914,4],[1097815.826122681,63],[1097816.059928265,10],[1097816.24403174,31],[1097816.423221967,29],[1097816.614212863,22],[1097816.796778892,14],[1097816.980325773,24],[1097817.160293206,4],[1097817.306058287,13],[1097817.494355376,21],[1097817.678271706,2],[1097817.858406665,18],[1097818.039630302,29],[1097818.278030267,27],[1097818.517995923,24],[1097818.668965583,4],[1097818.816428302,16],[1097819.055044629,10],[1097819.23635885,31],[1097819.384657175,8],[1097819.573248562,11],[1097819.753452404,16],[1097819.937050641,12],[1097820.122480686,39],[1097820.294416596,18],[1097820.480993672,11],[1097820.667377534,16],[1097820.850170488,16],[1097820.999038649,11],[1097821.181550622,6],[1097821.364889188,9],[1097821.554266926,29],[1097821.736524891,22],[1097821.918799087,8],[1097822.068084125,18],[1097822.24826428,18],[1097822.433709742,34],[1097822.626316467,10],[1097822.808070829,14],[1097822.989034938,13],[1097823.17253482,2],[1097823.3563071,36],[1097823.54671728,42],[1097823.728927888,26],[1097823.879155756,17],[1097824.063962769,34],[1097824.299780097,12],[1097824.488907983,36],[1097824.776520948,50],[1097824.963517499,2],[1097825.330445311,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-cdb057c3b0eafb22f97822482308396e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.13019048,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.874832337,"time_to_first_token":1.145727365,"token_sha":"2b28aa2108de","tokens_per_second":23.44863737644951},"seed":1234,"sent":1097813.30806663,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.1476543638855219,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":21.68437404048179,"done":1097825.330844318,"finish_reason":"length","first":1097813.57118392,"full_response_s":12.025723682017997,"last":1097825.330803982,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097813.57118392,2],[1097815.47067897,7],[1097815.624238646,4],[1097815.827054113,63],[1097816.060553008,10],[1097816.242802568,31],[1097816.423102205,29],[1097816.616236593,22],[1097816.796936943,14],[1097816.979569569,24],[1097817.161376795,4],[1097817.307487092,13],[1097817.494650363,21],[1097817.679932303,2],[1097817.859153087,18],[1097818.041088748,29],[1097818.277773456,27],[1097818.517882948,24],[1097818.667787958,4],[1097818.818038344,16],[1097819.056787061,10],[1097819.236997786,31],[1097819.385498802,8],[1097819.573488869,11],[1097819.754632716,16],[1097819.936544209,12],[1097820.122664122,39],[1097820.294321869,18],[1097820.482529824,11],[1097820.670524571,16],[1097820.850499566,16],[1097820.999091166,11],[1097821.181077804,6],[1097821.365183468,9],[1097821.553978457,29],[1097821.738423923,22],[1097821.919377562,8],[1097822.065612913,18],[1097822.249295803,18],[1097822.432731857,34],[1097822.625030216,10],[1097822.808530952,14],[1097822.989316524,13],[1097823.174813325,2],[1097823.358899844,36],[1097823.547241635,42],[1097823.73009476,26],[1097823.880339112,17],[1097824.064176279,34],[1097824.300380482,12],[1097824.488286933,36],[1097824.778785202,50],[1097824.960010273,2],[1097825.330803982,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-64ab14b09d3262d586e4309422afc52e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.134891261,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.760456558,"time_to_first_token":0.263637256,"token_sha":"2b28aa2108de","tokens_per_second":21.682831677698545},"seed":1234,"sent":1097813.305120636,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.2660632838960737,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.402647125376273,"done":1097825.331565903,"finish_reason":"length","first":1097813.948944023,"full_response_s":12.025744568090886,"last":1097825.331527455,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097813.948944023,2],[1097815.471368957,7],[1097815.625065603,4],[1097815.828150589,63],[1097816.061772624,10],[1097816.241674942,31],[1097816.423127304,29],[1097816.615535419,22],[1097816.798817397,14],[1097816.978139484,24],[1097817.160842059,4],[1097817.307604117,13],[1097817.495166386,21],[1097817.680696671,2],[1097817.858581522,18],[1097818.03966342,29],[1097818.276485497,27],[1097818.518406669,24],[1097818.667883278,4],[1097818.816987413,16],[1097819.057471104,10],[1097819.237307994,31],[1097819.385265362,8],[1097819.573746144,11],[1097819.753122556,16],[1097819.936667219,12],[1097820.120703764,39],[1097820.294626618,18],[1097820.482258016,11],[1097820.66748966,16],[1097820.851340982,16],[1097820.998564791,11],[1097821.181612807,6],[1097821.36477551,9],[1097821.555218627,29],[1097821.73873453,22],[1097821.919945569,8],[1097822.06658963,18],[1097822.248113082,18],[1097822.432989001,34],[1097822.625342359,10],[1097822.806966156,14],[1097822.989345112,13],[1097823.172809427,2],[1097823.356826318,36],[1097823.546618664,42],[1097823.730538331,26],[1097823.878722782,17],[1097824.063221095,34],[1097824.299268899,12],[1097824.487944219,36],[1097824.777567753,50],[1097824.963870856,2],[1097825.331527455,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-c50ffb536355b4e8d37b8121be211a2a","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.124878687,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.382194576,"time_to_first_token":0.640666664,"token_sha":"2b28aa2108de","tokens_per_second":22.403412478792262},"seed":1234,"sent":1097813.305821335,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.6431226881686598,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":25.21020241545584,"done":1097825.331470296,"finish_reason":"length","first":1097815.216492984,"full_response_s":12.021940720966086,"last":1097825.331445486,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097815.216492984,2],[1097815.472068581,7],[1097815.624693446,4],[1097815.829426037,63],[1097816.060391244,10],[1097816.241014281,31],[1097816.422650191,29],[1097816.615176164,22],[1097816.795104461,14],[1097816.980077782,24],[1097817.159564886,4],[1097817.305957253,13],[1097817.494286179,21],[1097817.679241871,2],[1097817.858240276,18],[1097818.040084923,29],[1097818.275770384,27],[1097818.519090265,24],[1097818.66821644,4],[1097818.815891793,16],[1097819.056699969,10],[1097819.235917596,31],[1097819.384861227,8],[1097819.573880775,11],[1097819.753349289,16],[1097819.937308363,12],[1097820.119785851,39],[1097820.293105705,18],[1097820.480891727,11],[1097820.668411991,16],[1097820.850315587,16],[1097820.997636828,11],[1097821.180580193,6],[1097821.365479749,9],[1097821.554482855,29],[1097821.737849833,22],[1097821.918444175,8],[1097822.066741003,18],[1097822.248534373,18],[1097822.432238072,34],[1097822.624214509,10],[1097822.80760989,14],[1097822.98909245,13],[1097823.171850736,2],[1097823.354559274,36],[1097823.547194304,42],[1097823.731353477,26],[1097823.879403616,17],[1097824.062369553,34],[1097824.300241514,12],[1097824.487613606,36],[1097824.77546758,50],[1097824.959194763,2],[1097825.331445486,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-24ec789dad56fa0cfdf2ee74a22cb7b3","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.122931924,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.114481546,"time_to_first_token":1.903328514,"token_sha":"2b28aa2108de","tokens_per_second":25.21137626681869},"seed":1234,"sent":1097813.309529575,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.9069634089246392,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.604959594111655,"done":1097825.331926415,"finish_reason":"length","first":1097814.968115851,"full_response_s":12.027463051956147,"last":1097825.331880076,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097814.968115851,2],[1097815.47005826,7],[1097815.625979125,4],[1097815.828610863,63],[1097816.061366982,10],[1097816.245551226,31],[1097816.4227769,29],[1097816.615098483,22],[1097816.798227485,14],[1097816.98037501,24],[1097817.160912473,4],[1097817.308882155,13],[1097817.493561012,21],[1097817.679707506,2],[1097817.860217289,18],[1097818.039363087,29],[1097818.278386799,27],[1097818.517937146,24],[1097818.668162193,4],[1097818.817247521,16],[1097819.056312081,10],[1097819.237710304,31],[1097819.385577219,8],[1097819.572828693,11],[1097819.75372322,16],[1097819.936078418,12],[1097820.120870249,39],[1097820.294954561,18],[1097820.480763738,11],[1097820.671105065,16],[1097820.851507498,16],[1097820.9991348,11],[1097821.181154732,6],[1097821.364518525,9],[1097821.553764673,29],[1097821.737058309,22],[1097821.919622894,8],[1097822.067596711,18],[1097822.249642616,18],[1097822.43240947,34],[1097822.626657485,10],[1097822.807427608,14],[1097822.989392811,13],[1097823.175248237,2],[1097823.357174667,36],[1097823.546124048,42],[1097823.730757844,26],[1097823.88041788,17],[1097824.063706618,34],[1097824.300882748,12],[1097824.487834415,36],[1097824.77735626,50],[1097824.960997772,2],[1097825.331880076,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-994c6c5fa54b92949e088d24c6dd5aeb","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126499671,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.363571663,"time_to_first_token":1.654441742,"token_sha":"2b28aa2108de","tokens_per_second":24.60541677059082},"seed":1234,"sent":1097813.304463363,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.6636524880304933,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"A","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":21.917138929671605,"done":1097825.331057283,"finish_reason":"length","first":1097813.696312099,"full_response_s":12.026805482106283,"last":1097825.331042381,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097813.696312099,2],[1097815.471292269,7],[1097815.626088963,4],[1097815.828410875,63],[1097816.063127081,10],[1097816.243366101,31],[1097816.423462676,29],[1097816.614781982,22],[1097816.795306722,14],[1097816.9779863,24],[1097817.159646536,4],[1097817.307207664,13],[1097817.493915368,21],[1097817.677708158,2],[1097817.858489612,18],[1097818.03993302,29],[1097818.276249975,27],[1097818.518564127,24],[1097818.669755912,4],[1097818.817802198,16],[1097819.056392882,10],[1097819.237796484,31],[1097819.384281739,8],[1097819.573624525,11],[1097819.754286349,16],[1097819.936497486,12],[1097820.120437639,39],[1097820.296171733,18],[1097820.481984944,11],[1097820.667799067,16],[1097820.850420894,16],[1097820.998191903,11],[1097821.180468884,6],[1097821.36423713,9],[1097821.55440383,29],[1097821.739427854,22],[1097821.918853573,8],[1097822.06629924,18],[1097822.248730981,18],[1097822.432472167,34],[1097822.62594121,10],[1097822.806845531,14],[1097822.989667498,13],[1097823.173340042,2],[1097823.357409802,36],[1097823.544673445,42],[1097823.73277069,26],[1097823.879323488,17],[1097824.064515648,34],[1097824.298532251,12],[1097824.488985534,36],[1097824.7774256,50],[1097824.961208272,2],[1097825.331042381,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-93fd733a046524242f5eb469054d297f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125672683,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.634313078,"time_to_first_token":0.388836649,"token_sha":"2b28aa2108de","tokens_per_second":21.917924873639024},"seed":1234,"sent":1097813.304251801,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.3920602980069816,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":49.48511617606272,"done":1097830.62616265,"finish_reason":"length","first":1097825.473018507,"full_response_s":5.28832744108513,"last":1097830.626083099,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097825.473018507,3],[1097825.557266528,26],[1097825.63984172,15],[1097825.720144067,34],[1097825.801372608,21],[1097825.881702269,25],[1097825.962232806,14],[1097826.042489473,14],[1097826.124520366,18],[1097826.204807366,26],[1097826.285857581,13],[1097826.3663962,13],[1097826.448022485,13],[1097826.537625237,23],[1097826.621120264,32],[1097826.702098114,7],[1097826.78492979,23],[1097826.865433317,9],[1097826.945813893,13],[1097827.026373645,15],[1097827.108197896,19],[1097827.188663274,12],[1097827.268789974,5],[1097827.350723085,23],[1097827.431029776,9],[1097827.518356316,17],[1097827.601520089,3],[1097827.68189436,21],[1097827.76231636,10],[1097827.843881125,24],[1097827.924532149,19],[1097828.004761696,14],[1097828.086317124,13],[1097828.167030076,5],[1097828.247246513,22],[1097828.32837667,13],[1097828.40967294,51],[1097828.497644107,15],[1097828.582834373,6],[1097828.663439475,8],[1097828.744038911,9],[1097828.826167941,16],[1097828.906957697,16],[1097828.987627854,9],[1097829.070448245,32],[1097829.151539095,18],[1097829.231979915,13],[1097829.313394156,11],[1097829.393864491,15],[1097829.478397328,8],[1097829.564592463,18],[1097829.638742757,25],[1097829.719498709,19],[1097829.800874422,8],[1097829.881698122,40],[1097829.962634198,16],[1097830.043824965,64],[1097830.124552416,6],[1097830.20599565,6],[1097830.287399412,11],[1097830.36780581,15],[1097830.44935763,17],[1097830.538614704,21],[1097830.626083099,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-cd5f7cbca5340bb86fbbdd979840b5fc","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":14,"prefill_raised":[],"prefill_seconds":0.134090989,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.153253101,"time_to_first_token":0.134471077,"token_sha":"8a91591c94a3","tokens_per_second":49.4833059821022},"seed":1234,"sent":1097825.337835209,"speculative":{"acceptance_rate":0.206163655685441,"accepted":194,"drafted":941,"rounds":63,"tokens_per_round":4.063492063492063},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.13518329802900553,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":194,"drafted":941,"rounds":63},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":49.547654253108774,"done":1097835.909330996,"finish_reason":"length","first":1097830.762479288,"full_response_s":5.278693869011477,"last":1097835.909039783,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097830.762479288,3],[1097830.844509832,26],[1097830.925485292,15],[1097831.005619722,34],[1097831.08801477,21],[1097831.168677021,25],[1097831.249227692,14],[1097831.330525516,14],[1097831.411115642,18],[1097831.49581155,26],[1097831.580557541,13],[1097831.661276565,13],[1097831.742072947,13],[1097831.823267618,23],[1097831.90399259,32],[1097831.984598764,7],[1097832.065909194,23],[1097832.146157118,9],[1097832.228699494,13],[1097832.309836225,15],[1097832.390297175,19],[1097832.473870844,12],[1097832.561684258,5],[1097832.644773964,23],[1097832.725323445,9],[1097832.80724443,17],[1097832.887659124,3],[1097832.968173514,21],[1097833.048671994,10],[1097833.130188673,24],[1097833.210517504,19],[1097833.292481074,14],[1097833.372819685,13],[1097833.455128218,5],[1097833.542160881,22],[1097833.625897098,13],[1097833.706389144,51],[1097833.787915106,15],[1097833.867831552,6],[1097833.948535771,8],[1097834.029308049,9],[1097834.110774847,16],[1097834.191129701,16],[1097834.272571034,9],[1097834.353555056,32],[1097834.433858066,18],[1097834.522721106,13],[1097834.60760503,11],[1097834.68808979,15],[1097834.768461179,8],[1097834.850288369,18],[1097834.924550769,25],[1097835.005021537,19],[1097835.086421634,8],[1097835.16722869,40],[1097835.248310699,16],[1097835.329690308,64],[1097835.41049738,6],[1097835.49583889,6],[1097835.580649143,11],[1097835.66196532,15],[1097835.742630627,17],[1097835.824103459,21],[1097835.909039783,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-8641928807b59b531362facdfb7e5811","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":14,"prefill_raised":[],"prefill_seconds":0.128805571,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.147614604,"time_to_first_token":0.130307516,"token_sha":"8a91591c94a3","tokens_per_second":49.537508072544895},"seed":1234,"sent":1097830.630637127,"speculative":{"acceptance_rate":0.206163655685441,"accepted":194,"drafted":941,"rounds":63,"tokens_per_round":4.063492063492063},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.13184216106310487,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":194,"drafted":941,"rounds":63},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.98975962730893,"done":1097851.453318088,"finish_reason":"length","first":1097836.443913804,"full_response_s":15.527400407008827,"last":1097851.452954873,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.443913804,3],[1097838.134426397,11],[1097838.317265354,30],[1097838.502099344,19],[1097838.651782077,21],[1097838.834616346,15],[1097838.979704298,19],[1097839.125779561,17],[1097839.274713276,6],[1097839.422018322,8],[1097839.611485943,3],[1097839.759063971,5],[1097839.941992448,27],[1097840.122825218,12],[1097840.303968775,13],[1097840.490689319,5],[1097840.64231947,16],[1097840.824188515,5],[1097841.009513356,14],[1097841.191699477,9],[1097841.337865409,22],[1097841.489684266,10],[1097841.680329733,7],[1097841.859592767,23],[1097842.006866906,5],[1097842.15982413,9],[1097842.327240123,14],[1097842.47904246,20],[1097842.634789839,8],[1097842.779729573,12],[1097842.930357718,5],[1097843.078403091,14],[1097843.261587347,14],[1097843.40694864,8],[1097843.596357511,13],[1097843.769578145,3],[1097843.948861811,10],[1097844.096782526,11],[1097844.277613475,9],[1097844.459202843,10],[1097844.655420027,20],[1097844.802997978,14],[1097844.983925392,14],[1097845.211745147,13],[1097845.359450693,5],[1097845.548741997,22],[1097845.779193259,3],[1097846.014835353,32],[1097846.19920381,33],[1097846.38233392,11],[1097846.57472288,6],[1097846.72256529,8],[1097846.870671957,9],[1097847.021145257,13],[1097847.167767778,8],[1097847.318951635,20],[1097847.465770162,17],[1097847.623579114,20],[1097847.77445062,13],[1097847.948263884,6],[1097848.12998513,13],[1097848.279823932,9],[1097848.429213455,11],[1097848.5896184,8],[1097848.773203427,15],[1097848.962312311,16],[1097849.170929077,27],[1097849.410613798,4],[1097849.566333122,8],[1097849.752138425,25],[1097849.901547523,18],[1097850.051029694,13],[1097850.234419929,55],[1097850.385365765,11],[1097850.576306743,10],[1097850.75146899,11],[1097850.8986453,15],[1097851.049215425,17],[1097851.199354343,14],[1097851.452954873,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-11922824e7d01351aa6fb877c3a69b33","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.128583306,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.008236287,"time_to_first_token":0.515484344,"token_sha":"8a91591c94a3","tokens_per_second":16.99067066400592},"seed":1234,"sent":1097835.925917681,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.5179961228277534,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.95777103001177,"done":1097851.456487722,"finish_reason":"length","first":1097838.005195485,"full_response_s":15.52367174392566,"last":1097851.45614389,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097838.005195485,3],[1097838.131772495,11],[1097838.313520709,30],[1097838.503044961,19],[1097838.651153602,21],[1097838.833514273,15],[1097838.979437383,19],[1097839.126154772,17],[1097839.273751335,6],[1097839.42223141,8],[1097839.612540558,3],[1097839.760559742,5],[1097839.939853146,27],[1097840.125718862,12],[1097840.303480099,13],[1097840.491279329,5],[1097840.641911719,16],[1097840.82707087,5],[1097841.006030702,14],[1097841.188438919,9],[1097841.337375918,22],[1097841.490661841,10],[1097841.675705059,7],[1097841.857921997,23],[1097842.00703516,5],[1097842.154886486,9],[1097842.327367369,14],[1097842.479923182,20],[1097842.633104263,8],[1097842.78108434,12],[1097842.930791284,5],[1097843.078600187,14],[1097843.261071607,14],[1097843.406075219,8],[1097843.59664904,13],[1097843.768185271,3],[1097843.948178748,10],[1097844.097307227,11],[1097844.276570906,9],[1097844.458757658,10],[1097844.653877341,20],[1097844.802399793,14],[1097844.983313892,14],[1097845.211070184,13],[1097845.358900255,5],[1097845.547535851,22],[1097845.779602542,3],[1097846.017267599,32],[1097846.200050785,33],[1097846.382722795,11],[1097846.573551532,6],[1097846.725251611,8],[1097846.872285828,9],[1097847.019217963,13],[1097847.169158097,8],[1097847.319156028,20],[1097847.46570782,17],[1097847.623164998,20],[1097847.773462428,13],[1097847.947934567,6],[1097848.13043601,13],[1097848.280283279,9],[1097848.428725138,11],[1097848.590310181,8],[1097848.77365333,15],[1097848.960157696,16],[1097849.170814716,27],[1097849.410128555,4],[1097849.566464913,8],[1097849.750783195,25],[1097849.908236348,18],[1097850.051715326,13],[1097850.235576878,55],[1097850.382877548,11],[1097850.57661578,10],[1097850.750312536,11],[1097850.900783262,15],[1097851.04964199,17],[1097851.198606354,14],[1097851.45614389,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-54465c7ef17c7a09026632a57af07a68","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126011631,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.451648138,"time_to_first_token":2.069888617,"token_sha":"8a91591c94a3","tokens_per_second":18.95678487750822},"seed":1234,"sent":1097835.932815978,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":2.0723795068915933,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.565850491788495,"done":1097851.453435025,"finish_reason":"length","first":1097836.05864831,"full_response_s":15.528424261836335,"last":1097851.451760793,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.05864831,3],[1097838.134096834,11],[1097838.314811127,30],[1097838.501696201,19],[1097838.652221409,21],[1097838.833013367,15],[1097838.97874463,19],[1097839.12575019,17],[1097839.273946159,6],[1097839.423291339,8],[1097839.612426404,3],[1097839.75862877,5],[1097839.942207247,27],[1097840.124261108,12],[1097840.303385456,13],[1097840.489549667,5],[1097840.642183083,16],[1097840.825997516,5],[1097841.00919119,14],[1097841.190383698,9],[1097841.337508558,22],[1097841.490900377,10],[1097841.676765032,7],[1097841.85873122,23],[1097842.007111396,5],[1097842.159249312,9],[1097842.326919302,14],[1097842.480144991,20],[1097842.634038732,8],[1097842.779552164,12],[1097842.929104943,5],[1097843.078889172,14],[1097843.259886185,14],[1097843.405865415,8],[1097843.59659214,13],[1097843.767162595,3],[1097843.94671217,10],[1097844.094329254,11],[1097844.276687332,9],[1097844.460049373,10],[1097844.652064174,20],[1097844.801123668,14],[1097844.983018682,14],[1097845.211273666,13],[1097845.360331858,5],[1097845.547987966,22],[1097845.776961416,3],[1097846.013314921,32],[1097846.200460004,33],[1097846.383152341,11],[1097846.573478802,6],[1097846.722626944,8],[1097846.8703327,9],[1097847.019736228,13],[1097847.168394161,8],[1097847.319345391,20],[1097847.465905762,17],[1097847.624759246,20],[1097847.774861342,13],[1097847.948532315,6],[1097848.134658146,13],[1097848.278873804,9],[1097848.429537618,11],[1097848.587960564,8],[1097848.773552414,15],[1097848.958364856,16],[1097849.17033157,27],[1097849.409967763,4],[1097849.565682237,8],[1097849.749672564,25],[1097849.901171183,18],[1097850.05085544,13],[1097850.234840188,55],[1097850.38455969,11],[1097850.576486374,10],[1097850.750045419,11],[1097850.898189445,15],[1097851.049914309,17],[1097851.197759099,14],[1097851.451760793,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-e324dfa426991e7c1e63a162d54c1830","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.130437877,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.393141923,"time_to_first_token":0.130953624,"token_sha":"8a91591c94a3","tokens_per_second":16.565818809153328},"seed":1234,"sent":1097835.925010763,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.13363754679448903,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.300595583247983,"done":1097851.453514669,"finish_reason":"length","first":1097836.714113776,"full_response_s":15.524969837162644,"last":1097851.45349074,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.714113776,3],[1097838.132178214,11],[1097838.314288756,30],[1097838.502069156,19],[1097838.651323185,21],[1097838.835081431,15],[1097838.979337746,19],[1097839.126286949,17],[1097839.273626232,6],[1097839.425255364,8],[1097839.613081031,3],[1097839.75958306,5],[1097839.943168722,27],[1097840.124533577,12],[1097840.304019306,13],[1097840.490761154,5],[1097840.642022384,16],[1097840.824286728,5],[1097841.008959617,14],[1097841.189663913,9],[1097841.337172003,22],[1097841.489735773,10],[1097841.675043696,7],[1097841.859828998,23],[1097842.007329331,5],[1097842.156248791,9],[1097842.327960658,14],[1097842.47886121,20],[1097842.63380652,8],[1097842.780969706,12],[1097842.931478429,5],[1097843.079033032,14],[1097843.259061517,14],[1097843.407728812,8],[1097843.595909828,13],[1097843.766401823,3],[1097843.949309285,10],[1097844.09683805,11],[1097844.277978695,9],[1097844.458634333,10],[1097844.655713125,20],[1097844.79990678,14],[1097844.981791441,14],[1097845.212551869,13],[1097845.359513403,5],[1097845.547643106,22],[1097845.777823214,3],[1097846.013808089,32],[1097846.198375658,33],[1097846.381438784,11],[1097846.574630319,6],[1097846.722691047,8],[1097846.871553247,9],[1097847.021236378,13],[1097847.16809804,8],[1097847.318382505,20],[1097847.466017178,17],[1097847.624452641,20],[1097847.772201019,13],[1097847.948007329,6],[1097848.130218429,13],[1097848.279094458,9],[1097848.430534275,11],[1097848.588796028,8],[1097848.77349649,15],[1097848.96049303,16],[1097849.172283442,27],[1097849.411654565,4],[1097849.565215208,8],[1097849.75041092,25],[1097849.902414995,18],[1097850.0497475,13],[1097850.234635781,55],[1097850.382802834,11],[1097850.57595041,10],[1097850.750269977,11],[1097850.899559587,15],[1097851.047496873,17],[1097851.200712129,14],[1097851.45349074,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-0288282024ffe9456e2b6afbd467c230","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127171253,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.738962128,"time_to_first_token":0.78391144,"token_sha":"8a91591c94a3","tokens_per_second":17.301082517579015},"seed":1234,"sent":1097835.928544832,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.7855689441785216,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.08878097272672,"done":1097851.455059431,"finish_reason":"length","first":1097837.35789974,"full_response_s":15.530284916982055,"last":1097851.455035439,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.35789974,3],[1097838.134954994,11],[1097838.317796464,30],[1097838.502210873,19],[1097838.651610381,21],[1097838.834385701,15],[1097838.98064207,19],[1097839.125506835,17],[1097839.274276217,6],[1097839.422130204,8],[1097839.611857697,3],[1097839.759409763,5],[1097839.939942571,27],[1097840.123486326,12],[1097840.303559536,13],[1097840.489912393,5],[1097840.641644725,16],[1097840.825728985,5],[1097841.006528212,14],[1097841.190267655,9],[1097841.337738835,22],[1097841.489819115,10],[1097841.678395647,7],[1097841.859955172,23],[1097842.00632871,5],[1097842.157790459,9],[1097842.327556911,14],[1097842.477778223,20],[1097842.636488619,8],[1097842.782171233,12],[1097842.928109173,5],[1097843.078258142,14],[1097843.259822578,14],[1097843.406593423,8],[1097843.596288493,13],[1097843.769186643,3],[1097843.948221836,10],[1097844.095924071,11],[1097844.276945185,9],[1097844.459310113,10],[1097844.652760778,20],[1097844.800693624,14],[1097844.982583838,14],[1097845.213350188,13],[1097845.359886818,5],[1097845.54856126,22],[1097845.778772019,3],[1097846.016208484,32],[1097846.200433818,33],[1097846.38192499,11],[1097846.574915285,6],[1097846.723492582,8],[1097846.870880415,9],[1097847.018501468,13],[1097847.167677986,8],[1097847.317331618,20],[1097847.466473533,17],[1097847.623239057,20],[1097847.773155823,13],[1097847.948397675,6],[1097848.134993144,13],[1097848.281380996,9],[1097848.429442704,11],[1097848.589216369,8],[1097848.776125117,15],[1097848.961031844,16],[1097849.169659061,27],[1097849.410893209,4],[1097849.56658742,8],[1097849.749720133,25],[1097849.901088578,18],[1097850.052990797,13],[1097850.235132291,55],[1097850.384987888,11],[1097850.575196946,10],[1097850.753969562,11],[1097850.900341556,15],[1097851.049006856,17],[1097851.200469244,14],[1097851.455035439,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-1c3874fbbf6a49c6fad98be362c069c2","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.128371707,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.09836138,"time_to_first_token":1.423469172,"token_sha":"8a91591c94a3","tokens_per_second":18.08720837314783},"seed":1234,"sent":1097835.924774514,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.4331252260599285,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.783519765445448,"done":1097851.455629534,"finish_reason":"length","first":1097837.879472666,"full_response_s":15.528187453979626,"last":1097851.45520305,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.879472666,3],[1097838.134582695,11],[1097838.316312407,30],[1097838.502446961,19],[1097838.650513539,21],[1097838.834320076,15],[1097838.981669604,19],[1097839.125232222,17],[1097839.274645859,6],[1097839.424056599,8],[1097839.61113258,3],[1097839.761333373,5],[1097839.940103911,27],[1097840.127708589,12],[1097840.303708823,13],[1097840.489228876,5],[1097840.640610343,16],[1097840.824410597,5],[1097841.008739152,14],[1097841.189429747,9],[1097841.337476963,22],[1097841.489069208,10],[1097841.680895461,7],[1097841.858321423,23],[1097842.005471629,5],[1097842.155352864,9],[1097842.327859757,14],[1097842.478240727,20],[1097842.633634329,8],[1097842.783668036,12],[1097842.928914826,5],[1097843.07846625,14],[1097843.260650975,14],[1097843.406007546,8],[1097843.59516767,13],[1097843.76584242,3],[1097843.946549967,10],[1097844.096942039,11],[1097844.276078072,9],[1097844.459835119,10],[1097844.652263462,20],[1097844.799097655,14],[1097844.984630926,14],[1097845.210663206,13],[1097845.360234543,5],[1097845.547819105,22],[1097845.777664069,3],[1097846.016969077,32],[1097846.198590247,33],[1097846.381856806,11],[1097846.573397797,6],[1097846.722463317,8],[1097846.871127271,9],[1097847.021387952,13],[1097847.168508874,8],[1097847.317950847,20],[1097847.465826502,17],[1097847.623819231,20],[1097847.772616014,13],[1097847.948067782,6],[1097848.133180726,13],[1097848.281233984,9],[1097848.429334554,11],[1097848.588080639,8],[1097848.774614566,15],[1097848.961288046,16],[1097849.170879411,27],[1097849.409546206,4],[1097849.566254871,8],[1097849.749579683,25],[1097849.900063883,18],[1097850.050377049,13],[1097850.23620177,55],[1097850.384748012,11],[1097850.575449531,10],[1097850.749755845,11],[1097850.898954144,15],[1097851.047158708,17],[1097851.197651093,14],[1097851.45520305,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-bb1dc84cf200b7528fddab5cb869d65e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.128139037,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.576605433,"time_to_first_token":1.944225614,"token_sha":"8a91591c94a3","tokens_per_second":18.782309116841816},"seed":1234,"sent":1097835.92744208,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.9520305858459324,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.437000548235517,"done":1097851.454956147,"finish_reason":"length","first":1097837.623978064,"full_response_s":15.529120256891474,"last":1097851.454861202,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.623978064,3],[1097838.134318453,11],[1097838.314907707,30],[1097838.501570635,19],[1097838.65236551,21],[1097838.832784931,15],[1097838.980961229,19],[1097839.125452639,17],[1097839.275323438,6],[1097839.423382365,8],[1097839.613416083,3],[1097839.75799391,5],[1097839.944263462,27],[1097840.12398331,12],[1097840.303627033,13],[1097840.489636995,5],[1097840.640527944,16],[1097840.826479773,5],[1097841.008068474,14],[1097841.189906386,9],[1097841.337987694,22],[1097841.48990782,10],[1097841.67460395,7],[1097841.859490666,23],[1097842.007579471,5],[1097842.155680776,9],[1097842.327045988,14],[1097842.480022258,20],[1097842.635446191,8],[1097842.781750279,12],[1097842.929078789,5],[1097843.07875237,14],[1097843.260274358,14],[1097843.406881608,8],[1097843.596509454,13],[1097843.767817057,3],[1097843.947929458,10],[1097844.09514094,11],[1097844.278056035,9],[1097844.458810413,10],[1097844.652505309,20],[1097844.800749373,14],[1097844.98282498,14],[1097845.211488959,13],[1097845.359400323,5],[1097845.547897485,22],[1097845.777605071,3],[1097846.018794482,32],[1097846.200346955,33],[1097846.382245473,11],[1097846.573679914,6],[1097846.724649027,8],[1097846.870544072,9],[1097847.020773141,13],[1097847.168135893,8],[1097847.319046645,20],[1097847.466283593,17],[1097847.622656321,20],[1097847.773357863,13],[1097847.94811239,6],[1097848.130771841,13],[1097848.279225096,9],[1097848.429758288,11],[1097848.587828966,8],[1097848.77452844,15],[1097848.961941253,16],[1097849.171890664,27],[1097849.411473877,4],[1097849.565777198,8],[1097849.750890721,25],[1097849.902132304,18],[1097850.05274772,13],[1097850.235928602,55],[1097850.384288427,11],[1097850.576381858,10],[1097850.75237177,11],[1097850.899038061,15],[1097851.047716598,17],[1097851.198507888,14],[1097851.454861202,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-37939b94fbfa9d49aed10b6e5d0f3333","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.133013768,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.832076163,"time_to_first_token":1.689634376,"token_sha":"8a91591c94a3","tokens_per_second":18.435410345853228},"seed":1234,"sent":1097835.92583589,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.6981421739328653,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.925507467648497,"done":1097851.454227909,"finish_reason":"length","first":1097837.228583391,"full_response_s":15.529539122944698,"last":1097851.45412208,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.228583391,3],[1097838.131903856,11],[1097838.315557741,30],[1097838.502293256,19],[1097838.652437761,21],[1097838.833559858,15],[1097838.980071282,19],[1097839.125601383,17],[1097839.275076114,6],[1097839.421921246,8],[1097839.61087935,3],[1097839.760768668,5],[1097839.942772207,27],[1097840.127213526,12],[1097840.304946208,13],[1097840.489458945,5],[1097840.642378435,16],[1097840.825026487,5],[1097841.009565103,14],[1097841.19143682,9],[1097841.3372948,22],[1097841.491052112,10],[1097841.678790944,7],[1097841.86080385,23],[1097842.008758301,5],[1097842.158854095,9],[1097842.327303042,14],[1097842.480388216,20],[1097842.634407011,8],[1097842.783524143,12],[1097842.931828124,5],[1097843.079209192,14],[1097843.258711982,14],[1097843.405936993,8],[1097843.596948525,13],[1097843.767443866,3],[1097843.949684813,10],[1097844.094594614,11],[1097844.27856862,9],[1097844.459767303,10],[1097844.652379152,20],[1097844.799821741,14],[1097844.980797819,14],[1097845.210568164,13],[1097845.358649205,5],[1097845.548105704,22],[1097845.777107084,3],[1097846.013424208,32],[1097846.199801656,33],[1097846.382052187,11],[1097846.574180173,6],[1097846.7220694,8],[1097846.870201198,9],[1097847.020132529,13],[1097847.169363098,8],[1097847.317808188,20],[1097847.465548371,17],[1097847.624534894,20],[1097847.771913908,13],[1097847.948193875,6],[1097848.131379096,13],[1097848.279421358,9],[1097848.43106275,11],[1097848.590242012,8],[1097848.774673803,15],[1097848.958832285,16],[1097849.172576889,27],[1097849.409803593,4],[1097849.566855963,8],[1097849.750183608,25],[1097849.906864363,18],[1097850.052021513,13],[1097850.234764818,55],[1097850.382640185,11],[1097850.574877378,10],[1097850.749628152,11],[1097850.899188242,15],[1097851.049571917,17],[1097851.19784204,14],[1097851.45412208,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-1fd978558240988bed5f10a798c9760c","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.128466223,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.226014357,"time_to_first_token":1.295357259,"token_sha":"8a91591c94a3","tokens_per_second":17.924908101510923},"seed":1234,"sent":1097835.924688786,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.3038946050219238,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.60769417537957,"done":1097851.455150471,"finish_reason":"length","first":1097837.751106617,"full_response_s":15.522722847992554,"last":1097851.455115163,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.751106617,3],[1097838.133586435,11],[1097838.315464987,30],[1097838.503377981,19],[1097838.649991009,21],[1097838.834578412,15],[1097838.98058741,19],[1097839.126366082,17],[1097839.273473632,6],[1097839.424464062,8],[1097839.614061571,3],[1097839.761762493,5],[1097839.942849804,27],[1097840.126101868,12],[1097840.304360168,13],[1097840.491810213,5],[1097840.641757118,16],[1097840.827158342,5],[1097841.008678426,14],[1097841.190724655,9],[1097841.337103145,22],[1097841.489168909,10],[1097841.679421671,7],[1097841.859125557,23],[1097842.005711732,5],[1097842.155781325,9],[1097842.327412218,14],[1097842.479464695,20],[1097842.633231606,8],[1097842.780368511,12],[1097842.928617214,5],[1097843.079385976,14],[1097843.26033598,14],[1097843.405440188,8],[1097843.596689407,13],[1097843.76749017,3],[1097843.949148283,10],[1097844.095499854,11],[1097844.277745331,9],[1097844.458722205,10],[1097844.654232398,20],[1097844.801323308,14],[1097844.984115028,14],[1097845.214616404,13],[1097845.358986382,5],[1097845.549370783,22],[1097845.777897561,3],[1097846.016799864,32],[1097846.199307543,33],[1097846.382781761,11],[1097846.574473302,6],[1097846.723343792,8],[1097846.871952173,9],[1097847.019479832,13],[1097847.167834058,8],[1097847.318301228,20],[1097847.466532097,17],[1097847.624638595,20],[1097847.77375402,13],[1097847.947886486,6],[1097848.130726465,13],[1097848.281276783,9],[1097848.429713408,11],[1097848.588942896,8],[1097848.772832191,15],[1097848.961788607,16],[1097849.173229791,27],[1097849.411418946,4],[1097849.566398249,8],[1097849.751485475,25],[1097849.900638116,18],[1097850.050280135,13],[1097850.234537906,55],[1097850.383063966,11],[1097850.576910171,10],[1097850.750418541,11],[1097850.898753482,15],[1097851.04725719,17],[1097851.198974147,14],[1097851.455115163,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-564d3e77cfa0ae46c7dc9e2592d7322f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127545477,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.704629503,"time_to_first_token":1.815512842,"token_sha":"8a91591c94a3","tokens_per_second":18.606851060379228},"seed":1234,"sent":1097835.932427623,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.8186789939645678,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.76610854565073,"done":1097851.453980831,"finish_reason":"length","first":1097837.100785803,"full_response_s":15.528747215867043,"last":1097851.453957143,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.100785803,3],[1097838.13277354,11],[1097838.315074041,30],[1097838.502149538,19],[1097838.650930543,21],[1097838.834144235,15],[1097838.979651255,19],[1097839.125560151,17],[1097839.274071181,6],[1097839.423975465,8],[1097839.613560377,3],[1097839.759927604,5],[1097839.94098638,27],[1097840.123057944,12],[1097840.304692562,13],[1097840.491546692,5],[1097840.641541823,16],[1097840.825285206,5],[1097841.00702806,14],[1097841.190839993,9],[1097841.337232041,22],[1097841.489621251,10],[1097841.677159769,7],[1097841.859893709,23],[1097842.008119556,5],[1097842.156190226,9],[1097842.327983435,14],[1097842.478018118,20],[1097842.636843645,8],[1097842.783054436,12],[1097842.929906674,5],[1097843.079624175,14],[1097843.259709384,14],[1097843.405711295,8],[1097843.595758285,13],[1097843.767123717,3],[1097843.947499687,10],[1097844.09660587,11],[1097844.278796607,9],[1097844.459672132,10],[1097844.654594257,20],[1097844.801624585,14],[1097844.984768415,14],[1097845.212375806,13],[1097845.358763038,5],[1097845.54964056,22],[1097845.7779951,3],[1097846.015549032,32],[1097846.199907422,33],[1097846.383101171,11],[1097846.573770763,6],[1097846.723667044,8],[1097846.870939845,9],[1097847.018716921,13],[1097847.167552149,8],[1097847.321419442,20],[1097847.466725126,17],[1097847.623075382,20],[1097847.774553696,13],[1097847.947817294,6],[1097848.131790554,13],[1097848.279595515,9],[1097848.430757042,11],[1097848.588719569,8],[1097848.775890906,15],[1097848.96071807,16],[1097849.170731487,27],[1097849.410250646,4],[1097849.56587845,8],[1097849.750693596,25],[1097849.900232118,18],[1097850.051620925,13],[1097850.234246988,55],[1097850.383312933,11],[1097850.575352393,10],[1097850.752598154,11],[1097850.899700996,15],[1097851.047839937,17],[1097851.198455118,14],[1097851.453957143,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-9c2a94a76cb6d03d86bd54596be78f2d","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.128916408,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.354129002,"time_to_first_token":1.168036987,"token_sha":"8a91591c94a3","tokens_per_second":17.764923247134686},"seed":1234,"sent":1097835.925233615,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.1755521879531443,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.260272100531477,"done":1097851.454891677,"finish_reason":"length","first":1097837.490067543,"full_response_s":15.526649343082681,"last":1097851.45481016,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097837.490067543,3],[1097838.133238226,11],[1097838.3156952,30],[1097838.503479987,19],[1097838.652184403,21],[1097838.834842894,15],[1097838.980931986,19],[1097839.125657531,17],[1097839.274138086,6],[1097839.422721928,8],[1097839.612767154,3],[1097839.758732184,5],[1097839.944779493,27],[1097840.125417039,12],[1097840.303332909,13],[1097840.490087945,5],[1097840.64099903,16],[1097840.826731145,5],[1097841.00785049,14],[1097841.190137527,9],[1097841.337416156,22],[1097841.491193396,10],[1097841.675547,7],[1097841.858996085,23],[1097842.008389399,5],[1097842.157359485,9],[1097842.32717634,14],[1097842.480536142,20],[1097842.635928719,8],[1097842.781623209,12],[1097842.929617337,5],[1097843.080417247,14],[1097843.25920869,14],[1097843.4076395,8],[1097843.596751766,13],[1097843.766929551,3],[1097843.948684451,10],[1097844.096896103,11],[1097844.277409081,9],[1097844.459461496,10],[1097844.654506545,20],[1097844.800460964,14],[1097844.983190808,14],[1097845.214348804,13],[1097845.359089715,5],[1097845.548887425,22],[1097845.77975935,3],[1097846.018071215,32],[1097846.201182918,33],[1097846.382838421,11],[1097846.573303476,6],[1097846.722809873,8],[1097846.87204014,9],[1097847.01957537,13],[1097847.169279628,8],[1097847.31999916,20],[1097847.465630896,17],[1097847.624273649,20],[1097847.773886563,13],[1097847.948225662,6],[1097848.133763125,13],[1097848.280559553,9],[1097848.430065565,11],[1097848.588247514,8],[1097848.773423536,15],[1097848.960103293,16],[1097849.17157388,27],[1097849.409725917,4],[1097849.565936695,8],[1097849.752343393,25],[1097849.906245202,18],[1097850.051795834,13],[1097850.233827961,55],[1097850.383163248,11],[1097850.575283761,10],[1097850.750812792,11],[1097850.899126556,15],[1097851.05043958,17],[1097851.198657428,14],[1097851.45481016,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-12fea6ffe300e2a7abd45ed261c4d4cd","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.133271833,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.965052381,"time_to_first_token":1.555770352,"token_sha":"8a91591c94a3","tokens_per_second":18.259867062649725},"seed":1234,"sent":1097835.928242334,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.5618252090644091,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.15303035884963,"done":1097851.452456107,"finish_reason":"length","first":1097836.586256505,"full_response_s":15.527005689917132,"last":1097851.452434356,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.586256505,3],[1097838.132050647,11],[1097838.313208897,30],[1097838.501447869,19],[1097838.652298861,21],[1097838.833890109,15],[1097838.980868602,19],[1097839.125060926,17],[1097839.274308245,6],[1097839.422573008,8],[1097839.612715727,3],[1097839.760262848,5],[1097839.942063866,27],[1097840.125089014,12],[1097840.304420782,13],[1097840.489344487,5],[1097840.640358746,16],[1097840.82461464,5],[1097841.008348305,14],[1097841.188666539,9],[1097841.337818912,22],[1097841.490766519,10],[1097841.676430653,7],[1097841.859234892,23],[1097842.006068855,5],[1097842.156328613,9],[1097842.327922708,14],[1097842.478596697,20],[1097842.633969843,8],[1097842.7809101,12],[1097842.929029075,5],[1097843.079809538,14],[1097843.261323619,14],[1097843.40630215,8],[1097843.596414139,13],[1097843.766487566,3],[1097843.947189783,10],[1097844.095592239,11],[1097844.276792346,9],[1097844.458684879,10],[1097844.653111977,20],[1097844.800112998,14],[1097844.983477167,14],[1097845.213173309,13],[1097845.359196025,5],[1097845.548141813,22],[1097845.777761672,3],[1097846.018243677,32],[1097846.200937631,33],[1097846.382426385,11],[1097846.574247381,6],[1097846.722186658,8],[1097846.872188449,9],[1097847.020320676,13],[1097847.168457192,8],[1097847.321662504,20],[1097847.465267919,17],[1097847.623686352,20],[1097847.773549387,13],[1097847.947326112,6],[1097848.133615536,13],[1097848.280750116,9],[1097848.430221412,11],[1097848.589301791,8],[1097848.77372113,15],[1097848.962018816,16],[1097849.172922355,27],[1097849.411171819,4],[1097849.566112212,8],[1097849.750011307,25],[1097849.901308671,18],[1097850.050728676,13],[1097850.23365867,55],[1097850.382077492,11],[1097850.575802839,10],[1097850.753469243,11],[1097850.900252469,15],[1097851.047663364,17],[1097851.199953945,14],[1097851.452434356,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-df24491c0b84d7b30f74c63c00d5b23c","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.142346398,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.866074641,"time_to_first_token":0.656329553,"token_sha":"8a91591c94a3","tokens_per_second":17.153149446506937},"seed":1234,"sent":1097835.925450417,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.6608060880098492,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.608380214353286,"done":1097851.453778136,"finish_reason":"length","first":1097836.971999203,"full_response_s":15.52307196194306,"last":1097851.453740107,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.971999203,3],[1097838.132406827,11],[1097838.315966886,30],[1097838.503247516,19],[1097838.651955533,21],[1097838.834784361,15],[1097838.980188606,19],[1097839.125410608,17],[1097839.274017594,6],[1097839.423906559,8],[1097839.612256693,3],[1097839.75817609,5],[1097839.942583418,27],[1097840.126443146,12],[1097840.30412248,13],[1097840.490188815,5],[1097840.640819044,16],[1097840.827413796,5],[1097841.007294942,14],[1097841.191124642,9],[1097841.337688,22],[1097841.490492419,10],[1097841.679774121,7],[1097841.859670492,23],[1097842.006574527,5],[1097842.156537169,9],[1097842.327114333,14],[1097842.478441025,20],[1097842.637195022,8],[1097842.782262563,12],[1097842.93123879,5],[1097843.080065231,14],[1097843.259537386,14],[1097843.405803488,8],[1097843.597716805,13],[1097843.767022592,3],[1097843.946940332,10],[1097844.096058472,11],[1097844.276303129,9],[1097844.459415608,10],[1097844.652884855,20],[1097844.80199672,14],[1097844.984543374,14],[1097845.210717609,13],[1097845.359613568,5],[1097845.548653133,22],[1097845.777274985,3],[1097846.018389601,32],[1097846.199741235,33],[1097846.38214899,11],[1097846.574580957,6],[1097846.724000107,8],[1097846.871305047,9],[1097847.019077001,13],[1097847.166821409,8],[1097847.317703751,20],[1097847.466784059,17],[1097847.623757097,20],[1097847.775235539,13],[1097847.94705688,6],[1097848.132374089,13],[1097848.280080775,9],[1097848.429012856,11],[1097848.589942306,8],[1097848.775815744,15],[1097848.959763045,16],[1097849.172972805,27],[1097849.410557202,4],[1097849.566784321,8],[1097849.752264101,25],[1097849.90756861,18],[1097850.052374805,13],[1097850.234356755,55],[1097850.383247774,11],[1097850.576025044,10],[1097850.750949383,11],[1097850.901223015,15],[1097851.048252913,17],[1097851.198698531,14],[1097851.453740107,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-c36c91f1c59ab72e89ed8fa3eace69d6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.128137828,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.4830283,"time_to_first_token":1.039741357,"token_sha":"8a91591c94a3","tokens_per_second":17.606815005671155},"seed":1234,"sent":1097835.930706174,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.0412930289749056,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.703036129358644,"done":1097851.452771257,"finish_reason":"length","first":1097836.186049442,"full_response_s":15.526037155883387,"last":1097851.452734973,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.186049442,3],[1097838.133988842,11],[1097838.313026413,30],[1097838.502577858,19],[1097838.651703631,21],[1097838.833267286,15],[1097838.978890844,19],[1097839.125160755,17],[1097839.274877497,6],[1097839.422655471,8],[1097839.612888207,3],[1097839.758509846,5],[1097839.944196189,27],[1097840.125157983,12],[1097840.304072718,13],[1097840.491233984,5],[1097840.641392295,16],[1097840.825938102,5],[1097841.006300048,14],[1097841.188750202,9],[1097841.337762236,22],[1097841.490706273,10],[1097841.677963776,7],[1097841.860308725,23],[1097842.006965886,5],[1097842.155128574,9],[1097842.327503835,14],[1097842.478488338,20],[1097842.632902429,8],[1097842.779862918,12],[1097842.928367635,5],[1097843.077657508,14],[1097843.258299288,14],[1097843.405639061,8],[1097843.595595762,13],[1097843.766132845,3],[1097843.947813576,10],[1097844.094917339,11],[1097844.278366211,9],[1097844.459534002,10],[1097844.654851069,20],[1097844.804115053,14],[1097844.981177236,14],[1097845.212272409,13],[1097845.360574153,5],[1097845.549142893,22],[1097845.778607784,3],[1097846.016656549,32],[1097846.201481809,33],[1097846.382891304,11],[1097846.573638396,6],[1097846.724342678,8],[1097846.871671273,9],[1097847.019666587,13],[1097847.167894255,8],[1097847.317408174,20],[1097847.466088051,17],[1097847.622957548,20],[1097847.772084593,13],[1097847.948609142,6],[1097848.135308264,13],[1097848.280660949,9],[1097848.430270614,11],[1097848.588864548,8],[1097848.773071652,15],[1097848.958905015,16],[1097849.16976489,27],[1097849.410945275,4],[1097849.566526518,8],[1097849.750961578,25],[1097849.900173185,18],[1097850.049870519,13],[1097850.233463898,55],[1097850.383363511,11],[1097850.574945402,10],[1097850.753088758,11],[1097850.898498529,15],[1097851.047053887,17],[1097851.200130342,14],[1097851.452734973,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-6332b758b97cdee6f74d5ca821eff0db","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127240847,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.266028976,"time_to_first_token":0.258326217,"token_sha":"8a91591c94a3","tokens_per_second":16.703754486572123},"seed":1234,"sent":1097835.926734101,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.2593153410125524,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.846650007884822,"done":1097851.451912542,"finish_reason":"length","first":1097836.315333924,"full_response_s":15.524372840998694,"last":1097851.451874416,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.315333924,3],[1097838.132306934,11],[1097838.316886397,30],[1097838.503598095,19],[1097838.65209088,21],[1097838.834192989,15],[1097838.979927037,19],[1097839.12570326,17],[1097839.274591679,6],[1097839.42453945,8],[1097839.612029281,3],[1097839.760043407,5],[1097839.941419452,27],[1097840.124643745,12],[1097840.304459661,13],[1097840.490021601,5],[1097840.640903587,16],[1097840.825541508,5],[1097841.007635595,14],[1097841.190688962,9],[1097841.33792684,22],[1097841.49022813,10],[1097841.67759412,7],[1097841.861416986,23],[1097842.005883091,5],[1097842.157125992,9],[1097842.327453369,14],[1097842.47879397,20],[1097842.632526548,8],[1097842.780778917,12],[1097842.929344614,5],[1097843.077897627,14],[1097843.260981495,14],[1097843.407369658,8],[1097843.597470155,13],[1097843.767761629,3],[1097843.948922505,10],[1097844.096992233,11],[1097844.277110077,9],[1097844.458873203,10],[1097844.655085394,20],[1097844.801769485,14],[1097844.982392457,14],[1097845.214218629,13],[1097845.359831038,5],[1097845.548487794,22],[1097845.779675496,3],[1097846.015701999,32],[1097846.200400622,33],[1097846.382933751,11],[1097846.574332916,6],[1097846.725815381,8],[1097846.872530347,9],[1097847.019959139,13],[1097847.16834051,8],[1097847.318221247,20],[1097847.465952003,17],[1097847.622855752,20],[1097847.772696219,13],[1097847.947624169,6],[1097848.130667964,13],[1097848.279548651,9],[1097848.429079872,11],[1097848.588411076,8],[1097848.77523349,15],[1097848.959570033,16],[1097849.173092447,27],[1097849.411718668,4],[1097849.566032424,8],[1097849.751420284,25],[1097849.907297587,18],[1097850.050904626,13],[1097850.233753887,55],[1097850.382545256,11],[1097850.576703298,10],[1097850.75014102,11],[1097850.9009373,15],[1097851.047357321,17],[1097851.199411099,14],[1097851.451874416,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-c6f68e514a849665d6bc431bc1f049eb","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.129271885,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.136667075,"time_to_first_token":0.386641645,"token_sha":"8a91591c94a3","tokens_per_second":16.84650912492901},"seed":1234,"sent":1097835.927539701,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.3877942229155451,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"A","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.45427917178293,"done":1097851.45287571,"finish_reason":"length","first":1097836.843245017,"full_response_s":15.528284257045016,"last":1097851.452842898,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097836.843245017,3],[1097838.134844025,11],[1097838.313378112,30],[1097838.502360017,19],[1097838.649853373,21],[1097838.834259622,15],[1097838.981287878,19],[1097839.126113749,17],[1097839.274202734,6],[1097839.424777138,8],[1097839.612133143,3],[1097839.758374115,5],[1097839.94395879,27],[1097840.126814147,12],[1097840.30477168,13],[1097840.49052241,5],[1097840.64095479,16],[1097840.823788048,5],[1097841.006788788,14],[1097841.189120674,9],[1097841.337629995,22],[1097841.489289993,10],[1097841.675828704,7],[1097841.858937104,23],[1097842.006443023,5],[1097842.156831244,9],[1097842.327607537,14],[1097842.480291557,20],[1097842.632096455,8],[1097842.780255557,12],[1097842.928509206,5],[1097843.077844328,14],[1097843.25840984,14],[1097843.405558135,8],[1097843.597093106,13],[1097843.768592203,3],[1097843.950372469,10],[1097844.095044089,11],[1097844.277364537,9],[1097844.459375289,10],[1097844.653495683,20],[1097844.799247534,14],[1097844.984405741,14],[1097845.21161931,13],[1097845.359572225,5],[1097845.549769423,22],[1097845.778421541,3],[1097846.014325811,32],[1097846.198923006,33],[1097846.381329529,11],[1097846.573351877,6],[1097846.723267509,8],[1097846.872404526,9],[1097847.020852225,13],[1097847.167079645,8],[1097847.321051455,20],[1097847.465463285,17],[1097847.624708988,20],[1097847.771771922,13],[1097847.947525062,6],[1097848.13012955,13],[1097848.279036197,9],[1097848.428600822,11],[1097848.588311985,8],[1097848.772632104,15],[1097848.960014078,16],[1097849.172680014,27],[1097849.40966031,4],[1097849.565423266,8],[1097849.751885968,25],[1097849.901249803,18],[1097850.050609546,13],[1097850.23357941,55],[1097850.385777141,11],[1097850.575873455,10],[1097850.751065632,11],[1097850.899748069,15],[1097851.047770009,17],[1097851.198561587,14],[1097851.452842898,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-b0beb8598a1b4fa342ed333df78246f6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.129445264,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.609884464,"time_to_first_token":0.912077437,"token_sha":"8a91591c94a3","tokens_per_second":17.45393679384267},"seed":1234,"sent":1097835.924591453,"speculative":{"acceptance_rate":0.44110275689223055,"accepted":176,"drafted":399,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.9186535640619695,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":176,"drafted":399,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"A","case":"long","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":20.327690514575725,"done":1097856.173351475,"finish_reason":"length","first":1097854.648287503,"full_response_s":4.686749584041536,"last":1097856.173300882,"output_sha256":"5d2c18c1a3da0aab39441e582248a278a66d8473e846a8d2c48dfda87c21e891","pieces":[[1097854.648287503,3],[1097854.732921415,14],[1097854.818595173,11],[1097854.902164336,11],[1097854.985492696,13],[1097855.069413835,12],[1097855.151336434,12],[1097855.235444933,12],[1097855.319575839,12],[1097855.403793719,11],[1097855.488355781,12],[1097855.57892841,4],[1097855.664231237,12],[1097855.747499897,12],[1097855.831443632,9],[1097855.914568773,1],[1097855.998659042,7],[1097856.082660065,4],[1097856.173300882,17]],"prompt":"long_initial","prompt_tokens":4419,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"df5eb0c68030e305529b5525635e0602046d98ecf7a739e2e42243ec364b4adc","request_path":"/v1/chat/completions","response_id":"chatcmpl-36038094b4df6d71da9814e906848d25","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":3.155551844,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":1.525246292,"time_to_first_token":3.160914386,"token_sha":"8b52dc1b081a","tokens_per_second":20.324586371785784},"seed":1234,"sent":1097851.486601891,"speculative":{"acceptance_rate":0.06015037593984962,"accepted":16,"drafted":266,"rounds":18,"tokens_per_round":1.7777777777777777},"token_sha":"8b52dc1b081a","tokens":32,"ttft_s":3.161685612052679,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4419,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":4451},"work":{"accepted":16,"drafted":266,"rounds":18},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"A","case":"long","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":51.20395782545257,"done":1097859.956075963,"finish_reason":"length","first":1097859.350619426,"full_response_s":3.773332908982411,"last":1097859.956041376,"output_sha256":"dd1aacffd9ef2ff0a3531f264db2a68aa99f695724c7828277affb8cb3535eb1","pieces":[[1097859.350619426,3],[1097859.434754998,36],[1097859.524722717,45],[1097859.612858974,8],[1097859.697843967,30],[1097859.781059936,18],[1097859.865740062,13],[1097859.956041376,12]],"prompt":"long_followup","prompt_tokens":4445,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"},{"content":"Acknowledged.","role":"assistant"},{"content":"Summarize the document in one sentence.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"0f7b9a7949106fe7d7c1dee63210f9621564bc3cdf8167180dc2d64159ad2c22","request_path":"/v1/chat/completions","response_id":"chatcmpl-0aa5011cda1912b7dc6d7f37627ba3c0","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":14,"prefill_raised":[],"prefill_seconds":3.160762389,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":0.606300468,"time_to_first_token":3.166326744,"token_sha":"526434da61a6","tokens_per_second":51.12976426071305},"seed":1234,"sent":1097856.182743054,"speculative":{"acceptance_rate":0.23300970873786409,"accepted":24,"drafted":103,"rounds":7,"tokens_per_round":4.571428571428571},"token_sha":"526434da61a6","tokens":32,"ttft_s":3.1678763718809932,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4445,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":4477},"work":{"accepted":24,"drafted":103,"rounds":7},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"followup","stream_index":0} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":69.4767532146739,"done":1097684.648095958,"finish_reason":"length","first":1097680.97776106,"full_response_s":3.810370694845915,"last":1097684.64805347,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097680.97776106,2],[1097681.0593116,7],[1097681.14065384,4],[1097681.220614937,73],[1097681.300492096,51],[1097681.386991108,21],[1097681.469984548,14],[1097681.549755491,20],[1097681.630645032,16],[1097681.711060061,15],[1097681.791190203,23],[1097681.871558872,23],[1097681.951741283,25],[1097682.031959007,36],[1097682.113187634,14],[1097682.194362321,6],[1097682.274710028,18],[1097682.361058589,18],[1097682.445736935,19],[1097682.526141497,8],[1097682.606332844,11],[1097682.687794594,18],[1097682.768296867,15],[1097682.848156631,39],[1097682.929210776,30],[1097683.008947858,14],[1097683.090578719,15],[1097683.171290437,8],[1097683.251581318,6],[1097683.33375057,15],[1097683.419687426,23],[1097683.501746269,22],[1097683.5824347,26],[1097683.663284712,35],[1097683.743856055,31],[1097683.823793992,12],[1097683.904936233,13],[1097683.986181212,38],[1097684.067274734,47],[1097684.148517647,25],[1097684.228918361,11],[1097684.309312669,34],[1097684.397962361,12],[1097684.483138483,36],[1097684.56349373,60],[1097684.64805347,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-1e173134eb6e43045b1135b120efe954","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.12544788,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.671116483,"time_to_first_token":0.126372833,"token_sha":"2b28aa2108de","tokens_per_second":69.46115743829968},"seed":1234,"sent":1097680.837725263,"speculative":{"acceptance_rate":0.31555555555555553,"accepted":213,"drafted":675,"rounds":45,"tokens_per_round":5.688888888888889},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.14003579691052437,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":213,"drafted":675,"rounds":45},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":69.49836526023209,"done":1097688.443385283,"finish_reason":"length","first":1097684.774183832,"full_response_s":3.7947294760961086,"last":1097688.443334884,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097684.774183832,2],[1097684.853943533,7],[1097684.935232873,4],[1097685.01568146,73],[1097685.096907989,51],[1097685.177959454,21],[1097685.257800808,14],[1097685.340630894,20],[1097685.425402685,16],[1097685.507565496,15],[1097685.587653933,23],[1097685.668441163,23],[1097685.748716209,25],[1097685.828959983,36],[1097685.909277255,14],[1097685.990320476,6],[1097686.071447329,18],[1097686.15240607,18],[1097686.23236654,19],[1097686.312416006,8],[1097686.400095069,11],[1097686.483699704,18],[1097686.564081468,15],[1097686.645204333,39],[1097686.725345053,30],[1097686.805645044,14],[1097686.886752762,15],[1097686.96710419,8],[1097687.048322784,6],[1097687.129757437,15],[1097687.209917438,23],[1097687.290658736,22],[1097687.37672553,26],[1097687.460783718,35],[1097687.541600063,31],[1097687.621624208,12],[1097687.703184419,13],[1097687.7834227,38],[1097687.863969024,47],[1097687.944380108,25],[1097688.02474643,11],[1097688.106072271,34],[1097688.186947917,12],[1097688.267440597,36],[1097688.352425546,60],[1097688.443334884,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-81f4463ecd8370b5509eaa79b4c204bd","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.123531247,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.669959545,"time_to_first_token":0.124098799,"token_sha":"2b28aa2108de","tokens_per_second":69.48305475122069},"seed":1234,"sent":1097684.648655807,"speculative":{"acceptance_rate":0.31555555555555553,"accepted":213,"drafted":675,"rounds":45,"tokens_per_round":5.688888888888889},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.12552802520804107,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":213,"drafted":675,"rounds":45},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.161851725878968,"done":1097700.380906514,"finish_reason":"length","first":1097689.827030785,"full_response_s":11.933586584869772,"last":1097700.380857656,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.827030785,2],[1097690.600061682,7],[1097690.748909419,4],[1097690.951525811,63],[1097691.177577022,10],[1097691.363947159,31],[1097691.54643386,29],[1097691.727040266,22],[1097691.905246345,14],[1097692.085104494,24],[1097692.26661286,4],[1097692.419112323,13],[1097692.601819057,21],[1097692.783630226,2],[1097692.961114373,18],[1097693.141405171,29],[1097693.376475815,27],[1097693.609963351,24],[1097693.758619151,4],[1097693.903172037,16],[1097694.138628606,10],[1097694.316203733,31],[1097694.472239029,8],[1097694.655512218,11],[1097694.836131706,16],[1097695.012979612,12],[1097695.196199745,39],[1097695.371253383,18],[1097695.553489954,11],[1097695.735132668,16],[1097695.915703717,16],[1097696.061583833,11],[1097696.243745232,6],[1097696.432693394,9],[1097696.613646717,29],[1097696.798537054,22],[1097696.977583955,8],[1097697.122462791,18],[1097697.307231588,18],[1097697.495206623,34],[1097697.676429765,10],[1097697.858049132,14],[1097698.038571621,13],[1097698.221330299,2],[1097698.410504248,36],[1097698.593771817,42],[1097698.77532024,26],[1097698.925013658,17],[1097699.104909846,34],[1097699.339621073,12],[1097699.526502989,36],[1097699.805745023,50],[1097699.990065315,2],[1097700.380857656,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-e587146bae0258bbab9396b9fbf0790b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125731531,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.554226216,"time_to_first_token":1.374068913,"token_sha":"2b28aa2108de","tokens_per_second":24.160937503255806},"seed":1234,"sent":1097688.447319929,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.3797108558937907,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.7464838504702,"done":1097700.381166946,"finish_reason":"length","first":1097690.07660317,"full_response_s":11.93096879683435,"last":1097700.381097398,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097690.07660317,2],[1097690.600404253,7],[1097690.750538261,4],[1097690.949888507,63],[1097691.178997648,10],[1097691.362846858,31],[1097691.548051882,29],[1097691.725601943,22],[1097691.907505596,14],[1097692.085873736,24],[1097692.265834025,4],[1097692.418792977,13],[1097692.601740025,21],[1097692.782950009,2],[1097692.960601335,18],[1097693.14199471,29],[1097693.374020513,27],[1097693.611961825,24],[1097693.757525878,4],[1097693.903073324,16],[1097694.1398775,10],[1097694.317036206,31],[1097694.473868296,8],[1097694.656206408,11],[1097694.834114824,16],[1097695.013473443,12],[1097695.195867081,39],[1097695.369928445,18],[1097695.553757856,11],[1097695.734721951,16],[1097695.914467473,16],[1097696.062652361,11],[1097696.242321975,6],[1097696.43373979,9],[1097696.615299521,29],[1097696.797246858,22],[1097696.976387032,8],[1097697.123314466,18],[1097697.306587279,18],[1097697.4989685,34],[1097697.677617191,10],[1097697.858747021,14],[1097698.039991098,13],[1097698.222556376,2],[1097698.410257953,36],[1097698.591382487,42],[1097698.775126664,26],[1097698.921851897,17],[1097699.105458097,34],[1097699.340045456,12],[1097699.524890232,36],[1097699.806541841,50],[1097699.989831626,2],[1097700.381097398,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-15dac5a93bd36da8cad0d76661447a97","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125172964,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.305252143,"time_to_first_token":1.62305068,"token_sha":"2b28aa2108de","tokens_per_second":24.744663833695},"seed":1234,"sent":1097688.450198149,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.6264050209429115,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.602596128999192,"done":1097700.379577291,"finish_reason":"length","first":1097689.57565152,"full_response_s":11.932045293040574,"last":1097700.379547778,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.57565152,2],[1097690.601097163,7],[1097690.747594449,4],[1097690.95244931,63],[1097691.178753954,10],[1097691.363642689,31],[1097691.547943079,29],[1097691.72757493,22],[1097691.906592335,14],[1097692.085347082,24],[1097692.266162788,4],[1097692.41950359,13],[1097692.601912373,21],[1097692.781719852,2],[1097692.960401763,18],[1097693.140216595,29],[1097693.375565698,27],[1097693.610176487,24],[1097693.759279394,4],[1097693.902242564,16],[1097694.138023248,10],[1097694.315343619,31],[1097694.472644689,8],[1097694.655957369,11],[1097694.834164212,16],[1097695.014286422,12],[1097695.195983169,39],[1097695.3711057,18],[1097695.553396991,11],[1097695.735678674,16],[1097695.914048239,16],[1097696.063863768,11],[1097696.242483173,6],[1097696.433790219,9],[1097696.615132141,29],[1097696.796736968,22],[1097696.976207867,8],[1097697.123841248,18],[1097697.305767632,18],[1097697.496681085,34],[1097697.676613254,10],[1097697.859998015,14],[1097698.039540305,13],[1097698.222091394,2],[1097698.410075683,36],[1097698.592310192,42],[1097698.774125599,26],[1097698.923069719,17],[1097699.104631301,34],[1097699.339691582,12],[1097699.525665752,36],[1097699.805509715,50],[1097699.990788786,2],[1097700.379547778,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-5a32a85008059b0264b1c6bb04d6203c","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.123842163,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.804163991,"time_to_first_token":1.123204844,"token_sha":"2b28aa2108de","tokens_per_second":23.60201124422196},"seed":1234,"sent":1097688.447531998,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.1281195220071822,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":24.45073179479747,"done":1097700.38068072,"finish_reason":"length","first":1097689.951517038,"full_response_s":11.931374554987997,"last":1097700.380652744,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.951517038,2],[1097690.600761238,7],[1097690.750683002,4],[1097690.953462508,63],[1097691.179606589,10],[1097691.363687882,31],[1097691.546127484,29],[1097691.727368918,22],[1097691.904504264,14],[1097692.085281422,24],[1097692.265426952,4],[1097692.41962988,13],[1097692.60288476,21],[1097692.78109631,2],[1097692.962673667,18],[1097693.140306629,29],[1097693.373636823,27],[1097693.609884144,24],[1097693.758051664,4],[1097693.903700542,16],[1097694.139484092,10],[1097694.316882901,31],[1097694.473969107,8],[1097694.654896131,11],[1097694.833418589,16],[1097695.01388187,12],[1097695.19656913,39],[1097695.369449875,18],[1097695.553268621,11],[1097695.735327081,16],[1097695.91527369,16],[1097696.063218045,11],[1097696.243606373,6],[1097696.433944867,9],[1097696.614477691,29],[1097696.797440261,22],[1097696.977812876,8],[1097697.123005667,18],[1097697.307803274,18],[1097697.49776088,34],[1097697.676727146,10],[1097697.86054566,14],[1097698.039344663,13],[1097698.221725619,2],[1097698.40954253,36],[1097698.592032461,42],[1097698.774779193,26],[1097698.924017836,17],[1097699.106497669,34],[1097699.339755525,12],[1097699.526981963,36],[1097699.807062932,50],[1097699.991567695,2],[1097700.380652744,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-f7dcc525aecf633c4aadf71c128cff0c","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.124548069,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.429656062,"time_to_first_token":1.498301156,"token_sha":"2b28aa2108de","tokens_per_second":24.449511899925586},"seed":1234,"sent":1097688.449306165,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.502210872946307,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":25.3595119804598,"done":1097700.381756301,"finish_reason":"length","first":1097690.326316852,"full_response_s":11.928787312004715,"last":1097700.381715402,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097690.326316852,2],[1097690.603920801,7],[1097690.748114063,4],[1097690.951384433,63],[1097691.179864456,10],[1097691.364692865,31],[1097691.546605729,29],[1097691.725882151,22],[1097691.906169263,14],[1097692.08421676,24],[1097692.266262959,4],[1097692.418712632,13],[1097692.604902832,21],[1097692.782194075,2],[1097692.962791213,18],[1097693.140851119,29],[1097693.374514943,27],[1097693.609338724,24],[1097693.757151188,4],[1097693.902562162,16],[1097694.139109821,10],[1097694.31541634,31],[1097694.472309156,8],[1097694.654131382,11],[1097694.832786459,16],[1097695.014964257,12],[1097695.196703089,39],[1097695.369777127,18],[1097695.55395086,11],[1097695.735052357,16],[1097695.914781687,16],[1097696.061940469,11],[1097696.244361248,6],[1097696.43258565,9],[1097696.614895932,29],[1097696.796206339,22],[1097696.979641776,8],[1097697.122955431,18],[1097697.305584239,18],[1097697.499027319,34],[1097697.677949851,10],[1097697.857749015,14],[1097698.038221924,13],[1097698.222348931,2],[1097698.410420879,36],[1097698.59265912,42],[1097698.775388924,26],[1097698.924934998,17],[1097699.103793696,34],[1097699.339251921,12],[1097699.527586295,36],[1097699.805698423,50],[1097699.990502571,2],[1097700.381715402,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-936310db90260798e58c6cedb2df6b1e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126229312,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.055107659,"time_to_first_token":1.871691617,"token_sha":"2b28aa2108de","tokens_per_second":25.360245623204023},"seed":1234,"sent":1097688.452968989,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.8733478628564626,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.300907746322864,"done":1097700.381018018,"finish_reason":"length","first":1097688.946450551,"full_response_s":11.9333104158286,"last":1097700.380962675,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097688.946450551,2],[1097690.601792539,7],[1097690.750792991,4],[1097690.954022716,63],[1097691.180681363,10],[1097691.36330478,31],[1097691.547676515,29],[1097691.72517589,22],[1097691.904814177,14],[1097692.087097246,24],[1097692.265222478,4],[1097692.420199726,13],[1097692.604544763,21],[1097692.782629478,2],[1097692.962857336,18],[1097693.141948124,29],[1097693.373571324,27],[1097693.609186507,24],[1097693.757902825,4],[1097693.903782775,16],[1097694.139946474,10],[1097694.316120186,31],[1097694.47172388,8],[1097694.655826916,11],[1097694.833685306,16],[1097695.014449175,12],[1097695.195239083,39],[1097695.369347657,18],[1097695.55399333,11],[1097695.736239493,16],[1097695.914837961,16],[1097696.063951845,11],[1097696.24298046,6],[1097696.433844074,9],[1097696.614290758,29],[1097696.796606693,22],[1097696.978518743,8],[1097697.122574761,18],[1097697.306348141,18],[1097697.496317566,34],[1097697.676868086,10],[1097697.858799258,14],[1097698.039417158,13],[1097698.220232862,2],[1097698.40886839,36],[1097698.591766838,42],[1097698.775446046,26],[1097698.922694514,17],[1097699.104988186,34],[1097699.339305231,12],[1097699.52505604,36],[1097699.806315821,50],[1097699.989530231,2],[1097700.380962675,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-b15052f4d11bf6e3a3903a1a15ba90c7","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.123841363,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.433126527,"time_to_first_token":0.495585767,"token_sha":"2b28aa2108de","tokens_per_second":22.303610425180068},"seed":1234,"sent":1097688.447707602,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.49874294898472726,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.3351345602107,"done":1097700.379501995,"finish_reason":"length","first":1097689.451730339,"full_response_s":11.932473505847156,"last":1097700.379458197,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.451730339,2],[1097690.602674838,7],[1097690.749194337,4],[1097690.951909025,63],[1097691.177795478,10],[1097691.363186816,31],[1097691.548597004,29],[1097691.727748112,22],[1097691.905403928,14],[1097692.085584914,24],[1097692.266029403,4],[1097692.419319198,13],[1097692.602234394,21],[1097692.782345076,2],[1097692.961189545,18],[1097693.14149357,29],[1097693.376233132,27],[1097693.610499627,24],[1097693.759028768,4],[1097693.903837448,16],[1097694.14011786,10],[1097694.316510007,31],[1097694.472906219,8],[1097694.654428752,11],[1097694.834386954,16],[1097695.015086894,12],[1097695.195574853,39],[1097695.370111968,18],[1097695.554621889,11],[1097695.735901496,16],[1097695.915849574,16],[1097696.063401023,11],[1097696.244150326,6],[1097696.432463799,9],[1097696.615738437,29],[1097696.797198095,22],[1097696.975882878,8],[1097697.12265781,18],[1097697.307604171,18],[1097697.497561265,34],[1097697.677333054,10],[1097697.86030726,14],[1097698.039469011,13],[1097698.220477348,2],[1097698.410683604,36],[1097698.593705968,42],[1097698.77498289,26],[1097698.925205872,17],[1097699.106039688,34],[1097699.339809092,12],[1097699.524346482,36],[1097699.805846311,50],[1097699.989249414,2],[1097700.379458197,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-5fc3a50edddd59d6e93664613523fa1b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.132582515,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.927653966,"time_to_first_token":0.999813309,"token_sha":"2b28aa2108de","tokens_per_second":23.335292350343444},"seed":1234,"sent":1097688.447028489,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.0047018500044942,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":25.739171147520477,"done":1097700.383448556,"finish_reason":"length","first":1097690.475833796,"full_response_s":11.93540034815669,"last":1097700.38291272,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097690.475833796,2],[1097690.599916317,7],[1097690.746862511,4],[1097690.950973578,63],[1097691.178579186,10],[1097691.364589234,31],[1097691.547121624,29],[1097691.726193204,22],[1097691.907191342,14],[1097692.084861842,24],[1097692.268227109,4],[1097692.419176012,13],[1097692.603893323,21],[1097692.781919858,2],[1097692.961063671,18],[1097693.14128708,29],[1097693.375422994,27],[1097693.609650974,24],[1097693.758310841,4],[1097693.903519291,16],[1097694.137808175,10],[1097694.315265389,31],[1097694.473468002,8],[1097694.654763436,11],[1097694.833537078,16],[1097695.014590501,12],[1097695.194738798,39],[1097695.370332821,18],[1097695.554655791,11],[1097695.735170718,16],[1097695.91434852,16],[1097696.061275097,11],[1097696.244552025,6],[1097696.433456147,9],[1097696.615467717,29],[1097696.796317653,22],[1097696.976645771,8],[1097697.122750948,18],[1097697.306644241,18],[1097697.498338331,34],[1097697.67621223,10],[1097697.857952792,14],[1097698.039113549,13],[1097698.221383962,2],[1097698.409184842,36],[1097698.591136384,42],[1097698.774482791,26],[1097698.924091852,17],[1097699.104571889,34],[1097699.339052405,12],[1097699.524768063,36],[1097699.805602804,50],[1097699.989365113,2],[1097700.38291272,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-69d7aa61c240d3faa8aea723b0db9137","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.149419846,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.907351384,"time_to_first_token":2.021420038,"token_sha":"2b28aa2108de","tokens_per_second":25.738463300273715},"seed":1234,"sent":1097688.448048208,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":2.027785588055849,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":21.827450419404045,"done":1097700.379948282,"finish_reason":"length","first":1097688.697382871,"full_response_s":11.932149890810251,"last":1097700.379920002,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097688.697382871,2],[1097690.602356273,7],[1097690.748363284,4],[1097690.952524643,63],[1097691.178281554,10],[1097691.363045615,31],[1097691.546775468,29],[1097691.72533727,22],[1097691.904342533,14],[1097692.086988123,24],[1097692.268293377,4],[1097692.419252322,13],[1097692.604755242,21],[1097692.781576218,2],[1097692.960667458,18],[1097693.141885732,29],[1097693.374885165,27],[1097693.612088371,24],[1097693.759542223,4],[1097693.902786283,16],[1097694.140184271,10],[1097694.316153288,31],[1097694.471841425,8],[1097694.653865928,11],[1097694.834730652,16],[1097695.013527844,12],[1097695.196245562,39],[1097695.369991653,18],[1097695.554848202,11],[1097695.736514265,16],[1097695.914252291,16],[1097696.062611173,11],[1097696.243279284,6],[1097696.43353601,9],[1097696.614212208,29],[1097696.796529665,22],[1097696.978402145,8],[1097697.123890924,18],[1097697.307862398,18],[1097697.497326675,34],[1097697.676547645,10],[1097697.858403361,14],[1097698.038950047,13],[1097698.220712673,2],[1097698.410970334,36],[1097698.590907481,42],[1097698.773573598,26],[1097698.923847128,17],[1097699.104866753,34],[1097699.339122401,12],[1097699.52485473,36],[1097699.80690638,50],[1097699.989454118,2],[1097700.379920002,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-4f7c079be98450bc85ef562dfa22b832","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125237415,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.681683835,"time_to_first_token":0.248413388,"token_sha":"2b28aa2108de","tokens_per_second":21.82904481937642},"seed":1234,"sent":1097688.447798391,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.24958447995595634,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.05536942973802,"done":1097700.379406072,"finish_reason":"length","first":1097689.319044656,"full_response_s":11.932472759857774,"last":1097700.379374909,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.319044656,2],[1097690.603679427,7],[1097690.748821882,4],[1097690.951446522,63],[1097691.180218028,10],[1097691.364470101,31],[1097691.548304977,29],[1097691.727249833,22],[1097691.904756653,14],[1097692.084953333,24],[1097692.266700492,4],[1097692.418538298,13],[1097692.602101185,21],[1097692.783557568,2],[1097692.961481088,18],[1097693.141756544,29],[1097693.375672579,27],[1097693.611055488,24],[1097693.75962987,4],[1097693.902710359,16],[1097694.139780646,10],[1097694.315644849,31],[1097694.473412496,8],[1097694.655433252,11],[1097694.835851553,16],[1097695.013702975,12],[1097695.196816918,39],[1097695.37005752,18],[1097695.553831986,11],[1097695.73541106,16],[1097695.916140775,16],[1097696.062706361,11],[1097696.242675503,6],[1097696.433892853,9],[1097696.613983799,29],[1097696.798996996,22],[1097696.977440693,8],[1097697.123220656,18],[1097697.30707899,18],[1097697.498905324,34],[1097697.677885138,10],[1097697.859354286,14],[1097698.039926722,13],[1097698.220901637,2],[1097698.411128744,36],[1097698.590244119,42],[1097698.774310096,26],[1097698.924601285,17],[1097699.106229436,34],[1097699.340559325,12],[1097699.526863301,36],[1097699.807309866,50],[1097699.990923109,2],[1097700.379374909,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-d6550f3c16fbfa661cc66c30e64f25f0","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.123673063,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.060361086,"time_to_first_token":0.86761733,"token_sha":"2b28aa2108de","tokens_per_second":23.055305158416054},"seed":1234,"sent":1097688.446933312,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.8721113437786698,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.54735717580797,"done":1097700.3802242,"finish_reason":"length","first":1097689.070665428,"full_response_s":11.931552706984803,"last":1097700.380194878,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.070665428,2],[1097690.599445387,7],[1097690.749873025,4],[1097690.951072388,63],[1097691.179961553,10],[1097691.363984954,31],[1097691.545917003,29],[1097691.727081792,22],[1097691.907820089,14],[1097692.087375595,24],[1097692.268077918,4],[1097692.418874347,13],[1097692.602748364,21],[1097692.780450524,2],[1097692.96074753,18],[1097693.140578185,29],[1097693.374956941,27],[1097693.60888385,24],[1097693.756815349,4],[1097693.903360589,16],[1097694.138991379,10],[1097694.316459786,31],[1097694.472163154,8],[1097694.654202678,11],[1097694.833931285,16],[1097695.014381339,12],[1097695.196361361,39],[1097695.370521885,18],[1097695.553148402,11],[1097695.73482023,16],[1097695.915787616,16],[1097696.06253991,11],[1097696.245247919,6],[1097696.43366297,9],[1097696.613801381,29],[1097696.798138638,22],[1097696.979355765,8],[1097697.122874384,18],[1097697.306422606,18],[1097697.496917612,34],[1097697.677571439,10],[1097697.858490413,14],[1097698.038886007,13],[1097698.219983237,2],[1097698.410358649,36],[1097698.59053932,42],[1097698.774063081,26],[1097698.92242364,17],[1097699.105103358,34],[1097699.338911338,12],[1097699.52582754,36],[1097699.806959931,50],[1097699.990821229,2],[1097700.380194878,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-bf276218dac6f33e4f6eb49527f6ecfc","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.124139928,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.30901672,"time_to_first_token":0.619639454,"token_sha":"2b28aa2108de","tokens_per_second":22.548379431523202},"seed":1234,"sent":1097688.448671493,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.6219939349684864,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":23.87465043870034,"done":1097700.381854081,"finish_reason":"length","first":1097689.701023132,"full_response_s":11.932415921008214,"last":1097700.381807785,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.701023132,2],[1097690.601912905,7],[1097690.746729829,4],[1097690.949964224,63],[1097691.181138773,10],[1097691.364183407,31],[1097691.545812652,29],[1097691.724985556,22],[1097691.905119254,14],[1097692.084087683,24],[1097692.266129798,4],[1097692.418643209,13],[1097692.601141754,21],[1097692.781986462,2],[1097692.961152391,18],[1097693.140079272,29],[1097693.375761315,27],[1097693.612171854,24],[1097693.75698451,4],[1097693.902858716,16],[1097694.138553547,10],[1097694.316686229,31],[1097694.47194041,8],[1097694.654042358,11],[1097694.832897277,16],[1097695.01336144,12],[1097695.196110962,39],[1097695.371332013,18],[1097695.553681532,11],[1097695.734185298,16],[1097695.914149016,16],[1097696.062799387,11],[1097696.242183724,6],[1097696.433296983,9],[1097696.614531851,29],[1097696.798918335,22],[1097696.976914455,8],[1097697.123676928,18],[1097697.307463952,18],[1097697.495885747,34],[1097697.677295853,10],[1097697.858607731,14],[1097698.03848364,13],[1097698.221434422,2],[1097698.4096688,36],[1097698.593274744,42],[1097698.774669273,26],[1097698.923224541,17],[1097699.105722822,34],[1097699.340741387,12],[1097699.526245774,36],[1097699.80723886,50],[1097699.989073983,2],[1097700.381807785,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-91267244023a159a21be1b9b3d0d67a9","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125113342,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.679901813,"time_to_first_token":1.248726196,"token_sha":"2b28aa2108de","tokens_per_second":23.87662400506378},"seed":1234,"sent":1097688.44943816,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.2515849720221013,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":25.047218877102907,"done":1097700.381654998,"finish_reason":"length","first":1097690.20054143,"full_response_s":11.930874434066936,"last":1097700.381312447,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097690.20054143,2],[1097690.603064957,7],[1097690.747116137,4],[1097690.952300999,63],[1097691.181388473,10],[1097691.36372707,31],[1097691.546334184,29],[1097691.72679316,22],[1097691.904607637,14],[1097692.083927169,24],[1097692.265542112,4],[1097692.418943722,13],[1097692.601382645,21],[1097692.781185575,2],[1097692.960848421,18],[1097693.141567205,29],[1097693.374455097,27],[1097693.611633961,24],[1097693.758839526,4],[1097693.903922324,16],[1097694.138336056,10],[1097694.315684708,31],[1097694.472043222,8],[1097694.654352172,11],[1097694.834342691,16],[1097695.014736102,12],[1097695.194577182,39],[1097695.370617874,18],[1097695.554278992,11],[1097695.736800504,16],[1097695.915363785,16],[1097696.062133743,11],[1097696.243950469,6],[1097696.432348529,9],[1097696.615507657,29],[1097696.796941132,22],[1097696.977517737,8],[1097697.123387234,18],[1097697.305823665,18],[1097697.497461674,34],[1097697.678247264,10],[1097697.858194299,14],[1097698.038653052,13],[1097698.219738896,2],[1097698.410882674,36],[1097698.592207558,42],[1097698.775676183,26],[1097698.924377539,17],[1097699.10413303,34],[1097699.340327332,12],[1097699.527270437,36],[1097699.806180376,50],[1097699.989950385,2],[1097700.381312447,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-324e95b32df8738b68530bce3b2b86dd","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.123969601,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.181281548,"time_to_first_token":1.746339664,"token_sha":"2b28aa2108de","tokens_per_second":25.045962907301384},"seed":1234,"sent":1097688.450780564,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":1.7497608659323305,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":21.595762060175442,"done":1097700.380382047,"finish_reason":"length","first":1097688.572465386,"full_response_s":11.932489376049489,"last":1097700.380337657,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097688.572465386,2],[1097690.599617914,7],[1097690.748584848,4],[1097690.950363901,63],[1097691.181607425,10],[1097691.36263918,31],[1097691.547403337,29],[1097691.7276727,22],[1097691.904420972,14],[1097692.086342065,24],[1097692.264974269,4],[1097692.419452519,13],[1097692.605107321,21],[1097692.780814198,2],[1097692.960298934,18],[1097693.141840379,29],[1097693.374387788,27],[1097693.609035395,24],[1097693.757708042,4],[1097693.903322796,16],[1097694.138712938,10],[1097694.315519184,31],[1097694.474406122,8],[1097694.653958523,11],[1097694.832645501,16],[1097695.014542025,12],[1097695.194933019,39],[1097695.369539715,18],[1097695.554080431,11],[1097695.733957559,16],[1097695.913874277,16],[1097696.063597484,11],[1097696.24515358,6],[1097696.432761374,9],[1097696.61471888,29],[1097696.796109341,22],[1097696.977990856,8],[1097697.123619277,18],[1097697.30776082,18],[1097697.498090802,34],[1097697.677434503,10],[1097697.858972913,14],[1097698.037903359,13],[1097698.219480927,2],[1097698.41106858,36],[1097698.592798857,42],[1097698.773855828,26],[1097698.923740364,17],[1097699.105222805,34],[1097699.339179154,12],[1097699.524116074,36],[1097699.805164921,50],[1097699.990734483,2],[1097700.380337657,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-007c2dae4ab5a8c3b6800a062b5f6dfb","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.123323214,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.80692721,"time_to_first_token":0.12376529,"token_sha":"2b28aa2108de","tokens_per_second":21.59749064803458},"seed":1234,"sent":1097688.447892671,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.12457271502353251,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.800261509633568,"done":1097700.380463718,"finish_reason":"length","first":1097689.195619198,"full_response_s":11.933355411048979,"last":1097700.379701446,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097689.195619198,2],[1097690.60201785,7],[1097690.747285461,4],[1097690.950513157,63],[1097691.179348305,10],[1097691.36437065,31],[1097691.547213772,29],[1097691.726563014,22],[1097691.905358543,14],[1097692.08595541,24],[1097692.268537215,4],[1097692.419983433,13],[1097692.604055038,21],[1097692.780704739,2],[1097692.960555485,18],[1097693.141169694,29],[1097693.375449193,27],[1097693.610677259,24],[1097693.757970631,4],[1097693.901937651,16],[1097694.138439076,10],[1097694.316609328,31],[1097694.472698657,8],[1097694.655367833,11],[1097694.83514273,16],[1097695.014921467,12],[1097695.196625356,39],[1097695.370729269,18],[1097695.554530736,11],[1097695.73489593,16],[1097695.91647395,16],[1097696.062364219,11],[1097696.244916026,6],[1097696.432995404,9],[1097696.614932861,29],[1097696.797767878,22],[1097696.978040483,8],[1097697.123438223,18],[1097697.306822317,18],[1097697.495535135,34],[1097697.678184953,10],[1097697.858458001,14],[1097698.038705786,13],[1097698.221096944,2],[1097698.41020389,36],[1097698.59005477,42],[1097698.772947276,26],[1097698.922957766,17],[1097699.105811571,34],[1097699.340421061,12],[1097699.524452013,36],[1097699.805950257,50],[1097699.988968404,2],[1097700.379701446,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-a44dfceacc200890d972b479dd9d76b0","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.125057449,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.184039178,"time_to_first_token":0.74427909,"token_sha":"2b28aa2108de","tokens_per_second":22.80034931401239},"seed":1234,"sent":1097688.447108307,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.7485108911059797,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"B","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":22.064943267933796,"done":1097700.379053945,"finish_reason":"length","first":1097688.822199289,"full_response_s":11.93223124789074,"last":1097700.378993109,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097688.822199289,2],[1097690.60446117,7],[1097690.750084868,4],[1097690.953717509,63],[1097691.177971447,10],[1097691.362742442,31],[1097691.546661988,29],[1097691.726730078,22],[1097691.908243898,14],[1097692.08434699,24],[1097692.26564121,4],[1097692.419200675,13],[1097692.603079497,21],[1097692.781450135,2],[1097692.961669081,18],[1097693.140422813,29],[1097693.373801276,27],[1097693.609752729,24],[1097693.757379618,4],[1097693.902106458,16],[1097694.139204482,10],[1097694.315719427,31],[1097694.474662559,8],[1097694.655326868,11],[1097694.833601424,16],[1097695.012774646,12],[1097695.1948357,39],[1097695.370285499,18],[1097695.554581421,11],[1097695.734627019,16],[1097695.91472908,16],[1097696.063352804,11],[1097696.24485711,6],[1097696.43221369,9],[1097696.615248035,29],[1097696.796878517,22],[1097696.975531914,8],[1097697.123087835,18],[1097697.305947982,18],[1097697.498611739,34],[1097697.676997111,10],[1097697.859613296,14],[1097698.039739167,13],[1097698.22280267,2],[1097698.409014838,36],[1097698.594102874,42],[1097698.77323431,26],[1097698.925386829,17],[1097699.104308301,34],[1097699.339955987,12],[1097699.525349254,36],[1097699.80525399,50],[1097699.991251872,2],[1097700.378993109,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-b3a4aba6a274e73ef982375dd18cbd47","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.124729234,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":11.556913252,"time_to_first_token":0.372945189,"token_sha":"2b28aa2108de","tokens_per_second":22.064715243568223},"seed":1234,"sent":1097688.446822697,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.37537659192457795,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":49.79677234376222,"done":1097705.640158323,"finish_reason":"length","first":1097700.519300179,"full_response_s":5.2520289570093155,"last":1097705.640113999,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097700.519300179,3],[1097700.599286481,26],[1097700.68149021,15],[1097700.761167975,34],[1097700.842025248,21],[1097700.922655977,25],[1097701.003316924,14],[1097701.084227348,14],[1097701.16511926,18],[1097701.245448515,26],[1097701.325469636,13],[1097701.411889325,13],[1097701.494955591,13],[1097701.574948447,23],[1097701.656482555,32],[1097701.736487614,7],[1097701.816730315,23],[1097701.897429666,9],[1097701.977237957,13],[1097702.058610968,15],[1097702.138490887,19],[1097702.219126488,12],[1097702.300324823,5],[1097702.387703633,23],[1097702.471840237,9],[1097702.552165639,17],[1097702.63249703,3],[1097702.713338918,21],[1097702.793443316,10],[1097702.873721357,24],[1097702.954353507,19],[1097703.034391428,14],[1097703.115762008,13],[1097703.196524187,5],[1097703.27692262,22],[1097703.360058255,13],[1097703.44442922,51],[1097703.526689798,15],[1097703.606749673,6],[1097703.687818191,8],[1097703.768093464,9],[1097703.848257468,16],[1097703.928826248,16],[1097704.00974625,9],[1097704.09039183,32],[1097704.171743857,18],[1097704.251664847,13],[1097704.332756192,11],[1097704.421033841,15],[1097704.504558408,8],[1097704.584804902,18],[1097704.659726409,25],[1097704.739764008,19],[1097704.820282257,8],[1097704.900737005,40],[1097704.981375906,16],[1097705.062503485,64],[1097705.142981324,6],[1097705.22439756,6],[1097705.304719894,11],[1097705.390680885,15],[1097705.475328431,17],[1097705.556114991,21],[1097705.640113999,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-6ef94387b20d63b61e6d3cb9461beaa8","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.129336932,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.121625787,"time_to_first_token":0.130229428,"token_sha":"8a91591c94a3","tokens_per_second":49.7888777128652},"seed":1234,"sent":1097700.388129366,"speculative":{"acceptance_rate":0.2052910052910053,"accepted":194,"drafted":945,"rounds":63,"tokens_per_round":4.063492063492063},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.13117081299424171,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":194,"drafted":945,"rounds":63},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":49.78045058425396,"done":1097710.893153593,"finish_reason":"length","first":1097705.770619554,"full_response_s":5.250370910856873,"last":1097710.89311236,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097705.770619554,3],[1097705.851630604,26],[1097705.932629933,15],[1097706.01271979,34],[1097706.093830988,21],[1097706.174562191,25],[1097706.255232176,14],[1097706.335663356,14],[1097706.424421318,18],[1097706.507193583,26],[1097706.587357966,13],[1097706.668410908,13],[1097706.748455576,13],[1097706.828460804,23],[1097706.909294545,32],[1097706.989285354,7],[1097707.070185629,23],[1097707.151610959,9],[1097707.231829459,13],[1097707.311756407,15],[1097707.397390695,19],[1097707.48129747,12],[1097707.561442101,5],[1097707.641622842,23],[1097707.722883461,9],[1097707.803204957,17],[1097707.883082369,3],[1097707.963305027,21],[1097708.044312822,10],[1097708.124843043,24],[1097708.206134941,19],[1097708.286208582,14],[1097708.37099842,13],[1097708.456428091,5],[1097708.538065567,22],[1097708.618446194,13],[1097708.698987496,51],[1097708.779980543,15],[1097708.859871269,6],[1097708.940162158,8],[1097709.020479452,9],[1097709.101937798,16],[1097709.183323412,16],[1097709.26393179,9],[1097709.34465109,32],[1097709.43159535,18],[1097709.513898347,13],[1097709.594083502,11],[1097709.675187177,15],[1097709.755392092,8],[1097709.836364591,18],[1097709.9098363,25],[1097709.990100977,19],[1097710.071334594,8],[1097710.151987311,40],[1097710.23315156,16],[1097710.313448535,64],[1097710.401211868,6],[1097710.485566379,6],[1097710.566485166,11],[1097710.647364738,15],[1097710.728461733,17],[1097710.809172016,21],[1097710.89311236,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-de4d6f3711587184bc78c2b31284826b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.125765645,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.123154866,"time_to_first_token":0.126467205,"token_sha":"8a91591c94a3","tokens_per_second":49.774017508687194},"seed":1234,"sent":1097705.642782682,"speculative":{"acceptance_rate":0.2052910052910053,"accepted":194,"drafted":945,"rounds":63,"tokens_per_round":4.063492063492063},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.12783687189221382,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":194,"drafted":945,"rounds":63},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.169518410684407,"done":1097726.524069553,"finish_reason":"length","first":1097711.672129903,"full_response_s":15.625110248103738,"last":1097726.524031622,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.672129903,3],[1097713.072729172,11],[1097713.25454783,30],[1097713.439152765,19],[1097713.619163854,21],[1097713.800235282,15],[1097713.94626445,19],[1097714.094835014,17],[1097714.273631015,12],[1097714.463016497,5],[1097714.610571016,5],[1097714.793425141,27],[1097714.971571145,12],[1097715.153400426,13],[1097715.331531533,5],[1097715.487761004,16],[1097715.669308705,5],[1097715.849515935,14],[1097716.030656378,9],[1097716.212694507,27],[1097716.362087221,11],[1097716.550464923,8],[1097716.697310836,16],[1097716.844388087,5],[1097716.992703503,9],[1097717.172755888,14],[1097717.320777271,20],[1097717.506918876,12],[1097717.656104977,8],[1097717.801832118,5],[1097717.951421049,14],[1097718.131244432,14],[1097718.315523199,8],[1097718.500277865,13],[1097718.670415458,3],[1097718.849159822,10],[1097718.99860464,11],[1097719.181568434,9],[1097719.361864637,10],[1097719.54567605,20],[1097719.730605301,14],[1097719.909653169,14],[1097720.140357511,13],[1097720.320897837,5],[1097720.511755485,22],[1097720.742614584,3],[1097720.973586697,32],[1097721.156172562,33],[1097721.339966163,11],[1097721.530415158,6],[1097721.701822131,8],[1097721.848507114,9],[1097721.996646421,13],[1097722.145940058,8],[1097722.293040866,20],[1097722.450544557,17],[1097722.63365906,20],[1097722.785111326,13],[1097722.95685417,6],[1097723.139781424,13],[1097723.287571773,9],[1097723.478340194,11],[1097723.624405668,8],[1097723.809233767,15],[1097723.990046019,16],[1097724.200067664,27],[1097724.447999681,4],[1097724.595163581,8],[1097724.779907097,25],[1097724.962271764,18],[1097725.10928229,13],[1097725.295098156,55],[1097725.447391332,11],[1097725.632114226,10],[1097725.816000144,11],[1097725.96477274,15],[1097726.11402224,17],[1097726.262815314,14],[1097726.524031622,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-50663f3f0e3fa7e8b887652ff9e5bf6a","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127732581,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.852239017,"time_to_first_token":0.768527768,"token_sha":"8a91591c94a3","tokens_per_second":17.169128486831166},"seed":1234,"sent":1097710.898959305,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.773170598084107,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.441293067265534,"done":1097726.523678822,"finish_reason":"length","first":1097712.695995327,"full_response_s":15.619322019862011,"last":1097726.523659096,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.695995327,3],[1097713.073266732,11],[1097713.254036675,30],[1097713.438606382,19],[1097713.618956618,21],[1097713.799662767,15],[1097713.945970556,19],[1097714.093340243,17],[1097714.273784447,12],[1097714.461327221,5],[1097714.609624179,5],[1097714.793359185,27],[1097714.97318123,12],[1097715.154087505,13],[1097715.332533961,5],[1097715.487573962,16],[1097715.670280422,5],[1097715.849829954,14],[1097716.030189041,9],[1097716.212655581,27],[1097716.362998902,11],[1097716.550651499,8],[1097716.697539797,16],[1097716.843705082,5],[1097716.993631416,9],[1097717.174273405,14],[1097717.318828045,20],[1097717.507056693,12],[1097717.656628323,8],[1097717.802169161,5],[1097717.950613809,14],[1097718.13080051,14],[1097718.314312627,8],[1097718.499092593,13],[1097718.67014261,3],[1097718.848415658,10],[1097718.99571091,11],[1097719.179503765,9],[1097719.362441411,10],[1097719.547055671,20],[1097719.728241749,14],[1097719.90904756,14],[1097720.138805126,13],[1097720.3206937,5],[1097720.511045078,22],[1097720.740443541,3],[1097720.974753634,32],[1097721.157184583,33],[1097721.340098518,11],[1097721.530798517,6],[1097721.703134826,8],[1097721.847480559,9],[1097721.996865592,13],[1097722.146305434,8],[1097722.296763008,20],[1097722.44970105,17],[1097722.633460401,20],[1097722.78323132,13],[1097722.955876824,6],[1097723.139574327,13],[1097723.287314959,9],[1097723.478948643,11],[1097723.624710534,8],[1097723.808349145,15],[1097723.990612883,16],[1097724.200152975,27],[1097724.448086609,4],[1097724.595597947,8],[1097724.778053737,25],[1097724.963584993,18],[1097725.109821227,13],[1097725.29290278,55],[1097725.448304142,11],[1097725.630691637,10],[1097725.818328422,11],[1097725.964052746,15],[1097726.114620946,17],[1097726.26413864,14],[1097726.523659096,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-2d28c5d0adf3712ee28cf4909cf9110e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127942378,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.827971645,"time_to_first_token":1.789735181,"token_sha":"8a91591c94a3","tokens_per_second":18.440882476946964},"seed":1234,"sent":1097710.904356802,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.7916385249700397,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.27293272178633,"done":1097726.524311776,"finish_reason":"length","first":1097712.568449263,"full_response_s":15.625103360041976,"last":1097726.52351624,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.568449263,3],[1097713.073840577,11],[1097713.252525131,30],[1097713.439437854,19],[1097713.619068563,21],[1097713.801898258,15],[1097713.946863928,19],[1097714.093527557,17],[1097714.274483347,12],[1097714.462377068,5],[1097714.609765569,5],[1097714.791512763,27],[1097714.97299097,12],[1097715.154579328,13],[1097715.333459481,5],[1097715.486444187,16],[1097715.66854086,5],[1097715.848863815,14],[1097716.029451094,9],[1097716.213687082,27],[1097716.362486217,11],[1097716.54947547,8],[1097716.696322408,16],[1097716.843047841,5],[1097716.991293877,9],[1097717.174655939,14],[1097717.319009497,20],[1097717.508192222,12],[1097717.654970682,8],[1097717.803155198,5],[1097717.950159399,14],[1097718.129918541,14],[1097718.31484569,8],[1097718.500461653,13],[1097718.669234317,3],[1097718.849621949,10],[1097718.996154719,11],[1097719.178473976,9],[1097719.361751191,10],[1097719.546620349,20],[1097719.729739301,14],[1097719.909597238,14],[1097720.139160898,13],[1097720.321493278,5],[1097720.512393966,22],[1097720.74067879,3],[1097720.974143818,32],[1097721.156977228,33],[1097721.339906102,11],[1097721.529696877,6],[1097721.702908032,8],[1097721.848789548,9],[1097721.996807869,13],[1097722.146257366,8],[1097722.293842614,20],[1097722.45073963,17],[1097722.635473193,20],[1097722.783470202,13],[1097722.955462294,6],[1097723.139956097,13],[1097723.287617391,9],[1097723.478679804,11],[1097723.624299654,8],[1097723.807635771,15],[1097723.98992758,16],[1097724.201919166,27],[1097724.446645792,4],[1097724.59436373,8],[1097724.779409916,25],[1097724.962191513,18],[1097725.109899668,13],[1097725.294439996,55],[1097725.448546606,11],[1097725.630439245,10],[1097725.81635802,11],[1097725.96444716,15],[1097726.113935537,17],[1097726.26401977,14],[1097726.52351624,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-a9088aeed7d73b969afce9f3876bb9af","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127066307,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.955915401,"time_to_first_token":1.662249135,"token_sha":"8a91591c94a3","tokens_per_second":18.271821852812906},"seed":1234,"sent":1097710.899208416,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.669240846997127,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.317463768914205,"done":1097726.523607715,"finish_reason":"length","first":1097711.798552221,"full_response_s":15.623217075131834,"last":1097726.523572201,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.798552221,3],[1097713.07306619,11],[1097713.253627745,30],[1097713.43887445,19],[1097713.618402373,21],[1097713.79951505,15],[1097713.946937537,19],[1097714.092747602,17],[1097714.274987582,12],[1097714.461220306,5],[1097714.610314287,5],[1097714.791926143,27],[1097714.972884327,12],[1097715.153124964,13],[1097715.333995755,5],[1097715.487522226,16],[1097715.670768097,5],[1097715.850866467,14],[1097716.029752511,9],[1097716.212729382,27],[1097716.362677549,11],[1097716.549411853,8],[1097716.698216768,16],[1097716.844271261,5],[1097716.992054701,9],[1097717.172952375,14],[1097717.319850943,20],[1097717.507194896,12],[1097717.656564482,8],[1097717.802585833,5],[1097717.951833572,14],[1097718.130597074,14],[1097718.312255314,8],[1097718.499771158,13],[1097718.671368057,3],[1097718.84827576,10],[1097718.995765672,11],[1097719.1807216,9],[1097719.362377138,10],[1097719.5456051,20],[1097719.731153525,14],[1097719.911097312,14],[1097720.140880273,13],[1097720.321978011,5],[1097720.512923885,22],[1097720.741372513,3],[1097720.97306855,32],[1097721.157337546,33],[1097721.338851136,11],[1097721.530177941,6],[1097721.70475527,8],[1097721.849060822,9],[1097721.997501665,13],[1097722.146527215,8],[1097722.294438977,20],[1097722.449375,17],[1097722.634798739,20],[1097722.784566601,13],[1097722.956429024,6],[1097723.140404877,13],[1097723.286940795,9],[1097723.477987469,11],[1097723.623322757,8],[1097723.80739713,15],[1097723.991313268,16],[1097724.200255065,27],[1097724.446927359,4],[1097724.596426795,8],[1097724.778851427,25],[1097724.963501444,18],[1097725.109698161,13],[1097725.293341019,55],[1097725.447583072,11],[1097725.632308287,10],[1097725.815540067,11],[1097725.96482593,15],[1097726.11374319,17],[1097726.262282785,14],[1097726.523572201,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-7b324838cc8d2705a439626a45f8087a","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126418123,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.72574852,"time_to_first_token":0.894538045,"token_sha":"8a91591c94a3","tokens_per_second":17.31660700667731},"seed":1234,"sent":1097710.90039064,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.8981615810189396,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.105323940755838,"done":1097726.525027991,"finish_reason":"length","first":1097712.440727396,"full_response_s":15.622680570930243,"last":1097726.524982438,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.440727396,3],[1097713.073213506,11],[1097713.252690413,30],[1097713.438475463,19],[1097713.619625097,21],[1097713.800420276,15],[1097713.945241488,19],[1097714.092796425,17],[1097714.275119173,12],[1097714.462960725,5],[1097714.610610647,5],[1097714.793079847,27],[1097714.972323795,12],[1097715.154330174,13],[1097715.332873105,5],[1097715.486334886,16],[1097715.669608763,5],[1097715.851485282,14],[1097716.031452674,9],[1097716.212479172,27],[1097716.361446245,11],[1097716.549967514,8],[1097716.696272705,16],[1097716.843226234,5],[1097716.993890175,9],[1097717.172364835,14],[1097717.319583089,20],[1097717.506989154,12],[1097717.656242587,8],[1097717.803916583,5],[1097717.950477817,14],[1097718.131605636,14],[1097718.311355011,8],[1097718.500206851,13],[1097718.669838041,3],[1097718.848796811,10],[1097718.997277586,11],[1097719.180155026,9],[1097719.362885827,10],[1097719.545869316,20],[1097719.730048575,14],[1097719.910380148,14],[1097720.140792799,13],[1097720.322033685,5],[1097720.512506146,22],[1097720.740741269,3],[1097720.973301876,32],[1097721.157276252,33],[1097721.339038316,11],[1097721.530968772,6],[1097721.702952497,8],[1097721.848682813,9],[1097721.996342628,13],[1097722.144966983,8],[1097722.295825191,20],[1097722.451279728,17],[1097722.635187508,20],[1097722.784061313,13],[1097722.955319036,6],[1097723.139281519,13],[1097723.286509709,9],[1097723.477448959,11],[1097723.623249935,8],[1097723.807846758,15],[1097723.990481394,16],[1097724.200843609,27],[1097724.447079054,4],[1097724.594221593,8],[1097724.779791684,25],[1097724.961509224,18],[1097725.109650558,13],[1097725.293999243,55],[1097725.449168468,11],[1097725.631539519,10],[1097725.816817648,11],[1097725.963457243,15],[1097726.113357086,17],[1097726.262325824,14],[1097726.524982438,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-3516c2cf33f3b8b198253b25d9a3696e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.137186004,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.082984638,"time_to_first_token":1.53519753,"token_sha":"8a91591c94a3","tokens_per_second":18.106957193714152},"seed":1234,"sent":1097710.90234742,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.5383799758274108,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.617949657071076,"done":1097726.525875197,"finish_reason":"length","first":1097712.051967451,"full_response_s":15.624743100022897,"last":1097726.525842389,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.051967451,3],[1097713.074477783,11],[1097713.251801312,30],[1097713.438546207,19],[1097713.619571375,21],[1097713.801020734,15],[1097713.945094077,19],[1097714.09615283,17],[1097714.275837128,12],[1097714.462611443,5],[1097714.610000473,5],[1097714.792198004,27],[1097714.970952872,12],[1097715.153207348,13],[1097715.331904303,5],[1097715.486530702,16],[1097715.668059653,5],[1097715.850345905,14],[1097716.030240328,9],[1097716.21174366,27],[1097716.362746515,11],[1097716.550539765,8],[1097716.698474664,16],[1097716.845252078,5],[1097716.991483014,9],[1097717.174808792,14],[1097717.3210237,20],[1097717.506393,12],[1097717.655214229,8],[1097717.802525883,5],[1097717.951648567,14],[1097718.133126893,14],[1097718.313899528,8],[1097718.499487054,13],[1097718.671261447,3],[1097718.849496013,10],[1097718.998130473,11],[1097719.179883781,9],[1097719.361917846,10],[1097719.546317127,20],[1097719.731595267,14],[1097719.911430651,14],[1097720.139976728,13],[1097720.321807114,5],[1097720.512106373,22],[1097720.743178977,3],[1097720.973479017,32],[1097721.156648342,33],[1097721.340190955,11],[1097721.530339021,6],[1097721.702699027,8],[1097721.848820307,9],[1097721.997676355,13],[1097722.144024939,8],[1097722.293952439,20],[1097722.450901078,17],[1097722.634535408,20],[1097722.782951945,13],[1097722.957749967,6],[1097723.140215425,13],[1097723.288269536,9],[1097723.478621185,11],[1097723.62435307,8],[1097723.808649575,15],[1097723.990404842,16],[1097724.201536774,27],[1097724.445963596,4],[1097724.59476524,8],[1097724.778441078,25],[1097724.961743491,18],[1097725.114883092,13],[1097725.293772662,55],[1097725.450065009,11],[1097725.631172163,10],[1097725.818613797,11],[1097725.963972432,15],[1097726.113172297,17],[1097726.263747253,14],[1097726.525842389,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-d58a86c91fea2d96bf543199e1d7132d","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.12762571,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.471725263,"time_to_first_token":1.147313164,"token_sha":"8a91591c94a3","tokens_per_second":17.620566681980964},"seed":1234,"sent":1097710.901132097,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.1508353541139513,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.610174164385988,"done":1097726.524556722,"finish_reason":"length","first":1097712.822340249,"full_response_s":15.621217501116917,"last":1097726.524522601,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.822340249,3],[1097713.072028236,11],[1097713.252026994,30],[1097713.439332507,19],[1097713.618862944,21],[1097713.799297695,15],[1097713.945190472,19],[1097714.092616267,17],[1097714.273698828,12],[1097714.461858053,5],[1097714.610362085,5],[1097714.79245571,27],[1097714.973398743,12],[1097715.152663148,13],[1097715.333689885,5],[1097715.487707794,16],[1097715.670644241,5],[1097715.849900617,14],[1097716.029985667,9],[1097716.212098206,27],[1097716.36193363,11],[1097716.550315944,8],[1097716.697876601,16],[1097716.843544735,5],[1097716.990910126,9],[1097717.175296623,14],[1097717.319130166,20],[1097717.507538536,12],[1097717.655849293,8],[1097717.803867665,5],[1097717.950388277,14],[1097718.132448919,14],[1097718.314212775,8],[1097718.500810559,13],[1097718.669130525,3],[1097718.849211253,10],[1097718.996838645,11],[1097719.181172662,9],[1097719.361339583,10],[1097719.546453871,20],[1097719.730529115,14],[1097719.912326029,14],[1097720.141334791,13],[1097720.322314872,5],[1097720.513265758,22],[1097720.742224803,3],[1097720.972963655,32],[1097721.156279586,33],[1097721.339211262,11],[1097721.531346415,6],[1097721.702243231,8],[1097721.848299376,9],[1097721.997285616,13],[1097722.145228762,8],[1097722.294670397,20],[1097722.450026092,17],[1097722.635906491,20],[1097722.784424399,13],[1097722.957360416,6],[1097723.140548503,13],[1097723.286570666,9],[1097723.47924041,11],[1097723.62348763,8],[1097723.808959308,15],[1097723.990719665,16],[1097724.200348206,27],[1097724.446370182,4],[1097724.594825076,8],[1097724.778742034,25],[1097724.962723373,18],[1097725.110318869,13],[1097725.29426177,55],[1097725.448813793,11],[1097725.632671976,10],[1097725.817179911,11],[1097725.96489185,15],[1097726.114960616,17],[1097726.262779592,14],[1097726.524522601,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-62fe48498f5c666615d7fee0f00f7bf2","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.12612014,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.701968099,"time_to_first_token":1.916219882,"token_sha":"8a91591c94a3","tokens_per_second":18.610465165118175},"seed":1234,"sent":1097710.903339221,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.9190010281745344,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.774704089578044,"done":1097726.525716426,"finish_reason":"length","first":1097712.179463554,"full_response_s":15.623842568136752,"last":1097726.525693945,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.179463554,3],[1097713.074153127,11],[1097713.252824855,30],[1097713.439010909,19],[1097713.619685081,21],[1097713.801615523,15],[1097713.944933023,19],[1097714.093057461,17],[1097714.274206442,12],[1097714.461569315,5],[1097714.610541457,5],[1097714.791128974,27],[1097714.973451328,12],[1097715.152190541,13],[1097715.332606722,5],[1097715.487637131,16],[1097715.669268594,5],[1097715.849581362,14],[1097716.029837329,9],[1097716.212274406,27],[1097716.362365116,11],[1097716.550055502,8],[1097716.697722931,16],[1097716.844239204,5],[1097716.991568632,9],[1097717.173843059,14],[1097717.319453678,20],[1097717.507788327,12],[1097717.657015468,8],[1097717.802638049,5],[1097717.950335101,14],[1097718.130304435,14],[1097718.314557262,8],[1097718.500140752,13],[1097718.671701303,3],[1097718.849995079,10],[1097718.998399861,11],[1097719.178628029,9],[1097719.360816611,10],[1097719.546823928,20],[1097719.7285981,14],[1097719.910513464,14],[1097720.141245332,13],[1097720.320278347,5],[1097720.511884542,22],[1097720.741526341,3],[1097720.974936667,32],[1097721.157673061,33],[1097721.339508492,11],[1097721.529895057,6],[1097721.702123509,8],[1097721.848236833,9],[1097721.99690812,13],[1097722.145732304,8],[1097722.294953487,20],[1097722.451430608,17],[1097722.634035756,20],[1097722.78475703,13],[1097722.956390019,6],[1097723.140471294,13],[1097723.287869929,9],[1097723.479055586,11],[1097723.624470388,8],[1097723.808402721,15],[1097723.98958749,16],[1097724.199611906,27],[1097724.44747416,4],[1097724.596488729,8],[1097724.779177634,25],[1097724.96320185,18],[1097725.109196275,13],[1097725.293207049,55],[1097725.448460399,11],[1097725.631715999,10],[1097725.815851301,11],[1097725.965067737,15],[1097726.113618075,17],[1097726.263455266,14],[1097726.525693945,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-5854059568d82b5eaae9fe0dbe1d1914","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127168484,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.34453987,"time_to_first_token":1.274156492,"token_sha":"8a91591c94a3","tokens_per_second":17.776798859425526},"seed":1234,"sent":1097710.901873858,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.277589695993811,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.455106766043293,"done":1097726.524436571,"finish_reason":"length","first":1097711.027693454,"full_response_s":15.625029355986044,"last":1097726.524402386,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.027693454,3],[1097713.073499155,11],[1097713.252183262,30],[1097713.439093871,19],[1097713.6205823,21],[1097713.799194061,15],[1097713.946796291,19],[1097714.09402401,17],[1097714.275409272,12],[1097714.462090379,5],[1097714.609196404,5],[1097714.792822109,27],[1097714.971843197,12],[1097715.154786081,13],[1097715.331471662,5],[1097715.487381348,16],[1097715.669542071,5],[1097715.850978138,14],[1097716.029301026,9],[1097716.212821517,27],[1097716.36354976,11],[1097716.549835412,8],[1097716.696156007,16],[1097716.845532373,5],[1097716.991896788,9],[1097717.175601769,14],[1097717.31949856,20],[1097717.506509713,12],[1097717.657486582,8],[1097717.802432371,5],[1097717.951140563,14],[1097718.130715229,14],[1097718.310971022,8],[1097718.498922911,13],[1097718.669866943,3],[1097718.84771343,10],[1097718.996206614,11],[1097719.180393351,9],[1097719.360615098,10],[1097719.546391504,20],[1097719.729548692,14],[1097719.910078577,14],[1097720.139777587,13],[1097720.319935705,5],[1097720.512997604,22],[1097720.742048766,3],[1097720.97286254,32],[1097721.15694173,33],[1097721.340043725,11],[1097721.529985557,6],[1097721.701471405,8],[1097721.848749871,9],[1097721.995752077,13],[1097722.146351356,8],[1097722.293290139,20],[1097722.451062078,17],[1097722.634403582,20],[1097722.78416166,13],[1097722.955696819,6],[1097723.140333784,13],[1097723.286229582,9],[1097723.478100144,11],[1097723.623701933,8],[1097723.808035073,15],[1097723.990253195,16],[1097724.200625273,27],[1097724.445694246,4],[1097724.596243509,8],[1097724.779833955,25],[1097724.961857013,18],[1097725.109947383,13],[1097725.295422312,55],[1097725.447475362,11],[1097725.631898388,10],[1097725.816466178,11],[1097725.96627895,15],[1097726.114133473,17],[1097726.263030574,14],[1097726.524402386,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-4c857fc606a5cfe53a421688c988850b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.12630464,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.496167408,"time_to_first_token":0.126998875,"token_sha":"8a91591c94a3","tokens_per_second":16.45568180093063},"seed":1234,"sent":1097710.899407215,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.12828623899258673,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":18.778301492101534,"done":1097726.526518462,"finish_reason":"length","first":1097712.946980922,"full_response_s":15.626335748005658,"last":1097726.526483845,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.946980922,3],[1097713.074060014,11],[1097713.251646086,30],[1097713.439843038,19],[1097713.618174401,21],[1097713.800676078,15],[1097713.945717109,19],[1097714.09570102,17],[1097714.274587509,12],[1097714.46256118,5],[1097714.610866784,5],[1097714.790950563,27],[1097714.971802974,12],[1097715.154022798,13],[1097715.332755958,5],[1097715.48674948,16],[1097715.668332554,5],[1097715.850190905,14],[1097716.031696398,9],[1097716.212424634,27],[1097716.362028167,11],[1097716.548085739,8],[1097716.696016267,16],[1097716.843874076,5],[1097716.993805005,9],[1097717.175364915,14],[1097717.319922342,20],[1097717.508103273,12],[1097717.657334738,8],[1097717.803616497,5],[1097717.94974702,14],[1097718.130053684,14],[1097718.311264862,8],[1097718.500052125,13],[1097718.6699732,3],[1097718.849848744,10],[1097718.995910134,11],[1097719.179662285,9],[1097719.361813495,10],[1097719.546069692,20],[1097719.729145955,14],[1097719.91212178,14],[1097720.141463528,13],[1097720.321251015,5],[1097720.512189988,22],[1097720.740209733,3],[1097720.974716631,32],[1097721.157237119,33],[1097721.339594396,11],[1097721.530084575,6],[1097721.702467718,8],[1097721.848096281,9],[1097721.995842224,13],[1097722.144714828,8],[1097722.294542302,20],[1097722.449272315,17],[1097722.634181352,20],[1097722.784958588,13],[1097722.955597978,6],[1097723.139866928,13],[1097723.286391237,9],[1097723.478556353,11],[1097723.623179851,8],[1097723.808111098,15],[1097723.989815546,16],[1097724.200390589,27],[1097724.446099824,4],[1097724.595327861,8],[1097724.779329041,25],[1097724.962071217,18],[1097725.10934006,13],[1097725.29515852,55],[1097725.44936879,11],[1097725.630283948,10],[1097725.817954312,11],[1097725.966346343,15],[1097726.113423919,17],[1097726.262488502,14],[1097726.526483845,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-042d34cba718284643257c5e399d8b67","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.124555806,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.577727241,"time_to_first_token":2.040488242,"token_sha":"8a91591c94a3","tokens_per_second":18.780757300086936},"seed":1234,"sent":1097710.900182714,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":2.0467982080299407,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.87937658447425,"done":1097726.524718455,"finish_reason":"length","first":1097711.41750285,"full_response_s":15.624450232135132,"last":1097726.524695863,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.41750285,3],[1097713.072471673,11],[1097713.25428154,30],[1097713.438699608,19],[1097713.618765796,21],[1097713.799833301,15],[1097713.946171433,19],[1097714.096100997,17],[1097714.274065484,12],[1097714.461984905,5],[1097714.61091503,5],[1097714.793533642,27],[1097714.973312869,12],[1097715.153710604,13],[1097715.331668472,5],[1097715.487102235,16],[1097715.668231627,5],[1097715.849727475,14],[1097716.029882227,9],[1097716.211859846,27],[1097716.361543729,11],[1097716.548433704,8],[1097716.69536498,16],[1097716.84380865,5],[1097716.992506904,9],[1097717.173447363,14],[1097717.31928936,20],[1097717.508362016,12],[1097717.656479521,8],[1097717.801719792,5],[1097717.94937187,14],[1097718.130663173,14],[1097718.311795763,8],[1097718.498994358,13],[1097718.669189242,3],[1097718.84893197,10],[1097718.998539487,11],[1097719.181282393,9],[1097719.360901395,10],[1097719.545725527,20],[1097719.728919591,14],[1097719.909143441,14],[1097720.13862957,13],[1097720.320966385,5],[1097720.510804257,22],[1097720.741034402,3],[1097720.974088993,32],[1097721.156575311,33],[1097721.340144536,11],[1097721.531149211,6],[1097721.702858267,8],[1097721.847693181,9],[1097721.996999692,13],[1097722.145888388,8],[1097722.29632417,20],[1097722.450662052,17],[1097722.633595413,20],[1097722.78518474,13],[1097722.956213169,6],[1097723.140375975,13],[1097723.287127285,9],[1097723.477665808,11],[1097723.623918589,8],[1097723.80774574,15],[1097723.992056165,16],[1097724.202161056,27],[1097724.446992991,4],[1097724.594915863,8],[1097724.779497821,25],[1097724.961618039,18],[1097725.110793055,13],[1097725.293518172,55],[1097725.448379557,11],[1097725.632502477,10],[1097725.817432734,11],[1097725.966610246,15],[1097726.113267374,17],[1097726.262138824,14],[1097726.524695863,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-eac446f7819580dc608b9b04bb2826b6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.135830925,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.106074229,"time_to_first_token":0.516253587,"token_sha":"8a91591c94a3","tokens_per_second":16.8806267024997},"seed":1234,"sent":1097710.900268223,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.5172346271574497,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.729833773500147,"done":1097726.524149259,"finish_reason":"length","first":1097711.281885744,"full_response_s":15.624810067005455,"last":1097726.524117284,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.281885744,3],[1097713.072590551,11],[1097713.252737699,30],[1097713.438792289,19],[1097713.618599168,21],[1097713.802250151,15],[1097713.945407491,19],[1097714.09460272,17],[1097714.274682783,12],[1097714.462186374,5],[1097714.609318147,5],[1097714.791683809,27],[1097714.972101974,12],[1097715.151757692,13],[1097715.33269186,5],[1097715.48746804,16],[1097715.669847365,5],[1097715.851104764,14],[1097716.029935292,9],[1097716.211962053,27],[1097716.363276973,11],[1097716.548786457,8],[1097716.696694519,16],[1097716.842945891,5],[1097716.990604291,9],[1097717.174603355,14],[1097717.319721388,20],[1097717.506285909,12],[1097717.656154711,8],[1097717.803193372,5],[1097717.949599387,14],[1097718.130765187,14],[1097718.312598856,8],[1097718.500878804,13],[1097718.669803535,3],[1097718.848229389,10],[1097718.996509676,11],[1097719.178773915,9],[1097719.360531099,10],[1097719.546907159,20],[1097719.729428713,14],[1097719.908856953,14],[1097720.141048528,13],[1097720.320565779,5],[1097720.512559723,22],[1097720.740930484,3],[1097720.973386756,32],[1097721.156891532,33],[1097721.339293259,11],[1097721.529581206,6],[1097721.705313592,8],[1097721.847025053,9],[1097721.997939142,13],[1097722.144116383,8],[1097722.294070975,20],[1097722.450102885,17],[1097722.635360789,20],[1097722.78363751,13],[1097722.955376311,6],[1097723.138987414,13],[1097723.287932343,9],[1097723.477171282,11],[1097723.623396043,8],[1097723.80888101,15],[1097723.991811874,16],[1097724.201087868,27],[1097724.445829225,4],[1097724.594607941,8],[1097724.779574949,25],[1097724.961070568,18],[1097725.108967324,13],[1097725.292740966,55],[1097725.450130545,11],[1097725.631367697,10],[1097725.818428687,11],[1097725.965144129,15],[1097726.113813001,17],[1097726.26185963,14],[1097726.524117284,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-9962f2aa44835ecc0a7d537a479aed06","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126632759,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.241850108,"time_to_first_token":0.380643575,"token_sha":"8a91591c94a3","tokens_per_second":16.730252442658387},"seed":1234,"sent":1097710.899339192,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.3825465519912541,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":16.590678269709507,"done":1097726.525222548,"finish_reason":"length","first":1097711.15511835,"full_response_s":15.626140009146184,"last":1097726.5251952,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.15511835,3],[1097713.07313008,11],[1097713.251929029,30],[1097713.439641695,19],[1097713.618663299,21],[1097713.799594185,15],[1097713.945843912,19],[1097714.093157844,17],[1097714.275077172,12],[1097714.463099554,5],[1097714.610987663,5],[1097714.790751799,27],[1097714.971109906,12],[1097715.152444594,13],[1097715.331376276,5],[1097715.487027425,16],[1097715.668906328,5],[1097715.850126391,14],[1097716.030416689,9],[1097716.211479984,27],[1097716.361724685,11],[1097716.54990798,8],[1097716.697068136,16],[1097716.84484077,5],[1097716.993535166,9],[1097717.17376447,14],[1097717.31940215,20],[1097717.507274749,12],[1097717.65561958,8],[1097717.802275947,5],[1097717.952102449,14],[1097718.129613156,14],[1097718.31317275,8],[1097718.498824884,13],[1097718.669617073,3],[1097718.847779304,10],[1097718.997392265,11],[1097719.179573562,9],[1097719.36250951,10],[1097719.545514999,20],[1097719.730372565,14],[1097719.909869507,14],[1097720.140149002,13],[1097720.320046364,5],[1097720.513533336,22],[1097720.741669473,3],[1097720.973207934,32],[1097721.156375642,33],[1097721.338959361,11],[1097721.529808096,6],[1097721.70259828,8],[1097721.84785631,9],[1097721.996055134,13],[1097722.144412907,8],[1097722.293557282,20],[1097722.449952341,17],[1097722.635722562,20],[1097722.784644563,13],[1097722.958045706,6],[1097723.140260594,13],[1097723.286309465,9],[1097723.477857469,11],[1097723.623867927,8],[1097723.808922897,15],[1097723.991120551,16],[1097724.199723556,27],[1097724.448159078,4],[1097724.594875641,8],[1097724.779059179,25],[1097724.961400777,18],[1097725.1106188,13],[1097725.292307898,55],[1097725.449780161,11],[1097725.631036624,10],[1097725.817699071,11],[1097725.965295886,15],[1097726.113663709,17],[1097726.261987579,14],[1097726.5251952,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-7c52219dab4ed8ded31547f4bf79b021","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.127628034,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":15.368518941,"time_to_first_token":0.254312562,"token_sha":"8a91591c94a3","tokens_per_second":16.592360069239543},"seed":1234,"sent":1097710.899082539,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.25603581103496253,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.932432435979045,"done":1097726.523941845,"finish_reason":"length","first":1097712.30386344,"full_response_s":15.622387105831876,"last":1097726.523908652,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097712.30386344,3],[1097713.072163398,11],[1097713.252079179,30],[1097713.439390713,19],[1097713.619110067,21],[1097713.800738447,15],[1097713.945296748,19],[1097714.093909505,17],[1097714.273860827,12],[1097714.461467251,5],[1097714.610151712,5],[1097714.791356513,27],[1097714.971405384,12],[1097715.155081927,13],[1097715.332213824,5],[1097715.487188878,16],[1097715.668650978,5],[1097715.849012634,14],[1097716.031064838,9],[1097716.212551452,27],[1097716.362274774,11],[1097716.550708855,8],[1097716.695673058,16],[1097716.843312573,5],[1097716.99080643,9],[1097717.174999098,14],[1097717.320266718,20],[1097717.507371751,12],[1097717.655535212,8],[1097717.803671739,5],[1097717.949467751,14],[1097718.130914821,14],[1097718.311535629,8],[1097718.49995666,13],[1097718.671040431,3],[1097718.848867088,10],[1097718.99559684,11],[1097719.178974147,9],[1097719.36262864,10],[1097719.546677481,20],[1097719.729904274,14],[1097719.909223421,14],[1097720.13897154,13],[1097720.32156482,5],[1097720.513383319,22],[1097720.742106394,3],[1097720.97390548,32],[1097721.157145722,33],[1097721.339452931,11],[1097721.53146889,6],[1097721.704487886,8],[1097721.848192784,9],[1097721.996935036,13],[1097722.14578375,8],[1097722.29460161,20],[1097722.450195145,17],[1097722.635244126,20],[1097722.782795669,13],[1097722.955651057,6],[1097723.139183287,13],[1097723.287250031,9],[1097723.478408372,11],[1097723.623774466,8],[1097723.808247102,15],[1097723.99124453,16],[1097724.200787024,27],[1097724.446240231,4],[1097724.594967004,8],[1097724.780170362,25],[1097724.962662801,18],[1097725.114671658,13],[1097725.295643065,55],[1097725.448261358,11],[1097725.630611995,10],[1097725.816738391,11],[1097725.964400374,15],[1097726.114389354,17],[1097726.262850011,14],[1097726.523908652,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-9a023ff06172749ee5baf7a21c790826","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.12440533,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.220137228,"time_to_first_token":1.40003282,"token_sha":"8a91591c94a3","tokens_per_second":17.932316398318235},"seed":1234,"sent":1097710.901554739,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.4023087008390576,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.020212946145758,"done":1097726.52618595,"finish_reason":"length","first":1097711.543969187,"full_response_s":15.62654502899386,"last":1097726.526155415,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.543969187,3],[1097713.072882381,11],[1097713.252312083,30],[1097713.438954801,19],[1097713.620409044,21],[1097713.802348965,15],[1097713.94593512,19],[1097714.092545428,17],[1097714.274130495,12],[1097714.462902728,5],[1097714.609708212,5],[1097714.791451851,27],[1097714.971744528,12],[1097715.1543789,13],[1097715.333641927,5],[1097715.487814918,16],[1097715.668833199,5],[1097715.850426607,14],[1097716.031218893,9],[1097716.212210501,27],[1097716.362444489,11],[1097716.549141836,8],[1097716.695524863,16],[1097716.8439363,5],[1097716.991403449,9],[1097717.174072467,14],[1097717.320604562,20],[1097717.506847028,12],[1097717.656912397,8],[1097717.802843759,5],[1097717.950530097,14],[1097718.129503776,14],[1097718.313522136,8],[1097718.498395148,13],[1097718.670754308,3],[1097718.849439666,10],[1097718.998306222,11],[1097719.179721177,9],[1097719.360199263,10],[1097719.545808022,20],[1097719.731525326,14],[1097719.910131753,14],[1097720.140712835,13],[1097720.320427771,5],[1097720.512695889,22],[1097720.742287506,3],[1097720.973516773,32],[1097721.156454549,33],[1097721.339121402,11],[1097721.53024407,6],[1097721.702361207,8],[1097721.847155534,9],[1097721.996123649,13],[1097722.144810243,8],[1097722.294714286,20],[1097722.449529947,17],[1097722.636143787,20],[1097722.783334501,13],[1097722.954826176,6],[1097723.139445817,13],[1097723.286639388,9],[1097723.478492322,11],[1097723.624107464,8],[1097723.808286795,15],[1097723.990352003,16],[1097724.199472828,27],[1097724.447304884,4],[1097724.595879338,8],[1097724.780560536,25],[1097724.961936031,18],[1097725.110707809,13],[1097725.294626644,55],[1097725.44873612,11],[1097725.632181154,10],[1097725.815330732,11],[1097725.963881022,15],[1097726.112929769,17],[1097726.262400053,14],[1097726.526155415,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-4154f284a1599691d85d92cb60b96a9f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126178427,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.979779396,"time_to_first_token":0.640806235,"token_sha":"8a91591c94a3","tokens_per_second":17.022947618847564},"seed":1234,"sent":1097710.899640921,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.6443282659165561,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"B","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":17.464392710257048,"done":1097726.525964781,"finish_reason":"length","first":1097711.924790944,"full_response_s":15.627354186959565,"last":1097726.525928483,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097711.924790944,3],[1097713.074842279,11],[1097713.253170216,30],[1097713.438380476,19],[1097713.619531408,21],[1097713.801680406,15],[1097713.946390613,19],[1097714.095382098,17],[1097714.275563297,12],[1097714.4628422,5],[1097714.609423205,5],[1097714.792129646,27],[1097714.970846853,12],[1097715.154997829,13],[1097715.331585799,5],[1097715.486907764,16],[1097715.670146109,5],[1097715.851551781,14],[1097716.030601039,9],[1097716.214117959,27],[1097716.361616393,11],[1097716.550237739,8],[1097716.698414137,16],[1097716.843754192,5],[1097716.992941798,9],[1097717.173193296,14],[1097717.318755862,20],[1097717.507128733,12],[1097717.655904807,8],[1097717.801608026,5],[1097717.949145425,14],[1097718.130175281,14],[1097718.31517354,8],[1097718.500741563,13],[1097718.669330214,3],[1097718.84787488,10],[1097718.996245572,11],[1097719.179250437,9],[1097719.361407171,10],[1097719.546133708,20],[1097719.727804154,14],[1097719.910805252,14],[1097720.139522082,13],[1097720.320159329,5],[1097720.51139025,22],[1097720.743474496,3],[1097720.974495233,32],[1097721.157578494,33],[1097721.339840549,11],[1097721.530296173,6],[1097721.70340879,8],[1097721.848992899,9],[1097721.996744494,13],[1097722.145138743,8],[1097722.295365137,20],[1097722.451179317,17],[1097722.633704966,20],[1097722.783071474,13],[1097722.955059981,6],[1097723.139356423,13],[1097723.287193606,9],[1097723.479570702,11],[1097723.624530208,8],[1097723.807447295,15],[1097723.991546512,16],[1097724.199813029,27],[1097724.447575082,4],[1097724.595399081,8],[1097724.779664263,25],[1097724.962148506,18],[1097725.110881263,13],[1097725.294819905,55],[1097725.447518401,11],[1097725.630810635,10],[1097725.818217878,11],[1097725.963562103,15],[1097726.114703662,17],[1097726.262214095,14],[1097726.525928483,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-f4f597b3a0d328b41f6e6ac2107ba789","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.126200489,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":14.599546396,"time_to_first_token":1.020403592,"token_sha":"8a91591c94a3","tokens_per_second":17.466296080942982},"seed":1234,"sent":1097710.898610594,"speculative":{"acceptance_rate":0.4031890660592255,"accepted":177,"drafted":439,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":1.0261803499888629,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":177,"drafted":439,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"B","case":"long","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":20.328164976244086,"done":1097731.213702258,"finish_reason":"length","first":1097729.688645169,"full_response_s":4.637223938014358,"last":1097731.213622954,"output_sha256":"5d2c18c1a3da0aab39441e582248a278a66d8473e846a8d2c48dfda87c21e891","pieces":[[1097729.688645169,3],[1097729.773731099,14],[1097729.857155056,11],[1097729.941119869,11],[1097730.02484408,13],[1097730.109783372,12],[1097730.193022008,12],[1097730.27760886,12],[1097730.362426883,12],[1097730.4529688,11],[1097730.538592959,12],[1097730.621806522,4],[1097730.705938349,12],[1097730.789416386,12],[1097730.872389383,9],[1097730.95566072,1],[1097731.039537265,7],[1097731.123042974,4],[1097731.213622954,17]],"prompt":"long_initial","prompt_tokens":4419,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"df5eb0c68030e305529b5525635e0602046d98ecf7a739e2e42243ec364b4adc","request_path":"/v1/chat/completions","response_id":"chatcmpl-a83a60acf356518285f4437c22d0a235","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":3.101604316,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":1.525631786,"time_to_first_token":3.111246285,"token_sha":"8b52dc1b081a","tokens_per_second":20.31945079046747},"seed":1234,"sent":1097726.57647832,"speculative":{"acceptance_rate":0.06015037593984962,"accepted":16,"drafted":266,"rounds":18,"tokens_per_round":1.7777777777777777},"token_sha":"8b52dc1b081a","tokens":32,"ttft_s":3.112166848964989,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4419,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":4451},"work":{"accepted":16,"drafted":266,"rounds":18},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"B","case":"long","request":{"cache_state":"reused","cached_tokens":4096,"case_id":null,"complete":true,"decode_tps":52.30957541802946,"done":1097732.166802882,"finish_reason":"length","first":1097731.574127139,"full_response_s":0.9438277550507337,"last":1097732.166752863,"output_sha256":"dd1aacffd9ef2ff0a3531f264db2a68aa99f695724c7828277affb8cb3535eb1","pieces":[[1097731.574127139,3],[1097731.6593033,34],[1097731.743551513,47],[1097731.827284156,8],[1097731.911077705,30],[1097731.993671522,18],[1097732.076370725,13],[1097732.166752863,12]],"prompt":"long_followup","prompt_tokens":4445,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"},{"content":"Acknowledged.","role":"assistant"},{"content":"Summarize the document in one sentence.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"0f7b9a7949106fe7d7c1dee63210f9621564bc3cdf8167180dc2d64159ad2c22","request_path":"/v1/chat/completions","response_id":"chatcmpl-5a2f70a869d9384d457acf90b7247010","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.335502866,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":0.593396121,"time_to_first_token":0.349722178,"token_sha":"526434da61a6","tokens_per_second":52.24166269870173},"seed":1234,"sent":1097731.222975127,"speculative":{"acceptance_rate":0.22857142857142856,"accepted":24,"drafted":105,"rounds":7,"tokens_per_round":4.571428571428571},"token_sha":"526434da61a6","tokens":32,"ttft_s":0.35115201212465763,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4445,"prompt_tokens_details":{"cached_tokens":4096},"total_tokens":4477},"work":{"accepted":24,"drafted":105,"rounds":7},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"followup","stream_index":0} +{"arm":"C","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":69.41609400179837,"done":1097876.408741989,"finish_reason":"length","first":1097872.735193433,"full_response_s":3.8164350448641926,"last":1097876.408693126,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097872.735193433,2],[1097872.82294078,7],[1097872.903365594,4],[1097872.983631158,73],[1097873.064762049,51],[1097873.145067162,21],[1097873.225286792,14],[1097873.306215572,20],[1097873.387920355,16],[1097873.468488878,15],[1097873.554276763,23],[1097873.638400159,23],[1097873.718871228,25],[1097873.799165761,36],[1097873.880213483,14],[1097873.960774477,6],[1097874.041756449,18],[1097874.122249549,18],[1097874.202606159,19],[1097874.282963729,8],[1097874.365597112,11],[1097874.4462183,18],[1097874.531655661,15],[1097874.618065965,39],[1097874.700287732,30],[1097874.780696246,14],[1097874.861943218,15],[1097874.942510639,8],[1097875.022879758,6],[1097875.10406926,15],[1097875.184946809,23],[1097875.265311447,22],[1097875.346298582,26],[1097875.427081206,35],[1097875.509654241,31],[1097875.596021941,12],[1097875.67833243,13],[1097875.758948298,38],[1097875.839915847,47],[1097875.920749057,25],[1097876.001424793,11],[1097876.082542411,34],[1097876.163202703,12],[1097876.24373217,36],[1097876.324941143,60],[1097876.408693126,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-b0287a5c83b80d2b2d927bc185151edc","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.127291119,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.674044713,"time_to_first_token":0.128154431,"token_sha":"2b28aa2108de","tokens_per_second":69.40579658644998},"seed":1234,"sent":1097872.592306944,"speculative":{"acceptance_rate":0.31555555555555553,"accepted":213,"drafted":675,"rounds":45,"tokens_per_round":5.688888888888889},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.14288648893125355,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":213,"drafted":675,"rounds":45},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":69.31634116319097,"done":1097880.097646699,"finish_reason":"length","first":1097876.41882006,"full_response_s":3.6882623040582985,"last":1097880.09760627,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097876.41882006,2],[1097876.507404414,7],[1097876.594508119,4],[1097876.677388537,73],[1097876.758276819,51],[1097876.839307871,21],[1097876.919871264,14],[1097876.999790844,20],[1097877.081290445,16],[1097877.161405525,15],[1097877.242198637,23],[1097877.323617797,23],[1097877.403828938,25],[1097877.484041056,36],[1097877.570416539,14],[1097877.654123394,6],[1097877.734349468,18],[1097877.815403598,18],[1097877.896416213,19],[1097877.976468214,8],[1097878.057552003,11],[1097878.138518492,18],[1097878.218718731,15],[1097878.299399228,39],[1097878.381226012,30],[1097878.461981989,14],[1097878.548797355,15],[1097878.633655722,8],[1097878.714146882,6],[1097878.795177374,15],[1097878.875898062,23],[1097878.956919412,22],[1097879.037351415,26],[1097879.119043194,35],[1097879.19978966,31],[1097879.279874165,12],[1097879.361562135,13],[1097879.441930822,38],[1097879.525790301,47],[1097879.611540498,25],[1097879.692224546,11],[1097879.772759236,34],[1097879.854020997,12],[1097879.934463898,36],[1097880.014914625,60],[1097880.09760627,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-c0a93ce7cfcfd098c46df916c802643b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.007574626,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.679357604,"time_to_first_token":0.007868429,"token_sha":"2b28aa2108de","tokens_per_second":69.30557652857055},"seed":1234,"sent":1097876.409384395,"speculative":{"acceptance_rate":0.31555555555555553,"accepted":213,"drafted":675,"rounds":45,"tokens_per_round":5.688888888888889},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.009435665095224977,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":213,"drafted":675,"rounds":45},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.198424387438234,"done":1097890.295665578,"finish_reason":"length","first":1097880.175943157,"full_response_s":10.19390987395309,"last":1097890.295623502,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.175943157,2],[1097880.491750088,7],[1097880.648324934,4],[1097880.85650777,63],[1097881.082876517,10],[1097881.264686466,31],[1097881.449814024,29],[1097881.6377459,22],[1097881.817788859,14],[1097881.99936671,24],[1097882.179082578,4],[1097882.326634822,13],[1097882.509346133,21],[1097882.698125522,2],[1097882.878850558,18],[1097883.06196953,29],[1097883.296178184,27],[1097883.532016798,24],[1097883.686944834,4],[1097883.831477657,16],[1097884.070761484,10],[1097884.252388033,31],[1097884.400018519,8],[1097884.591084101,11],[1097884.774142276,16],[1097884.957675882,12],[1097885.138744371,39],[1097885.311075746,18],[1097885.493765771,11],[1097885.684995125,16],[1097885.864067943,16],[1097886.011757726,11],[1097886.194521063,6],[1097886.376232714,9],[1097886.564969587,29],[1097886.751093722,22],[1097886.932668368,8],[1097887.079948938,18],[1097887.261918982,18],[1097887.446852038,34],[1097887.634027481,10],[1097887.81660323,14],[1097887.998945973,13],[1097888.186229438,2],[1097888.365490193,36],[1097888.554359834,42],[1097888.741478643,26],[1097888.890377288,17],[1097889.072787874,34],[1097889.30949471,12],[1097889.492464531,36],[1097889.785107598,50],[1097889.974378731,2],[1097890.295623502,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-8b66f4aa8fdfa8b17720e38e29dc3ff3","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010168745,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.120223131,"time_to_first_token":0.069201894,"token_sha":"2b28aa2108de","tokens_per_second":25.19707290038801},"seed":1234,"sent":1097880.101755704,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.07418745290488005,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.40919069338064,"done":1097890.294792734,"finish_reason":"length","first":1097880.259012117,"full_response_s":10.192669447045773,"last":1097890.294750881,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.259012117,2],[1097880.492553195,7],[1097880.64970889,4],[1097880.854529013,63],[1097881.083105514,10],[1097881.265476273,31],[1097881.446312328,29],[1097881.636072163,22],[1097881.816862709,14],[1097881.999787844,24],[1097882.178535485,4],[1097882.326788613,13],[1097882.511130589,21],[1097882.697410267,2],[1097882.878545955,18],[1097883.061174957,29],[1097883.294224481,27],[1097883.532633206,24],[1097883.686833345,4],[1097883.832735023,16],[1097884.07432035,10],[1097884.251954593,31],[1097884.40266169,8],[1097884.592041681,11],[1097884.773555429,16],[1097884.956277403,12],[1097885.139714994,39],[1097885.310111444,18],[1097885.491895441,11],[1097885.684440113,16],[1097885.864603428,16],[1097886.010224266,11],[1097886.193864021,6],[1097886.376868517,9],[1097886.566283062,29],[1097886.751915164,22],[1097886.934622045,8],[1097887.079112421,18],[1097887.262902614,18],[1097887.446732787,34],[1097887.635225592,10],[1097887.816994049,14],[1097887.999826678,13],[1097888.184115909,2],[1097888.366992696,36],[1097888.554973629,42],[1097888.741590948,26],[1097888.889497912,17],[1097889.073450499,34],[1097889.308614054,12],[1097889.491820695,36],[1097889.786413225,50],[1097889.971724331,2],[1097890.294750881,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-e7b22912b174d621f05a658d5f957618","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010132798,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.03671359,"time_to_first_token":0.149491901,"token_sha":"2b28aa2108de","tokens_per_second":25.406722799589225},"seed":1234,"sent":1097880.102123287,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.15688883000984788,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.36890911490009,"done":1097890.294487733,"finish_reason":"length","first":1097880.242788866,"full_response_s":10.189441511873156,"last":1097890.294462702,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.242788866,2],[1097880.491023083,7],[1097880.649095232,4],[1097880.856353307,63],[1097881.083823234,10],[1097881.266704483,31],[1097881.449337001,29],[1097881.638045959,22],[1097881.820172743,14],[1097881.999096981,24],[1097882.178795323,4],[1097882.326246178,13],[1097882.509524315,21],[1097882.698742795,2],[1097882.877320228,18],[1097883.061539033,29],[1097883.296088429,27],[1097883.531552579,24],[1097883.686052008,4],[1097883.831577111,16],[1097884.074151388,10],[1097884.25308845,31],[1097884.401122608,8],[1097884.589919866,11],[1097884.774599148,16],[1097884.955451191,12],[1097885.138487479,39],[1097885.311989712,18],[1097885.491165001,11],[1097885.682637819,16],[1097885.864348377,16],[1097886.012281497,11],[1097886.195640161,6],[1097886.378245611,9],[1097886.565207161,29],[1097886.751582346,22],[1097886.932458931,8],[1097887.080777389,18],[1097887.262808186,18],[1097887.444707617,34],[1097887.632790302,10],[1097887.816536027,14],[1097887.99919771,13],[1097888.185981174,2],[1097888.366030431,36],[1097888.555611927,42],[1097888.742722062,26],[1097888.889354623,17],[1097889.073523064,34],[1097889.308055588,12],[1097889.492768315,36],[1097889.784488362,50],[1097889.97377803,2],[1097890.294462702,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-4cc2a63bb436cce75af2dde1e8c6497d","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010120715,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.053022391,"time_to_first_token":0.134267656,"token_sha":"2b28aa2108de","tokens_per_second":25.36550602217792},"seed":1234,"sent":1097880.105046221,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.13774264487437904,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.44786183525434,"done":1097890.296686626,"finish_reason":"length","first":1097880.276168723,"full_response_s":10.19541926193051,"last":1097890.296656953,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.276168723,2],[1097880.492641766,7],[1097880.649924075,4],[1097880.855841121,63],[1097881.084626869,10],[1097881.263702058,31],[1097881.446448353,29],[1097881.638491133,22],[1097881.816767609,14],[1097881.999134736,24],[1097882.181644659,4],[1097882.326938113,13],[1097882.510696203,21],[1097882.696183403,2],[1097882.877473122,18],[1097883.061928654,29],[1097883.29390224,27],[1097883.532141603,24],[1097883.685240805,4],[1097883.831892196,16],[1097884.070290528,10],[1097884.253458928,31],[1097884.40176896,8],[1097884.590499111,11],[1097884.773051328,16],[1097884.955659124,12],[1097885.139625768,39],[1097885.311123776,18],[1097885.493581973,11],[1097885.68181079,16],[1097885.863769455,16],[1097886.010661499,11],[1097886.193612795,6],[1097886.378128184,9],[1097886.565331982,29],[1097886.75352996,22],[1097886.933664485,8],[1097887.080646055,18],[1097887.263297018,18],[1097887.445715849,34],[1097887.635034464,10],[1097887.815155287,14],[1097888.000586884,13],[1097888.184767251,2],[1097888.367650792,36],[1097888.554593358,42],[1097888.744083708,26],[1097888.890612172,17],[1097889.07551854,34],[1097889.311102137,12],[1097889.49189675,36],[1097889.786520024,50],[1097889.973363349,2],[1097890.296656953,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-a242aaa778b4014279e20749fef566f4","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010049301,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.019326552,"time_to_first_token":0.166768463,"token_sha":"2b28aa2108de","tokens_per_second":25.450812355157577},"seed":1234,"sent":1097880.101267364,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.1749013587832451,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.238247837212253,"done":1097890.296076401,"finish_reason":"length","first":1097880.192324568,"full_response_s":10.193436877802014,"last":1097890.296037062,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.192324568,2],[1097880.490891043,7],[1097880.647755146,4],[1097880.855211891,63],[1097881.08457202,10],[1097881.264580867,31],[1097881.447160184,29],[1097881.638309638,22],[1097881.818037782,14],[1097881.998376461,24],[1097882.180887568,4],[1097882.325730567,13],[1097882.509971809,21],[1097882.697892732,2],[1097882.877755847,18],[1097883.061047014,29],[1097883.295982878,27],[1097883.53193887,24],[1097883.684548983,4],[1097883.832272341,16],[1097884.07158272,10],[1097884.253291422,31],[1097884.399486889,8],[1097884.590989913,11],[1097884.772278794,16],[1097884.955719638,12],[1097885.13913892,39],[1097885.310495062,18],[1097885.491373079,11],[1097885.683047764,16],[1097885.865384859,16],[1097886.00970056,11],[1097886.19484876,6],[1097886.376145616,9],[1097886.564505546,29],[1097886.752428628,22],[1097886.933979633,8],[1097887.080040901,18],[1097887.262292403,18],[1097887.444100079,34],[1097887.6364798,10],[1097887.815775933,14],[1097887.999599284,13],[1097888.182601259,2],[1097888.367449917,36],[1097888.554462071,42],[1097888.74071594,26],[1097888.890017792,17],[1097889.074662819,34],[1097889.311665805,12],[1097889.492260119,36],[1097889.786588508,50],[1097889.969513915,2],[1097890.296037062,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-2a1aa640304ec52bd51af85face77c6e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010146434,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.103057553,"time_to_first_token":0.086257414,"token_sha":"2b28aa2108de","tokens_per_second":25.239883932392363},"seed":1234,"sent":1097880.102639523,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.08968504494987428,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.281031117457335,"done":1097890.294936455,"finish_reason":"length","first":1097880.20830677,"full_response_s":10.193071230081841,"last":1097890.294920675,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.20830677,2],[1097880.491193519,7],[1097880.648544201,4],[1097880.854462801,63],[1097881.083208121,10],[1097881.266263038,31],[1097881.447739543,29],[1097881.63935701,22],[1097881.81853121,14],[1097881.999314326,24],[1097882.180127195,4],[1097882.325581514,13],[1097882.508496566,21],[1097882.697045198,2],[1097882.87939232,18],[1097883.060464664,29],[1097883.294267181,27],[1097883.532067293,24],[1097883.687421009,4],[1097883.83167594,16],[1097884.07338548,10],[1097884.251897343,31],[1097884.400703395,8],[1097884.591702765,11],[1097884.773910655,16],[1097884.957494996,12],[1097885.140405343,39],[1097885.312487604,18],[1097885.492608693,11],[1097885.681185341,16],[1097885.864278309,16],[1097886.009907261,11],[1097886.193307025,6],[1097886.376702964,9],[1097886.566217507,29],[1097886.750817337,22],[1097886.931546487,8],[1097887.080393789,18],[1097887.262982846,18],[1097887.446161003,34],[1097887.635756484,10],[1097887.81787661,14],[1097887.999464364,13],[1097888.187148459,2],[1097888.367175277,36],[1097888.555261825,42],[1097888.742989756,26],[1097888.889425875,17],[1097889.074042799,34],[1097889.309092369,12],[1097889.492929914,36],[1097889.785855271,50],[1097889.974508081,2],[1097890.294920675,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-b5906e1314c4283fc201c527ebbc92a9","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010362595,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.086503316,"time_to_first_token":0.101817457,"token_sha":"2b28aa2108de","tokens_per_second":25.281308299923825},"seed":1234,"sent":1097880.101865225,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.10644154506735504,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.664045185380974,"done":1097890.29645259,"finish_reason":"length","first":1097880.36035663,"full_response_s":10.190275659086183,"last":1097890.296436393,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.36035663,2],[1097880.492372628,7],[1097880.649147744,4],[1097880.85832054,63],[1097881.084801145,10],[1097881.266824583,31],[1097881.446096967,29],[1097881.637122832,22],[1097881.817124868,14],[1097881.999968474,24],[1097882.179305589,4],[1097882.326704859,13],[1097882.50989425,21],[1097882.696329351,2],[1097882.87903266,18],[1097883.060104525,29],[1097883.293972869,27],[1097883.532503711,24],[1097883.684843871,4],[1097883.831849608,16],[1097884.071066567,10],[1097884.253693702,31],[1097884.399400576,8],[1097884.591315209,11],[1097884.774484922,16],[1097884.957026937,12],[1097885.140105222,39],[1097885.311221805,18],[1097885.491483912,11],[1097885.682819409,16],[1097885.864673209,16],[1097886.0120755,11],[1097886.194274158,6],[1097886.377913177,9],[1097886.56568727,29],[1097886.75032476,22],[1097886.933463962,8],[1097887.07965394,18],[1097887.262391648,18],[1097887.446405091,34],[1097887.636209033,10],[1097887.815736497,14],[1097888.000275481,13],[1097888.18317341,2],[1097888.368176257,36],[1097888.556296015,42],[1097888.744750383,26],[1097888.890258854,17],[1097889.075157683,34],[1097889.308504582,12],[1097889.492816617,36],[1097889.786728965,50],[1097889.969249839,2],[1097890.296436393,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-f61c24ce815b100b4281639ba9455093","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010117082,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.936786779,"time_to_first_token":0.248974042,"token_sha":"2b28aa2108de","tokens_per_second":25.662219153067326},"seed":1234,"sent":1097880.106176931,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.2541796991135925,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.494811268373255,"done":1097890.294566876,"finish_reason":"length","first":1097880.292518109,"full_response_s":10.189626944018528,"last":1097890.29455332,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.292518109,2],[1097880.492116455,7],[1097880.649832991,4],[1097880.855956019,63],[1097881.084185648,10],[1097881.266056287,31],[1097881.448448909,29],[1097881.639180205,22],[1097881.81810593,14],[1097881.999628132,24],[1097882.179580055,4],[1097882.326089043,13],[1097882.509430319,21],[1097882.699051576,2],[1097882.878486577,18],[1097883.061126462,29],[1097883.293802979,27],[1097883.533679518,24],[1097883.686510241,4],[1097883.831745929,16],[1097884.070925421,10],[1097884.252663139,31],[1097884.400229158,8],[1097884.592270037,11],[1097884.774673842,16],[1097884.955911759,12],[1097885.141110976,39],[1097885.312149584,18],[1097885.493299713,11],[1097885.684113184,16],[1097885.863882096,16],[1097886.0112764,11],[1097886.194659056,6],[1097886.375913116,9],[1097886.565250262,29],[1097886.753784531,22],[1097886.932293794,8],[1097887.080738034,18],[1097887.26210438,18],[1097887.445026319,34],[1097887.636095656,10],[1097887.81627307,14],[1097888.000820664,13],[1097888.18576071,2],[1097888.367847361,36],[1097888.55611473,42],[1097888.743608513,26],[1097888.889262788,17],[1097889.072954963,34],[1097889.309147777,12],[1097889.493163038,36],[1097889.785263771,50],[1097889.969433012,2],[1097890.29455332,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-6b4ed7adecff314357332761175990a8","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010059912,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.002625699,"time_to_first_token":0.182894837,"token_sha":"2b28aa2108de","tokens_per_second":25.4933062251338},"seed":1234,"sent":1097880.104939932,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.18757817707955837,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.071881899552263,"done":1097890.295576368,"finish_reason":"length","first":1097880.124771685,"full_response_s":10.192533409921452,"last":1097890.295527954,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.124771685,2],[1097880.490773056,7],[1097880.649387224,4],[1097880.856078472,63],[1097881.084680837,10],[1097881.267427821,31],[1097881.446990645,29],[1097881.635731757,22],[1097881.81993656,14],[1097881.998822355,24],[1097882.178663652,4],[1097882.326844069,13],[1097882.509036104,21],[1097882.697705476,2],[1097882.879283231,18],[1097883.06027027,29],[1097883.295275611,27],[1097883.533071801,24],[1097883.684391543,4],[1097883.833136855,16],[1097884.074992966,10],[1097884.253226555,31],[1097884.40309305,8],[1097884.591922766,11],[1097884.772446572,16],[1097884.956533783,12],[1097885.138231867,39],[1097885.310845549,18],[1097885.491251955,11],[1097885.685279097,16],[1097885.865862487,16],[1097886.010275914,11],[1097886.195916658,6],[1097886.375691786,9],[1097886.564677165,29],[1097886.750527043,22],[1097886.933720837,8],[1097887.082104179,18],[1097887.262588378,18],[1097887.44535864,34],[1097887.636343072,10],[1097887.817080842,14],[1097887.99952531,13],[1097888.183527641,2],[1097888.3652132,36],[1097888.554672965,42],[1097888.744437651,26],[1097888.890310645,17],[1097889.073224417,34],[1097889.309853214,12],[1097889.492395775,36],[1097889.785700522,50],[1097889.970354847,2],[1097890.295527954,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-07b3430a3440741197e20a1392ccb36f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010557999,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.170289141,"time_to_first_token":0.019685532,"token_sha":"2b28aa2108de","tokens_per_second":25.07303346686631},"seed":1234,"sent":1097880.103042958,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.021728727035224438,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.153608872323836,"done":1097890.296352145,"finish_reason":"length","first":1097880.158595213,"full_response_s":10.193825216032565,"last":1097890.296305523,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.158595213,2],[1097880.491950804,7],[1097880.648691334,4],[1097880.85662763,63],[1097881.084943444,10],[1097881.267588622,31],[1097881.44661085,29],[1097881.637460214,22],[1097881.817397622,14],[1097881.999584167,24],[1097882.178939767,4],[1097882.325801036,13],[1097882.508959041,21],[1097882.697626972,2],[1097882.878227556,18],[1097883.062247869,29],[1097883.294111343,27],[1097883.533278727,24],[1097883.683778784,4],[1097883.832521488,16],[1097884.072727204,10],[1097884.251744258,31],[1097884.403203995,8],[1097884.590049569,11],[1097884.772805286,16],[1097884.956082882,12],[1097885.138156165,39],[1097885.31062995,18],[1097885.492356746,11],[1097885.681540246,16],[1097885.863371193,16],[1097886.00998682,11],[1097886.193997645,6],[1097886.376967378,9],[1097886.56427924,29],[1097886.750660811,22],[1097886.932555919,8],[1097887.080812216,18],[1097887.263689869,18],[1097887.445981287,34],[1097887.635522015,10],[1097887.816120561,14],[1097887.999738924,13],[1097888.182812729,2],[1097888.367885548,36],[1097888.555923394,42],[1097888.743258427,26],[1097888.890865414,17],[1097889.073674308,34],[1097889.312043819,12],[1097889.491591237,36],[1097889.786796873,50],[1097889.971397388,2],[1097890.296305523,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-6d6043ecf8bb9cfa827523c57166824b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010209477,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.136809261,"time_to_first_token":0.052025842,"token_sha":"2b28aa2108de","tokens_per_second":25.155844747032774},"seed":1234,"sent":1097880.102526929,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.05606828397139907,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.325310644236286,"done":1097890.294334729,"finish_reason":"length","first":1097880.225315044,"full_response_s":10.189524035900831,"last":1097890.294293213,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.225315044,2],[1097880.491457599,7],[1097880.648898836,4],[1097880.857875494,63],[1097881.084077503,10],[1097881.266190264,31],[1097881.449455276,29],[1097881.638883123,22],[1097881.817894043,14],[1097881.99828637,24],[1097882.179812669,4],[1097882.326518916,13],[1097882.509840074,21],[1097882.697986904,2],[1097882.877851427,18],[1097883.062173351,29],[1097883.295774368,27],[1097883.531821011,24],[1097883.682973456,4],[1097883.833451092,16],[1097884.072638794,10],[1097884.253535543,31],[1097884.399688677,8],[1097884.592761255,11],[1097884.772348911,16],[1097884.95758888,12],[1097885.140717947,39],[1097885.312338423,18],[1097885.493826621,11],[1097885.684244775,16],[1097885.865028002,16],[1097886.011207612,11],[1097886.196203943,6],[1097886.377536426,9],[1097886.564126907,29],[1097886.751412632,22],[1097886.93203327,8],[1097887.081824545,18],[1097887.261535749,18],[1097887.44557704,34],[1097887.63599385,10],[1097887.815644438,14],[1097887.999674314,13],[1097888.18667288,2],[1097888.367805045,36],[1097888.555065944,42],[1097888.740627338,26],[1097888.891108973,17],[1097889.075933445,34],[1097889.311788705,12],[1097889.491718458,36],[1097889.785420505,50],[1097889.972630929,2],[1097890.294293213,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-9d376799c8f1f3129512ef18f6d566dc","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010169145,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.069606441,"time_to_first_token":0.118012027,"token_sha":"2b28aa2108de","tokens_per_second":25.32373052453441},"seed":1234,"sent":1097880.104810693,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.12050435086712241,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.53577851889982,"done":1097890.295346638,"finish_reason":"length","first":1097880.309344343,"full_response_s":10.18966238386929,"last":1097890.29533321,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.309344343,2],[1097880.491890914,7],[1097880.648804408,4],[1097880.857477102,63],[1097881.08269555,10],[1097881.267212076,31],[1097881.447644122,29],[1097881.634683184,22],[1097881.817660036,14],[1097881.998183635,24],[1097882.180260867,4],[1097882.325869121,13],[1097882.510866574,21],[1097882.697182167,2],[1097882.878659653,18],[1097883.06056868,29],[1097883.295220826,27],[1097883.531631114,24],[1097883.683151989,4],[1097883.832102739,16],[1097884.071272836,10],[1097884.254205934,31],[1097884.401836356,8],[1097884.590443542,11],[1097884.772942735,16],[1097884.95720419,12],[1097885.13780987,39],[1097885.31150376,18],[1097885.493170011,11],[1097885.682438416,16],[1097885.864455113,16],[1097886.011643053,11],[1097886.196525942,6],[1097886.378589792,9],[1097886.565808266,29],[1097886.75292455,22],[1097886.9329952,8],[1097887.080500524,18],[1097887.261400894,18],[1097887.445938042,34],[1097887.635670795,10],[1097887.817224836,14],[1097888.00012859,13],[1097888.184702512,2],[1097888.366855488,36],[1097888.554503123,42],[1097888.741297678,26],[1097888.890161561,17],[1097889.073025655,34],[1097889.308908412,12],[1097889.492154969,36],[1097889.786027209,50],[1097889.972718603,2],[1097890.29533321,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-107b6851ff67c63e4ca9f3f346c46187","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010205316,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.986213155,"time_to_first_token":0.19925946,"token_sha":"2b28aa2108de","tokens_per_second":25.535204991325863},"seed":1234,"sent":1097880.105684254,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.20366008882410824,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.621224070078252,"done":1097890.298787634,"finish_reason":"length","first":1097880.34380011,"full_response_s":10.195643045939505,"last":1097890.296486184,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.34380011,2],[1097880.491540872,7],[1097880.64902369,4],[1097880.855740994,63],[1097881.084737926,10],[1097881.267290964,31],[1097881.446710961,29],[1097881.636645776,22],[1097881.819552365,14],[1097881.998633338,24],[1097882.179752139,4],[1097882.326145492,13],[1097882.509114608,21],[1097882.697936297,2],[1097882.877624191,18],[1097883.062015352,29],[1097883.295115051,27],[1097883.531709538,24],[1097883.684762438,4],[1097883.832668667,16],[1097884.073234187,10],[1097884.252784984,31],[1097884.401593964,8],[1097884.591173407,11],[1097884.774045063,16],[1097884.956924299,12],[1097885.137414681,39],[1097885.310715015,18],[1097885.492532494,11],[1097885.683591829,16],[1097885.863618226,16],[1097886.010845441,11],[1097886.194562515,6],[1097886.377774928,9],[1097886.565144182,29],[1097886.751847048,22],[1097886.93327269,8],[1097887.08015239,18],[1097887.261727709,18],[1097887.444314206,34],[1097887.634613141,10],[1097887.817647503,14],[1097887.999327268,13],[1097888.184939142,2],[1097888.3675136,36],[1097888.554769553,42],[1097888.74120133,26],[1097888.889713495,17],[1097889.073882016,34],[1097889.308134747,12],[1097889.493035001,36],[1097889.785952771,50],[1097889.969772406,2],[1097890.296486184,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-7977773843cd48171511144d2d43d474","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010703803,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.953264735,"time_to_first_token":0.232788562,"token_sha":"2b28aa2108de","tokens_per_second":25.61973450814679},"seed":1234,"sent":1097880.103144588,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.2406555220950395,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.57758465368759,"done":1097890.296221947,"finish_reason":"length","first":1097880.32653195,"full_response_s":10.193473383085802,"last":1097890.296198885,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.32653195,2],[1097880.491829872,7],[1097880.648094177,4],[1097880.855037486,63],[1097881.083696348,10],[1097881.264334984,31],[1097881.445961134,29],[1097881.638773922,22],[1097881.816511964,14],[1097881.999050295,24],[1097882.180505165,4],[1097882.325958812,13],[1097882.510417159,21],[1097882.698438656,2],[1097882.877983211,18],[1097883.060772676,29],[1097883.293675773,27],[1097883.532742182,24],[1097883.686286334,4],[1097883.83243042,16],[1097884.071424049,10],[1097884.254515946,31],[1097884.400592178,8],[1097884.592525601,11],[1097884.773442036,16],[1097884.95529945,12],[1097885.139398981,39],[1097885.309808394,18],[1097885.491740275,11],[1097885.682345732,16],[1097885.864174998,16],[1097886.011818064,11],[1097886.19442425,6],[1097886.378018568,9],[1097886.566381523,29],[1097886.751962538,22],[1097886.932913608,8],[1097887.081215438,18],[1097887.264069789,18],[1097887.446916089,34],[1097887.633588616,10],[1097887.817291543,14],[1097888.000524882,13],[1097888.185480452,2],[1097888.36624787,36],[1097888.555383733,42],[1097888.742089861,26],[1097888.889781099,17],[1097889.072586856,34],[1097889.31082737,12],[1097889.492862358,36],[1097889.786272736,50],[1097889.973997709,2],[1097890.296198885,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-2f9513ea6cd19e743b5ee44ca644797b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010068891,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.969105216,"time_to_first_token":0.216138557,"token_sha":"2b28aa2108de","tokens_per_second":25.579025847849973},"seed":1234,"sent":1097880.102748564,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.2237833859398961,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.034441262499524,"done":1097890.29418771,"finish_reason":"length","first":1097880.108173824,"full_response_s":10.192624288843945,"last":1097890.294141121,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.108173824,2],[1097880.49281473,7],[1097880.647877647,4],[1097880.854248976,63],[1097881.08252353,10],[1097881.263602956,31],[1097881.445730761,29],[1097881.635123573,22],[1097881.820505163,14],[1097881.999428456,24],[1097882.181226181,4],[1097882.32530055,13],[1097882.508623517,21],[1097882.696690291,2],[1097882.87713286,18],[1097883.06036169,29],[1097883.294613797,27],[1097883.533732142,24],[1097883.683424807,4],[1097883.831198422,16],[1097884.070085587,10],[1097884.251594709,31],[1097884.399290863,8],[1097884.590144093,11],[1097884.771707552,16],[1097884.955094669,12],[1097885.136800339,39],[1097885.31224558,18],[1097885.492734634,11],[1097885.682105213,16],[1097885.863119407,16],[1097886.009477998,11],[1097886.193175995,6],[1097886.375551281,9],[1097886.566459146,29],[1097886.750141073,22],[1097886.931810037,8],[1097887.079376546,18],[1097887.26113059,18],[1097887.443885312,34],[1097887.63251286,10],[1097887.818091937,14],[1097887.998782165,13],[1097888.182327563,2],[1097888.364973802,36],[1097888.554846824,42],[1097888.740473694,26],[1097888.890561374,17],[1097889.07229818,34],[1097889.307930655,12],[1097889.491183662,36],[1097889.784319817,50],[1097889.969095746,2],[1097890.294141121,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-17119ffb9dd1246f817c009b8bb981a6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.002274497,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.187175285,"time_to_first_token":0.002927943,"token_sha":"2b28aa2108de","tokens_per_second":25.03147269640801},"seed":1234,"sent":1097880.101563421,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.006610402837395668,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"C","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.11520591430807,"done":1097890.29512491,"finish_reason":"length","first":1097880.141897578,"full_response_s":10.19397415802814,"last":1097890.295109177,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097880.141897578,2],[1097880.491994034,7],[1097880.647600123,4],[1097880.856032986,63],[1097881.08451061,10],[1097881.265570381,31],[1097881.447071245,29],[1097881.638238817,22],[1097881.818799787,14],[1097881.998975024,24],[1097882.178874899,4],[1097882.326886786,13],[1097882.508733438,21],[1097882.697796751,2],[1097882.879656687,18],[1097883.061620402,29],[1097883.294474779,27],[1097883.53247263,24],[1097883.684206064,4],[1097883.831351828,16],[1097884.072272925,10],[1097884.253771789,31],[1097884.403448116,8],[1097884.590622427,11],[1097884.772236653,16],[1097884.956396335,12],[1097885.137077878,39],[1097885.310281782,18],[1097885.491326777,11],[1097885.684719779,16],[1097885.866172259,16],[1097886.010740082,11],[1097886.196073633,6],[1097886.377445183,9],[1097886.565537098,29],[1097886.753659102,22],[1097886.93439942,8],[1097887.082459259,18],[1097887.262245157,18],[1097887.444815425,34],[1097887.63316766,10],[1097887.815032484,14],[1097888.000699157,13],[1097888.185182429,2],[1097888.366530449,36],[1097888.555736652,42],[1097888.742459057,26],[1097888.890791889,17],[1097889.074939412,34],[1097889.309053846,12],[1097889.491285724,36],[1097889.786110417,50],[1097889.969588481,2],[1097890.295109177,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-163165ee89dccac2f51955d125c85692","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010442428,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.153569192,"time_to_first_token":0.036921026,"token_sha":"2b28aa2108de","tokens_per_second":25.11432139556547},"seed":1234,"sent":1097880.101150752,"speculative":{"acceptance_rate":0.5607734806629834,"accepted":203,"drafted":362,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.04074682598002255,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":362,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"C","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":49.50709200497177,"done":1097895.61436442,"finish_reason":"length","first":1097890.463506701,"full_response_s":5.31293655000627,"last":1097895.614283886,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097890.463506701,3],[1097890.556292788,26],[1097890.642120301,15],[1097890.722885101,34],[1097890.802986541,21],[1097890.8840884,25],[1097890.965629159,14],[1097891.047128738,14],[1097891.127573709,18],[1097891.207771312,26],[1097891.288540489,13],[1097891.369529841,13],[1097891.449920508,13],[1097891.53397644,23],[1097891.620443994,32],[1097891.701482656,7],[1097891.781568404,23],[1097891.862891737,9],[1097891.943191747,13],[1097892.023901778,15],[1097892.105067509,19],[1097892.185296922,12],[1097892.266114377,5],[1097892.347085171,23],[1097892.427579701,9],[1097892.5081229,17],[1097892.597117388,3],[1097892.679723708,21],[1097892.760325836,10],[1097892.841473937,24],[1097892.922441347,19],[1097893.003160718,14],[1097893.084222923,13],[1097893.164682892,5],[1097893.245016617,22],[1097893.326325773,13],[1097893.406644182,51],[1097893.487461933,15],[1097893.573170297,6],[1097893.656867638,8],[1097893.737920081,9],[1097893.818325481,16],[1097893.900111693,16],[1097893.979995652,9],[1097894.062199039,32],[1097894.142967682,18],[1097894.223506099,13],[1097894.30420477,11],[1097894.385493161,15],[1097894.466044286,8],[1097894.551526301,18],[1097894.630543262,25],[1097894.712952499,19],[1097894.793034114,8],[1097894.875219992,40],[1097894.956053063,16],[1097895.036495224,64],[1097895.118342487,6],[1097895.199063127,6],[1097895.279747964,11],[1097895.361469463,15],[1097895.442323197,17],[1097895.52319007,21],[1097895.614283886,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-4cae315592f8552dbd2ff62f46c6d602","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.160610278,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.1513917,"time_to_first_token":0.1611025,"token_sha":"8a91591c94a3","tokens_per_second":49.501186252251024},"seed":1234,"sent":1097890.30142787,"speculative":{"acceptance_rate":0.2052910052910053,"accepted":194,"drafted":945,"rounds":63,"tokens_per_round":4.063492063492063},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.16207883087918162,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":194,"drafted":945,"rounds":63},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":49.59872418581017,"done":1097900.769453034,"finish_reason":"length","first":1097895.6281561,"full_response_s":5.150651934091002,"last":1097900.769417376,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097895.6281561,3],[1097895.716362923,26],[1097895.797094398,15],[1097895.878493592,34],[1097895.959403464,21],[1097896.040223391,25],[1097896.121305552,14],[1097896.201570955,14],[1097896.282144973,18],[1097896.363172478,26],[1097896.443643651,13],[1097896.525030426,13],[1097896.612601199,13],[1097896.695794979,23],[1097896.776496041,32],[1097896.857607543,7],[1097896.93784087,23],[1097897.018322682,9],[1097897.100245666,13],[1097897.180527599,15],[1097897.26113455,19],[1097897.3421943,12],[1097897.422675008,5],[1097897.502904189,23],[1097897.590062611,9],[1097897.673244172,17],[1097897.753520055,3],[1097897.834447787,21],[1097897.914619238,10],[1097897.995617959,24],[1097898.076797131,19],[1097898.157648116,14],[1097898.238014229,13],[1097898.318516716,5],[1097898.400184009,22],[1097898.480536182,13],[1097898.567479396,51],[1097898.652097897,15],[1097898.732665458,6],[1097898.813301996,8],[1097898.89459955,9],[1097898.975261424,16],[1097899.056693767,16],[1097899.138033141,9],[1097899.218429981,32],[1097899.299218081,18],[1097899.380543051,13],[1097899.461149549,11],[1097899.545025036,15],[1097899.630526303,8],[1097899.712401266,18],[1097899.786432095,25],[1097899.867490281,19],[1097899.948149289,8],[1097900.029237198,40],[1097900.110117371,16],[1097900.191449732,64],[1097900.272187065,6],[1097900.35354068,6],[1097900.434299395,11],[1097900.515323495,15],[1097900.604175171,17],[1097900.687827855,21],[1097900.769417376,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-c4f253da8649e66730b47b2628789e21","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":0.008288922,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.141190761,"time_to_first_token":0.008792681,"token_sha":"8a91591c94a3","tokens_per_second":49.59940446761415},"seed":1234,"sent":1097895.6188011,"speculative":{"acceptance_rate":0.2052910052910053,"accepted":194,"drafted":945,"rounds":63,"tokens_per_round":4.063492063492063},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.009354999987408519,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":194,"drafted":945,"rounds":63},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.59766953691715,"done":1097914.504965374,"finish_reason":"length","first":1097900.792784822,"full_response_s":13.731071799062192,"last":1097914.504180193,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.792784822,3],[1097901.136173742,11],[1097901.320324034,30],[1097901.501131388,19],[1097901.687956401,21],[1097901.869692886,15],[1097902.015237896,19],[1097902.160310409,17],[1097902.344384301,12],[1097902.527838167,5],[1097902.682392258,5],[1097902.865656326,27],[1097903.048721981,12],[1097903.230359821,13],[1097903.411983594,5],[1097903.562555359,16],[1097903.747353119,5],[1097903.932995014,14],[1097904.112554716,9],[1097904.296088187,27],[1097904.4431767,11],[1097904.632744086,8],[1097904.780638498,16],[1097904.928722936,5],[1097905.079412446,9],[1097905.261665933,14],[1097905.409355382,20],[1097905.559776178,8],[1097905.71071145,12],[1097905.860766586,5],[1097906.00893449,14],[1097906.190950023,14],[1097906.337707337,8],[1097906.522021042,13],[1097906.702362396,3],[1097906.883144312,10],[1097907.02980348,11],[1097907.213954131,9],[1097907.395573845,10],[1097907.583337498,20],[1097907.767406927,14],[1097907.949479005,14],[1097908.179788149,13],[1097908.362175783,5],[1097908.548104956,22],[1097908.784241515,3],[1097909.01841666,32],[1097909.201880567,33],[1097909.389395137,11],[1097909.573308204,6],[1097909.747741641,8],[1097909.896401491,9],[1097910.049051689,13],[1097910.194747835,8],[1097910.346013553,20],[1097910.493268848,17],[1097910.689461528,20],[1097910.83326212,13],[1097911.007546777,6],[1097911.189836814,13],[1097911.340950677,9],[1097911.485400123,11],[1097911.642549292,8],[1097911.827062297,15],[1097912.00870097,16],[1097912.220015979,27],[1097912.46533179,4],[1097912.621456197,8],[1097912.806528957,25],[1097912.989339535,18],[1097913.144214644,13],[1097913.321534611,55],[1097913.470966646,11],[1097913.665207254,10],[1097913.848722739,11],[1097913.996762937,15],[1097914.147423371,17],[1097914.297676803,14],[1097914.504180193,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-1f906f7adff6dc511ebd9b523d574fc4","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008446531,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.71127685,"time_to_first_token":0.017745442,"token_sha":"8a91591c94a3","tokens_per_second":18.59783029616239},"seed":1234,"sent":1097900.773893575,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.018891246989369392,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.618625978454485,"done":1097914.50424597,"finish_reason":"length","first":1097900.808261892,"full_response_s":13.72970189503394,"last":1097914.50422422,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.808261892,3],[1097901.136066209,11],[1097901.319123991,30],[1097901.500425881,19],[1097901.688072225,21],[1097901.869776541,15],[1097902.014234725,19],[1097902.160152862,17],[1097902.344720377,12],[1097902.527544374,5],[1097902.68504377,5],[1097902.865316777,27],[1097903.047362889,12],[1097903.230135072,13],[1097903.411779202,5],[1097903.562758822,16],[1097903.748935968,5],[1097903.931119079,14],[1097904.113240198,9],[1097904.294516111,27],[1097904.441653517,11],[1097904.6344276,8],[1097904.780458945,16],[1097904.927244952,5],[1097905.079787908,9],[1097905.262137741,14],[1097905.40808707,20],[1097905.559258838,8],[1097905.711998215,12],[1097905.860404729,5],[1097906.009665151,14],[1097906.191039439,14],[1097906.340363302,8],[1097906.520673079,13],[1097906.703521893,3],[1097906.883703294,10],[1097907.030251009,11],[1097907.212862557,9],[1097907.398051759,10],[1097907.582875838,20],[1097907.767648016,14],[1097907.949816071,14],[1097908.178555755,13],[1097908.361609407,5],[1097908.547642016,22],[1097908.782471714,3],[1097909.020439583,32],[1097909.201950586,33],[1097909.38773884,11],[1097909.572811096,6],[1097909.7501716,8],[1097909.896967241,9],[1097910.045476709,13],[1097910.1967886,8],[1097910.344012959,20],[1097910.494158284,17],[1097910.689859361,20],[1097910.836466663,13],[1097911.007065866,6],[1097911.189933015,13],[1097911.341476191,9],[1097911.485679733,11],[1097911.642795646,8],[1097911.825895382,15],[1097912.008827259,16],[1097912.222293954,27],[1097912.463057031,4],[1097912.619869683,8],[1097912.805269505,25],[1097912.989444507,18],[1097913.139058045,13],[1097913.321474563,55],[1097913.473241258,11],[1097913.664475559,10],[1097913.850059488,11],[1097913.9968017,15],[1097914.146867994,17],[1097914.297161404,14],[1097914.50422422,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-fdd7a1ce258c6b0e6bbf2f4c05d0d2e9","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008338021,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.696437299,"time_to_first_token":0.031990971,"token_sha":"8a91591c94a3","tokens_per_second":18.61798031365558},"seed":1234,"sent":1097900.774544075,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.03371781716123223,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.741467063850862,"done":1097914.505221328,"finish_reason":"length","first":1097900.898997663,"full_response_s":13.73043774603866,"last":1097914.505189704,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.898997663,3],[1097901.137047663,11],[1097901.32111502,30],[1097901.49957291,19],[1097901.688594897,21],[1097901.867435434,15],[1097902.015426316,19],[1097902.159428759,17],[1097902.3436521,12],[1097902.527688701,5],[1097902.68273989,5],[1097902.865765652,27],[1097903.04758423,12],[1097903.230262419,13],[1097903.414037051,5],[1097903.562103843,16],[1097903.747625209,5],[1097903.932605964,14],[1097904.1122776,9],[1097904.294962713,27],[1097904.442333718,11],[1097904.631954159,8],[1097904.781239397,16],[1097904.927551963,5],[1097905.079201957,9],[1097905.261615343,14],[1097905.408680223,20],[1097905.560207415,8],[1097905.710265858,12],[1097905.859953199,5],[1097906.009209076,14],[1097906.190491435,14],[1097906.338056295,8],[1097906.521955921,13],[1097906.702521399,3],[1097906.882750174,10],[1097907.030315874,11],[1097907.213843045,9],[1097907.3976251,10],[1097907.584167466,20],[1097907.768935738,14],[1097907.949611968,14],[1097908.179685417,13],[1097908.363986283,5],[1097908.547811101,22],[1097908.782838292,3],[1097909.02029324,32],[1097909.202693888,33],[1097909.387406207,11],[1097909.573545163,6],[1097909.74915008,8],[1097909.89981779,9],[1097910.046788113,13],[1097910.196303751,8],[1097910.345428518,20],[1097910.493560958,17],[1097910.687152499,20],[1097910.835789668,13],[1097911.007828292,6],[1097911.191172078,13],[1097911.339437525,9],[1097911.487685719,11],[1097911.643749274,8],[1097911.82797769,15],[1097912.009618907,16],[1097912.221347641,27],[1097912.463655221,4],[1097912.620350194,8],[1097912.805966841,25],[1097912.990491133,18],[1097913.139849581,13],[1097913.321869323,55],[1097913.47106424,11],[1097913.663987191,10],[1097913.8495225,11],[1097913.99985981,15],[1097914.14859763,17],[1097914.297776221,14],[1097914.505189704,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-ecbf581680fc2c50903424532c8573b2","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008257061,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.606209595,"time_to_first_token":0.119632173,"token_sha":"8a91591c94a3","tokens_per_second":18.74144288455671},"seed":1234,"sent":1097900.774783582,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.12421408109366894,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.78203458320297,"done":1097914.504923683,"finish_reason":"length","first":1097900.928085426,"full_response_s":13.727121248841286,"last":1097914.504889306,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.928085426,3],[1097901.136409935,11],[1097901.318488616,30],[1097901.499507916,19],[1097901.687779921,21],[1097901.869651818,15],[1097902.013484695,19],[1097902.159545736,17],[1097902.343247717,12],[1097902.527234481,5],[1097902.683056473,5],[1097902.865388637,27],[1097903.047097489,12],[1097903.230717007,13],[1097903.411644669,5],[1097903.562621153,16],[1097903.748112254,5],[1097903.93031502,14],[1097904.111311241,9],[1097904.294281296,27],[1097904.442025667,11],[1097904.633678436,8],[1097904.779548025,16],[1097904.929466738,5],[1097905.077860249,9],[1097905.262045092,14],[1097905.408567808,20],[1097905.561307473,8],[1097905.713360152,12],[1097905.859678709,5],[1097906.008762811,14],[1097906.191507853,14],[1097906.339023853,8],[1097906.520337436,13],[1097906.703803313,3],[1097906.883407135,10],[1097907.029350319,11],[1097907.213088346,9],[1097907.397995728,10],[1097907.584582713,20],[1097907.770634931,14],[1097907.951903894,14],[1097908.179153227,13],[1097908.362874594,5],[1097908.548487746,22],[1097908.782653602,3],[1097909.020368828,32],[1097909.20330325,33],[1097909.388081795,11],[1097909.573157844,6],[1097909.749240872,8],[1097909.899259817,9],[1097910.046654797,13],[1097910.19557391,8],[1097910.344838794,20],[1097910.49466181,17],[1097910.685461554,20],[1097910.83593345,13],[1097911.009245226,6],[1097911.19127561,13],[1097911.337989526,9],[1097911.486248667,11],[1097911.643223183,8],[1097911.828167692,15],[1097912.009911096,16],[1097912.221131472,27],[1097912.463428057,4],[1097912.620674055,8],[1097912.809152847,25],[1097912.990077232,18],[1097913.139401047,13],[1097913.321263628,55],[1097913.47166102,11],[1097913.663836751,10],[1097913.849805566,11],[1097913.997809403,15],[1097914.148968222,17],[1097914.297324567,14],[1097914.504889306,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-14b3e784a947dfee90b4e5d9d4aef629","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008175438,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.576847019,"time_to_first_token":0.147847855,"token_sha":"8a91591c94a3","tokens_per_second":18.781974905008685},"seed":1234,"sent":1097900.777802434,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.1502829920500517,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.678330441233193,"done":1097914.505728533,"finish_reason":"length","first":1097900.853507377,"full_response_s":13.732180780963972,"last":1097914.505691163,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.853507377,3],[1097901.137297492,11],[1097901.317649809,30],[1097901.499441866,19],[1097901.687152884,21],[1097901.869586312,15],[1097902.014135866,19],[1097902.160239558,17],[1097902.342868429,12],[1097902.527399374,5],[1097902.681972591,5],[1097902.865811504,27],[1097903.046982545,12],[1097903.231395016,13],[1097903.413164621,5],[1097903.561949001,16],[1097903.746834865,5],[1097903.931484571,14],[1097904.111534134,9],[1097904.294899576,27],[1097904.443025379,11],[1097904.632300989,8],[1097904.779662296,16],[1097904.927624799,5],[1097905.07764275,9],[1097905.25946817,14],[1097905.40794916,20],[1097905.559911991,8],[1097905.711207089,12],[1097905.858879293,5],[1097906.008294493,14],[1097906.191565501,14],[1097906.342275562,8],[1097906.521694362,13],[1097906.704473029,3],[1097906.88188751,10],[1097907.03065803,11],[1097907.213364708,9],[1097907.397366103,10],[1097907.584511734,20],[1097907.767527744,14],[1097907.952245873,14],[1097908.177785661,13],[1097908.3619716,5],[1097908.54831626,22],[1097908.784572516,3],[1097909.018633358,32],[1097909.203439654,33],[1097909.386810369,11],[1097909.574381066,6],[1097909.748590171,8],[1097909.900212454,9],[1097910.04741015,13],[1097910.195379426,8],[1097910.346431472,20],[1097910.493035378,17],[1097910.686349422,20],[1097910.834210979,13],[1097911.00670081,6],[1097911.190969433,13],[1097911.337734531,9],[1097911.486970538,11],[1097911.644556143,8],[1097911.825638051,15],[1097912.010270519,16],[1097912.220116037,27],[1097912.463223011,4],[1097912.621369646,8],[1097912.804891486,25],[1097912.989096623,18],[1097913.138585057,13],[1097913.321962563,55],[1097913.472310342,11],[1097913.662909164,10],[1097913.848877964,11],[1097913.999052751,15],[1097914.147256048,17],[1097914.296791964,14],[1097914.505691163,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-8b698b8dea60f80dda0276ee1fb31caf","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008244769,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.651433123,"time_to_first_token":0.075308578,"token_sha":"8a91591c94a3","tokens_per_second":18.6793575225721},"seed":1234,"sent":1097900.773547752,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.07995962491258979,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.841830760501427,"done":1097914.50715785,"finish_reason":"length","first":1097900.973322219,"full_response_s":13.730410475982353,"last":1097914.507038939,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.973322219,3],[1097901.135981753,11],[1097901.319708664,30],[1097901.50050155,19],[1097901.690450081,21],[1097901.868966318,15],[1097902.01463213,19],[1097902.16005906,17],[1097902.344255562,12],[1097902.528656679,5],[1097902.685359857,5],[1097902.864751967,27],[1097903.048628867,12],[1097903.23109311,13],[1097903.412739256,5],[1097903.563907488,16],[1097903.749894005,5],[1097903.931393202,14],[1097904.112484041,9],[1097904.297417301,27],[1097904.443131295,11],[1097904.634182285,8],[1097904.780701091,16],[1097904.928163665,5],[1097905.07934614,9],[1097905.262967326,14],[1097905.409111427,20],[1097905.559112318,8],[1097905.711891034,12],[1097905.860652748,5],[1097906.009160551,14],[1097906.19078617,14],[1097906.341338413,8],[1097906.521763917,13],[1097906.701311264,3],[1097906.883052287,10],[1097907.030211622,11],[1097907.212341697,9],[1097907.396995587,10],[1097907.583257173,20],[1097907.769635606,14],[1097907.949066606,14],[1097908.177975648,13],[1097908.36214203,5],[1097908.547731208,22],[1097908.782402783,3],[1097909.019023686,32],[1097909.203912068,33],[1097909.385907664,11],[1097909.574304726,6],[1097909.748197922,8],[1097909.896748639,9],[1097910.045978314,13],[1097910.195152086,8],[1097910.346312881,20],[1097910.493729787,17],[1097910.685739548,20],[1097910.834413224,13],[1097911.006209256,6],[1097911.190051222,13],[1097911.338245946,9],[1097911.485906609,11],[1097911.644072239,8],[1097911.825827156,15],[1097912.010450534,16],[1097912.220305223,27],[1097912.46314315,4],[1097912.620034511,8],[1097912.808259252,25],[1097912.991132069,18],[1097913.138085678,13],[1097913.32170592,55],[1097913.473365659,11],[1097913.66374063,10],[1097913.848338863,11],[1097913.997040626,15],[1097914.146204461,17],[1097914.297731233,14],[1097914.507038939,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-08054176868ac4de7f7790631eaaf5ae","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008344157,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.534764689,"time_to_first_token":0.190974463,"token_sha":"8a91591c94a3","tokens_per_second":18.840371876375812},"seed":1234,"sent":1097900.776747374,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.1965748448856175,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.70089644156093,"done":1097914.504457337,"finish_reason":"length","first":1097900.868714177,"full_response_s":13.729327019071206,"last":1097914.504424144,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.868714177,3],[1097901.136510475,11],[1097901.317547732,30],[1097901.499649187,19],[1097901.688284044,21],[1097901.869112278,15],[1097902.014383518,19],[1097902.159726217,17],[1097902.345533785,12],[1097902.528574129,5],[1097902.685196836,5],[1097902.864188997,27],[1097903.047868323,12],[1097903.229879242,13],[1097903.415032475,5],[1097903.563165269,16],[1097903.747463389,5],[1097903.931924642,14],[1097904.112033262,9],[1097904.297182501,27],[1097904.441973669,11],[1097904.631896591,8],[1097904.781583923,16],[1097904.927382909,5],[1097905.079586381,9],[1097905.260326034,14],[1097905.408348069,20],[1097905.560492372,8],[1097905.710563346,12],[1097905.862716265,5],[1097906.007838482,14],[1097906.191270093,14],[1097906.337451812,8],[1097906.521419104,13],[1097906.70269759,3],[1097906.88221259,10],[1097907.030168891,11],[1097907.212178902,9],[1097907.396896088,10],[1097907.584130192,20],[1097907.768880747,14],[1097907.949699208,14],[1097908.183589294,13],[1097908.361735633,5],[1097908.548760939,22],[1097908.78396637,3],[1097909.019189762,32],[1097909.202381517,33],[1097909.3869127,11],[1097909.574461567,6],[1097909.748836093,8],[1097909.897162829,9],[1097910.047249643,13],[1097910.195782077,8],[1097910.345314136,20],[1097910.493151217,17],[1097910.688835265,20],[1097910.834133806,13],[1097911.007372091,6],[1097911.190878417,13],[1097911.340338467,9],[1097911.486343236,11],[1097911.642203777,8],[1097911.826453433,15],[1097912.009491818,16],[1097912.221748755,27],[1097912.465064135,4],[1097912.621147556,8],[1097912.808729056,25],[1097912.990220134,18],[1097913.144266914,13],[1097913.321734247,55],[1097913.472835536,11],[1097913.66424273,10],[1097913.847806996,11],[1097913.997212063,15],[1097914.147127886,17],[1097914.29761994,14],[1097914.504424144,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-e086c326eb815ac02fbc7108f09170d7","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008618345,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.636464513,"time_to_first_token":0.090523989,"token_sha":"8a91591c94a3","tokens_per_second":18.699861665529347},"seed":1234,"sent":1097900.775130318,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.09358385903760791,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.720776514130286,"done":1097914.504841693,"finish_reason":"length","first":1097900.883586406,"full_response_s":13.730579775990918,"last":1097914.504816263,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.883586406,3],[1097901.138135019,11],[1097901.317418257,30],[1097901.50099279,19],[1097901.689092987,21],[1097901.868357335,15],[1097902.0138362,19],[1097902.161112293,17],[1097902.345176503,12],[1097902.528319131,5],[1097902.684048905,5],[1097902.863982989,27],[1097903.046725483,12],[1097903.230661007,13],[1097903.411093671,5],[1097903.563231015,16],[1097903.747704463,5],[1097903.930141165,14],[1097904.112132377,9],[1097904.295793979,27],[1097904.441460409,11],[1097904.634329717,8],[1097904.781651685,16],[1097904.929778134,5],[1097905.078531183,9],[1097905.259651931,14],[1097905.40782279,20],[1097905.559180177,8],[1097905.714060902,12],[1097905.862657737,5],[1097906.008981558,14],[1097906.191325436,14],[1097906.340821089,8],[1097906.520457308,13],[1097906.702052408,3],[1097906.882653908,10],[1097907.029204967,11],[1097907.21264802,9],[1097907.398361442,10],[1097907.584949916,20],[1097907.768445446,14],[1097907.950011403,14],[1097908.183521228,13],[1097908.363456269,5],[1097908.549373151,22],[1097908.783466684,3],[1097909.018880656,32],[1097909.203731204,33],[1097909.388683412,11],[1097909.572974579,6],[1097909.74831549,8],[1097909.898518277,9],[1097910.046112446,13],[1097910.195439762,8],[1097910.344934179,20],[1097910.494061987,17],[1097910.685790105,20],[1097910.836022113,13],[1097911.00797241,6],[1097911.191379557,13],[1097911.340410486,9],[1097911.485808231,11],[1097911.642315023,8],[1097911.825957927,15],[1097912.009253675,16],[1097912.220872236,27],[1097912.464363503,4],[1097912.621297387,8],[1097912.80719326,25],[1097912.990666219,18],[1097913.144110377,13],[1097913.32178386,55],[1097913.472079418,11],[1097913.663485491,10],[1097913.849341237,11],[1097913.997523488,15],[1097914.149207917,17],[1097914.298209838,14],[1097914.504816263,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-896dfb83d5b62449b5d9f9fea24eb5d6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.0082259,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.621327237,"time_to_first_token":0.104914645,"token_sha":"8a91591c94a3","tokens_per_second":18.72064267770737},"seed":1234,"sent":1097900.774261917,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.1093244890216738,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.638739599147726,"done":1097914.50514302,"finish_reason":"length","first":1097900.823937752,"full_response_s":13.731682860990986,"last":1097914.505120358,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.823937752,3],[1097901.136636158,11],[1097901.321042024,30],[1097901.500604331,19],[1097901.690103537,21],[1097901.867912749,15],[1097902.015287446,19],[1097902.160562126,17],[1097902.344423144,12],[1097902.52817786,5],[1097902.684462571,5],[1097902.864998242,27],[1097903.047655305,12],[1097903.231441925,13],[1097903.415566781,5],[1097903.562322287,16],[1097903.748056287,5],[1097903.931188506,14],[1097904.11353876,9],[1097904.29619388,27],[1097904.443425648,11],[1097904.632426751,8],[1097904.780517649,16],[1097904.928061557,5],[1097905.080101192,9],[1097905.262431037,14],[1097905.408315164,20],[1097905.560664483,8],[1097905.714346963,12],[1097905.860160007,5],[1097906.008627287,14],[1097906.19052674,14],[1097906.341710577,8],[1097906.520271386,13],[1097906.702641383,3],[1097906.883480066,10],[1097907.03059883,11],[1097907.213904774,9],[1097907.397069959,10],[1097907.583165356,20],[1097907.770964379,14],[1097907.951065749,14],[1097908.182697231,13],[1097908.363551895,5],[1097908.547878063,22],[1097908.782923258,3],[1097909.019944731,32],[1097909.202754832,33],[1097909.386130828,11],[1097909.574129095,6],[1097909.74989359,8],[1097909.898270803,9],[1097910.048108015,13],[1097910.195659949,8],[1097910.344748002,20],[1097910.493966361,17],[1097910.68591129,20],[1097910.837400461,13],[1097911.008700042,6],[1097911.192073437,13],[1097911.339493492,9],[1097911.486910474,11],[1097911.644309229,8],[1097911.826920212,15],[1097912.010214568,16],[1097912.221053356,27],[1097912.465640687,4],[1097912.621213077,8],[1097912.807057208,25],[1097912.990176874,18],[1097913.139311648,13],[1097913.32121259,55],[1097913.472489813,11],[1097913.665353853,10],[1097913.849300282,11],[1097913.999381413,15],[1097914.147822644,17],[1097914.297402379,14],[1097914.505120358,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-56ca0a1b011997b50a95117247ca44aa","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008860603,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.681518985,"time_to_first_token":0.046610923,"token_sha":"8a91591c94a3","tokens_per_second":18.63828133992828},"seed":1234,"sent":1097900.773460159,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.050477592973038554,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.659784364511957,"done":1097914.504545152,"finish_reason":"length","first":1097900.83876512,"full_response_s":13.731171241030097,"last":1097914.504517897,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.83876512,3],[1097901.136949924,11],[1097901.318723641,30],[1097901.49984713,19],[1097901.689413715,21],[1097901.867969612,15],[1097902.014577747,19],[1097902.159632879,17],[1097902.344106097,12],[1097902.528427272,5],[1097902.686077606,5],[1097902.865197977,27],[1097903.047480953,12],[1097903.230529259,13],[1097903.411856663,5],[1097903.562689667,16],[1097903.749700384,5],[1097903.930903965,14],[1097904.113160833,9],[1097904.294825652,27],[1097904.442245919,11],[1097904.634689815,8],[1097904.780577713,16],[1097904.928547593,5],[1097905.077410335,9],[1097905.261147969,14],[1097905.408260829,20],[1097905.559654177,8],[1097905.711432062,12],[1097905.858967749,5],[1097906.009046216,14],[1097906.191621148,14],[1097906.342574907,8],[1097906.521344908,13],[1097906.702151731,3],[1097906.882953493,10],[1097907.029507417,11],[1097907.214197556,9],[1097907.395751701,10],[1097907.583532207,20],[1097907.767859625,14],[1097907.949373489,14],[1097908.177902684,13],[1097908.362035233,5],[1097908.547955348,22],[1097908.782269371,3],[1097909.019429682,32],[1097909.202109876,33],[1097909.385466875,11],[1097909.572898159,6],[1097909.748943641,8],[1097909.898388338,9],[1097910.047748191,13],[1097910.194205659,8],[1097910.345933036,20],[1097910.494478337,17],[1097910.688061428,20],[1097910.835299042,13],[1097911.005698097,6],[1097911.191678292,13],[1097911.338868767,9],[1097911.48810519,11],[1097911.643020585,8],[1097911.825299658,15],[1097912.009077645,16],[1097912.219801426,27],[1097912.463271776,4],[1097912.620397726,8],[1097912.80565007,25],[1097912.991049583,18],[1097913.140093597,13],[1097913.321582912,55],[1097913.472246405,11],[1097913.664828163,10],[1097913.847688517,11],[1097913.99977846,15],[1097914.148275578,17],[1097914.297225853,14],[1097914.504517897,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-b3e796164056f305c7c53055ce5f4dc3","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008275145,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.666141035,"time_to_first_token":0.060928214,"token_sha":"8a91591c94a3","tokens_per_second":18.65925423621241},"seed":1234,"sent":1097900.773373911,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.06539120897650719,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.821878877223416,"done":1097914.506385984,"finish_reason":"length","first":1097900.958281304,"full_response_s":13.729410114930943,"last":1097914.506344261,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.958281304,3],[1097901.137513023,11],[1097901.320391973,30],[1097901.500079338,19],[1097901.687541279,21],[1097901.868649047,15],[1097902.013723065,19],[1097902.159968828,17],[1097902.344161904,12],[1097902.527765218,5],[1097902.682166949,5],[1097902.865700802,27],[1097903.047802017,12],[1097903.231331511,13],[1097903.414276685,5],[1097903.563729072,16],[1097903.747539186,5],[1097903.930790574,14],[1097904.112236565,9],[1097904.296021481,27],[1097904.44238226,11],[1097904.633754025,8],[1097904.781866768,16],[1097904.929528483,5],[1097905.078430052,9],[1097905.261827785,14],[1097905.408395665,20],[1097905.561062959,8],[1097905.71114544,12],[1097905.859068192,5],[1097906.008233452,14],[1097906.190559997,14],[1097906.340700449,8],[1097906.521090231,13],[1097906.703107349,3],[1097906.882856187,10],[1097907.029925657,11],[1097907.213226463,9],[1097907.396704917,10],[1097907.584244334,20],[1097907.768992377,14],[1097907.951414883,14],[1097908.181211394,13],[1097908.363235474,5],[1097908.548718704,22],[1097908.783339002,3],[1097909.019477503,32],[1097909.204192575,33],[1097909.386566672,11],[1097909.5726012,6],[1097909.7500496,8],[1097909.898357274,9],[1097910.049585703,13],[1097910.196228307,8],[1097910.343749081,20],[1097910.493859133,17],[1097910.686096891,20],[1097910.835732389,13],[1097911.007103348,6],[1097911.190308074,13],[1097911.337548834,9],[1097911.486694385,11],[1097911.643527551,8],[1097911.825698915,15],[1097912.00892999,16],[1097912.220669911,27],[1097912.462971777,4],[1097912.619736368,8],[1097912.806916339,25],[1097912.990888789,18],[1097913.140390314,13],[1097913.321401824,55],[1097913.47085945,11],[1097913.663380584,10],[1097913.848029422,11],[1097913.996960989,15],[1097914.147320288,17],[1097914.2980328,14],[1097914.506344261,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-91a1a0229bab1fc39eedcadd886f3799","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008207527,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.548809902,"time_to_first_token":0.176514045,"token_sha":"8a91591c94a3","tokens_per_second":18.8208412284505},"seed":1234,"sent":1097900.776975869,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.18130543502047658,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.88312179872644,"done":1097914.50781783,"finish_reason":"length","first":1097901.003504412,"full_response_s":13.729137440910563,"last":1097914.507627444,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097901.003504412,3],[1097901.138587016,11],[1097901.318847115,30],[1097901.500806066,19],[1097901.689156636,21],[1097901.867071222,15],[1097902.014306425,19],[1097902.16051112,17],[1097902.343991185,12],[1097902.527485622,5],[1097902.684272022,5],[1097902.866201643,27],[1097903.048107061,12],[1097903.230908755,13],[1097903.411917896,5],[1097903.564121275,16],[1097903.748814062,5],[1097903.931831273,14],[1097904.111721721,9],[1097904.295118627,27],[1097904.442172379,11],[1097904.631644859,8],[1097904.779981711,16],[1097904.929163584,5],[1097905.078190755,9],[1097905.263384367,14],[1097905.408791805,20],[1097905.559344749,8],[1097905.713137932,12],[1097905.860907536,5],[1097906.009478653,14],[1097906.193334266,14],[1097906.34291423,8],[1097906.52059701,13],[1097906.702465111,3],[1097906.882996528,10],[1097907.029723427,11],[1097907.212777239,9],[1097907.396115319,10],[1097907.583743384,20],[1097907.768821131,14],[1097907.953071806,14],[1097908.183630057,13],[1097908.362828565,5],[1097908.548050573,22],[1097908.784032947,3],[1097909.02052543,32],[1097909.202290164,33],[1097909.387331419,11],[1097909.573621103,6],[1097909.74906044,8],[1097909.897787827,9],[1097910.045561995,13],[1097910.196696704,8],[1097910.345243653,20],[1097910.495593001,17],[1097910.687580356,20],[1097910.834995138,13],[1097911.006575128,6],[1097911.192406437,13],[1097911.338399619,9],[1097911.486420392,11],[1097911.645102927,8],[1097911.828510022,15],[1097912.00985745,16],[1097912.221966908,27],[1097912.464874661,4],[1097912.620902835,8],[1097912.805047559,25],[1097912.989641518,18],[1097913.138240263,13],[1097913.321837074,55],[1097913.471741249,11],[1097913.664166021,10],[1097913.848996971,11],[1097913.997474323,15],[1097914.14607678,17],[1097914.296900808,14],[1097914.507627444,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-a4de8e09d6154ce28e23f5d500f3b751","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008145798,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.505226504,"time_to_first_token":0.220496794,"token_sha":"8a91591c94a3","tokens_per_second":18.881578914983297},"seed":1234,"sent":1097900.778680389,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.22482402296736836,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.86255407097117,"done":1097914.507500243,"finish_reason":"length","first":1097900.988609201,"full_response_s":13.728687272872776,"last":1097914.507457128,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.988609201,3],[1097901.138390914,11],[1097901.319572147,30],[1097901.501199215,19],[1097901.688835155,21],[1097901.868014057,15],[1097902.014801761,19],[1097902.160343458,17],[1097902.344325949,12],[1097902.526813117,5],[1097902.683610833,5],[1097902.865089483,27],[1097903.049162405,12],[1097903.231279529,13],[1097903.413368212,5],[1097903.563039699,16],[1097903.749379593,5],[1097903.932034448,14],[1097904.111943253,9],[1097904.295206587,27],[1097904.441765164,11],[1097904.632227481,8],[1097904.780375914,16],[1097904.927948679,5],[1097905.07872546,9],[1097905.261464551,14],[1097905.408732381,20],[1097905.559546788,8],[1097905.712834586,12],[1097905.862069114,5],[1097906.009853778,14],[1097906.190679998,14],[1097906.339675644,8],[1097906.520407183,13],[1097906.701969682,3],[1097906.883329802,10],[1097907.029095306,11],[1097907.213053297,9],[1097907.398092618,10],[1097907.583817131,20],[1097907.768265558,14],[1097907.950107173,14],[1097908.180378179,13],[1097908.362667706,5],[1097908.548359536,22],[1097908.78324312,3],[1097909.019095753,32],[1097909.203043437,33],[1097909.387656306,11],[1097909.57401188,6],[1097909.748783086,8],[1097909.897371877,9],[1097910.0461969,13],[1097910.195702232,8],[1097910.344329058,20],[1097910.495208995,17],[1097910.685669129,20],[1097910.835235713,13],[1097911.00676854,6],[1097911.191516922,13],[1097911.34115209,9],[1097911.486309547,11],[1097911.643944813,8],[1097911.826520203,15],[1097912.009177687,16],[1097912.220997949,27],[1097912.46522781,4],[1097912.619956458,8],[1097912.808598221,25],[1097912.991641018,18],[1097913.139140179,13],[1097913.321908501,55],[1097913.473015055,11],[1097913.665257299,10],[1097913.848918022,11],[1097913.998827972,15],[1097914.147934529,17],[1097914.298491015,14],[1097914.507457128,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-67c51f7b03ff3f0d484bb688b88b0fd9","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008614951,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.520018106,"time_to_first_token":0.206050251,"token_sha":"8a91591c94a3","tokens_per_second":18.860921486993753},"seed":1234,"sent":1097900.77881297,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.20979623100720346,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.759924372189918,"done":1097914.506029603,"finish_reason":"length","first":1097900.913201179,"full_response_s":13.729403095087036,"last":1097914.506006509,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.913201179,3],[1097901.137961179,11],[1097901.320695592,30],[1097901.500179029,19],[1097901.688908536,21],[1097901.870037764,15],[1097902.015479595,19],[1097902.160613373,17],[1097902.344793837,12],[1097902.527028264,5],[1097902.686367238,5],[1097902.865876098,27],[1097903.048876103,12],[1097903.23156111,13],[1097903.413425124,5],[1097903.563690757,16],[1097903.747234398,5],[1097903.931277218,14],[1097904.112191689,9],[1097904.295565309,27],[1097904.442910307,11],[1097904.63284925,8],[1097904.781164401,16],[1097904.928633728,5],[1097905.078613253,9],[1097905.260820792,14],[1097905.408629441,20],[1097905.559712049,8],[1097905.712722139,12],[1097905.862356263,5],[1097906.008904418,14],[1097906.190446383,14],[1097906.338155506,8],[1097906.520949041,13],[1097906.701783039,3],[1097906.883249252,10],[1097907.029642445,11],[1097907.214131138,9],[1097907.397928494,10],[1097907.583450201,20],[1097907.769111081,14],[1097907.949852705,14],[1097908.179884175,13],[1097908.363329723,5],[1097908.5489443,22],[1097908.783567079,3],[1097909.018765537,32],[1097909.204131407,33],[1097909.386436941,11],[1097909.573814243,6],[1097909.750366804,8],[1097909.899537523,9],[1097910.048685384,13],[1097910.196064952,8],[1097910.344557103,20],[1097910.493815105,17],[1097910.685348388,20],[1097910.837107872,13],[1097911.006639209,6],[1097911.190561021,13],[1097911.33880475,9],[1097911.48611748,11],[1097911.644372734,8],[1097911.827493723,15],[1097912.009323918,16],[1097912.220470178,27],[1097912.463746461,4],[1097912.619648217,8],[1097912.805828116,25],[1097912.990589366,18],[1097913.139471257,13],[1097913.321676472,55],[1097913.471545966,11],[1097913.664561885,10],[1097913.848810106,11],[1097913.998471895,15],[1097914.148859394,17],[1097914.297539135,14],[1097914.506006509,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-45a03543b26ae8c798a199a91442a43e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008200901,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.591507925,"time_to_first_token":0.133182388,"token_sha":"8a91591c94a3","tokens_per_second":18.76171513912427},"seed":1234,"sent":1097900.776626508,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.13657467090524733,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.80189481826569,"done":1097914.506149459,"finish_reason":"length","first":1097900.943651494,"full_response_s":13.731752015883103,"last":1097914.506114345,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.943651494,3],[1097901.136129058,11],[1097901.319023499,30],[1097901.500935254,19],[1097901.68870664,21],[1097901.870159574,15],[1097902.014447535,19],[1097902.160856911,17],[1097902.343876722,12],[1097902.527287407,5],[1097902.685476929,5],[1097902.864835958,27],[1097903.048317726,12],[1097903.230583322,13],[1097903.412287453,5],[1097903.563491487,16],[1097903.748377815,5],[1097903.930686353,14],[1097904.112628256,9],[1097904.29501159,27],[1097904.442075536,11],[1097904.633241245,8],[1097904.780970988,16],[1097904.929106224,5],[1097905.080025252,9],[1097905.260224166,14],[1097905.409293093,20],[1097905.559453595,8],[1097905.711630195,12],[1097905.859740902,5],[1097906.007971334,14],[1097906.191689278,14],[1097906.338611885,8],[1097906.522096391,13],[1097906.70155653,3],[1097906.883195206,10],[1097907.029279964,11],[1097907.212828212,9],[1097907.396776024,10],[1097907.582708978,20],[1097907.768098361,14],[1097907.9526306,14],[1097908.182316938,13],[1097908.363855096,5],[1097908.549151748,22],[1097908.783905233,3],[1097909.019825051,32],[1097909.203787267,33],[1097909.389799037,11],[1097909.572655102,6],[1097909.749549787,8],[1097909.897432645,9],[1097910.046540575,13],[1097910.195244862,8],[1097910.344513988,20],[1097910.494234417,17],[1097910.686873625,20],[1097910.835498967,13],[1097911.00742945,6],[1097911.19232019,13],[1097911.339122193,9],[1097911.486008236,11],[1097911.644893336,8],[1097911.827872911,15],[1097912.009698448,16],[1097912.221382978,27],[1097912.463856361,4],[1097912.620460159,8],[1097912.805194286,25],[1097912.990539097,18],[1097913.144014848,13],[1097913.32163859,55],[1097913.472451211,11],[1097913.666223473,10],[1097913.848580334,11],[1097913.997332558,15],[1097914.147743487,17],[1097914.298592898,14],[1097914.506114345,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-1a23411e1179761fe8f5fd03baf43e64","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.008129794,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.56271553,"time_to_first_token":0.161847794,"token_sha":"8a91591c94a3","tokens_per_second":18.80154453110468},"seed":1234,"sent":1097900.774397443,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.1692540510557592,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"C","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.58067091536602,"done":1097914.503995168,"finish_reason":"length","first":1097900.780016375,"full_response_s":13.730703806970268,"last":1097914.503955686,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097900.780016375,3],[1097901.136705793,11],[1097901.317993855,30],[1097901.499329803,19],[1097901.686992888,21],[1097901.866982622,15],[1097902.01573989,19],[1097902.16091904,17],[1097902.343426278,12],[1097902.526906071,5],[1097902.685811757,5],[1097902.863699087,27],[1097903.047892522,12],[1097903.231520522,13],[1097903.412930829,5],[1097903.564013181,16],[1097903.748450251,5],[1097903.932132331,14],[1097904.11182178,9],[1097904.296764147,27],[1097904.442953775,11],[1097904.63382486,8],[1097904.779292387,16],[1097904.928973708,5],[1097905.079499445,9],[1097905.260553168,14],[1097905.407892489,20],[1097905.559220844,8],[1097905.713748033,12],[1097905.86155864,5],[1097906.00866965,14],[1097906.1906155,14],[1097906.337363757,8],[1097906.521162267,13],[1097906.704091326,3],[1097906.883299506,10],[1097907.029987002,11],[1097907.212700098,9],[1097907.396183145,10],[1097907.584065086,20],[1097907.767770689,14],[1097907.948958033,14],[1097908.178427673,13],[1097908.361426926,5],[1097908.547541637,22],[1097908.7834083,3],[1097909.018839029,32],[1097909.202184376,33],[1097909.385802596,11],[1097909.573745393,6],[1097909.750277356,8],[1097909.898106872,9],[1097910.045826594,13],[1097910.1943774,8],[1097910.343893791,20],[1097910.493336514,17],[1097910.685191707,20],[1097910.836107704,13],[1097911.005605672,6],[1097911.191597615,13],[1097911.338991087,9],[1097911.487618613,11],[1097911.644673198,8],[1097911.827721687,15],[1097912.008773037,16],[1097912.221304734,27],[1097912.462867621,4],[1097912.619508212,8],[1097912.805611083,25],[1097912.991324983,18],[1097913.144352921,13],[1097913.321315097,55],[1097913.47346959,11],[1097913.663281166,10],[1097913.8486335,11],[1097913.996671602,15],[1097914.146490488,17],[1097914.297120898,14],[1097914.503955686,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-881c5b3f550919db44db35d9dac44775","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.0022697,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.726400894,"time_to_first_token":0.002703357,"token_sha":"8a91591c94a3","tokens_per_second":18.577338806377426},"seed":1234,"sent":1097900.773291361,"speculative":{"acceptance_rate":0.41451990632318503,"accepted":177,"drafted":427,"rounds":78,"tokens_per_round":3.282051282051282},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.006725013954564929,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":177,"drafted":427,"rounds":78},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"C","case":"long","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":20.192112908134092,"done":1097919.301918227,"finish_reason":"length","first":1097917.766617774,"full_response_s":4.759321738034487,"last":1097919.301870679,"output_sha256":"5d2c18c1a3da0aab39441e582248a278a66d8473e846a8d2c48dfda87c21e891","pieces":[[1097917.766617774,3],[1097917.859128158,14],[1097917.942963038,11],[1097918.02676695,11],[1097918.110924073,13],[1097918.194011545,12],[1097918.277494111,12],[1097918.361663829,12],[1097918.445029372,12],[1097918.528727001,11],[1097918.621092823,12],[1097918.708791639,4],[1097918.792055988,12],[1097918.876024226,12],[1097918.95973257,9],[1097919.043923918,1],[1097919.127528428,7],[1097919.211030527,4],[1097919.301870679,17]],"prompt":"long_initial","prompt_tokens":4419,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"df5eb0c68030e305529b5525635e0602046d98ecf7a739e2e42243ec364b4adc","request_path":"/v1/chat/completions","response_id":"chatcmpl-deb8b4eaf25617e9070b8b08ab2c4b03","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":3.218115809,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":1.535686778,"time_to_first_token":3.223522323,"token_sha":"8b52dc1b081a","tokens_per_second":20.186408090569625},"seed":1234,"sent":1097914.542596489,"speculative":{"acceptance_rate":0.05925925925925926,"accepted":16,"drafted":270,"rounds":18,"tokens_per_round":1.7777777777777777},"token_sha":"8b52dc1b081a","tokens":32,"ttft_s":3.2240212850738317,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4419,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":4451},"work":{"accepted":16,"drafted":270,"rounds":18},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"C","case":"long","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":50.27620319279011,"done":1097923.096508287,"finish_reason":"length","first":1097922.479886624,"full_response_s":3.78918566997163,"last":1097923.09648052,"output_sha256":"dd1aacffd9ef2ff0a3531f264db2a68aa99f695724c7828277affb8cb3535eb1","pieces":[[1097922.479886624,3],[1097922.574874744,36],[1097922.665374258,45],[1097922.752333822,8],[1097922.836277189,30],[1097922.919843716,18],[1097923.003608709,13],[1097923.09648052,12]],"prompt":"long_followup","prompt_tokens":4445,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"},{"content":"Acknowledged.","role":"assistant"},{"content":"Summarize the document in one sentence.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"0f7b9a7949106fe7d7c1dee63210f9621564bc3cdf8167180dc2d64159ad2c22","request_path":"/v1/chat/completions","response_id":"chatcmpl-57ada1401c87b966c7499451160b0cb6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":16,"prefill_raised":[],"prefill_seconds":3.166362934,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":0.616481101,"time_to_first_token":3.17145366,"token_sha":"526434da61a6","tokens_per_second":50.2854020175389},"seed":1234,"sent":1097919.307322617,"speculative":{"acceptance_rate":0.22857142857142856,"accepted":24,"drafted":105,"rounds":7,"tokens_per_round":4.571428571428571},"token_sha":"526434da61a6","tokens":32,"ttft_s":3.172564007109031,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4445,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":4477},"work":{"accepted":24,"drafted":105,"rounds":7},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"followup","stream_index":0} +{"arm":"D","case":"code","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":65.44392056729643,"done":1097748.853311911,"finish_reason":"length","first":1097744.956759775,"full_response_s":4.03748605796136,"last":1097748.853225601,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097744.956759775,2],[1097745.044091083,7],[1097745.124069213,4],[1097745.197267541,72],[1097745.277363587,1],[1097745.35692148,51],[1097745.443424256,21],[1097745.526260971,14],[1097745.606436043,20],[1097745.685430247,14],[1097745.766670001,4],[1097745.845844255,29],[1097745.926205522,5],[1097746.006294264,6],[1097746.087274869,33],[1097746.167518573,22],[1097746.247033253,26],[1097746.327540469,17],[1097746.412284639,20],[1097746.498372962,18],[1097746.577781862,19],[1097746.657516218,8],[1097746.738240452,11],[1097746.818210721,18],[1097746.898986472,15],[1097746.978064828,39],[1097747.059390383,30],[1097747.139631316,14],[1097747.220712092,15],[1097747.30107182,8],[1097747.381823379,6],[1097747.468870139,15],[1097747.55076058,23],[1097747.630969492,22],[1097747.711915479,18],[1097747.792161679,38],[1097747.872262115,22],[1097747.952911353,10],[1097748.032982435,14],[1097748.114103938,15],[1097748.194415075,38],[1097748.275134309,47],[1097748.354047351,25],[1097748.442826391,11],[1097748.526923189,34],[1097748.607423838,12],[1097748.68765715,36],[1097748.768565687,60],[1097748.853225601,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-437a6eea774eebb0c59af46543a376fa","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":0.125908275,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.897344583,"time_to_first_token":0.126860886,"token_sha":"2b28aa2108de","tokens_per_second":65.42916454251846},"seed":1234,"sent":1097744.815825853,"speculative":{"acceptance_rate":0.3088235294117647,"accepted":210,"drafted":680,"rounds":48,"tokens_per_round":5.333333333333333},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.1409339220263064,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":270},"work":{"accepted":210,"drafted":680,"rounds":48},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":65.35232133194735,"done":1097752.768737928,"finish_reason":"length","first":1097748.866735485,"full_response_s":3.914394620107487,"last":1097752.768662682,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097748.866735485,2],[1097748.955035971,7],[1097749.035069674,4],[1097749.108865558,72],[1097749.18952189,1],[1097749.271739374,51],[1097749.351628026,21],[1097749.437757779,14],[1097749.521234834,20],[1097749.599866252,14],[1097749.680351341,4],[1097749.760179107,29],[1097749.840428061,5],[1097749.920981918,6],[1097750.001059591,33],[1097750.082200564,22],[1097750.161218506,26],[1097750.242417099,17],[1097750.321483984,20],[1097750.406550611,18],[1097750.492605052,19],[1097750.572939364,8],[1097750.653592866,11],[1097750.734432084,18],[1097750.814450327,15],[1097750.893257015,39],[1097750.973962268,30],[1097751.055290338,14],[1097751.135568511,15],[1097751.216670696,8],[1097751.296860543,6],[1097751.377666258,15],[1097751.464469627,23],[1097751.546619259,22],[1097751.62645588,18],[1097751.707887805,38],[1097751.788149024,22],[1097751.868330548,10],[1097751.948629208,14],[1097752.029226804,15],[1097752.110482883,38],[1097752.190170817,47],[1097752.270130677,25],[1097752.350507497,11],[1097752.438496651,34],[1097752.522610163,12],[1097752.602629515,36],[1097752.683081342,60],[1097752.768662682,48]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-bf0b393bca28f9867630698343ae9f36","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":0.011056334,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":3.902011203,"time_to_first_token":0.011594447,"token_sha":"2b28aa2108de","tokens_per_second":65.35091437050393},"seed":1234,"sent":1097748.854343308,"speculative":{"acceptance_rate":0.3088235294117647,"accepted":210,"drafted":680,"rounds":48,"tokens_per_round":5.333333333333333},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.012392177013680339,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":210,"drafted":680,"rounds":48},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.552354830260196,"done":1097762.995091337,"finish_reason":"length","first":1097753.01555477,"full_response_s":10.220493270084262,"last":1097762.995065531,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097753.01555477,2],[1097753.184857694,7],[1097753.330771194,4],[1097753.540208933,63],[1097753.775036437,10],[1097753.953257213,31],[1097754.134108186,29],[1097754.314688931,22],[1097754.502733621,14],[1097754.682833755,24],[1097754.863545261,4],[1097755.009906135,13],[1097755.191514002,21],[1097755.373541625,2],[1097755.560434386,18],[1097755.740358727,29],[1097755.973071133,27],[1097756.206969891,24],[1097756.355558847,4],[1097756.511074554,16],[1097756.749407789,10],[1097756.930774085,31],[1097757.078555769,8],[1097757.262623715,11],[1097757.446589488,16],[1097757.630438823,12],[1097757.809985505,39],[1097757.983818758,18],[1097758.166466707,11],[1097758.346478555,16],[1097758.538518373,16],[1097758.685892489,11],[1097758.864904009,6],[1097759.04704389,9],[1097759.231482303,29],[1097759.415612088,22],[1097759.603366561,8],[1097759.746867833,18],[1097759.929806456,18],[1097760.110473969,34],[1097760.296549838,10],[1097760.483706595,14],[1097760.66620015,13],[1097760.849625978,2],[1097761.030102683,36],[1097761.216184992,42],[1097761.396372554,26],[1097761.551667374,17],[1097761.735126465,34],[1097761.96805263,12],[1097762.15121496,36],[1097762.440140926,50],[1097762.625827158,2],[1097762.995065531,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-dfd93d641d3b1df7b15fd4d69f41fd0a","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010991612,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.977774446,"time_to_first_token":0.235495948,"token_sha":"2b28aa2108de","tokens_per_second":25.556801406973797},"seed":1234,"sent":1097752.774598067,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.24095670296810567,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.201615147368138,"done":1097762.993771846,"finish_reason":"length","first":1097752.875346011,"full_response_s":10.220659330952913,"last":1097762.99374511,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.875346011,2],[1097753.183494367,7],[1097753.33161835,4],[1097753.539378955,63],[1097753.773246418,10],[1097753.953934696,31],[1097754.13618474,29],[1097754.315370512,22],[1097754.502980699,14],[1097754.684263965,24],[1097754.866136357,4],[1097755.009090107,13],[1097755.191614558,21],[1097755.372565102,2],[1097755.56160802,18],[1097755.743354131,29],[1097755.974469681,27],[1097756.208433631,24],[1097756.354831051,4],[1097756.510612325,16],[1097756.749638567,10],[1097756.930252431,31],[1097757.079107665,8],[1097757.261486399,11],[1097757.448722852,16],[1097757.629715305,12],[1097757.81048945,39],[1097757.982925318,18],[1097758.164640076,11],[1097758.346712885,16],[1097758.538114231,16],[1097758.685558308,11],[1097758.866016086,6],[1097759.046975449,9],[1097759.229123991,29],[1097759.416544267,22],[1097759.602304177,8],[1097759.746560308,18],[1097759.928373572,18],[1097760.110867463,34],[1097760.293837728,10],[1097760.485122269,14],[1097760.667876872,13],[1097760.849918068,2],[1097761.031277253,36],[1097761.215958204,42],[1097761.396091655,26],[1097761.551843108,17],[1097761.73475854,34],[1097761.967748839,12],[1097762.150163506,36],[1097762.441555085,50],[1097762.625362298,2],[1097762.99374511,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-24c0def73bf4592049d0d5820f6a6f4e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010990539,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.118564618,"time_to_first_token":0.097539451,"token_sha":"2b28aa2108de","tokens_per_second":25.201202900496217},"seed":1234,"sent":1097752.773112515,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.10223349602892995,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.157045781257473,"done":1097762.994429198,"finish_reason":"length","first":1097752.858057409,"full_response_s":10.221697524189949,"last":1097762.994382724,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.858057409,2],[1097753.18326008,7],[1097753.331300284,4],[1097753.540532907,63],[1097753.773771538,10],[1097753.954050222,31],[1097754.134838404,29],[1097754.316063956,22],[1097754.505076785,14],[1097754.683711023,24],[1097754.866234544,4],[1097755.009318581,13],[1097755.192280607,21],[1097755.371918969,2],[1097755.561177585,18],[1097755.742542234,29],[1097755.972478794,27],[1097756.206287915,24],[1097756.354057629,4],[1097756.5108375,16],[1097756.748493786,10],[1097756.930722742,31],[1097757.079013225,8],[1097757.262557387,11],[1097757.447473021,16],[1097757.629782113,12],[1097757.811792108,39],[1097757.982714425,18],[1097758.164958968,11],[1097758.346291036,16],[1097758.537969761,16],[1097758.684499634,11],[1097758.865307046,6],[1097759.048139957,9],[1097759.229598573,29],[1097759.414398672,22],[1097759.601109316,8],[1097759.745832905,18],[1097759.928266389,18],[1097760.111628627,34],[1097760.294502404,10],[1097760.483960903,14],[1097760.667376484,13],[1097760.850708702,2],[1097761.031766464,36],[1097761.21420538,42],[1097761.396466992,26],[1097761.552338867,17],[1097761.734185407,34],[1097761.969303815,12],[1097762.150741503,36],[1097762.440062689,50],[1097762.625479294,2],[1097762.994382724,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-e15be585d289610723001b1e992482a5","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010957527,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.136345878,"time_to_first_token":0.079786182,"token_sha":"2b28aa2108de","tokens_per_second":25.156994746346797},"seed":1234,"sent":1097752.772731674,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.08532573515549302,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.46787808849683,"done":1097762.993500714,"finish_reason":"length","first":1097752.980214284,"full_response_s":10.220290106954053,"last":1097762.992827,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.980214284,2],[1097753.185051492,7],[1097753.331039869,4],[1097753.541339879,63],[1097753.773882118,10],[1097753.953811501,31],[1097754.135566013,29],[1097754.314989145,22],[1097754.504526499,14],[1097754.6836547,24],[1097754.864256553,4],[1097755.009235859,13],[1097755.191931293,21],[1097755.372332241,2],[1097755.559942746,18],[1097755.740562898,29],[1097755.972538734,27],[1097756.206846986,24],[1097756.355015386,4],[1097756.512104066,16],[1097756.750365322,10],[1097756.931481898,31],[1097757.08170577,8],[1097757.26278461,11],[1097757.447832739,16],[1097757.630947862,12],[1097757.809850881,39],[1097757.983991708,18],[1097758.164040826,11],[1097758.347284407,16],[1097758.537487812,16],[1097758.685078776,11],[1097758.86567443,6],[1097759.047623798,9],[1097759.231743865,29],[1097759.416291783,22],[1097759.600510564,8],[1097759.74756485,18],[1097759.927619804,18],[1097760.110716943,34],[1097760.294796079,10],[1097760.484400215,14],[1097760.666442579,13],[1097760.850570814,2],[1097761.031218178,36],[1097761.214906795,42],[1097761.397623183,26],[1097761.552800367,17],[1097761.735828152,34],[1097761.96843778,12],[1097762.15136716,36],[1097762.441203586,50],[1097762.625900032,2],[1097762.992827,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-78fc0a4fd0aa0ac69f475e05a05455dd","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010719814,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.01281265,"time_to_first_token":0.200483822,"token_sha":"2b28aa2108de","tokens_per_second":25.467369550752554},"seed":1234,"sent":1097752.773210607,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.2070036770310253,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.066102681637254,"done":1097762.995451336,"finish_reason":"length","first":1097752.822303664,"full_response_s":10.222623843932524,"last":1097762.995404893,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.822303664,2],[1097753.182889951,7],[1097753.330029862,4],[1097753.540868183,63],[1097753.774944141,10],[1097753.95317875,31],[1097754.135892227,29],[1097754.314265314,22],[1097754.50372262,14],[1097754.683545626,24],[1097754.86489124,4],[1097755.009812862,13],[1097755.192824551,21],[1097755.372195279,2],[1097755.560697472,18],[1097755.7402396,29],[1097755.972804461,27],[1097756.206583901,24],[1097756.355773295,4],[1097756.512455428,16],[1097756.748881779,10],[1097756.930663411,31],[1097757.079851259,8],[1097757.261779437,11],[1097757.448406792,16],[1097757.630251992,12],[1097757.809781511,39],[1097757.984483983,18],[1097758.165884027,11],[1097758.34622053,16],[1097758.537602824,16],[1097758.684121107,11],[1097758.866936467,6],[1097759.047959435,9],[1097759.230126675,29],[1097759.41499483,22],[1097759.601884203,8],[1097759.746735723,18],[1097759.928559536,18],[1097760.111322784,34],[1097760.294312053,10],[1097760.48596769,14],[1097760.668464351,13],[1097760.849448035,2],[1097761.030373512,36],[1097761.214685946,42],[1097761.396051535,26],[1097761.552428455,17],[1097761.734682127,34],[1097761.967658883,12],[1097762.149851118,36],[1097762.441091137,50],[1097762.62665951,2],[1097762.995404893,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-e3a9b04764d252ac35700d24eae35362","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010864607,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.171639983,"time_to_first_token":0.046660495,"token_sha":"2b28aa2108de","tokens_per_second":25.069703649183904},"seed":1234,"sent":1097752.772827492,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.04947617184370756,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.329910557609953,"done":1097762.994948295,"finish_reason":"length","first":1097752.927755733,"full_response_s":10.221254302887246,"last":1097762.994905375,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.927755733,2],[1097753.185468402,7],[1097753.331504825,4],[1097753.540458237,63],[1097753.774869231,10],[1097753.952889791,31],[1097754.134622097,29],[1097754.313653296,22],[1097754.505514442,14],[1097754.682932999,24],[1097754.864611759,4],[1097755.009773399,13],[1097755.191984461,21],[1097755.372623585,2],[1097755.559607104,18],[1097755.74061288,29],[1097755.97539052,27],[1097756.207090059,24],[1097756.355129486,4],[1097756.512647159,16],[1097756.749829241,10],[1097756.930390321,31],[1097757.081099185,8],[1097757.262455119,11],[1097757.448034219,16],[1097757.631049394,12],[1097757.810892921,39],[1097757.98235996,18],[1097758.16444007,11],[1097758.346916782,16],[1097758.53666435,16],[1097758.684216827,11],[1097758.866163901,6],[1097759.048081395,9],[1097759.231219621,29],[1097759.414917264,22],[1097759.601617262,8],[1097759.748198518,18],[1097759.92945686,18],[1097760.11167563,34],[1097760.296316342,10],[1097760.485687142,14],[1097760.668846334,13],[1097760.849147989,2],[1097761.030027808,36],[1097761.215419716,42],[1097761.397869102,26],[1097761.552493565,17],[1097761.733696243,34],[1097761.969043025,12],[1097762.150710301,36],[1097762.440837951,50],[1097762.625563406,2],[1097762.994905375,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-e4b7d21680c9335481f38827721250fb","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011337807,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.065889753,"time_to_first_token":0.149337821,"token_sha":"2b28aa2108de","tokens_per_second":25.333080955312543},"seed":1234,"sent":1097752.773693992,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.15406174096278846,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.290285419263782,"done":1097762.995020961,"finish_reason":"length","first":1097752.909930388,"full_response_s":10.22049890900962,"last":1097762.992853367,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.909930388,2],[1097753.182460489,7],[1097753.331660952,4],[1097753.539894742,63],[1097753.775471583,10],[1097753.952285265,31],[1097754.137614294,29],[1097754.313822854,22],[1097754.501734824,14],[1097754.682668087,24],[1097754.86366972,4],[1097755.009674219,13],[1097755.192615401,21],[1097755.373077346,2],[1097755.561137545,18],[1097755.740138131,29],[1097755.972288937,27],[1097756.207479572,24],[1097756.355214817,4],[1097756.512037242,16],[1097756.748833366,10],[1097756.932570129,31],[1097757.079785156,8],[1097757.262405681,11],[1097757.44885112,16],[1097757.630176651,12],[1097757.81113255,39],[1097757.982112662,18],[1097758.165592062,11],[1097758.347195475,16],[1097758.5385766,16],[1097758.683349499,11],[1097758.865920542,6],[1097759.047285728,9],[1097759.228975311,29],[1097759.415184109,22],[1097759.601740967,8],[1097759.748596336,18],[1097759.929873823,18],[1097760.111122764,34],[1097760.295308346,10],[1097760.485068189,14],[1097760.666735502,13],[1097760.849340629,2],[1097761.031151708,36],[1097761.213265779,42],[1097761.398068689,26],[1097761.551418782,17],[1097761.734368137,34],[1097761.967877297,12],[1097762.149730376,36],[1097762.44027694,50],[1097762.6265522,2],[1097762.992853367,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-80dcf08860608e389b26fdf2353f63a9","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011086918,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.083621862,"time_to_first_token":0.131918597,"token_sha":"2b28aa2108de","tokens_per_second":25.288532581826004},"seed":1234,"sent":1097752.774522052,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.13540833606384695,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.416768982516135,"done":1097762.995767869,"finish_reason":"length","first":1097752.962975825,"full_response_s":10.220042359083891,"last":1097762.995722323,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.962975825,2],[1097753.184727503,7],[1097753.330331149,4],[1097753.540779473,63],[1097753.773106685,10],[1097753.952346198,31],[1097754.135087819,29],[1097754.315089414,22],[1097754.502502729,14],[1097754.684032673,24],[1097754.863466014,4],[1097755.009959783,13],[1097755.191702675,21],[1097755.372751422,2],[1097755.559458679,18],[1097755.739626207,29],[1097755.974910089,27],[1097756.206533743,24],[1097756.355486643,4],[1097756.510932117,16],[1097756.748937332,10],[1097756.931159514,31],[1097757.078901831,8],[1097757.262334679,11],[1097757.446354116,16],[1097757.630832769,12],[1097757.813454226,39],[1097757.98196483,18],[1097758.164537823,11],[1097758.346377519,16],[1097758.537242339,16],[1097758.683764816,11],[1097758.864991181,6],[1097759.047171661,9],[1097759.230382217,29],[1097759.414159411,22],[1097759.602934692,8],[1097759.748535212,18],[1097759.929152506,18],[1097760.110120291,34],[1097760.295575846,10],[1097760.485367868,14],[1097760.669504894,13],[1097760.850959792,2],[1097761.029874119,36],[1097761.216052371,42],[1097761.396155692,26],[1097761.552129162,17],[1097761.734121514,34],[1097761.969629763,12],[1097762.150228695,36],[1097762.43996159,50],[1097762.626397791,2],[1097762.995722323,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-0d3accb0f29c5a22fe446f7eeab871a4","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010758942,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.029932866,"time_to_first_token":0.183763441,"token_sha":"2b28aa2108de","tokens_per_second":25.423898983851885},"seed":1234,"sent":1097752.77572551,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.18725031497888267,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.37224450496651,"done":1097762.995350862,"finish_reason":"length","first":1097752.944959803,"full_response_s":10.221578165888786,"last":1097762.995312264,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.944959803,2],[1097753.184133509,7],[1097753.333586878,4],[1097753.540033034,63],[1097753.772957506,10],[1097753.954235438,31],[1097754.13433554,29],[1097754.314013498,22],[1097754.503454602,14],[1097754.68456205,24],[1097754.865130055,4],[1097755.009996493,13],[1097755.19372759,21],[1097755.37303364,2],[1097755.561535,18],[1097755.741579078,29],[1097755.973615458,27],[1097756.205991082,24],[1097756.354935194,4],[1097756.512727319,16],[1097756.748656332,10],[1097756.930121009,31],[1097757.079625349,8],[1097757.262143733,11],[1097757.447056966,16],[1097757.630309498,12],[1097757.812178496,39],[1097757.983728689,18],[1097758.163763542,11],[1097758.345880121,16],[1097758.536117979,16],[1097758.683124518,11],[1097758.865219651,6],[1097759.047085578,9],[1097759.232285287,29],[1097759.414644736,22],[1097759.599192398,8],[1097759.747798956,18],[1097759.927766978,18],[1097760.110547692,34],[1097760.294091444,10],[1097760.486228018,14],[1097760.66912384,13],[1097760.850899165,2],[1097761.030063204,36],[1097761.21343956,42],[1097761.397266304,26],[1097761.551914653,17],[1097761.736443853,34],[1097761.968620862,12],[1097762.151098541,36],[1097762.440746891,50],[1097762.627376916,2],[1097762.995312264,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-ba98fd34014f8c2c5bdef0aaf46a8002","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.01105469,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.04780917,"time_to_first_token":0.16582662,"token_sha":"2b28aa2108de","tokens_per_second":25.37866670093218},"seed":1234,"sent":1097752.773772696,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.1711871069855988,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.599510920874007,"done":1097762.993999976,"finish_reason":"length","first":1097753.032835191,"full_response_s":10.219923877855763,"last":1097762.993962995,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097753.032835191,2],[1097753.185128067,7],[1097753.331567103,4],[1097753.540734918,63],[1097753.774543657,10],[1097753.95245897,31],[1097754.136840193,29],[1097754.316483458,22],[1097754.502865605,14],[1097754.684507441,24],[1097754.86648263,4],[1097755.009475331,13],[1097755.194216236,21],[1097755.372069427,2],[1097755.560324736,18],[1097755.742609859,29],[1097755.975652997,27],[1097756.208321596,24],[1097756.354194831,4],[1097756.511792647,16],[1097756.750092375,10],[1097756.929865401,31],[1097757.079720958,8],[1097757.261928149,11],[1097757.446906668,16],[1097757.631266675,12],[1097757.810339842,39],[1097757.98222175,18],[1097758.166129324,11],[1097758.345659223,16],[1097758.536805569,16],[1097758.685978636,11],[1097758.867442252,6],[1097759.047117709,9],[1097759.230336543,29],[1097759.415761024,22],[1097759.60082121,8],[1097759.745527766,18],[1097759.928844695,18],[1097760.110045063,34],[1097760.292670118,10],[1097760.484704441,14],[1097760.670128938,13],[1097760.848592771,2],[1097761.030629757,36],[1097761.216324977,42],[1097761.398316641,26],[1097761.550342526,17],[1097761.734832006,34],[1097761.967537149,12],[1097762.151134145,36],[1097762.440670511,50],[1097762.627748826,2],[1097762.993962995,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-5c771e56abec995d08413d90fb6ad430","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010819011,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.96033182,"time_to_first_token":0.253175109,"token_sha":"2b28aa2108de","tokens_per_second":25.601556716009085},"seed":1234,"sent":1097752.774076098,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.25875909300521016,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":24.983958699425145,"done":1097762.992785809,"finish_reason":"length","first":1097752.786200511,"full_response_s":10.219328698003665,"last":1097762.992749564,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.786200511,2],[1097753.184227742,7],[1097753.330693962,4],[1097753.540431485,63],[1097753.774106078,10],[1097753.952532823,31],[1097754.138074108,29],[1097754.313191272,22],[1097754.501366409,14],[1097754.683758171,24],[1097754.863595067,4],[1097755.00985929,13],[1097755.191441638,21],[1097755.371982271,2],[1097755.56108521,18],[1097755.739956325,29],[1097755.971914505,27],[1097756.207529218,24],[1097756.353056229,4],[1097756.51121792,16],[1097756.749703086,10],[1097756.931362163,31],[1097757.08023576,8],[1097757.261348733,11],[1097757.44645544,16],[1097757.629110641,12],[1097757.809696965,39],[1097757.983062599,18],[1097758.165038407,11],[1097758.347164736,16],[1097758.538787572,16],[1097758.685207957,11],[1097758.865128861,6],[1097759.046903247,9],[1097759.229849072,29],[1097759.414265137,22],[1097759.599085743,8],[1097759.745941481,18],[1097759.927453018,18],[1097760.109860523,34],[1097760.292504213,10],[1097760.483559965,14],[1097760.667020949,13],[1097760.848912476,2],[1097761.029312577,36],[1097761.214033968,42],[1097761.396511642,26],[1097761.55114712,17],[1097761.734979404,34],[1097761.967388663,12],[1097762.150524914,36],[1097762.441627222,50],[1097762.625142348,2],[1097762.992749564,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-47d004c321d14d40517cf93287e4b23e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011297425,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.20637956,"time_to_first_token":0.011941661,"token_sha":"2b28aa2108de","tokens_per_second":24.984373597017196},"seed":1234,"sent":1097752.773457111,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.012743399944156408,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.23908834926805,"done":1097762.996028178,"finish_reason":"length","first":1097752.892616202,"full_response_s":10.222405381966382,"last":1097762.995992222,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.892616202,2],[1097753.185787028,7],[1097753.33084367,4],[1097753.539640155,63],[1097753.775832763,10],[1097753.953036007,31],[1097754.135807167,29],[1097754.316398847,22],[1097754.502236136,14],[1097754.684214976,24],[1097754.865798666,4],[1097755.009720327,13],[1097755.191805265,21],[1097755.372677457,2],[1097755.560250819,18],[1097755.740526957,29],[1097755.973154719,27],[1097756.206774783,24],[1097756.353405735,4],[1097756.510752409,16],[1097756.748597833,10],[1097756.932395129,31],[1097757.078817893,8],[1097757.26251136,11],[1097757.447299125,16],[1097757.630499195,12],[1097757.810547052,39],[1097757.984432881,18],[1097758.165125178,11],[1097758.346674575,16],[1097758.537720414,16],[1097758.685345638,11],[1097758.865046301,6],[1097759.048337705,9],[1097759.230765499,29],[1097759.41663629,22],[1097759.601794454,8],[1097759.746309377,18],[1097759.929956655,18],[1097760.1122962,34],[1097760.296068117,10],[1097760.48383539,14],[1097760.669549384,13],[1097760.850070476,2],[1097761.029580156,36],[1097761.213153826,42],[1097761.396438976,26],[1097761.550954801,17],[1097761.736837649,34],[1097761.968814893,12],[1097762.150000036,36],[1097762.440203762,50],[1097762.626992438,2],[1097762.995992222,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-a534cb28ec9f7924e4cdab6d252cb2bd","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010924451,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.101237782,"time_to_first_token":0.114902931,"token_sha":"2b28aa2108de","tokens_per_second":25.24443097997354},"seed":1234,"sent":1097752.773622796,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.11899340595118701,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.513669821074863,"done":1097762.992961429,"finish_reason":"length","first":1097752.99829316,"full_response_s":10.21680944901891,"last":1097762.992935318,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.99829316,2],[1097753.182581587,7],[1097753.331244906,4],[1097753.540388995,63],[1097753.775174345,10],[1097753.952078371,31],[1097754.13494751,29],[1097754.314420753,22],[1097754.503611609,14],[1097754.684286987,24],[1097754.863783709,4],[1097755.009615111,13],[1097755.192035836,21],[1097755.373444175,2],[1097755.559761197,18],[1097755.740066696,29],[1097755.972229397,27],[1097756.208060736,24],[1097756.353879682,4],[1097756.511456927,16],[1097756.749346633,10],[1097756.930943866,31],[1097757.079214049,8],[1097757.261846533,11],[1097757.447788633,16],[1097757.629315259,12],[1097757.812601786,39],[1097757.983568146,18],[1097758.164868163,11],[1097758.347493795,16],[1097758.538258412,16],[1097758.683569132,11],[1097758.866868731,6],[1097759.046791725,9],[1097759.230260738,29],[1097759.414728081,22],[1097759.601201033,8],[1097759.746813273,18],[1097759.929565867,18],[1097760.111743478,34],[1097760.293736501,10],[1097760.485431377,14],[1097760.669815411,13],[1097760.849812646,2],[1097761.031483612,36],[1097761.21568539,42],[1097761.398464823,26],[1097761.551622036,17],[1097761.735992183,34],[1097761.968772549,12],[1097762.150456955,36],[1097762.441260611,50],[1097762.62598352,2],[1097762.992935318,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-0baa1b38672132090ad2540f0fda4dfb","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.010841744,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.99542051,"time_to_first_token":0.218072367,"token_sha":"2b28aa2108de","tokens_per_second":25.511683049740945},"seed":1234,"sent":1097752.77615198,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.22214117995463312,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.113111220624308,"done":1097762.994225802,"finish_reason":"length","first":1097752.840131555,"full_response_s":10.219554041977972,"last":1097762.994190037,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.840131555,2],[1097753.184431114,7],[1097753.332911682,4],[1097753.541025303,63],[1097753.773947549,10],[1097753.953698088,31],[1097754.137258478,29],[1097754.313412766,22],[1097754.502082363,14],[1097754.683172056,24],[1097754.864143381,4],[1097755.009381371,13],[1097755.191350751,21],[1097755.37183511,2],[1097755.560520166,18],[1097755.741453915,29],[1097755.974815921,27],[1097756.208555272,24],[1097756.354564444,4],[1097756.511539728,16],[1097756.748747794,10],[1097756.932080174,31],[1097757.080695858,8],[1097757.262683094,11],[1097757.44794543,16],[1097757.630539043,12],[1097757.81403356,39],[1097757.982880219,18],[1097758.165829851,11],[1097758.34575997,16],[1097758.536359898,16],[1097758.68386694,11],[1097758.86543669,6],[1097759.047209875,9],[1097759.230452723,29],[1097759.416164669,22],[1097759.599416738,8],[1097759.745632355,18],[1097759.928951861,18],[1097760.112046583,34],[1097760.293174477,10],[1097760.486510167,14],[1097760.667287888,13],[1097760.850768353,2],[1097761.030233159,36],[1097761.215165184,42],[1097761.397446584,26],[1097761.550664696,17],[1097761.733542618,34],[1097761.967940085,12],[1097762.150588311,36],[1097762.441346629,50],[1097762.625247512,2],[1097762.994190037,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-a9886e31621b18c0562e34cd6a32acce","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.01084152,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.153844544,"time_to_first_token":0.062927412,"token_sha":"2b28aa2108de","tokens_per_second":25.113640345289888},"seed":1234,"sent":1097752.77467176,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.06545979506336153,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.64552840557727,"done":1097762.994832069,"finish_reason":"length","first":1097753.051550518,"full_response_s":10.2184520768933,"last":1097762.994804405,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097753.051550518,2],[1097753.182726443,7],[1097753.331107174,4],[1097753.539206835,63],[1097753.775721383,10],[1097753.951707331,31],[1097754.13649243,29],[1097754.314076576,22],[1097754.504671451,14],[1097754.683350468,24],[1097754.864398063,4],[1097755.009158613,13],[1097755.193423695,21],[1097755.373613621,2],[1097755.560895239,18],[1097755.741667948,29],[1097755.972848135,27],[1097756.2077668,24],[1097756.353131459,4],[1097756.511719596,16],[1097756.74975755,10],[1097756.929701367,31],[1097757.078699054,8],[1097757.262092039,11],[1097757.448468941,16],[1097757.631413787,12],[1097757.810962914,39],[1097757.984231386,18],[1097758.16432192,11],[1097758.346419352,16],[1097758.537143961,16],[1097758.684809961,11],[1097758.867163048,6],[1097759.046857908,9],[1097759.231998094,29],[1097759.414846599,22],[1097759.599771954,8],[1097759.746428023,18],[1097759.929016939,18],[1097760.110598362,34],[1097760.293927252,10],[1097760.484998773,14],[1097760.668125209,13],[1097760.850139012,2],[1097761.031065498,36],[1097761.215475284,42],[1097761.396878927,26],[1097761.553254518,17],[1097761.73575547,34],[1097761.968012591,12],[1097762.151034856,36],[1097762.440600054,50],[1097762.627447388,2],[1097762.994804405,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-d5b2cbce2c66b87ceb6175803a156502","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.01110757,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":9.942954555,"time_to_first_token":0.271236405,"token_sha":"2b28aa2108de","tokens_per_second":25.64630046224726},"seed":1234,"sent":1097752.776379992,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.2751705260016024,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"D","case":"code","request":{"cache_state":"reused","cached_tokens":14,"case_id":null,"complete":true,"decode_tps":25.027628509471302,"done":1097762.993312815,"finish_reason":"length","first":1097752.80453025,"full_response_s":10.220698029035702,"last":1097762.993270262,"output_sha256":"aafe5ed4d4636e0ee4268bf2173d5de3929d2acfae0d404b79b67567f06e98dd","pieces":[[1097752.80453025,2],[1097753.183390544,7],[1097753.329760273,4],[1097753.540142829,63],[1097753.772804277,10],[1097753.953095387,31],[1097754.133955373,29],[1097754.315632479,22],[1097754.505442414,14],[1097754.68380849,24],[1097754.864048428,4],[1097755.009539418,13],[1097755.192561544,21],[1097755.373721029,2],[1097755.561438734,18],[1097755.742838092,29],[1097755.972055437,27],[1097756.208499062,24],[1097756.353795056,4],[1097756.512238386,16],[1097756.749280849,10],[1097756.932528232,31],[1097757.07934062,8],[1097757.261594223,11],[1097757.447601625,16],[1097757.629608746,12],[1097757.80992478,39],[1097757.982278152,18],[1097758.164597571,11],[1097758.347725099,16],[1097758.537650805,16],[1097758.684727673,11],[1097758.866609155,6],[1097759.047584655,9],[1097759.23222261,29],[1097759.416375096,22],[1097759.600179362,8],[1097759.74618865,18],[1097759.928478769,18],[1097760.111843296,34],[1097760.293315711,10],[1097760.484653259,14],[1097760.669294804,13],[1097760.849259526,2],[1097761.031540173,36],[1097761.21386744,42],[1097761.396415811,26],[1097761.551208116,17],[1097761.736343523,34],[1097761.968232206,12],[1097762.150664162,36],[1097762.439872579,50],[1097762.62577825,2],[1097762.993270262,56]],"prompt":"code","prompt_tokens":14,"request_body":{"ignore_eos":true,"max_tokens":256,"model":"qwen27-overlap","prompt":"Write a short Python function that computes the Fibonacci sequence and explain it.","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"d499e4f8773449fd492de563e5fa1dc12064536d55b33ea1f2e88522b3f632aa","request_path":"/v1/completions","response_id":"cmpl-8f1bbd94d48623fc86f1b91ba025961d","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011009457,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":10.189265029,"time_to_first_token":0.028935988,"token_sha":"2b28aa2108de","tokens_per_second":25.026338923782646},"seed":1234,"sent":1097752.772614786,"speculative":{"acceptance_rate":0.559228650137741,"accepted":203,"drafted":363,"rounds":53,"tokens_per_round":4.830188679245283},"token_sha":"2b28aa2108de","tokens":256,"ttft_s":0.03191546397283673,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":14,"prompt_tokens_details":{"cached_tokens":14},"total_tokens":270},"work":{"accepted":203,"drafted":363,"rounds":53},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"D","case":"chat","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":49.15328105985827,"done":1097768.320954669,"finish_reason":"length","first":1097763.13303044,"full_response_s":5.320269583025947,"last":1097768.320883509,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097763.13303044,3],[1097763.220954906,26],[1097763.301595647,15],[1097763.380288553,19],[1097763.467082515,21],[1097763.550085986,15],[1097763.630094525,25],[1097763.709269643,14],[1097763.790193379,14],[1097763.869923247,18],[1097763.950678971,26],[1097764.030929287,13],[1097764.111821761,13],[1097764.192088006,13],[1097764.272894767,23],[1097764.353095182,32],[1097764.439320805,7],[1097764.52329308,23],[1097764.60409872,9],[1097764.683886293,13],[1097764.764337827,15],[1097764.84421708,19],[1097764.923581255,12],[1097765.002708208,5],[1097765.083218872,23],[1097765.162850366,9],[1097765.244610841,17],[1097765.32362738,3],[1097765.404267333,21],[1097765.490904632,10],[1097765.57183651,24],[1097765.650571065,19],[1097765.731885031,14],[1097765.810818464,13],[1097765.890487784,5],[1097765.970768488,22],[1097766.051400558,13],[1097766.132064262,51],[1097766.211525001,15],[1097766.293137158,6],[1097766.371654647,8],[1097766.460102521,9],[1097766.54437411,16],[1097766.625314385,16],[1097766.706568485,9],[1097766.788122065,32],[1097766.867351985,14],[1097766.947418774,10],[1097767.028049287,18],[1097767.108816388,15],[1097767.188587572,8],[1097767.270131299,18],[1097767.343791867,25],[1097767.427558437,19],[1097767.512359697,8],[1097767.591989234,40],[1097767.672178396,16],[1097767.753298991,64],[1097767.832642419,6],[1097767.913273666,6],[1097767.993282262,11],[1097768.074712689,15],[1097768.155539356,17],[1097768.236072597,21],[1097768.320883509,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-e1cb1149b5df20bf995fc3e500d12e14","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":0.130843901,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.187903583,"time_to_first_token":0.131561881,"token_sha":"8a91591c94a3","tokens_per_second":49.152802460631236},"seed":1234,"sent":1097763.000685086,"speculative":{"acceptance_rate":0.22441860465116278,"accepted":193,"drafted":860,"rounds":64,"tokens_per_round":4.0},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.13234535395167768,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":287},"work":{"accepted":193,"drafted":860,"rounds":64},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":49.084485304040726,"done":1097773.532574597,"finish_reason":"length","first":1097768.33737083,"full_response_s":5.208992755971849,"last":1097773.532495082,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097768.33737083,3],[1097768.429782798,26],[1097768.515335867,15],[1097768.594694551,19],[1097768.673796532,21],[1097768.75451698,15],[1097768.834224227,25],[1097768.914503631,14],[1097768.994768772,14],[1097769.075825261,18],[1097769.155895094,26],[1097769.236847093,13],[1097769.31739908,13],[1097769.397979322,13],[1097769.484749943,23],[1097769.56675438,32],[1097769.646941765,7],[1097769.727723591,23],[1097769.807910865,9],[1097769.887945697,13],[1097769.968036697,15],[1097770.048708415,19],[1097770.127961611,12],[1097770.207042809,5],[1097770.287148825,23],[1097770.368059843,9],[1097770.455283078,17],[1097770.537850702,3],[1097770.617276809,21],[1097770.697541296,10],[1097770.777333028,24],[1097770.856603874,19],[1097770.936783712,14],[1097771.015811866,13],[1097771.096286617,5],[1097771.176442138,22],[1097771.257505515,13],[1097771.337953739,51],[1097771.421503336,15],[1097771.507235147,6],[1097771.586833821,8],[1097771.66667056,9],[1097771.74719878,16],[1097771.827629369,16],[1097771.907867855,9],[1097771.988280691,32],[1097772.068252953,14],[1097772.148917013,10],[1097772.229645765,18],[1097772.309338547,15],[1097772.390105911,8],[1097772.477960159,18],[1097772.554695961,25],[1097772.634915765,19],[1097772.714954126,8],[1097772.796046947,40],[1097772.876845201,16],[1097772.957415315,64],[1097773.036682823,6],[1097773.117614396,6],[1097773.197532799,11],[1097773.278773899,15],[1097773.35854086,17],[1097773.444362268,21],[1097773.532495082,14]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-7ada1d8160bdf7fb64dc8cbae177c0e5","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":0.011929748,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":5.196005609,"time_to_first_token":0.012901392,"token_sha":"8a91591c94a3","tokens_per_second":49.076159494191955},"seed":1234,"sent":1097768.323581841,"speculative":{"acceptance_rate":0.22441860465116278,"accepted":193,"drafted":860,"rounds":64,"tokens_per_round":4.0},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.013788988813757896,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":193,"drafted":860,"rounds":64},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"repeat","stream_index":0} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.696154440069108,"done":1097787.285717897,"finish_reason":"length","first":1097773.646512264,"full_response_s":13.744158088928089,"last":1097787.285680726,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.646512264,3],[1097773.964783836,11],[1097774.146336829,30],[1097774.32629492,19],[1097774.48105415,21],[1097774.661709238,15],[1097774.80894447,19],[1097774.956542688,17],[1097775.10760217,6],[1097775.2518936,8],[1097775.436856416,3],[1097775.589452668,5],[1097775.766808057,27],[1097775.948555842,12],[1097776.130087467,13],[1097776.310848492,5],[1097776.467187502,16],[1097776.652587927,5],[1097776.8312244,14],[1097777.012818522,9],[1097777.161288016,22],[1097777.309867488,10],[1097777.499235476,7],[1097777.678094736,23],[1097777.826443765,5],[1097777.973089454,9],[1097778.143578173,14],[1097778.291858989,20],[1097778.445312318,8],[1097778.597625948,12],[1097778.74589663,5],[1097778.893049269,14],[1097779.075815871,14],[1097779.224157808,8],[1097779.405794158,13],[1097779.583116682,3],[1097779.761298036,10],[1097779.908759309,11],[1097780.090910531,9],[1097780.273832343,10],[1097780.461056875,20],[1097780.60931298,14],[1097780.790964808,14],[1097781.019818016,13],[1097781.168234773,5],[1097781.350860751,22],[1097781.58703579,3],[1097781.819448167,32],[1097782.004367223,33],[1097782.184982034,11],[1097782.367873335,6],[1097782.523419439,8],[1097782.671895726,9],[1097782.820066402,13],[1097782.967488264,8],[1097783.116893724,20],[1097783.268200919,17],[1097783.417292172,20],[1097783.570316078,13],[1097783.746805512,6],[1097783.927820413,13],[1097784.075183915,9],[1097784.225575681,11],[1097784.37390295,8],[1097784.56580853,15],[1097784.748669289,16],[1097784.960476336,27],[1097785.199948599,4],[1097785.351543994,8],[1097785.537966028,25],[1097785.68784011,18],[1097785.837248922,13],[1097786.066157674,64],[1097786.215943398,2],[1097786.398102723,10],[1097786.583102728,11],[1097786.729551284,15],[1097786.878680864,17],[1097787.028491841,14],[1097787.285680726,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-0077e0e8f869ef3ce37a4fb657b6b60c","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011912405,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.63716952,"time_to_first_token":0.103290168,"token_sha":"8a91591c94a3","tokens_per_second":18.698894930214227},"seed":1234,"sent":1097773.541559808,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.10495245596393943,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":0} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.826232091159802,"done":1097787.28478044,"finish_reason":"length","first":1097773.739833715,"full_response_s":13.745393611025065,"last":1097787.284763936,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.739833715,3],[1097773.966798628,11],[1097774.146105745,30],[1097774.326614579,19],[1097774.481405426,21],[1097774.661426863,15],[1097774.80917613,19],[1097774.957991171,17],[1097775.10686589,6],[1097775.251682448,8],[1097775.439461868,3],[1097775.589665022,5],[1097775.767377369,27],[1097775.949225396,12],[1097776.131365151,13],[1097776.312797412,5],[1097776.467663306,16],[1097776.651177308,5],[1097776.833278313,14],[1097777.01182512,9],[1097777.160142229,22],[1097777.306920929,10],[1097777.495598683,7],[1097777.677957751,23],[1097777.825393904,5],[1097777.971771053,9],[1097778.144440368,14],[1097778.292933311,20],[1097778.445196786,8],[1097778.597695136,12],[1097778.745376774,5],[1097778.894486752,14],[1097779.07732997,14],[1097779.222453562,8],[1097779.404423366,13],[1097779.581708126,3],[1097779.762511403,10],[1097779.911300574,11],[1097780.092465414,9],[1097780.273072478,10],[1097780.459942891,20],[1097780.610073197,14],[1097780.790000005,14],[1097781.018954501,13],[1097781.169486984,5],[1097781.350649971,22],[1097781.587165825,3],[1097781.820938224,32],[1097782.003390645,33],[1097782.188054419,11],[1097782.368583041,6],[1097782.524653606,8],[1097782.672324729,9],[1097782.820614696,13],[1097782.968115414,8],[1097783.116711024,20],[1097783.266842021,17],[1097783.415779949,20],[1097783.573589196,13],[1097783.746319633,6],[1097783.926642394,13],[1097784.075410925,9],[1097784.223998133,11],[1097784.373346765,8],[1097784.565931232,15],[1097784.749623755,16],[1097784.960432426,27],[1097785.202027658,4],[1097785.349075508,8],[1097785.537649277,25],[1097785.687339011,18],[1097785.837508171,13],[1097786.064720829,64],[1097786.214048869,2],[1097786.39788391,10],[1097786.580998511,11],[1097786.731457083,15],[1097786.87950087,17],[1097787.030315609,14],[1097787.284763936,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-607d18572593f5b71375f5c3b7c4a2e8","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.012262657,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.544724951,"time_to_first_token":0.194406655,"token_sha":"8a91591c94a3","tokens_per_second":18.826517402346624},"seed":1234,"sent":1097773.539386829,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.200446886010468,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":1} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.624408028554743,"done":1097787.283334587,"finish_reason":"length","first":1097773.591583074,"full_response_s":13.744430786930025,"last":1097787.283293415,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.591583074,3],[1097773.96738262,11],[1097774.145840035,30],[1097774.326025159,19],[1097774.47953332,21],[1097774.660958004,15],[1097774.809589968,19],[1097774.955931194,17],[1097775.106659316,6],[1097775.251994566,8],[1097775.436644895,3],[1097775.587236977,5],[1097775.767531451,27],[1097775.949719962,12],[1097776.12869529,13],[1097776.310355767,5],[1097776.466126252,16],[1097776.652485264,5],[1097776.83344067,14],[1097777.01160177,9],[1097777.159224642,22],[1097777.308106609,10],[1097777.495539403,7],[1097777.679910139,23],[1097777.826929539,5],[1097777.971129773,9],[1097778.145023105,14],[1097778.291084984,20],[1097778.444543948,8],[1097778.596672713,12],[1097778.744830232,5],[1097778.894141677,14],[1097779.076577611,14],[1097779.223990409,8],[1097779.40539411,13],[1097779.582787791,3],[1097779.762765375,10],[1097779.910897518,11],[1097780.091499747,9],[1097780.271354934,10],[1097780.460887875,20],[1097780.61148415,14],[1097780.790295886,14],[1097781.018264482,13],[1097781.16887936,5],[1097781.351297631,22],[1097781.585399269,3],[1097781.819981447,32],[1097782.001040981,33],[1097782.185720891,11],[1097782.367940489,6],[1097782.523193964,8],[1097782.671422444,9],[1097782.819156692,13],[1097782.966813089,8],[1097783.115790701,20],[1097783.265991765,17],[1097783.417200126,20],[1097783.571505354,13],[1097783.746889619,6],[1097783.928182147,13],[1097784.073882602,9],[1097784.224329356,11],[1097784.373784906,8],[1097784.565529077,15],[1097784.748938449,16],[1097784.959889402,27],[1097785.20098624,4],[1097785.348600741,8],[1097785.538668444,25],[1097785.692553499,18],[1097785.836283945,13],[1097786.063842279,64],[1097786.215381181,2],[1097786.398241224,10],[1097786.582329271,11],[1097786.728901712,15],[1097786.878935166,17],[1097787.02991,14],[1097787.283293415,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-1af46645fbe4de09b41ecd652366e2f1","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011985196,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.69203844,"time_to_first_token":0.049463348,"token_sha":"8a91591c94a3","tokens_per_second":18.623961736408916},"seed":1234,"sent":1097773.5389038,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.052679273998364806,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":2} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.725196736367675,"done":1097787.284875159,"finish_reason":"length","first":1097773.665557681,"full_response_s":13.746070964029059,"last":1097787.283572145,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.665557681,3],[1097773.965894783,11],[1097774.146038141,30],[1097774.326131744,19],[1097774.479661676,21],[1097774.662047307,15],[1097774.809853885,19],[1097774.955738948,17],[1097775.104185663,6],[1097775.253935375,8],[1097775.438400038,3],[1097775.58784751,5],[1097775.768040906,27],[1097775.948300955,12],[1097776.131243247,13],[1097776.310187582,5],[1097776.465646174,16],[1097776.649604059,5],[1097776.831995654,14],[1097777.013048467,9],[1097777.159645232,22],[1097777.306874905,10],[1097777.496527123,7],[1097777.678221954,23],[1097777.826367822,5],[1097777.971983996,9],[1097778.145461222,14],[1097778.292336423,20],[1097778.443835626,8],[1097778.595634522,12],[1097778.744352863,5],[1097778.894064437,14],[1097779.07807266,14],[1097779.221735269,8],[1097779.405842856,13],[1097779.580911407,3],[1097779.762845609,10],[1097779.909463432,11],[1097780.089425715,9],[1097780.271979769,10],[1097780.460071741,20],[1097780.612219314,14],[1097780.790642785,14],[1097781.018013745,13],[1097781.168492745,5],[1097781.349738718,22],[1097781.587300438,3],[1097781.817929007,32],[1097782.001281103,33],[1097782.185451634,11],[1097782.368373702,6],[1097782.523281033,8],[1097782.67082233,9],[1097782.821099998,13],[1097782.968190716,8],[1097783.11684562,20],[1097783.266424737,17],[1097783.41591296,20],[1097783.573972222,13],[1097783.745576792,6],[1097783.926037865,13],[1097784.074384584,9],[1097784.224197498,11],[1097784.374053506,8],[1097784.566745867,15],[1097784.749562716,16],[1097784.960209883,27],[1097785.201755554,4],[1097785.351035819,8],[1097785.538762347,25],[1097785.689007352,18],[1097785.837415853,13],[1097786.063781064,64],[1097786.213850402,2],[1097786.397378506,10],[1097786.582752186,11],[1097786.729448209,15],[1097786.879199969,17],[1097787.031651904,14],[1097787.283572145,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-ed8577f5c025e3a9c8952eccd28a8d86","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.01196653,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.618624214,"time_to_first_token":0.122139144,"token_sha":"8a91591c94a3","tokens_per_second":18.724358348757356},"seed":1234,"sent":1097773.538804195,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.1267534859944135,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":3} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.92584537454418,"done":1097787.286953005,"finish_reason":"length","first":1097773.813260131,"full_response_s":13.747486853040755,"last":1097787.28689869,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.813260131,3],[1097773.965736955,11],[1097774.145453284,30],[1097774.329192344,19],[1097774.481222064,21],[1097774.662405531,15],[1097774.809929077,19],[1097774.957549112,17],[1097775.10501809,6],[1097775.252131759,8],[1097775.436556512,3],[1097775.586666817,5],[1097775.76813534,27],[1097775.949450924,12],[1097776.128575899,13],[1097776.311588372,5],[1097776.466049443,16],[1097776.649849517,5],[1097776.834112416,14],[1097777.014083524,9],[1097777.160606586,22],[1097777.306633994,10],[1097777.49763806,7],[1097777.679073506,23],[1097777.825850127,5],[1097777.972177729,9],[1097778.144950203,14],[1097778.291277789,20],[1097778.444182143,8],[1097778.597438393,12],[1097778.746399869,5],[1097778.893732776,14],[1097779.077636065,14],[1097779.222819352,8],[1097779.404517655,13],[1097779.582610291,3],[1097779.763404475,10],[1097779.908887567,11],[1097780.090215003,9],[1097780.274949704,10],[1097780.460566941,20],[1097780.610855537,14],[1097780.793168923,14],[1097781.0185716,13],[1097781.168277915,5],[1097781.351022322,22],[1097781.587111285,3],[1097781.817836015,32],[1097782.00322027,33],[1097782.184604993,11],[1097782.367348251,6],[1097782.525202574,8],[1097782.673452377,9],[1097782.821053655,13],[1097782.967181068,8],[1097783.116511242,20],[1097783.266782567,17],[1097783.417246101,20],[1097783.574428582,13],[1097783.744546045,6],[1097783.926957017,13],[1097784.074018527,9],[1097784.225023194,11],[1097784.374687233,8],[1097784.566525804,15],[1097784.749933671,16],[1097784.962200049,27],[1097785.198809708,4],[1097785.348791829,8],[1097785.539448963,25],[1097785.687182917,18],[1097785.836514412,13],[1097786.065316166,64],[1097786.216147804,2],[1097786.398627721,10],[1097786.58284905,11],[1097786.729927472,15],[1097786.877976642,17],[1097787.031974383,14],[1097787.28689869,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-bb4209a7e67c594c36122ac4adcaac50","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011771475,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.474186133,"time_to_first_token":0.266877117,"token_sha":"8a91591c94a3","tokens_per_second":18.92507625195057},"seed":1234,"sent":1097773.539466152,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.27379397908225656,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":4} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.950311746952966,"done":1097787.287903396,"finish_reason":"length","first":1097773.831572998,"full_response_s":13.743951275013387,"last":1097787.287816009,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.831572998,3],[1097773.966905245,11],[1097774.14676453,30],[1097774.327037301,19],[1097774.480290957,21],[1097774.662910826,15],[1097774.809246664,19],[1097774.95734947,17],[1097775.104358315,6],[1097775.252903466,8],[1097775.437962429,3],[1097775.586779197,5],[1097775.76706429,27],[1097775.950022891,12],[1097776.130672064,13],[1097776.312044469,5],[1097776.466656597,16],[1097776.648555104,5],[1097776.831466479,14],[1097777.012393883,9],[1097777.159482426,22],[1097777.309335256,10],[1097777.498716851,7],[1097777.677649333,23],[1097777.82450608,5],[1097777.972384541,9],[1097778.14290017,14],[1097778.291901459,20],[1097778.444625623,8],[1097778.597315433,12],[1097778.745808232,5],[1097778.894436806,14],[1097779.0771339,14],[1097779.224473332,8],[1097779.405439046,13],[1097779.581183373,3],[1097779.761398905,10],[1097779.908644433,11],[1097780.091819384,9],[1097780.274216221,10],[1097780.460130252,20],[1097780.610670401,14],[1097780.793752184,14],[1097781.01904325,13],[1097781.169863946,5],[1097781.351081232,22],[1097781.584844408,3],[1097781.818396142,32],[1097782.001361624,33],[1097782.185759711,11],[1097782.36723336,6],[1097782.524090038,8],[1097782.671221509,9],[1097782.819876129,13],[1097782.967332104,8],[1097783.11692731,20],[1097783.267526817,17],[1097783.416096493,20],[1097783.574825247,13],[1097783.745093763,6],[1097783.928874273,13],[1097784.074942769,9],[1097784.223535436,11],[1097784.375501917,8],[1097784.567538683,15],[1097784.74947608,16],[1097784.960515892,27],[1097785.199085815,4],[1097785.34941611,8],[1097785.538072577,25],[1097785.68750535,18],[1097785.837952185,13],[1097786.064093588,64],[1097786.214811793,2],[1097786.397673421,10],[1097786.581427173,11],[1097786.729809157,15],[1097786.878473705,17],[1097787.029238419,14],[1097787.287816009,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-bde77a03f5d5947eb89beb957d0feaa8","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011782568,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.456756407,"time_to_first_token":0.284731224,"token_sha":"8a91591c94a3","tokens_per_second":18.949588763258944},"seed":1234,"sent":1097773.543952121,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.28762087714858353,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":5} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.598240008057427,"done":1097787.285788812,"finish_reason":"length","first":1097773.57287808,"full_response_s":13.746190124889836,"last":1097787.283852876,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.57287808,3],[1097773.965837168,11],[1097774.14516521,30],[1097774.327909145,19],[1097774.48101581,21],[1097774.661500983,15],[1097774.80869536,19],[1097774.957068696,17],[1097775.106243959,6],[1097775.25570968,8],[1097775.43691243,3],[1097775.588104959,5],[1097775.767675479,27],[1097775.949264281,12],[1097776.12946922,13],[1097776.30987396,5],[1097776.466597446,16],[1097776.650627593,5],[1097776.832439791,14],[1097777.012876889,9],[1097777.159939483,22],[1097777.310209299,10],[1097777.49660803,7],[1097777.677498198,23],[1097777.826172487,5],[1097777.972486579,9],[1097778.143482827,14],[1097778.291791226,20],[1097778.443958362,8],[1097778.595506663,12],[1097778.747052259,5],[1097778.893675435,14],[1097779.076639947,14],[1097779.223650456,8],[1097779.405664251,13],[1097779.583587935,3],[1097779.763741306,10],[1097779.90908266,11],[1097780.091191956,9],[1097780.273502684,10],[1097780.461380594,20],[1097780.610437993,14],[1097780.794362051,14],[1097781.020568226,13],[1097781.167961224,5],[1097781.350338387,22],[1097781.586360355,3],[1097781.821018201,32],[1097782.000613562,33],[1097782.186160892,11],[1097782.368329728,6],[1097782.52432912,8],[1097782.673192308,9],[1097782.820495724,13],[1097782.969524977,8],[1097783.116083537,20],[1097783.265280188,17],[1097783.417495347,20],[1097783.573228068,13],[1097783.743721514,6],[1097783.92618109,13],[1097784.074850034,9],[1097784.225307754,11],[1097784.375909114,8],[1097784.568233497,15],[1097784.749077351,16],[1097784.962011763,27],[1097785.200681623,4],[1097785.34899329,8],[1097785.538814117,25],[1097785.687441446,18],[1097785.838799984,13],[1097786.064043756,64],[1097786.217586217,2],[1097786.397577453,10],[1097786.580749203,11],[1097786.731892132,15],[1097786.878066159,17],[1097787.030032045,14],[1097787.283852876,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-4a222673736e5700083be7446cf43e62","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.012048494,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.71055402,"time_to_first_token":0.031144563,"token_sha":"8a91591c94a3","tokens_per_second":18.598810786786864},"seed":1234,"sent":1097773.539598687,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.03327939286828041,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":6} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.774560560974507,"done":1097787.284852748,"finish_reason":"length","first":1097773.702613189,"full_response_s":13.745004727039486,"last":1097787.2848219,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.702613189,3],[1097773.965492824,11],[1097774.145049612,30],[1097774.326907408,19],[1097774.479338751,21],[1097774.662234864,15],[1097774.809784456,19],[1097774.956296221,17],[1097775.105931713,6],[1097775.252304875,8],[1097775.438553288,3],[1097775.588919505,5],[1097775.767167529,27],[1097775.948691722,12],[1097776.129838663,13],[1097776.311395742,5],[1097776.465327653,16],[1097776.65071945,5],[1097776.831081317,14],[1097777.012963415,9],[1097777.159345794,22],[1097777.306426093,10],[1097777.496180797,7],[1097777.677147357,23],[1097777.826721158,5],[1097777.97173124,9],[1097778.143920752,14],[1097778.290652166,20],[1097778.444243872,8],[1097778.597089443,12],[1097778.74553585,5],[1097778.893452647,14],[1097779.076866225,14],[1097779.22270294,8],[1097779.404788003,13],[1097779.58383536,3],[1097779.761675785,10],[1097779.908455792,11],[1097780.092122533,9],[1097780.272171228,10],[1097780.459821164,20],[1097780.611949767,14],[1097780.789825947,14],[1097781.020165347,13],[1097781.167145989,5],[1097781.349804945,22],[1097781.586887074,3],[1097781.818035317,32],[1097782.001192626,33],[1097782.183815294,11],[1097782.36848267,6],[1097782.52376043,8],[1097782.673504547,9],[1097782.819657458,13],[1097782.968053366,8],[1097783.116321914,20],[1097783.265364375,17],[1097783.415839163,20],[1097783.571361777,13],[1097783.746569616,6],[1097783.925854157,13],[1097784.074218292,9],[1097784.22287491,11],[1097784.372638648,8],[1097784.568536274,15],[1097784.748755301,16],[1097784.959646,27],[1097785.202316155,4],[1097785.348140126,8],[1097785.538985643,25],[1097785.688521412,18],[1097785.836128924,13],[1097786.063724989,64],[1097786.217149391,2],[1097786.396989672,10],[1097786.58207825,11],[1097786.731657679,15],[1097786.878159517,17],[1097787.02826448,14],[1097787.2848219,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-dcc7d8f69399af54ea9c9b83acebb54f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011948216,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.581709422,"time_to_first_token":0.157473516,"token_sha":"8a91591c94a3","tokens_per_second":18.775250749139463},"seed":1234,"sent":1097773.539848021,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.16276516788639128,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":7} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.648942514670406,"done":1097787.284519991,"finish_reason":"length","first":1097773.610784575,"full_response_s":13.74415189679712,"last":1097787.284482148,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.610784575,3],[1097773.966288258,11],[1097774.146384151,30],[1097774.330645859,19],[1097774.480228428,21],[1097774.661925098,15],[1097774.809086578,19],[1097774.95617446,17],[1097775.106168287,6],[1097775.252200643,8],[1097775.437440407,3],[1097775.586933711,5],[1097775.767594748,27],[1097775.950113386,12],[1097776.129949074,13],[1097776.31328154,5],[1097776.466692568,16],[1097776.650403554,5],[1097776.833571379,14],[1097777.014608696,9],[1097777.161934996,22],[1097777.308673355,10],[1097777.497063645,7],[1097777.679734382,23],[1097777.825489778,5],[1097777.972559321,9],[1097778.143197254,14],[1097778.291423481,20],[1097778.444065874,8],[1097778.595319397,12],[1097778.745437702,5],[1097778.893989966,14],[1097779.076223379,14],[1097779.225053714,8],[1097779.404845969,13],[1097779.583734603,3],[1097779.763132382,10],[1097779.908224104,11],[1097780.090679371,9],[1097780.275011336,10],[1097780.460416416,20],[1097780.610193131,14],[1097780.792685633,14],[1097781.018844476,13],[1097781.169528941,5],[1097781.351637694,22],[1097781.586222957,3],[1097781.818885784,32],[1097782.001430668,33],[1097782.184441342,11],[1097782.368292525,6],[1097782.524408008,8],[1097782.671675517,9],[1097782.820815502,13],[1097782.967646472,8],[1097783.117105912,20],[1097783.26717475,17],[1097783.416911419,20],[1097783.571732269,13],[1097783.743979133,6],[1097783.928960396,13],[1097784.075464569,9],[1097784.224117073,11],[1097784.37493728,8],[1097784.567039231,15],[1097784.750102669,16],[1097784.960713511,27],[1097785.200532652,4],[1097785.349644865,8],[1097785.538866383,25],[1097785.693829707,18],[1097785.836727078,13],[1097786.065595697,64],[1097786.214595013,2],[1097786.397518111,10],[1097786.581897551,11],[1097786.731819695,15],[1097786.879429042,17],[1097787.031332961,14],[1097787.284482148,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-dea231f8125ff4f8d8b4f2d976c28736","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011869789,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.673632232,"time_to_first_token":0.068134708,"token_sha":"8a91591c94a3","tokens_per_second":18.649031630617575},"seed":1234,"sent":1097773.540368094,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.0704164810013026,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":8} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.90107320787797,"done":1097787.286690507,"finish_reason":"length","first":1097773.795340112,"full_response_s":13.744313603034243,"last":1097787.286637521,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.795340112,3],[1097773.965624639,11],[1097774.144879905,30],[1097774.327821097,19],[1097774.480376987,21],[1097774.66198164,15],[1097774.809712754,19],[1097774.956468345,17],[1097775.104699504,6],[1097775.252559891,8],[1097775.437792739,3],[1097775.588440882,5],[1097775.768207459,27],[1097775.94793673,12],[1097776.131137367,13],[1097776.310074034,5],[1097776.465444019,16],[1097776.651097394,5],[1097776.833832572,14],[1097777.014271271,9],[1097777.161783445,22],[1097777.306775189,10],[1097777.496919858,7],[1097777.679319236,23],[1097777.826499074,5],[1097777.972520164,9],[1097778.14572829,14],[1097778.292056788,20],[1097778.445028426,8],[1097778.595887502,12],[1097778.745611937,5],[1097778.892905354,14],[1097779.07590563,14],[1097779.221849328,8],[1097779.404719328,13],[1097779.581949723,3],[1097779.762323545,10],[1097779.908030356,11],[1097780.092820414,9],[1097780.271555886,10],[1097780.460340744,20],[1097780.609844311,14],[1097780.793293467,14],[1097781.019507313,13],[1097781.167867768,5],[1097781.352227869,22],[1097781.585478654,3],[1097781.81916146,32],[1097782.001736312,33],[1097782.185889697,11],[1097782.36811866,6],[1097782.524023332,8],[1097782.670626309,9],[1097782.820714923,13],[1097782.968164607,8],[1097783.116219206,20],[1097783.266647506,17],[1097783.416293313,20],[1097783.571144851,13],[1097783.744828765,6],[1097783.926474694,13],[1097784.075359283,9],[1097784.223312443,11],[1097784.374555823,8],[1097784.567492547,15],[1097784.750021107,16],[1097784.961924199,27],[1097785.199214423,4],[1097785.34850631,8],[1097785.537873757,25],[1097785.687733465,18],[1097785.836879026,13],[1097786.063915548,64],[1097786.213777822,2],[1097786.397937232,10],[1097786.581129376,11],[1097786.730877674,15],[1097786.878377561,17],[1097787.031108546,14],[1097787.286637521,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-51b10c24d9a54eed2f167ae5c082643b","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011789106,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.49209839,"time_to_first_token":0.248818398,"token_sha":"8a91591c94a3","tokens_per_second":18.899951114275858},"seed":1234,"sent":1097773.542376904,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.2529632078949362,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":9} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.877793852627793,"done":1097787.285087584,"finish_reason":"length","first":1097773.777117394,"full_response_s":13.74038758710958,"last":1097787.285051742,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.777117394,3],[1097773.966018786,11],[1097774.145930612,30],[1097774.327401208,19],[1097774.479091323,21],[1097774.66179069,15],[1097774.809639514,19],[1097774.957832799,17],[1097775.105624269,6],[1097775.254509553,8],[1097775.436494864,3],[1097775.589355929,5],[1097775.768472416,27],[1097775.948047493,12],[1097776.130170615,13],[1097776.313711015,5],[1097776.466258594,16],[1097776.651765443,5],[1097776.832978795,14],[1097777.012203399,9],[1097777.161990097,22],[1097777.307580268,10],[1097777.495093633,7],[1097777.679027434,23],[1097777.82472152,5],[1097777.971358693,9],[1097778.144312333,14],[1097778.29163945,20],[1097778.445393209,8],[1097778.596198353,12],[1097778.744619898,5],[1097778.894833621,14],[1097779.076954815,14],[1097779.223023522,8],[1097779.40491924,13],[1097779.583324598,3],[1097779.762452156,10],[1097779.910693492,11],[1097780.092768467,9],[1097780.272073066,10],[1097780.460500379,20],[1097780.610549331,14],[1097780.794046368,14],[1097781.018462904,13],[1097781.168668868,5],[1097781.351336275,22],[1097781.586554759,3],[1097781.820318675,32],[1097782.002014982,33],[1097782.1865286,11],[1097782.367785354,6],[1097782.525091797,8],[1097782.671347158,9],[1097782.820541379,13],[1097782.96825398,8],[1097783.117237611,20],[1097783.266913273,17],[1097783.417132971,20],[1097783.570179561,13],[1097783.74617079,6],[1097783.928501621,13],[1097784.074327147,9],[1097784.223658554,11],[1097784.374507863,8],[1097784.566223363,15],[1097784.750503767,16],[1097784.959967201,27],[1097785.200758926,4],[1097785.349926671,8],[1097785.538909637,25],[1097785.694157119,18],[1097785.838487204,13],[1097786.064997319,64],[1097786.216235399,2],[1097786.398686502,10],[1097786.582520758,11],[1097786.729225602,15],[1097786.878228527,17],[1097787.028596805,14],[1097787.285051742,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-9e9964430c47f0dc0051773354861a9f","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011804011,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.508010447,"time_to_first_token":0.23074925,"token_sha":"8a91591c94a3","tokens_per_second":18.87768750257615},"seed":1234,"sent":1097773.544699997,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.2324173969682306,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":10} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.671481789630537,"done":1097787.285449556,"finish_reason":"length","first":1097773.628234092,"full_response_s":13.746388070983812,"last":1097787.285425464,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.628234092,3],[1097773.964405329,11],[1097774.146986009,30],[1097774.330262597,19],[1097774.480750388,21],[1097774.662818857,15],[1097774.809343276,19],[1097774.957618621,17],[1097775.107111765,6],[1097775.255995608,8],[1097775.438704728,3],[1097775.587027761,5],[1097775.768549529,27],[1097775.949644274,12],[1097776.12920969,13],[1097776.312399809,5],[1097776.467365772,16],[1097776.649247807,5],[1097776.831688324,14],[1097777.014680046,9],[1097777.160355173,22],[1097777.306247023,10],[1097777.49570802,7],[1097777.679986947,23],[1097777.826011683,5],[1097777.971663812,9],[1097778.144495757,14],[1097778.291944762,20],[1097778.443894441,8],[1097778.597546594,12],[1097778.745203932,5],[1097778.893514919,14],[1097779.076517244,14],[1097779.223432775,8],[1097779.405195799,13],[1097779.582284329,3],[1097779.762097076,10],[1097779.910075013,11],[1097780.089664303,9],[1097780.273449873,10],[1097780.460960585,20],[1097780.609202427,14],[1097780.794759121,14],[1097781.019723456,13],[1097781.167534093,5],[1097781.350146848,22],[1097781.585554165,3],[1097781.818593906,32],[1097782.001883859,33],[1097782.186934951,11],[1097782.36909311,6],[1097782.523490884,8],[1097782.672695782,9],[1097782.819528144,13],[1097782.969866191,8],[1097783.117401438,20],[1097783.266543181,17],[1097783.415556859,20],[1097783.571795901,13],[1097783.744646016,6],[1097783.929146266,13],[1097784.074496896,9],[1097784.224171805,11],[1097784.373106996,8],[1097784.566048619,15],[1097784.75044609,16],[1097784.96124874,27],[1097785.20119785,4],[1097785.350711176,8],[1097785.539059888,25],[1097785.693697273,18],[1097785.837360914,13],[1097786.06413292,64],[1097786.214523265,2],[1097786.397119384,10],[1097786.580617297,11],[1097786.730031364,15],[1097786.877739053,17],[1097787.030895913,14],[1097787.285425464,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-018517881c9f68a64dbc2dfb521175b5","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011908851,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.655514916,"time_to_first_token":0.085688713,"token_sha":"8a91591c94a3","tokens_per_second":18.673774044303496},"seed":1234,"sent":1097773.539061485,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.08917260682210326,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":11} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.85267099356745,"done":1097787.284601263,"finish_reason":"length","first":1097773.758632183,"full_response_s":13.740820514038205,"last":1097787.284567054,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.758632183,3],[1097773.965139451,11],[1097774.145717505,30],[1097774.327567618,19],[1097774.47988879,21],[1097774.661856965,15],[1097774.80884837,19],[1097774.956790596,17],[1097775.105284776,6],[1097775.252728893,8],[1097775.437281971,3],[1097775.588039228,5],[1097775.76730786,27],[1097775.948137284,12],[1097776.128824542,13],[1097776.310960183,5],[1097776.465917774,16],[1097776.648838582,5],[1097776.831279453,14],[1097777.013762075,9],[1097777.160865746,22],[1097777.308410593,10],[1097777.497541257,7],[1097777.678963465,23],[1097777.826987762,5],[1097777.971816196,9],[1097778.143975693,14],[1097778.292547045,20],[1097778.445253407,8],[1097778.596290241,12],[1097778.745835398,5],[1097778.893156877,14],[1097779.077590714,14],[1097779.224849368,8],[1097779.405073288,13],[1097779.582458372,3],[1097779.76317205,10],[1097779.911432546,11],[1097780.090418564,9],[1097780.273261232,10],[1097780.460814525,20],[1097780.612037588,14],[1097780.791779997,14],[1097781.019346014,13],[1097781.167714697,5],[1097781.352096041,22],[1097781.586827075,3],[1097781.817741855,32],[1097782.002493739,33],[1097782.185353583,11],[1097782.368249911,6],[1097782.52422241,8],[1097782.672440148,9],[1097782.819778784,13],[1097782.96743065,8],[1097783.117658241,20],[1097783.267933775,17],[1097783.41603507,20],[1097783.572463952,13],[1097783.745871086,6],[1097783.926713806,13],[1097784.075759806,9],[1097784.224576681,11],[1097784.376784164,8],[1097784.566312208,15],[1097784.74925765,16],[1097784.962431253,27],[1097785.202687542,4],[1097785.349857148,8],[1097785.538363587,25],[1097785.688291137,18],[1097785.837114711,13],[1097786.064505522,64],[1097786.214698777,2],[1097786.397198896,10],[1097786.581627657,11],[1097786.730313264,15],[1097786.878734922,17],[1097787.029017285,14],[1097787.284567054,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-717248c671ee79775bad4d3d5b49b0fc","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011791475,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.526137379,"time_to_first_token":0.213166237,"token_sha":"8a91591c94a3","tokens_per_second":18.85238873855445},"seed":1234,"sent":1097773.543780749,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.2148514341097325,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":12} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.750895902876326,"done":1097787.283796128,"finish_reason":"length","first":1097773.684423946,"full_response_s":13.744119030889124,"last":1097787.283774149,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.684423946,3],[1097773.967038996,11],[1097774.146571979,30],[1097774.330758015,19],[1097774.479936496,21],[1097774.663150795,15],[1097774.809018334,19],[1097774.958510536,17],[1097775.10643097,6],[1097775.256163602,8],[1097775.439180438,3],[1097775.588181079,5],[1097775.766936222,27],[1097775.949182894,12],[1097776.129528483,13],[1097776.310453371,5],[1097776.466799024,16],[1097776.651477707,5],[1097776.83226951,14],[1097777.01266196,9],[1097777.161234196,22],[1097777.308918076,10],[1097777.496323368,7],[1097777.679589314,23],[1097777.825337475,5],[1097777.971476883,9],[1097778.143742307,14],[1097778.290909596,20],[1097778.445453016,8],[1097778.595996183,12],[1097778.746903621,5],[1097778.894784363,14],[1097779.076310176,14],[1097779.223103228,8],[1097779.404828376,13],[1097779.582721581,3],[1097779.7639749,10],[1097779.908343013,11],[1097780.092262397,9],[1097780.272656295,10],[1097780.461307468,20],[1097780.609930883,14],[1097780.791970672,14],[1097781.01942844,13],[1097781.169047782,5],[1097781.350771073,22],[1097781.586434281,3],[1097781.821368908,32],[1097782.003849983,33],[1097782.184915872,11],[1097782.367681301,6],[1097782.525286105,8],[1097782.673763832,9],[1097782.820102564,13],[1097782.968657433,8],[1097783.117337774,20],[1097783.266120006,17],[1097783.416717208,20],[1097783.572862251,13],[1097783.745707787,6],[1097783.927630909,13],[1097784.074625457,9],[1097784.224885781,11],[1097784.376265934,8],[1097784.569104881,15],[1097784.748859897,16],[1097784.960945756,27],[1097785.20033617,4],[1097785.351838478,8],[1097785.538431621,25],[1097785.692978416,18],[1097785.839116461,13],[1097786.06458585,64],[1097786.214764073,2],[1097786.397447804,10],[1097786.580867262,11],[1097786.730617256,15],[1097786.878591219,17],[1097787.029537912,14],[1097787.283774149,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-d83ea752599b7a17ff29b15d54892899","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.011911396,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.600252453,"time_to_first_token":0.140575912,"token_sha":"8a91591c94a3","tokens_per_second":18.749651955449625},"seed":1234,"sent":1097773.539677097,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.1447468490805477,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":13} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.80232504268924,"done":1097787.283457944,"finish_reason":"length","first":1097773.721255157,"full_response_s":13.74329046998173,"last":1097787.283407679,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.721255157,3],[1097773.965370647,11],[1097774.146284273,30],[1097774.329727719,19],[1097774.479444952,21],[1097774.662687971,15],[1097774.809426552,19],[1097774.958242858,17],[1097775.10398983,6],[1097775.253110872,8],[1097775.437862232,3],[1097775.588874905,5],[1097775.768733419,27],[1097775.948202759,12],[1097776.130480587,13],[1097776.312470567,5],[1097776.467729837,16],[1097776.650238491,5],[1097776.8327229,14],[1097777.013216188,9],[1097777.162063624,22],[1097777.307434656,10],[1097777.494677958,7],[1097777.678688552,23],[1097777.825939421,5],[1097777.972438137,9],[1097778.143657879,14],[1097778.291548619,20],[1097778.444712116,8],[1097778.596982523,12],[1097778.745580257,5],[1097778.894727405,14],[1097779.076066609,14],[1097779.223343945,8],[1097779.404373596,13],[1097779.584012732,3],[1097779.763484517,10],[1097779.910379298,11],[1097780.091896176,9],[1097780.271498321,10],[1097780.461237432,20],[1097780.611376846,14],[1097780.792361914,14],[1097781.019679177,13],[1097781.16930369,5],[1097781.350464355,22],[1097781.586060505,3],[1097781.820212429,32],[1097782.004189917,33],[1097782.18419456,11],[1097782.368015343,6],[1097782.523062665,8],[1097782.670918203,9],[1097782.820975919,13],[1097782.967562461,8],[1097783.116658853,20],[1097783.26629275,17],[1097783.415709353,20],[1097783.571941575,13],[1097783.744270449,6],[1097783.928675853,13],[1097784.074551548,9],[1097784.223787931,11],[1097784.373968247,8],[1097784.566144795,15],[1097784.750226267,16],[1097784.960279966,27],[1097785.200603248,4],[1097785.350155473,8],[1097785.539294166,25],[1097785.688164369,18],[1097785.837788855,13],[1097786.064430861,64],[1097786.214322381,2],[1097786.398053098,10],[1097786.582236376,11],[1097786.731168683,15],[1097786.877874047,17],[1097787.028449436,14],[1097787.283407679,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-d037af07c35196a684a4b436e8554f44","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.012203874,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.563074637,"time_to_first_token":0.176362336,"token_sha":"8a91591c94a3","tokens_per_second":18.80104672611336},"seed":1234,"sent":1097773.540167474,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.18108768295496702,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":14} +{"arm":"D","case":"chat","request":{"cache_state":"reused","cached_tokens":31,"case_id":null,"complete":true,"decode_tps":18.57025993579315,"done":1097787.285163014,"finish_reason":"length","first":1097773.553508216,"full_response_s":13.746437581954524,"last":1097787.285141531,"output_sha256":"d5e2af618b7bab2c585fd951191098691c8095c39e2449b19d713334b28c5503","pieces":[[1097773.553508216,3],[1097773.964127276,11],[1097774.144698624,30],[1097774.328876671,19],[1097774.479246494,21],[1097774.661071409,15],[1097774.80951159,19],[1097774.956028158,17],[1097775.106490362,6],[1097775.253073476,8],[1097775.43719506,3],[1097775.587442318,5],[1097775.768624065,27],[1097775.948780441,12],[1097776.128992984,13],[1097776.310426781,5],[1097776.465563666,16],[1097776.652034033,5],[1097776.832624032,14],[1097777.013674749,9],[1097777.161150087,22],[1097777.308011487,10],[1097777.494393665,7],[1097777.678507273,23],[1097777.825252342,5],[1097777.972115809,9],[1097778.143122495,14],[1097778.292096377,20],[1097778.445123259,8],[1097778.596077666,12],[1097778.745307714,5],[1097778.894200219,14],[1097779.077678952,14],[1097779.224076005,8],[1097779.404993694,13],[1097779.581341791,3],[1097779.76192838,10],[1097779.910493149,11],[1097780.089883152,9],[1097780.272584834,10],[1097780.460252747,20],[1097780.609053294,14],[1097780.791386482,14],[1097781.019760051,13],[1097781.168172981,5],[1097781.352147876,22],[1097781.585710085,3],[1097781.817552815,32],[1097782.00209283,33],[1097782.187646098,11],[1097782.367069452,6],[1097782.52356846,8],[1097782.672591729,9],[1097782.819398959,13],[1097782.968704785,8],[1097783.11785322,20],[1097783.265676805,17],[1097783.416513985,20],[1097783.570909019,13],[1097783.745970992,6],[1097783.927416353,13],[1097784.074899275,9],[1097784.222998349,11],[1097784.376902031,8],[1097784.568832312,15],[1097784.749200805,16],[1097784.959774896,27],[1097785.199731018,4],[1097785.349603596,8],[1097785.537755122,25],[1097785.692797509,18],[1097785.837016726,13],[1097786.063970327,64],[1097786.215127839,2],[1097786.398189998,10],[1097786.580489889,11],[1097786.729712452,15],[1097786.878540714,17],[1097787.028372086,14],[1097787.285141531,21]],"prompt":"chat","prompt_tokens":31,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":256,"messages":[{"content":"Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"8eb6c97ea5354f07fa7753b7dbfcc84d9833a58b472bea950648a3bf6a58e35f","request_path":"/v1/chat/completions","response_id":"chatcmpl-a07047934f476dbfdcb52ab0e5d91ff6","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":4,"prefill_raised":[],"prefill_seconds":0.0126683,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":13.729579747,"time_to_first_token":0.013455284,"token_sha":"8a91591c94a3","tokens_per_second":18.573037536397944},"seed":1234,"sent":1097773.538725432,"speculative":{"acceptance_rate":0.44,"accepted":176,"drafted":400,"rounds":79,"tokens_per_round":3.240506329113924},"token_sha":"8a91591c94a3","tokens":256,"ttft_s":0.014782784041017294,"usage":{"completion_tokens":256,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":31,"prompt_tokens_details":{"cached_tokens":31},"total_tokens":287},"work":{"accepted":176,"drafted":400,"rounds":79},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"burst","stream_index":15} +{"arm":"D","case":"long","request":{"cache_state":"cold","cached_tokens":0,"case_id":null,"complete":true,"decode_tps":20.359685500823876,"done":1097791.966970091,"finish_reason":"length","first":1097790.444276222,"full_response_s":4.630935040069744,"last":1097791.966893062,"output_sha256":"5d2c18c1a3da0aab39441e582248a278a66d8473e846a8d2c48dfda87c21e891","pieces":[[1097790.444276222,3],[1097790.539777574,14],[1097790.623614702,11],[1097790.707115505,11],[1097790.790428024,13],[1097790.873501591,12],[1097790.955251821,12],[1097791.037845164,12],[1097791.12031078,12],[1097791.20319985,11],[1097791.286862895,12],[1097791.368898399,4],[1097791.45631555,12],[1097791.544150407,12],[1097791.628240474,9],[1097791.711863659,1],[1097791.796021455,7],[1097791.877951808,4],[1097791.966893062,17]],"prompt":"long_initial","prompt_tokens":4419,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"df5eb0c68030e305529b5525635e0602046d98ecf7a739e2e42243ec364b4adc","request_path":"/v1/chat/completions","response_id":"chatcmpl-fec2a9b8d305bd9e1c6f27227f6f450e","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":3.098456802,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":1.522637092,"time_to_first_token":3.107443113,"token_sha":"8b52dc1b081a","tokens_per_second":20.359414704183497},"seed":1234,"sent":1097787.336035051,"speculative":{"acceptance_rate":0.06578947368421052,"accepted":15,"drafted":228,"rounds":18,"tokens_per_round":1.7777777777777777},"token_sha":"8b52dc1b081a","tokens":32,"ttft_s":3.1082411711104214,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4419,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":4451},"work":{"accepted":15,"drafted":228,"rounds":18},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"initial","stream_index":0} +{"arm":"D","case":"long","request":{"cache_state":"reused","cached_tokens":4096,"case_id":null,"complete":true,"decode_tps":45.119286189341054,"done":1097793.001558073,"finish_reason":"length","first":1097792.314451167,"full_response_s":1.0253900331445038,"last":1097793.001518774,"output_sha256":"dd1aacffd9ef2ff0a3531f264db2a68aa99f695724c7828277affb8cb3535eb1","pieces":[[1097792.314451167,3],[1097792.404280497,32],[1097792.496630048,20],[1097792.582536945,29],[1097792.664518144,8],[1097792.748245463,30],[1097792.830784185,18],[1097792.912854333,13],[1097793.001518774,12]],"prompt":"long_followup","prompt_tokens":4445,"request_body":{"chat_template_kwargs":{"enable_thinking":false},"ignore_eos":true,"max_tokens":32,"messages":[{"content":"atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta atlas beacon cedar delta\nWhich word occurs first?","role":"user"},{"content":"Acknowledged.","role":"assistant"},{"content":"Summarize the document in one sentence.","role":"user"}],"model":"qwen27-overlap","seed":1234,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0},"request_body_sha256":"0f7b9a7949106fe7d7c1dee63210f9621564bc3cdf8167180dc2d64159ad2c22","request_path":"/v1/chat/completions","response_id":"chatcmpl-2436c52afa7735aa92b7e915d3711e87","runtime":{"drafts":true,"enable_thinking":false,"engine":"lanes","min_rows":12,"prefill_raised":[],"prefill_seconds":0.322744574,"prefill_widths":[],"reasoning_effort":"none","sampling":"greedy","seconds":0.687692276,"time_to_first_token":0.336940593,"token_sha":"526434da61a6","tokens_per_second":45.0783018537204},"seed":1234,"sent":1097791.97616804,"speculative":{"acceptance_rate":0.2169811320754717,"accepted":23,"drafted":106,"rounds":8,"tokens_per_round":4.0},"token_sha":"526434da61a6","tokens":32,"ttft_s":0.33828312694095075,"usage":{"completion_tokens":32,"completion_tokens_details":{"reasoning_tokens":0},"prompt_tokens":4445,"prompt_tokens_details":{"cached_tokens":4096},"total_tokens":4477},"work":{"accepted":23,"drafted":106,"rounds":8},"work_source":"speculative"},"schema":"pr726_sanitized_request_v1","stage":"followup","stream_index":0} diff --git a/validation/pr726-ttft/run_overlap.py b/validation/pr726-ttft/run_overlap.py new file mode 100644 index 0000000000..3c59a46f4e --- /dev/null +++ b/validation/pr726-ttft/run_overlap.py @@ -0,0 +1,269 @@ +"""Collect a bounded BF16 text overlap panel from one already-running server.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +import re +import signal +from typing import Any +import urllib.request + +import bench_concurrent as bench + + +SCHEMA = "pr726_pr608_overlap_text_r3" +SHORT_NAMES = ("code", "chat") +PROMPT_TOKENS = {"code": 14, "chat": 31} +PROMPTS = { + "code": ("completion", "Write a short Python function that computes the Fibonacci sequence and explain it."), + "chat": ("chat", "Explain how matrix multiplication uses a GPU in plain English, then give a small numerical example."), +} +SEED = 1234 +SHORT_OUTPUT = 256 +LONG_OUTPUT = 32 + + +def digest(path: Path) -> str: + """Hash exact source or input bytes.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def expected_cache(arm: str, case: str, stage: str) -> int: + """Return the source-derived cached depth for this admitted request.""" + if stage == "initial" or arm in ("A", "B") and case in SHORT_NAMES: + return 0 + if case in SHORT_NAMES: + return PROMPT_TOKENS[case] + return 4096 if arm in ("B", "D") else 0 + + +def long_inputs(path: Path) -> tuple[dict[str, Any], dict[str, Any]]: + """Require rendered chat history exposing PR726's planned 4096 mark.""" + data = json.loads(path.read_text()) + if data.get("schema") != "pr726_chat_history_fixture_v1": + raise ValueError("unrecognized long-input manifest") + messages, rendered = data.get("messages"), data.get("rendered") + if not isinstance(messages, dict) or not isinstance(rendered, dict): + raise ValueError("missing literal messages or rendered IDs") + first, next_ = rendered["initial"], rendered["followup"] + histories = (first["history"], next_["history"]) + full = (first["generation"], next_["generation"]) + vocab_size = data.get("vocab_size") + if type(vocab_size) is not int or vocab_size <= 0: + raise ValueError("missing tokenizer vocabulary size") + if any(not isinstance(ids, list) or any(type(x) is not int or x < 0 or x >= vocab_size + for x in ids) for ids in (*histories, *full)): + raise ValueError("rendered token IDs are invalid") + if (not 4096 <= len(histories[0]) < 8192 or len(full[1]) >= 8192 + or histories[0] != full[0][:len(histories[0])] + or histories[1] != full[1][:len(histories[1])] + or histories[0] != histories[1][:len(histories[0])] + or full[0][:4096] != full[1][:4096] + or data.get("planned_mark") != 4096 or data.get("lookahead") != 0 + or data.get("history_len") != len(histories[0]) + or data.get("followup_history_len") != len(histories[1])): + raise ValueError("chat history does not expose the 4096 mark") + for name in ("initial", "followup"): + if not isinstance(messages[name], list) or not messages[name]: + raise ValueError("missing chat messages") + for kind in ("history", "generation"): + digest = hashlib.sha256(",".join(map(str, rendered[name][kind])).encode()).hexdigest() + if data["id_sha256"][name][kind] != digest: + raise ValueError("rendered token hash drift") + if messages["followup"][:len(messages["initial"])] != messages["initial"]: + raise ValueError("follow-up changed the prior chat messages") + if any(not isinstance(data.get(key), str) or not re.fullmatch(r"[0-9a-f]{64}", data[key]) + for key in ("tokenizer_sha256", "tokenizer_config_sha256", "chat_template_sha256")): + raise ValueError("missing pinned tokenizer/template identity") + return ( + {"name": "long_initial", "kind": "chat", "messages": messages["initial"], + "prompt_tokens": len(full[0])}, + {"name": "long_followup", "kind": "chat", "messages": messages["followup"], + "prompt_tokens": len(full[1])}, + ) + + +def tokenize_preflight(base: str, manifest: Path) -> dict[str, Any]: + """Require the live server's exact chat rendering to match the manifest.""" + data = json.loads(manifest.read_text()) + result: dict[str, Any] = {} + for name in ("initial", "followup"): + result[name] = {} + for generation, kind in ((False, "history"), (True, "generation")): + body = { + "messages": data["messages"][name], + "add_generation_prompt": generation, + "chat_template_kwargs": {"enable_thinking": False}, + } + req = urllib.request.Request( + base + "/tokenize", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=60) as response: + actual = json.load(response) + expected = data["rendered"][name][kind] + if actual.get("tokens") != expected or actual.get("count") != len(expected): + raise ValueError(f"{name}/{kind}: live /tokenize IDs differ from pinned fixture") + result[name][kind] = { + "count": len(expected), + "sha256": hashlib.sha256(",".join(map(str, expected)).encode()).hexdigest(), + } + return result + + +def validate_row( + row: dict[str, Any], *, item: dict[str, Any], tokens: int, cached: int, +) -> None: + """Fail on incomplete work, wrong cache depth, changed body or SSE clocks.""" + if row.get("error") or row.get("complete") is not True: + raise ValueError(f"{item['name']}: incomplete SSE response") + # The 32-token long-prefix check is a correctness cell: a speculative + # response may arrive in fewer than four text pieces. Short 256-token + # cells need multiple pieces to support their decode-timing diagnostic. + if tokens == SHORT_OUTPUT and row.get("unmeasured"): + raise ValueError(f"{item['name']}: unmeasured short response") + if row.get("tokens") != tokens or row.get("finish_reason") != "length": + raise ValueError(f"{item['name']}: completion count or finish reason drift") + body = row.get("request_body") or {} + expected = { + "model": body.get("model"), "max_tokens": tokens, "temperature": 0.0, + "stream": True, "stream_options": {"include_usage": True}, + "ignore_eos": True, "seed": SEED, + } + if item["kind"] == "chat": + expected["messages"] = item.get("messages") or [{"role": "user", "content": item["prompt"]}] + expected["chat_template_kwargs"] = {"enable_thinking": False} + else: + expected["prompt"] = item["prompt"] + if body != expected or row.get("request_body_sha256") != hashlib.sha256( + json.dumps(expected).encode() + ).hexdigest(): + raise ValueError(f"{item['name']}: literal body drift") + route = "/v1/chat/completions" if item["kind"] == "chat" else "/v1/completions" + if not isinstance(row.get("request_url"), str) or not row["request_url"].endswith(route): + raise ValueError(f"{item['name']}: request route drift") + if not isinstance(row.get("response_id"), str) or not row["response_id"]: + raise ValueError(f"{item['name']}: missing response ID") + usage = row.get("usage") or {} + details = usage.get("prompt_tokens_details") or {} + prompt_tokens = item.get("prompt_tokens", PROMPT_TOKENS.get(item["name"])) + if (usage.get("prompt_tokens"), usage.get("completion_tokens"), + details.get("cached_tokens"), row.get("cached_tokens")) != ( + prompt_tokens, tokens, cached, cached + ): + raise ValueError(f"{item['name']}: prompt/output/cache usage drift") + if row.get("cache_state") != ("reused" if cached else "cold"): + raise ValueError(f"{item['name']}: cache-state label drift") + runtime = row.get("runtime") or {} + if runtime.get("token_sha") != row.get("token_sha"): + raise ValueError(f"{item['name']}: token fingerprint drift") + for field, widths in (("output_sha256", {64}), ("token_sha", {12, 64})): + value = row.get(field) + if not isinstance(value, str) or len(value) not in widths or not re.fullmatch(r"[0-9a-f]+", value): + raise ValueError(f"{item['name']}: missing {field}") + clocks = [row.get(key) for key in ("sent", "first", "last", "done")] + if any(type(value) not in (int, float) or not math.isfinite(value) for value in clocks): + raise ValueError(f"{item['name']}: missing clocks") + sent, first, last, done = clocks + if not sent < first <= last <= done or (tokens == SHORT_OUTPUT and first == last): + raise ValueError(f"{item['name']}: clock order drift") + if not math.isclose(row.get("ttft_s", -1), first - sent, abs_tol=1e-7): + raise ValueError(f"{item['name']}: TTFT drift") + if not math.isclose(row.get("full_response_s", -1), done - sent, abs_tol=1e-7): + raise ValueError(f"{item['name']}: full-response drift") + + +def save(path: Path, report: dict[str, Any]) -> None: + """Keep partial evidence atomically after each completed stage.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n") + temporary.replace(path) + + +def collect(args: argparse.Namespace) -> dict[str, Any]: + """Run one arm's short and optional grid-prefix panels.""" + root = Path(__file__).resolve().parent + report: dict[str, Any] = { + "schema": SCHEMA, "complete": False, "arm": args.arm, "suite": args.suite, + "model": args.model, "base": args.base, + "source_sha256": {name: digest(root / name) for name in ( + "run_overlap.py", "bench_concurrent.py", "bench_evidence.py")}, + "short": {}, "long": {}, + } + if args.suite in ("short", "both"): + items = {item["name"]: item for item in bench.PROMPTS} + if tuple(items) != SHORT_NAMES or any( + (items[name].get("kind"), items[name].get("prompt")) != PROMPTS[name] + for name in SHORT_NAMES + ): + raise ValueError("upstream short-prompt order changed") + for name in SHORT_NAMES: + item = items[name] + stages: dict[str, Any] = {} + for stage in ("initial", "repeat"): + row = bench.stream(args.base, args.model, item, SHORT_OUTPUT, 0.0, SEED) + stages[stage] = row + report["short"][name] = stages + save(args.output, report) + validate_row(row, item=item, tokens=SHORT_OUTPUT, + cached=expected_cache(args.arm, name, stage)) + burst = bench.together( + args.base, args.model, [(item, SEED)] * 16, SHORT_OUTPUT, 0.0, + ) + stages["burst"] = burst + save(args.output, report) + if len(burst) != 16: + raise ValueError(f"{name}: incomplete concurrent burst") + for row in burst: + validate_row(row, item=item, tokens=SHORT_OUTPUT, + cached=expected_cache(args.arm, name, "burst")) + if (row["output_sha256"], row["token_sha"]) != ( + stages["repeat"]["output_sha256"], stages["repeat"]["token_sha"] + ): + raise ValueError(f"{name}: concurrent reply differs from solo") + if args.suite in ("long", "both"): + if args.long_manifest is None: + raise ValueError("long panel requires --long-manifest") + initial, followup = long_inputs(args.long_manifest) + report["long_manifest_sha256"] = digest(args.long_manifest) + report["tokenize_preflight"] = tokenize_preflight(args.base, args.long_manifest) + save(args.output, report) + for stage, item in (("initial", initial), ("followup", followup)): + row = bench.stream(args.base, args.model, item, LONG_OUTPUT, 0.0, SEED) + report["long"][stage] = row + save(args.output, report) + validate_row(row, item=item, tokens=LONG_OUTPUT, + cached=expected_cache(args.arm, "long", stage)) + report["complete"] = True + save(args.output, report) + return report + + +def main() -> None: + """Parse one arm and report a complete or partial private receipt.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--arm", choices=("A", "B", "C", "D"), required=True) + parser.add_argument("--suite", choices=("short", "long", "both"), default="both") + parser.add_argument("--long-manifest", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-seconds", type=int, default=900) + args = parser.parse_args() + if args.max_seconds <= 0: + parser.error("--max-seconds must be positive") + signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(TimeoutError("arm deadline"))) + signal.alarm(args.max_seconds) + try: + collect(args) + finally: + signal.alarm(0) + + +if __name__ == "__main__": + main() diff --git a/validation/pr726-ttft/summary.json b/validation/pr726-ttft/summary.json new file mode 100644 index 0000000000..2b799c5073 --- /dev/null +++ b/validation/pr726-ttft/summary.json @@ -0,0 +1,177 @@ +{ + "complete": true, + "metrics": { + "A": { + "chat": { + "burst_median_decode_tps": 17.687244380002006, + "burst_median_full_response_s": 15.52720304846298, + "burst_median_ttft_s": 1.108422608464025, + "burst_slowest_ttft_s": 2.0723795068915933, + "burst_wall_s": 15.531896268948913, + "initial_decode_tps": 49.48511617606272, + "initial_full_response_s": 5.28832744108513, + "initial_ttft_s": 0.13518329802900553, + "repeat_cached_tokens": 0, + "repeat_decode_tps": 49.547654253108774, + "repeat_full_response_s": 5.278693869011477, + "repeat_ttft_s": 0.13184216106310487 + }, + "code": { + "burst_median_decode_tps": 23.310635187040624, + "burst_median_full_response_s": 12.026004086947069, + "burst_median_ttft_s": 1.0838163003791124, + "burst_slowest_ttft_s": 2.0320792738348246, + "burst_wall_s": 12.028927627019584, + "initial_decode_tps": 69.46525076037533, + "initial_full_response_s": 3.8124594420660287, + "initial_ttft_s": 0.1415074539836496, + "repeat_cached_tokens": 0, + "repeat_decode_tps": 69.55600513446439, + "repeat_full_response_s": 3.7954030090477318, + "repeat_ttft_s": 0.1292427540756762 + }, + "long": { + "followup": { + "cached_tokens": 0, + "full_response_s": 3.773332908982411, + "ttft_s": 3.1678763718809932 + }, + "initial": { + "cached_tokens": 0, + "full_response_s": 4.686749584041536, + "ttft_s": 3.161685612052679 + } + } + }, + "B": { + "chat": { + "burst_median_decode_tps": 17.541171183664062, + "burst_median_full_response_s": 15.624776583514176, + "burst_median_ttft_s": 1.088507852051407, + "burst_slowest_ttft_s": 2.0467982080299407, + "burst_wall_s": 15.62790786800906, + "initial_decode_tps": 49.79677234376222, + "initial_full_response_s": 5.2520289570093155, + "initial_ttft_s": 0.13117081299424171, + "repeat_cached_tokens": 0, + "repeat_decode_tps": 49.78045058425396, + "repeat_full_response_s": 5.250370910856873, + "repeat_ttft_s": 0.12783687189221382 + }, + "code": { + "burst_median_decode_tps": 23.468865344604946, + "burst_median_full_response_s": 11.932323584449477, + "burst_median_ttft_s": 1.0664106860058382, + "burst_slowest_ttft_s": 2.027785588055849, + "burst_wall_s": 11.936625859001651, + "initial_decode_tps": 69.4767532146739, + "initial_full_response_s": 3.810370694845915, + "initial_ttft_s": 0.14003579691052437, + "repeat_cached_tokens": 0, + "repeat_decode_tps": 69.49836526023209, + "repeat_full_response_s": 3.7947294760961086, + "repeat_ttft_s": 0.12552802520804107 + }, + "long": { + "followup": { + "cached_tokens": 4096, + "full_response_s": 0.9438277550507337, + "ttft_s": 0.35115201212465763 + }, + "initial": { + "cached_tokens": 0, + "full_response_s": 4.637223938014358, + "ttft_s": 3.112166848964989 + } + } + }, + "C": { + "chat": { + "burst_median_decode_tps": 18.731121788990574, + "burst_median_full_response_s": 13.730424111010507, + "burst_median_ttft_s": 0.11676928505767137, + "burst_slowest_ttft_s": 0.22482402296736836, + "burst_wall_s": 13.734526468906552, + "initial_decode_tps": 49.50709200497177, + "initial_full_response_s": 5.31293655000627, + "initial_ttft_s": 0.16207883087918162, + "repeat_cached_tokens": 31, + "repeat_decode_tps": 49.59872418581017, + "repeat_full_response_s": 5.150651934091002, + "repeat_ttft_s": 0.009354999987408519 + }, + "code": { + "burst_median_decode_tps": 25.347109879568187, + "burst_median_full_response_s": 10.192870338563807, + "burst_median_ttft_s": 0.12912349787075073, + "burst_slowest_ttft_s": 0.2541796991135925, + "burst_wall_s": 10.19763688184321, + "initial_decode_tps": 69.41609400179837, + "initial_full_response_s": 3.8164350448641926, + "initial_ttft_s": 0.14288648893125355, + "repeat_cached_tokens": 14, + "repeat_decode_tps": 69.31634116319097, + "repeat_full_response_s": 3.6882623040582985, + "repeat_ttft_s": 0.009435665095224977 + }, + "long": { + "followup": { + "cached_tokens": 0, + "full_response_s": 3.78918566997163, + "ttft_s": 3.172564007109031 + }, + "initial": { + "cached_tokens": 0, + "full_response_s": 4.759321738034487, + "ttft_s": 3.2240212850738317 + } + } + }, + "D": { + "chat": { + "burst_median_decode_tps": 18.762728231925415, + "burst_median_full_response_s": 13.744372194982134, + "burst_median_ttft_s": 0.1537560084834695, + "burst_slowest_ttft_s": 0.28762087714858353, + "burst_wall_s": 13.749177963938564, + "initial_decode_tps": 49.15328105985827, + "initial_full_response_s": 5.320269583025947, + "initial_ttft_s": 0.13234535395167768, + "repeat_cached_tokens": 31, + "repeat_decode_tps": 49.084485304040726, + "repeat_full_response_s": 5.208992755971849, + "repeat_ttft_s": 0.013788988813757896 + }, + "code": { + "burst_median_decode_tps": 25.310097988436866, + "burst_median_full_response_s": 10.220496089546941, + "burst_median_ttft_s": 0.1447350385133177, + "burst_slowest_ttft_s": 0.2751705260016024, + "burst_wall_s": 10.223413391970098, + "initial_decode_tps": 65.44392056729643, + "initial_full_response_s": 4.03748605796136, + "initial_ttft_s": 0.1409339220263064, + "repeat_cached_tokens": 14, + "repeat_decode_tps": 65.35232133194735, + "repeat_full_response_s": 3.914394620107487, + "repeat_ttft_s": 0.012392177013680339 + }, + "long": { + "followup": { + "cached_tokens": 4096, + "full_response_s": 1.0253900331445038, + "ttft_s": 0.33828312694095075 + }, + "initial": { + "cached_tokens": 0, + "full_response_s": 4.630935040069744, + "ttft_s": 3.1082411711104214 + } + } + } + }, + "parity_matches": true, + "requests_per_arm": 38, + "schema": "pr726_portable_summary_v1", + "total_requests": 152 +} diff --git a/validation/pr726-ttft/test_verify.py b/validation/pr726-ttft/test_verify.py new file mode 100644 index 0000000000..f5f0a260ee --- /dev/null +++ b/validation/pr726-ttft/test_verify.py @@ -0,0 +1,95 @@ +"""Mutation checks for the actual portable overlap panel.""" +from __future__ import annotations + +import copy +from pathlib import Path +import unittest +from typing import Any + +import verify + +ROOT = Path(__file__).resolve().parent + + +class PortableTests(unittest.TestCase): + """Reject changed evidence without depending on a GPU or private file.""" + + @classmethod + def setUpClass(cls) -> None: + """Load the frozen sanitized panel once.""" + cls.data = verify.load(ROOT) + + def check(self, records: list[dict[str, Any]] | None = None, + provenance: dict[str, Any] | None = None, + counters: dict[str, Any] | None = None) -> dict[str, Any]: + """Validate supplied copies against the actual long fixture.""" + rows, prov, count = self.data + return verify.verify_data(rows if records is None else records, + prov if provenance is None else provenance, + count if counters is None else counters, + ROOT / "long-manifest.json") + + def test_actual_panel(self) -> None: + """All 152 actual requests reconstruct the saved summary.""" + self.assertEqual(self.check()["total_requests"], 152) + + def test_missing_request(self) -> None: + """A missing request cannot pass complete-count gates.""" + with self.assertRaises(ValueError): + self.check(records=self.data[0][1:]) + + def test_changed_output(self) -> None: + """A valid-looking output hash still fails paired literal parity.""" + records = copy.deepcopy(self.data[0]) + records[-1]["request"]["output_sha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "parity"): + self.check(records=records) + + def test_wrong_long_cache(self) -> None: + """The long host-prefix hit must be the actual 4096-token mark.""" + records = copy.deepcopy(self.data[0]) + row = next(r["request"] for r in records if r["arm"] == "D" + and r["case"] == "long" and r["stage"] == "followup") + row["cached_tokens"] = 0 + with self.assertRaises(ValueError): + self.check(records=records) + + def test_body_and_generation(self) -> None: + """Changed body or incomplete generation cannot pass.""" + for field, value in (("request_body_sha256", "0" * 64), ("tokens", 255)): + records = copy.deepcopy(self.data[0]) + records[0]["request"][field] = value + with self.assertRaises(ValueError): + self.check(records=records) + + def test_changed_decode_rate(self) -> None: + """Client decode rate must reconstruct from literal SSE clocks.""" + records = copy.deepcopy(self.data[0]) + records[0]["request"]["decode_tps"] += 1 + with self.assertRaisesRegex(ValueError, "decode rate"): + self.check(records=records) + + def test_changed_work(self) -> None: + """Speculative counters must agree with the original raw fields.""" + records = copy.deepcopy(self.data[0]) + records[0]["request"]["work"]["rounds"] += 1 + with self.assertRaisesRegex(ValueError, "work"): + self.check(records=records) + + def test_changed_model(self) -> None: + """A wrong model owner must fail provenance validation.""" + prov = copy.deepcopy(self.data[1]) + prov["models"][0]["repo"] = "TensorFold/Qwen3.8-27B-MLX-4bit" + with self.assertRaisesRegex(ValueError, "model identity"): + self.check(provenance=prov) + + def test_changed_host_hook_exposure(self) -> None: + """The observed hook limitation cannot become a pool-reuse claim.""" + counters = copy.deepcopy(self.data[2]) + counters["arms"]["D"]["seq_pool"]["reuse_hits"] = 1 + with self.assertRaisesRegex(ValueError, "exposure"): + self.check(counters=counters) + + +if __name__ == "__main__": + unittest.main() diff --git a/validation/pr726-ttft/verify.py b/validation/pr726-ttft/verify.py new file mode 100644 index 0000000000..0bf6ee95e4 --- /dev/null +++ b/validation/pr726-ttft/verify.py @@ -0,0 +1,199 @@ +"""Verify the sanitized PR726 overlap panel without a model, GPU or private receipts.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +import statistics +from typing import Any + +import run_overlap as client + +ARMS = ("A", "B", "C", "D") +FLAGS = ( + "TF_PR608_EXACT_PROMPT_CACHE", "TF_PR608_EXACT_CACHE_SEQ_REUSE", + "TF_PR608_EXACT_CACHE_LOGITS", "TF_PR608_EXACT_CACHE_DRAFT_REUSE", + "TF_PR608_EXACT_CACHE_SKIP_RESET", +) +BASE = "d49a906ffc168d707f62be598d329bf8dd8f5e2d" +CANDIDATE = "5e08e130bfb32e944390c2b9dfa3359a8859ea76" +BINARIES = { + "base": "97b1a443773157d4534d6005ab5baae6506dba7f739dad590324efbe3968f313", + "candidate": "d2c6d38c020580c5673a76a355ca1d3720078c18ad6d6f0116ee18c184abb976", +} +MODELS = { + "target": ("Vontra/Qwen3.8-27B-MLX-4bit", "70ae7fac63274ff2eac54152031433374cb80f2f"), + "drafter": ("z-lab/Qwen3.8-27B-DFlash2", "50307d4c4cde6860d4eee73e2547cd786fe8e8a4"), +} + + +def sha(path: Path) -> str: + """Hash exact payload bytes.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def load(root: Path) -> tuple[list[dict[str, Any]], dict[str, Any], dict[str, Any]]: + """Read only the portable payloads.""" + records = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] + return records, json.loads((root / "provenance.json").read_text()), json.loads((root / "counters.json").read_text()) + + +def validate_numeric(row: dict[str, Any], tokens: int) -> None: + """Check SSE-derived rates and actual speculative work, without estimating tokens from characters.""" + pieces = row.get("pieces") + if not isinstance(pieces, list) or not pieces: + raise ValueError("missing nonempty SSE pieces") + if any(not isinstance(p, list) or len(p) != 2 or type(p[0]) not in (int, float) + or not math.isfinite(p[0]) or type(p[1]) is not int or p[1] <= 0 for p in pieces): + raise ValueError("invalid nonempty SSE piece") + if pieces[0][0] != row["first"] or pieces[-1][0] != row["last"]: + raise ValueError("SSE piece boundaries disagree with first/last") + if any(b[0] < a[0] for a, b in zip(pieces, pieces[1:])): + raise ValueError("SSE piece order drift") + if row["last"] > row["first"]: + rate = (tokens - 1) / (row["last"] - row["first"]) + if not math.isclose(row.get("decode_tps", -1), rate, rel_tol=1e-9, abs_tol=1e-7): + raise ValueError("client decode rate drift") + if row.get("prompt_tokens") != row["usage"]["prompt_tokens"]: + raise ValueError("prompt count drift") + work = row.get("work") or {} + spec = row.get("speculative") or {} + if row.get("work_source") != "speculative" or set(work) != {"rounds", "accepted", "drafted"}: + raise ValueError("missing actual speculative work") + if any(type(v) is not int or v < 0 for v in work.values()) or work["rounds"] == 0: + raise ValueError("invalid speculative work count") + if work != {k: spec.get(k) for k in work} or work["accepted"] > work["drafted"]: + raise ValueError("speculative work fields disagree") + + +def verify_data(records: list[dict[str, Any]], provenance: dict[str, Any], + counters: dict[str, Any], manifest: Path) -> dict[str, Any]: + """Reconstruct all 152 aligned requests and the independently accepted timing table.""" + if len(records) != 152: + raise ValueError("expected 152 request records") + if provenance.get("base_commit") != BASE or provenance.get("candidate_commit") != CANDIDATE: + raise ValueError("source revision drift") + if provenance.get("binary_sha256") != BINARIES: + raise ValueError("binary identity drift") + actual_models = {m["name"]: (m["repo"], m["revision"]) for m in provenance["models"]} + if actual_models != MODELS: + raise ValueError("model identity drift") + if provenance["aot"]["aot_sha256"] != "dce85c91013171fd92c66c9401dfce3998df4f7753a3a3c2e22efd8911b22d37": + raise ValueError("AOT identity drift") + if len(provenance.get("runtime_source_sha256", {})) != 31: + raise ValueError("missing 31 runtime source identities") + for arm in ARMS: + settings = provenance["arms"][arm] + if settings["host_cache_gib"] != int(arm in "BD") or settings["flags"] != { + f: str(int(arm in "CD")) for f in FLAGS + }: + raise ValueError("arm settings drift") + initial, followup = client.long_inputs(manifest) + long_items = {"initial": initial, "followup": followup} + groups: dict[str, dict[tuple[str, str, int], dict[str, Any]]] = {a: {} for a in ARMS} + expected_keys = {(n, s, 0) for n in client.SHORT_NAMES for s in ("initial", "repeat")} + expected_keys |= {(n, "burst", i) for n in client.SHORT_NAMES for i in range(16)} + expected_keys |= {("long", s, 0) for s in ("initial", "followup")} + for record in records: + if record.get("schema") != "pr726_sanitized_request_v1" or record.get("arm") not in ARMS: + raise ValueError("request schema/arm drift") + arm, name, stage, index = (record[k] for k in ("arm", "case", "stage", "stream_index")) + key = (name, stage, index) + if type(index) is not int or key not in expected_keys or key in groups[arm]: + raise ValueError("duplicate or unexpected request identity") + row = dict(record["request"]) + route = row.pop("request_path", None) + kind = "completion" if name == "code" else "chat" + if route != ("/v1/completions" if kind == "completion" else "/v1/chat/completions"): + raise ValueError("request route drift") + row["request_url"] = route + if row.get("request_body", {}).get("model") != "qwen27-overlap": + raise ValueError("model alias drift") + item = long_items[stage] if name == "long" else { + "name": name, "kind": kind, "prompt": client.PROMPTS[name][1] + } + tokens = client.LONG_OUTPUT if name == "long" else client.SHORT_OUTPUT + client.validate_row(row, item=item, tokens=tokens, + cached=client.expected_cache(arm, name, stage)) + validate_numeric(row, tokens) + groups[arm][key] = row + for arm in ARMS: + if set(groups[arm]) != expected_keys: + raise ValueError("missing complete 38-request arm") + if len({r["response_id"] for r in groups[arm].values()}) != 38: + raise ValueError("duplicate response identity") + for key in expected_keys: + reference = groups["A"][key] + for arm in ARMS[1:]: + row = groups[arm][key] + if any(row[k] != reference[k] for k in ("request_body", "output_sha256", "token_sha")): + raise ValueError("literal input or output parity drift") + for arm in "CD": + stats = counters["arms"][arm] + if stats["exact_cache"]["hits"] != 34 or stats["logits"]["hits"] != 34: + raise ValueError("missing short cache/logit exposure") + if any(stats[name].get("failed") for name in ("exact_cache", "logits", "seq_pool", "draft_pool")): + raise ValueError("failed cache owner") + d = counters["arms"]["D"] + if d["seq_pool"]["reuse_hits"] or d["draft_pool"]["reuse_hits"] or d["reset_skip"]["avoided_fill_calls"]: + raise ValueError("host-hook treatment exposure drift") + metrics: dict[str, Any] = {} + for arm, group in groups.items(): + cells = {} + for name in client.SHORT_NAMES: + cold, repeat = group[(name, "initial", 0)], group[(name, "repeat", 0)] + burst = [group[(name, "burst", i)] for i in range(16)] + cells[name] = { + "initial_ttft_s": cold["ttft_s"], "initial_full_response_s": cold["full_response_s"], + "initial_decode_tps": cold["decode_tps"], "repeat_ttft_s": repeat["ttft_s"], + "repeat_full_response_s": repeat["full_response_s"], "repeat_decode_tps": repeat["decode_tps"], + "burst_median_ttft_s": statistics.median(r["ttft_s"] for r in burst), + "burst_slowest_ttft_s": max(r["ttft_s"] for r in burst), + "burst_median_decode_tps": statistics.median(r["decode_tps"] for r in burst), + "burst_median_full_response_s": statistics.median(r["full_response_s"] for r in burst), + "burst_wall_s": max(r["done"] for r in burst) - min(r["sent"] for r in burst), + "repeat_cached_tokens": repeat["cached_tokens"], + } + cells["long"] = {s: {k: group[("long", s, 0)][k] for k in ( + "ttft_s", "full_response_s", "cached_tokens")} for s in ("initial", "followup")} + metrics[arm] = cells + return {"schema": "pr726_portable_summary_v1", "complete": True, + "total_requests": 152, "requests_per_arm": 38, "parity_matches": True, "metrics": metrics} + + +def verify(root: Path, source_root: Path | None = None) -> dict[str, Any]: + """Verify file identities, numerical gates and optional candidate runtime files.""" + bundle = json.loads((root / "bundle-manifest.json").read_text()) + for name, digest in bundle["files_sha256"].items(): + if sha(root / name) != digest: + raise ValueError(f"bundle hash drift: {name}") + records, provenance, counters = load(root) + if source_root is not None: + for name, digest in provenance["runtime_source_sha256"].items(): + if sha(source_root / name) != digest: + raise ValueError(f"runtime source drift: {name}") + result = verify_data(records, provenance, counters, root / "long-manifest.json") + if result != json.loads((root / "summary.json").read_text()): + raise ValueError("saved summary differs from recomputation") + return result + + +def main() -> None: + """Print recomputed tables or write the complete portable result.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", type=Path, default=Path(__file__).resolve().parent) + parser.add_argument("--source-root", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + result = verify(args.bundle, args.source_root) + text = json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n" + if args.output: + args.output.write_text(text) + else: + print(text, end="") + + +if __name__ == "__main__": + main() From c7c77bc7dd9e4158665a5419873949f0f81cefdd Mon Sep 17 00:00:00 2001 From: Jon Regan-Beasley Date: Wed, 14 Oct 2026 08:46:14 +1100 Subject: [PATCH 14/19] test(qwen27): integrate current native CUDA with PR726 1.0.9 --- zig/build/cuda.zig | 12 +- zig/kernels/cuda/act_rows.cu | 142 +++++ zig/kernels/cuda/block_attention.cu | 203 +++++++ zig/kernels/cuda/dense_rows.cu | 70 +++ zig/kernels/cuda/gdn_prep.cu | 99 +++ zig/kernels/cuda/moe_glue.cu | 184 ++++++ zig/kernels/cuda/qk_rows.cu | 194 ++++++ zig/kernels/cuda/tree_attention.cu | 407 +++++++++++++ zig/kernels/cuda/window_attention.cu | 570 ++++++++++++++++++ zig/src/cli/qwen27_cuda.zig | 148 ++++- zig/src/core/act_ref.zig | 11 + zig/src/core/attention_ref.zig | 38 ++ zig/src/core/dense_ref.zig | 24 + zig/src/core/gdn_prep_ref.zig | 45 ++ zig/src/core/moe_ref.zig | 55 ++ zig/src/core/qk_ref.zig | 30 + zig/src/core/root.zig | 12 + zig/src/cuda/act_rows.zig | 97 +++ zig/src/cuda/block_attention.zig | 48 ++ zig/src/cuda/dense_rows.zig | 51 ++ zig/src/cuda/gdn_prep.zig | 73 +++ zig/src/cuda/kernels.zig | 10 + zig/src/cuda/launch.zig | 8 + zig/src/cuda/moe_glue.zig | 76 +++ zig/src/cuda/norms.zig | 5 +- zig/src/cuda/qk_rows.zig | 107 ++++ zig/src/cuda/root.zig | 18 + zig/src/cuda/tree_attention.zig | 129 ++++ zig/src/cuda/window_attention.zig | 115 ++++ zig/src/families/qwen27/cuda.zig | 1 + zig/src/families/qwen27/cuda/draft.zig | 21 +- zig/src/families/qwen27/cuda/engine.zig | 5 +- zig/src/families/qwen27/cuda/forward.zig | 23 + zig/src/families/qwen27/cuda/kernels.zig | 9 +- zig/src/families/qwen27/cuda/native.zig | 7 +- .../qwen27/cuda/native_attention_plan.zig | 51 ++ zig/src/families/qwen27/cuda/round.zig | 7 +- zig/src/families/qwen27/cuda/state.zig | 16 +- zig/src/families/qwen27/cuda/triton.zig | 140 +++-- zig/tests/cuda/act_rows_tests.zig | 330 ++++++++++ zig/tests/cuda/block_tests.zig | 152 +++++ zig/tests/cuda/dense_rows_tests.zig | 79 +++ zig/tests/cuda/gdn_prep_tests.zig | 156 +++++ zig/tests/cuda/main.zig | 17 + zig/tests/cuda/moe_glue_tests.zig | 161 +++++ zig/tests/cuda/qk_rows_tests.zig | 226 +++++++ zig/tests/cuda/tree_tests.zig | 217 +++++++ zig/tests/cuda/window_tests.zig | 217 +++++++ 48 files changed, 4732 insertions(+), 84 deletions(-) create mode 100644 zig/kernels/cuda/act_rows.cu create mode 100644 zig/kernels/cuda/block_attention.cu create mode 100644 zig/kernels/cuda/dense_rows.cu create mode 100644 zig/kernels/cuda/gdn_prep.cu create mode 100644 zig/kernels/cuda/moe_glue.cu create mode 100644 zig/kernels/cuda/qk_rows.cu create mode 100644 zig/kernels/cuda/tree_attention.cu create mode 100644 zig/kernels/cuda/window_attention.cu create mode 100644 zig/src/core/attention_ref.zig create mode 100644 zig/src/core/dense_ref.zig create mode 100644 zig/src/core/gdn_prep_ref.zig create mode 100644 zig/src/core/moe_ref.zig create mode 100644 zig/src/core/qk_ref.zig create mode 100644 zig/src/cuda/act_rows.zig create mode 100644 zig/src/cuda/block_attention.zig create mode 100644 zig/src/cuda/dense_rows.zig create mode 100644 zig/src/cuda/gdn_prep.zig create mode 100644 zig/src/cuda/moe_glue.zig create mode 100644 zig/src/cuda/qk_rows.zig create mode 100644 zig/src/cuda/tree_attention.zig create mode 100644 zig/src/cuda/window_attention.zig create mode 100644 zig/src/families/qwen27/cuda/native_attention_plan.zig create mode 100644 zig/tests/cuda/act_rows_tests.zig create mode 100644 zig/tests/cuda/block_tests.zig create mode 100644 zig/tests/cuda/dense_rows_tests.zig create mode 100644 zig/tests/cuda/gdn_prep_tests.zig create mode 100644 zig/tests/cuda/moe_glue_tests.zig create mode 100644 zig/tests/cuda/qk_rows_tests.zig create mode 100644 zig/tests/cuda/tree_tests.zig create mode 100644 zig/tests/cuda/window_tests.zig diff --git a/zig/build/cuda.zig b/zig/build/cuda.zig index 33c40f5633..b918136b3d 100644 --- a/zig/build/cuda.zig +++ b/zig/build/cuda.zig @@ -34,13 +34,21 @@ const kernels = [_]Kernel{ .{ .name = "train", .flags = &.{"-O3"} }, // ours: Sliding Weights' learning step (train.metal's kernels) .{ .name = "qwen_ops", .flags = &.{"-O3"} }, // ours: the 27B's batched state copies, taps, drafter norms and rotary .{ .name = "qwen_vision", .flags = &.{ "-O3", "--fmad=false" } }, // ours: the Qwen3.5 vision tower's glue and attention + .{ .name = "norms", .flags = &.{"-O3"} }, + .{ .name = "moe_glue", .flags = &.{"-O3"} }, + .{ .name = "dense_rows", .flags = &.{"-O3"} }, + .{ .name = "qk_rows", .flags = &.{"-O3"} }, + .{ .name = "window_attention", .flags = &.{"-O3"} }, + .{ .name = "tree_attention", .flags = &.{"-O3"} }, + .{ .name = "act_rows", .flags = &.{"-O3"} }, + .{ .name = "gdn_prep", .flags = &.{"-O3"} }, + .{ .name = "embed_rows", .flags = &.{"-O3"} }, + .{ .name = "block_attention", .flags = &.{"-O3"} }, .{ .name = "nemotron_norms", .flags = glue }, // ours, each with a host reference (glue_ref.zig): Nemotron's glue .{ .name = "nemotron_route", .flags = glue }, .{ .name = "nemotron_mamba", .flags = glue }, .{ .name = "nemotron_attention", .flags = glue }, .{ .name = "nemotron_keyed", .flags = glue }, - .{ .name = "norms", .flags = &.{"-O3"} }, // ours: RMS norm rows for every family (cuda/norms.zig) - .{ .name = "embed_rows", .flags = &.{"-O3"} }, // ours: 4-bit affine and bf16 token rows (cuda/embed_rows.zig) .{ .name = "qmmf", .flags = &.{"-O3"} }, // nvfp4/qmmf.cu's device code (FP8G, FP4), tensorfold_nvfp4_v3 .{ .name = "prompt16", .flags = &.{"-O3"} }, // nvfp4/prompt.cu's FP4 and FP8 tile 4, tensorfold_nvfp4_prompt_v1 .{ .name = "modelopt_pack", .flags = &.{"-O3"} }, // ours: ModelOpt tensors into the lane, prompt and expert layouts diff --git a/zig/kernels/cuda/act_rows.cu b/zig/kernels/cuda/act_rows.cu new file mode 100644 index 0000000000..9fc24a05b6 --- /dev/null +++ b/zig/kernels/cuda/act_rows.cu @@ -0,0 +1,142 @@ +// Activation rows for every CUDA family (float64 references: core/act_ref.zig): SwiGLU, output gates, group sums. + +// A thread 8 values with 16-byte loads; a row's bits depend on that row alone. + +#include +#include + +namespace tf_act { + +constexpr int NT = 256; + +__device__ __forceinline__ void unpack8(const uint4& u, float* f) { + const __nv_bfloat162* h = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) { + const float2 t = __bfloat1622float2(h[i]); + f[2 * i] = t.x; + f[2 * i + 1] = t.y; + } +} + +__device__ __forceinline__ uint4 pack8(const float* f) { + uint4 u; + __nv_bfloat162* h = reinterpret_cast<__nv_bfloat162*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) h[i] = __floats2bfloat162_rn(f[2 * i], f[2 * i + 1]); + return u; +} + +__device__ __forceinline__ float sigmoid(float x) { return __fdiv_rn(1.f, __fadd_rn(1.f, expf(-x))); } + +// A thread's 8 stored values in order, then the group's 8 lanes in a fixed tree (every lane of the warp calls). +__device__ __forceinline__ float group_sum8(const uint4& packed, bool live) { + float f[8]; + unpack8(packed, f); + float s = 0.f; + if (live) +#pragma unroll + for (int i = 0; i < 8; ++i) s = __fadd_rn(s, f[i]); + s = __fadd_rn(s, __shfl_xor_sync(0xFFFFFFFFu, s, 4)); + s = __fadd_rn(s, __shfl_xor_sync(0xFFFFFFFFu, s, 2)); + return __fadd_rn(s, __shfl_xor_sync(0xFFFFFFFFu, s, 1)); +} + +} // namespace tf_act + +using namespace tf_act; + +// out[r, i] = bf16(silu(gate[r * g_ld + i]) * up[r * u_ld + i]) for i < n (n % 64 == 0); xs (or null) out's sums. +extern "C" __global__ void __launch_bounds__(NT) tf_swiglu_rows(const __nv_bfloat16* __restrict__ gate, long long g_ld, + const __nv_bfloat16* __restrict__ up, long long u_ld, + __nv_bfloat16* __restrict__ out, float* __restrict__ xs, + int n) { + const int64_t r = blockIdx.x; + const int c = blockIdx.y * blockDim.x + threadIdx.x; // 8 values + const bool live = 8 * c < n; + float g[8], u[8], o[8]; + if (live) { + unpack8(*reinterpret_cast(gate + r * g_ld + 8 * c), g); + unpack8(*reinterpret_cast(up + r * u_ld + 8 * c), u); + } +#pragma unroll + for (int i = 0; i < 8; ++i) o[i] = live ? __fmul_rn(__fmul_rn(g[i], sigmoid(g[i])), u[i]) : 0.f; + const uint4 op = pack8(o); + if (live) *reinterpret_cast(out + r * n + 8 * c) = op; + if (xs != nullptr) { + const float s = group_sum8(op, live); + if (live && (c & 7) == 0) xs[r * (n / 64) + c / 8] = s; + } +} + +// out[r, h * d + e] = bf16(o[r, h * d + e] * sigmoid(g[r * g_ld + h * g_hs + e])) (d % 64 == 0); xs (or null) its sums. +extern "C" __global__ void __launch_bounds__(NT) tf_gate_rows(const __nv_bfloat16* __restrict__ o, + const __nv_bfloat16* __restrict__ g, long long g_ld, + long long g_hs, __nv_bfloat16* __restrict__ out, + float* __restrict__ xs, int heads, int d) { + const int64_t r = blockIdx.x; + const int c = blockIdx.y * blockDim.x + threadIdx.x; // 8 values + const int n = heads * d; + const bool live = 8 * c < n; + const int h = 8 * c / d, e = 8 * c % d; + float ov[8], gv[8], y[8]; + if (live) { + unpack8(*reinterpret_cast(o + r * n + 8 * c), ov); + unpack8(*reinterpret_cast(g + r * g_ld + h * g_hs + e), gv); + } +#pragma unroll + for (int i = 0; i < 8; ++i) y[i] = live ? __fmul_rn(ov[i], sigmoid(gv[i])) : 0.f; + const uint4 yp = pack8(y); + if (live) *reinterpret_cast(out + r * n + 8 * c) = yp; + if (xs != nullptr) { + const float s = group_sum8(yp, live); + if (live && (c & 7) == 0) xs[r * (n / 64) + c / 8] = s; + } +} + +// xs[r, j] = the sum of x[r * ld + 64 j ..][0 .. 64] (k % 64 == 0), in group_sum8's order. +extern "C" __global__ void __launch_bounds__(NT) tf_group_sums_rows(const __nv_bfloat16* __restrict__ x, long long ld, + float* __restrict__ xs, int k) { + const int64_t r = blockIdx.x; + const int c = blockIdx.y * blockDim.x + threadIdx.x; + const bool live = 8 * c < k; + const uint4 v = live ? *reinterpret_cast(x + r * ld + 8 * c) : make_uint4(0, 0, 0, 0); + const float s = group_sum8(v, live); + if (live && (c & 7) == 0) xs[r * (k / 64) + c / 8] = s; +} + +// DFlash2's two-tap conv: out = bf16(x k0 + x[r - 1] k1) [+ res], no r - 1 at a seg block's start, k = bf16(b + dyn). +extern "C" __global__ void __launch_bounds__(NT) tf_dconv_rows(const __nv_bfloat16* __restrict__ x, + const __nv_bfloat16* __restrict__ dyn, + const __nv_bfloat16* __restrict__ base, + const __nv_bfloat16* __restrict__ res, + __nv_bfloat16* __restrict__ out, int d, int gs, int branch, + int seg) { + const int64_t r = blockIdx.x; + const int c = blockIdx.y * blockDim.x + threadIdx.x; + if (8 * c >= d) return; + const int g = d / gs, grp = 8 * c / gs; + float xv[8], pv[8], b0[8], b1[8], y[8]; + unpack8(*reinterpret_cast(x + r * d + 8 * c), xv); + if (r % seg > 0) unpack8(*reinterpret_cast(x + (r - 1) * d + 8 * c), pv); + else +#pragma unroll + for (int i = 0; i < 8; ++i) pv[i] = 0.f; + unpack8(*reinterpret_cast(base + (int64_t)(branch * 2) * d + 8 * c), b0); + unpack8(*reinterpret_cast(base + (int64_t)(branch * 2 + 1) * d + 8 * c), b1); + const float d0 = __bfloat162float(dyn[((r * 2 + branch) * 2) * g + grp]); + const float d1 = __bfloat162float(dyn[((r * 2 + branch) * 2 + 1) * g + grp]); +#pragma unroll + for (int i = 0; i < 8; ++i) { + const float k0 = __bfloat162float(__float2bfloat16_rn(__fadd_rn(b0[i], d0))); + const float k1 = __bfloat162float(__float2bfloat16_rn(__fadd_rn(b1[i], d1))); + y[i] = __fmaf_rn(pv[i], k1, __fmul_rn(xv[i], k0)); + } + if (res != nullptr) { + float rv[8]; + unpack8(*reinterpret_cast(res + r * d + 8 * c), rv); +#pragma unroll + for (int i = 0; i < 8; ++i) y[i] = __fadd_rn(rv[i], __bfloat162float(__float2bfloat16_rn(y[i]))); + } + *reinterpret_cast(out + r * d + 8 * c) = pack8(y); +} diff --git a/zig/kernels/cuda/block_attention.cu b/zig/kernels/cuda/block_attention.cu new file mode 100644 index 0000000000..e3cd06b121 --- /dev/null +++ b/zig/kernels/cuda/block_attention.cu @@ -0,0 +1,203 @@ +// DFlash-style block attention for every CUDA family (float64 reference: core/attention_ref.zig): context, then block. + +// Block (stream, key head): a warp a query head, its 16 mma rows the block's rows; key tiles of 32 shared by the warps. + +#include +#include +#include +#include + +#include "qmm_frag.cuh" + +namespace tf_block { + +using qmm_frag::commit; +using qmm_frag::cp16z; +using qmm_frag::ldmatrix4; +using qmm_frag::mma; +using qmm_frag::mma0; + +constexpr int D = 128, C = D / 8, K16 = D / 16, NB = D / 8, KT = 32; +constexpr float LOG2E = 1.4426950408889634f; + +__device__ __forceinline__ void wait_all() { asm volatile("cp.async.wait_group 0;\n" ::); } + +__device__ __forceinline__ void ldmatrix4t(uint32_t (&r)[4], const void* p) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(qmm_frag::smem(p))); +} + +__device__ __forceinline__ float texp(float x) { + float y; + asm("ex2.approx.f32 %0, %1;\n" : "=f"(y) : "f"(__fmul_rn(x, LOG2E))); + return y; +} + +__device__ __forceinline__ uint32_t pack(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} + +__device__ __forceinline__ int sw(int r, int c) { return r * C + (c ^ (r & 7)); } + +// One warp's running fold over a 32-key tile: scores masked by `vis(row, key)`, online softmax per row, P x V. +struct Fold { + float o[NB][4]; + float m0 = -INFINITY, m1 = -INFINITY, l0 = 0.f, l1 = 0.f; + + __device__ __forceinline__ Fold() { +#pragma unroll + for (int i = 0; i < NB; ++i) o[i][0] = o[i][1] = o[i][2] = o[i][3] = 0.f; + } + + template + __device__ __forceinline__ void tile(const uint32_t (&qa)[K16][4], const uint4* ks, const uint4* vs, float scale, + int lane, int k0, Vis vis) { + const int g8 = lane / 4, qd = lane % 4, mt = lane / 8; + float sc[4][4]; +#pragma unroll + for (int k = 0; k < K16; ++k) { +#pragma unroll + for (int n = 0; n < 4; n += 2) { + uint32_t b[4]; + ldmatrix4(b, ks + sw(8 * (n + mt / 2) + lane % 8, 2 * k + mt % 2)); + if (k == 0) { + mma0(sc[n], qa[k], b[0], b[1]); + mma0(sc[n + 1], qa[k], b[2], b[3]); + } else { + mma(sc[n], qa[k], b[0], b[1]); + mma(sc[n + 1], qa[k], b[2], b[3]); + } + } + } + float tm0 = -INFINITY, tm1 = -INFINITY; +#pragma unroll + for (int n = 0; n < 4; ++n) { + const int c = k0 + 8 * n + 2 * qd; + sc[n][0] = vis(g8, c) ? __fmul_rn(sc[n][0], scale) : -INFINITY; + sc[n][1] = vis(g8, c + 1) ? __fmul_rn(sc[n][1], scale) : -INFINITY; + sc[n][2] = vis(g8 + 8, c) ? __fmul_rn(sc[n][2], scale) : -INFINITY; + sc[n][3] = vis(g8 + 8, c + 1) ? __fmul_rn(sc[n][3], scale) : -INFINITY; + tm0 = fmaxf(tm0, fmaxf(sc[n][0], sc[n][1])); + tm1 = fmaxf(tm1, fmaxf(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + tm0 = fmaxf(tm0, __shfl_xor_sync(0xffffffffu, tm0, x)); + tm1 = fmaxf(tm1, __shfl_xor_sync(0xffffffffu, tm1, x)); + } + const float n0 = fmaxf(m0, tm0), n1 = fmaxf(m1, tm1); + const float a0 = tm0 == -INFINITY ? 1.f : (m0 == -INFINITY ? 0.f : texp(__fsub_rn(m0, n0))); + const float a1 = tm1 == -INFINITY ? 1.f : (m1 == -INFINITY ? 0.f : texp(__fsub_rn(m1, n1))); + float c0 = 0.f, c1 = 0.f; +#pragma unroll + for (int n = 0; n < 4; ++n) { + sc[n][0] = sc[n][0] != -INFINITY ? texp(__fsub_rn(sc[n][0], n0)) : 0.f; + sc[n][1] = sc[n][1] != -INFINITY ? texp(__fsub_rn(sc[n][1], n0)) : 0.f; + sc[n][2] = sc[n][2] != -INFINITY ? texp(__fsub_rn(sc[n][2], n1)) : 0.f; + sc[n][3] = sc[n][3] != -INFINITY ? texp(__fsub_rn(sc[n][3], n1)) : 0.f; + c0 = __fadd_rn(c0, __fadd_rn(sc[n][0], sc[n][1])); + c1 = __fadd_rn(c1, __fadd_rn(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + c0 = __fadd_rn(c0, __shfl_xor_sync(0xffffffffu, c0, x)); + c1 = __fadd_rn(c1, __shfl_xor_sync(0xffffffffu, c1, x)); + } + l0 = __fmaf_rn(l0, a0, c0); + l1 = __fmaf_rn(l1, a1, c1); + m0 = n0; + m1 = n1; +#pragma unroll + for (int i = 0; i < NB; ++i) { + o[i][0] = __fmul_rn(o[i][0], a0); + o[i][1] = __fmul_rn(o[i][1], a0); + o[i][2] = __fmul_rn(o[i][2], a1); + o[i][3] = __fmul_rn(o[i][3], a1); + } + uint32_t pa[2][4]; +#pragma unroll + for (int j = 0; j < 2; ++j) { + pa[j][0] = pack(sc[2 * j][0], sc[2 * j][1]); + pa[j][1] = pack(sc[2 * j][2], sc[2 * j][3]); + pa[j][2] = pack(sc[2 * j + 1][0], sc[2 * j + 1][1]); + pa[j][3] = pack(sc[2 * j + 1][2], sc[2 * j + 1][3]); + } +#pragma unroll + for (int j = 0; j < 2; ++j) +#pragma unroll + for (int i = 0; i < NB; i += 2) { + uint32_t b[4]; + ldmatrix4t(b, vs + sw(16 * j + 8 * (mt % 2) + lane % 8, i + mt / 2)); + mma(o[i], pa[j], b[0], b[1]); + mma(o[i + 1], pa[j], b[2], b[3]); + } + } +}; + +} // namespace tf_block + +using namespace tf_block; + +// Block rows r see context key k when lens[j] + r - k <= window, then block keys c < L (c <= r if causal). +extern "C" __global__ void __launch_bounds__(256) tf_block_attention( + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kb, const __nv_bfloat16* __restrict__ vb, + const long long* __restrict__ table, const int* __restrict__ lens, __nv_bfloat16* __restrict__ out, float scale, + int window, int rows, int h, int hk, int L, int causal) { + extern __shared__ uint4 smem[]; + uint4* ks = smem; + uint4* vs = ks + KT * C; + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32, g8 = lane / 4, qd = lane % 4; + const int j = blockIdx.x, g = blockIdx.y, G = h / hk; + const int head = g * G + warp; + const int s = lens[j]; + const __nv_bfloat16* kc = reinterpret_cast(table[2 * j]) + (int64_t)g * s * D; + const __nv_bfloat16* vc = reinterpret_cast(table[2 * j + 1]) + (int64_t)g * s * D; + uint32_t qa[K16][4]; + { + const int r0 = g8, r1 = g8 + 8; + const uint32_t* q0 = reinterpret_cast(q + ((int64_t)head * rows + j * L + min(r0, L - 1)) * D); + const uint32_t* q1 = reinterpret_cast(q + ((int64_t)head * rows + j * L + min(r1, L - 1)) * D); +#pragma unroll + for (int k = 0; k < K16; ++k) { + qa[k][0] = r0 < L ? q0[8 * k + qd] : 0u; + qa[k][1] = r1 < L ? q1[8 * k + qd] : 0u; + qa[k][2] = r0 < L ? q0[8 * k + qd + 4] : 0u; + qa[k][3] = r1 < L ? q1[8 * k + qd + 4] : 0u; + } + } + Fold f; + const int lo = max(0, s - window); // the earliest context key any row sees (row 0: s - k <= window) + for (int k0 = lo / KT * KT; k0 < s; k0 += KT) { + for (int i = threadIdx.x; i < KT * C; i += blockDim.x) { + const int key = k0 + i / C; + const bool in = key < s; + cp16z(ks + sw(i / C, i % C), kc + (int64_t)(in ? key : 0) * D + (i % C) * 8, in); + cp16z(vs + sw(i / C, i % C), vc + (int64_t)(in ? key : 0) * D + (i % C) * 8, in); + } + commit(); + wait_all(); + __syncthreads(); + f.tile(qa, ks, vs, scale, lane, k0, [&](int r, int key) { return r < L && key < s && s + r - key <= window; }); + __syncthreads(); + } + for (int i = threadIdx.x; i < KT * C; i += blockDim.x) { // the block's own keys + const int c = i / C; + const bool in = c < L; + const int64_t at = ((int64_t)g * rows + j * L + (in ? c : 0)) * D + (i % C) * 8; + cp16z(ks + sw(c, i % C), kb + at, in); + cp16z(vs + sw(c, i % C), vb + at, in); + } + commit(); + wait_all(); + __syncthreads(); + f.tile(qa, ks, vs, scale, lane, 0, [&](int r, int c) { return r < L && c < L && (!causal || c <= r); }); + if (g8 < L) +#pragma unroll + for (int i = 0; i < NB; ++i) + *reinterpret_cast(out + ((int64_t)(j * L + g8) * h + head) * D + 8 * i + 2 * qd) = + pack(__fdiv_rn(f.o[i][0], f.l0), __fdiv_rn(f.o[i][1], f.l0)); + if (g8 + 8 < L) +#pragma unroll + for (int i = 0; i < NB; ++i) + *reinterpret_cast(out + ((int64_t)(j * L + g8 + 8) * h + head) * D + 8 * i + 2 * qd) = + pack(__fdiv_rn(f.o[i][2], f.l1), __fdiv_rn(f.o[i][3], f.l1)); +} diff --git a/zig/kernels/cuda/dense_rows.cu b/zig/kernels/cuda/dense_rows.cu new file mode 100644 index 0000000000..acfc61efe3 --- /dev/null +++ b/zig/kernels/cuda/dense_rows.cu @@ -0,0 +1,70 @@ +// bf16 rows times bf16 weight rows into fp32 for every CUDA family: routers and output heads (core/dense_ref.zig). + +// A warp an output column, RT rows a launch tile; each row's sum runs in one order whatever RT and the row count. + +#include +#include + +namespace tf_dense { + +constexpr int WARPS = 8; +constexpr int VEC = 8; // bf16 a 16-byte load + +__device__ __forceinline__ void unpack(const uint4& u, float* f) { + const __nv_bfloat162* h = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) { + const float2 t = __bfloat1622float2(h[i]); + f[2 * i] = t.x; + f[2 * i + 1] = t.y; + } +} + +// out[r, n] = x[r] . w[n] for a tile's rows: lane l sums k = 8 l + 256 j in order, then the fixed shuffle tree. +template +__device__ __forceinline__ void tile(const __nv_bfloat16* x, int64_t x_stride, const __nv_bfloat16* w, float* out, + int64_t out_stride, int rows, int n, int k) { + const int lane = threadIdx.x & 31; + const int col = blockIdx.x * WARPS + (threadIdx.x >> 5); + const int r0 = blockIdx.y * RT; + if (col >= n) return; + float acc[RT]; +#pragma unroll + for (int r = 0; r < RT; ++r) acc[r] = 0.f; + const __nv_bfloat16* wr = w + static_cast(col) * k; +#pragma unroll 4 // several 16-byte weight loads in flight a lane (memory-level parallelism) + for (int k0 = lane * VEC; k0 < k; k0 += 32 * VEC) { + float wf[VEC]; + unpack(*reinterpret_cast(wr + k0), wf); +#pragma unroll + for (int r = 0; r < RT; ++r) { + if (r0 + r >= rows) break; + float xf[VEC]; + unpack(*reinterpret_cast(x + (r0 + r) * x_stride + k0), xf); +#pragma unroll + for (int i = 0; i < VEC; ++i) acc[r] = __fmaf_rn(xf[i], wf[i], acc[r]); + } + } +#pragma unroll + for (int r = 0; r < RT; ++r) { + float v = acc[r]; + for (int m = 16; m > 0; m >>= 1) v = __fadd_rn(v, __shfl_xor_sync(0xFFFFFFFFu, v, m)); + if (lane == 0 && r0 + r < rows) out[(r0 + r) * out_stride + col] = v; + } +} + +} // namespace tf_dense + +using namespace tf_dense; + +#define TF_DENSE_ROWS(RT) \ + extern "C" __global__ void __launch_bounds__(WARPS * 32) tf_dense_rows_##RT( \ + const __nv_bfloat16* x, long long x_stride, const __nv_bfloat16* w, float* out, long long out_stride, \ + int rows, int n, int k) { \ + tile(x, x_stride, w, out, out_stride, rows, n, k); \ + } + +TF_DENSE_ROWS(1) +TF_DENSE_ROWS(4) +TF_DENSE_ROWS(16) +TF_DENSE_ROWS(32) diff --git a/zig/kernels/cuda/gdn_prep.cu b/zig/kernels/cuda/gdn_prep.cu new file mode 100644 index 0000000000..04f330f4e7 --- /dev/null +++ b/zig/kernels/cuda/gdn_prep.cu @@ -0,0 +1,99 @@ +// Gated DeltaNet inputs for every CUDA family (float64 reference: core/gdn_prep_ref.zig): conv, norms, g, beta. + +// A warp a head (q heads, k heads, then v heads), lane l holding channels J l .. J l + J - 1; a row's bits are its own. + +#include +#include +#include + +namespace tf_gdn_prep { + +constexpr int MAXT = 8; // conv taps at most + +struct Io { + const __nv_bfloat16* qkv; // [rows, qkv_ld]: q (kh dk), k (kh dk), v (vh dk) channels + long long qkv_ld; + const __nv_bfloat16* state; // [streams * keep, c]: each stream's kept conv rows + const __nv_bfloat16* conv; // [c, keep + 1] taps a channel + const int* win; // [rows, keep + 1]: < keep a kept row, else qkv row win - keep + const int* sid; // [rows] stream (null: stream 0) + const __nv_bfloat16* a; // [rows, a_ld] the v heads' dt inputs + long long a_ld; + const __nv_bfloat16* b; // [rows, b_ld] the v heads' beta inputs + long long b_ld; + const float* a_log; // [vh] + const float* dt_bias; // [vh] + __nv_bfloat16* q; // [rows, kh, dk] + __nv_bfloat16* k; // [rows, kh, dk] + __nv_bfloat16* v; // [rows, vh, dk] + float* g; // [rows, vh] + float* beta; // [rows, vh] +}; + +__device__ __forceinline__ float sigmoid(float x) { return __fdiv_rn(1.f, __fadd_rn(1.f, expf(-x))); } + +template +__device__ __forceinline__ void prep(const Io& io, int c, int kh, int vh, int keep, float eps) { + constexpr int DK = 32 * J; + const int lane = threadIdx.x & 31; + const int64_t r = blockIdx.x; + const int hd = blockIdx.y * (blockDim.x / 32) + threadIdx.x / 32; + if (hd >= 2 * kh + vh) return; + const int ch = hd * DK + J * lane; + const int64_t srow = (io.sid != nullptr ? static_cast(io.sid[r]) : 0) * keep; + int src[MAXT]; +#pragma unroll + for (int t = 0; t < MAXT; ++t) src[t] = t <= keep ? io.win[r * (keep + 1) + t] : 0; + float acc[J]; +#pragma unroll + for (int i = 0; i < J; ++i) acc[i] = 0.f; +#pragma unroll + for (int t = 0; t < MAXT; ++t) { + if (t > keep) break; + const __nv_bfloat16* x = src[t] < keep ? io.state + (srow + src[t]) * c : io.qkv + (src[t] - keep) * io.qkv_ld; +#pragma unroll + for (int i = 0; i < J; ++i) + acc[i] = __fmaf_rn(__bfloat162float(x[ch + i]), __bfloat162float(io.conv[(ch + i) * (keep + 1) + t]), acc[i]); + } + float cv[J]; +#pragma unroll + for (int i = 0; i < J; ++i) cv[i] = __bfloat162float(__float2bfloat16_rn(__fmul_rn(acc[i], sigmoid(acc[i])))); + if (hd < 2 * kh) { // q: l2norm / sqrt(dk), k: l2norm (rms with eps, scaled) + float part = 0.f; +#pragma unroll + for (int i = 0; i < J; ++i) part = __fmaf_rn(cv[i], cv[i], part); + for (int m = 16; m > 0; m >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xFFFFFFFFu, part, m)); + const float inv = rsqrtf(__fadd_rn(__fdiv_rn(part, static_cast(DK)), eps)); + const bool is_q = hd < kh; + const float scale = is_q ? 1.f / DK : rsqrtf(static_cast(DK)); + __nv_bfloat16* out = (is_q ? io.q : io.k) + (r * kh + (is_q ? hd : hd - kh)) * DK + J * lane; +#pragma unroll + for (int i = 0; i < J; ++i) out[i] = __float2bfloat16_rn(__fmul_rn(__fmul_rn(cv[i], inv), scale)); + return; + } + const int hv = hd - 2 * kh; +#pragma unroll + for (int i = 0; i < J; ++i) io.v[(r * vh + hv) * DK + J * lane + i] = __float2bfloat16_rn(cv[i]); + if (lane == 0) { + const float x = __fadd_rn(__bfloat162float(io.a[r * io.a_ld + hv]), io.dt_bias[hv]); + const float sp = x > 20.f ? x : log1pf(expf(x)); + io.g[r * vh + hv] = expf(-__fmul_rn(expf(io.a_log[hv]), sp)); + io.beta[r * vh + hv] = sigmoid(__bfloat162float(io.b[r * io.b_ld + hv])); + } +} + +} // namespace tf_gdn_prep + +using namespace tf_gdn_prep; + +extern "C" __global__ void __launch_bounds__(128) tf_gdn_prep_64(const Io io, int c, int kh, int vh, int keep, float eps) { + prep<2>(io, c, kh, vh, keep, eps); +} + +extern "C" __global__ void __launch_bounds__(128) tf_gdn_prep_128(const Io io, int c, int kh, int vh, int keep, float eps) { + prep<4>(io, c, kh, vh, keep, eps); +} + +extern "C" __global__ void __launch_bounds__(128) tf_gdn_prep_256(const Io io, int c, int kh, int vh, int keep, float eps) { + prep<8>(io, c, kh, vh, keep, eps); +} diff --git a/zig/kernels/cuda/moe_glue.cu b/zig/kernels/cuda/moe_glue.cu new file mode 100644 index 0000000000..f4beca0f82 --- /dev/null +++ b/zig/kernels/cuda/moe_glue.cu @@ -0,0 +1,184 @@ +// MoE glue for every CUDA family (float64 references: core/moe_ref.zig): sigmoid top-k and the weighted combine. + +// Every row's bits depend on that row alone; kinds 0 fp32, 1 fp16, 2 bf16 (as norms.cu). + +#include +#include +#include +#include + +namespace tf_moe { + +constexpr int NT = 256; +constexpr int MOST = 4096; // experts a row ranks + +__device__ __forceinline__ float load(const void* p, int kind, int64_t i) { + if (kind == 0) return static_cast(p)[i]; + if (kind == 1) return __half2float(static_cast(p)[i]); + return __bfloat162float(static_cast(p)[i]); +} + +__device__ __forceinline__ void store(void* p, int kind, int64_t i, float v) { + if (kind == 0) static_cast(p)[i] = v; + else if (kind == 1) static_cast<__half*>(p)[i] = __float2half_rn(v); + else static_cast<__nv_bfloat16*>(p)[i] = __float2bfloat16_rn(v); +} + +// (value, id) wins over (other value, other id): the larger value, the lower id among equals. +__device__ __forceinline__ bool beats(float v, int i, float w, int j) { + return v > w || (v == w && i < j); +} + +} // namespace tf_moe + +using namespace tf_moe; + +// Top `topk` by logit + bias (ties: lower id), weights sigmoid(logit) [normalized] * scale; `shared`: slot topk = ne. +extern "C" __global__ void __launch_bounds__(NT) tf_moe_topk_sigmoid(const float* logits, const float* bias, int* pick, + float* wts, int ne, int topk, int shared, + int normalize, float scale) { + __shared__ float v[MOST]; + __shared__ float best_v[NT / 32]; + __shared__ int best_i[NT / 32]; + __shared__ int picked[64]; + const int64_t r = blockIdx.x; + const float* lg = logits + r * ne; + for (int e = threadIdx.x; e < ne; e += NT) v[e] = __fadd_rn(lg[e], bias != nullptr ? bias[e] : 0.f); + __syncthreads(); + for (int k = 0; k < topk; ++k) { + float bv = -INFINITY; + int bi = ne; + for (int e = threadIdx.x; e < ne; e += NT) + if (beats(v[e], e, bv, bi)) bv = v[e], bi = e; + for (int m = 16; m > 0; m >>= 1) { + const float ov = __shfl_xor_sync(0xFFFFFFFFu, bv, m); + const int oi = __shfl_xor_sync(0xFFFFFFFFu, bi, m); + if (beats(ov, oi, bv, bi)) bv = ov, bi = oi; + } + if ((threadIdx.x & 31) == 0) best_v[threadIdx.x >> 5] = bv, best_i[threadIdx.x >> 5] = bi; + __syncthreads(); + if (threadIdx.x == 0) { + float tv = best_v[0]; + int ti = best_i[0]; + for (int w = 1; w < NT / 32; ++w) + if (beats(best_v[w], best_i[w], tv, ti)) tv = best_v[w], ti = best_i[w]; + picked[k] = ti; + v[ti] = -INFINITY; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + const int slots = topk + (shared ? 1 : 0); + float sum = 0.f; + float w[64]; + for (int k = 0; k < topk; ++k) { + w[k] = __fdiv_rn(1.f, __fadd_rn(1.f, expf(-lg[picked[k]]))); + sum = __fadd_rn(sum, w[k]); + } + for (int k = 0; k < topk; ++k) { + pick[r * slots + k] = picked[k]; + wts[r * slots + k] = __fmul_rn(normalize ? __fdiv_rn(w[k], sum) : w[k], scale); + } + if (shared) pick[r * slots + topk] = ne, wts[r * slots + topk] = 1.f; + } +} + +// A float's order as an unsigned (-0 as +0, so equal values stay equal). +__device__ __forceinline__ unsigned ordered(float v) { + const unsigned u = __float_as_uint(__fadd_rn(v, 0.f)); + return u & 0x80000000u ? ~u : u | 0x80000000u; +} + +// tf_moe_topk_sigmoid with a warp a row (up to 32 * PER experts): values in registers, no block barrier. +constexpr int PER = 16; + +extern "C" __global__ void __launch_bounds__(256) tf_moe_topk_sigmoid_warp(const float* logits, const float* bias, + int* pick, float* wts, int rows, int ne, + int topk, int shared, int normalize, + float scale) { + const int lane = threadIdx.x & 31; + const int64_t r = static_cast(blockIdx.x) * 8 + (threadIdx.x >> 5); + if (r >= rows) return; + const float* lg = logits + r * ne; + float v[PER], x[PER]; +#pragma unroll + for (int j = 0; j < PER; ++j) { + const int e = lane + 32 * j; + x[j] = e < ne ? lg[e] : 0.f; + v[j] = e < ne ? __fadd_rn(x[j], bias != nullptr ? bias[e] : 0.f) : -INFINITY; + } + const int slots = topk + (shared ? 1 : 0); + float xk = 0.f; // lane k's pick's logit + int pk = 0; // lane k's pick + for (int k = 0; k < topk; ++k) { + float bv = -INFINITY; + int bi = ne; +#pragma unroll + for (int j = 0; j < PER; ++j) { + const int e = lane + 32 * j; + if (e < ne && beats(v[j], e, bv, bi)) bv = v[j], bi = e; + } + { // the warp's best: the largest value (redux on an order-keeping key), then the lowest id among equals + const unsigned key = ordered(bv); + const unsigned top = __reduce_max_sync(0xFFFFFFFFu, key); + bi = static_cast(__reduce_min_sync(0xFFFFFFFFu, key == top ? static_cast(bi) : 0xFFFFFFFFu)); + } + float xb = 0.f; +#pragma unroll + for (int j = 0; j < PER; ++j) + if (lane + 32 * j == bi) v[j] = -INFINITY, xb = x[j]; + xb = __shfl_sync(0xFFFFFFFFu, xb, bi & 31); + if (lane == k) xk = xb, pk = bi; + } + // the weights after the picks, each in its lane; the sum in pick order + const float wk = __fdiv_rn(1.f, __fadd_rn(1.f, expf(-xk))); + float sum = 0.f; + for (int k = 0; k < topk; ++k) sum = __fadd_rn(sum, __shfl_sync(0xFFFFFFFFu, wk, k)); + if (lane < topk) { + pick[r * slots + lane] = pk; + wts[r * slots + lane] = __fmul_rn(normalize ? __fdiv_rn(wk, sum) : wk, scale); + } + if (shared && lane == 0) pick[r * slots + topk] = ne, wts[r * slots + topk] = 1.f; +} + +// out[r, d] = sum over the row's slots, in slot order, of w[r, k] * y[r, k, d] (fp32 fma), rounded once to out's kind. +template +__device__ __forceinline__ void combine_row(const void* y, const float* w, void* out, int slots, int dims, int64_t r, int d) { + float acc = 0.f; + int k = 0; + for (; k + 8 <= slots; k += 8) { // eight slots' loads in flight, then folded in order + float v[8], wk[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = load(y, YK, (r * slots + k + i) * dims + d), wk[i] = w[r * slots + k + i]; +#pragma unroll + for (int i = 0; i < 8; ++i) acc = __fmaf_rn(wk[i], v[i], acc); + } + if (k < slots) { + float v[8], wk[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + v[i] = k + i < slots ? load(y, YK, (r * slots + k + i) * dims + d) : 0.f; + wk[i] = k + i < slots ? w[r * slots + k + i] : 0.f; + } +#pragma unroll + for (int i = 0; i < 8; ++i) + if (k + i < slots) acc = __fmaf_rn(wk[i], v[i], acc); + } + store(out, OK, r * dims + d, acc); +} + +extern "C" __global__ void __launch_bounds__(NT) tf_moe_combine(const void* y, int y_kind, const float* w, void* out, + int out_kind, int slots, int dims) { + const int64_t r = blockIdx.x; + const int d = blockIdx.y * NT + threadIdx.x; + if (d >= dims) return; + // the kinds picked once a thread (a branch an element costs instruction fetch on small calls) + if (y_kind == 0 && out_kind == 2) combine_row<0, 2>(y, w, out, slots, dims, r, d); + else if (y_kind == 2 && out_kind == 2) combine_row<2, 2>(y, w, out, slots, dims, r, d); + else if (y_kind == 0 && out_kind == 0) combine_row<0, 0>(y, w, out, slots, dims, r, d); + else { + float acc = 0.f; + for (int k = 0; k < slots; ++k) acc = __fmaf_rn(w[r * slots + k], load(y, y_kind, (r * slots + k) * dims + d), acc); + store(out, out_kind, r * dims + d, acc); + } +} diff --git a/zig/kernels/cuda/qk_rows.cu b/zig/kernels/cuda/qk_rows.cu new file mode 100644 index 0000000000..146e14017c --- /dev/null +++ b/zig/kernels/cuda/qk_rows.cu @@ -0,0 +1,194 @@ +// Attention inputs for every CUDA family (float64 references: core/qk_ref.zig): QK norm, RoPE and the cache write. + +// A block a (row, head), a thread an element; a head's norm sums in one fixed order, never depending on the row count. + +#include +#include + +namespace tf_qk { + +constexpr int MOST = 256; // head_dim at most + +// Where a row's heads come from: q, k and v (null: none) at row stride *_ld and head stride *_hs (elements). +struct QkSrc { + const __nv_bfloat16* q; + long long q_ld, q_hs; + const __nv_bfloat16* k; + long long k_ld, k_hs; + const __nv_bfloat16* v; + long long v_ld, v_hs; +}; + +// The q and k norm weights: fp32 or bf16 (null q: no norm). +struct QkW { + const void* q; + const void* k; + int bf16; +}; + +__device__ __forceinline__ float wv(const QkW& w, bool is_k, int i) { + const void* p = is_k ? w.k : w.q; + return w.bf16 ? __bfloat162float(static_cast(p)[i]) : static_cast(p)[i]; +} + +// cos and sin of p's angle on rotated dim d: from tables [positions, rot], or computed from inv [rot / 2]. +__device__ __forceinline__ void rope_at(const float* cosv, const float* sinv, const float* inv, int p, int rot, int d, + float& c, float& sn) { + if (inv != nullptr) { + const float a = __fmul_rn(static_cast(p), inv[d < rot / 2 ? d : d - rot / 2]); + c = cosf(a); + sn = sinf(a); + } else { + c = cosv[static_cast(p) * rot + d]; + sn = sinv[static_cast(p) * rot + d]; + } +} + +// Where a row's q, k and v (null: none) go, at row stride *_ld and head stride *_hs (elements). +struct QkDst { + __nv_bfloat16* q; + long long q_ld, q_hs; + __nv_bfloat16* k; + long long k_ld, k_hs; + __nv_bfloat16* v; + long long v_ld, v_hs; +}; + +__device__ __forceinline__ const __nv_bfloat16* head_src(const QkSrc& s, int64_t r, int hd, int h, int hk) { + if (hd < h) return s.q + r * s.q_ld + hd * s.q_hs; + if (hd < h + hk) return s.k + r * s.k_ld + (hd - h) * s.k_hs; + return s.v + r * s.v_ld + (hd - h - hk) * s.v_hs; +} + +} // namespace tf_qk + +using namespace tf_qk; + +// QkSrc's heads -> q, k, v (bf16); q, k normed by nw, rotate-half RoPE on `rot` dims; k, v also into kc/vc. +extern "C" __global__ void __launch_bounds__(MOST) tf_qk_rows(const QkSrc in, const QkW nw, const float* cosv, + const float* sinv, const float* inv, int rot, const int* pos, + const int* slot, const QkDst dst, __nv_bfloat16* kc, __nv_bfloat16* vc, + long long len, int h, int hk, float eps) { + __shared__ float warps[MOST / 32]; + const int d = threadIdx.x; + const int dim = blockDim.x; + const int64_t r = blockIdx.x; + const int hd = blockIdx.y; + const __nv_bfloat16* src = head_src(in, r, hd, h, hk); + const float x = __bfloat162float(src[d]); + const int p = pos[r]; + const int64_t pl = p < len ? p : p % len; // a 64-bit modulo only past the cache's end + const int64_t at = slot != nullptr ? static_cast(slot[r]) * len + pl : pl; + if (hd >= h + hk) { // v: as is + const int k = hd - h - hk; + if (dst.v != nullptr) dst.v[r * dst.v_ld + k * dst.v_hs + d] = src[d]; + if (vc != nullptr) vc[(at * hk + k) * dim + d] = src[d]; + return; + } + float y = x; + float yp = 0.f; + const int half = rot / 2; + const int partner = d < half ? d + half : d - half; + if (nw.q != nullptr) { + float part = __fmul_rn(x, x); + for (int m = 16; m > 0; m >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xFFFFFFFFu, part, m)); + if ((d & 31) == 0) warps[d >> 5] = part; + __syncthreads(); + float s = 0.f; + for (int w = 0; w < dim / 32; ++w) s = __fadd_rn(s, warps[w]); + const float rstd = rsqrtf(__fadd_rn(__fdiv_rn(s, static_cast(dim)), eps)); + y = __fmul_rn(__fmul_rn(x, rstd), wv(nw, hd >= h, d)); + if (d < rot) yp = __fmul_rn(__fmul_rn(__bfloat162float(src[partner]), rstd), wv(nw, hd >= h, partner)); + } else if (d < rot) { + yp = __bfloat162float(src[partner]); + } + if ((cosv != nullptr || inv != nullptr) && d < rot) { + float c, sn; + rope_at(cosv, sinv, inv, p, rot, d, c, sn); + y = __fadd_rn(__fmul_rn(y, c), __fmul_rn(d < half ? -yp : yp, sn)); + } + const __nv_bfloat16 out = __float2bfloat16_rn(y); + if (hd < h) { + dst.q[r * dst.q_ld + hd * dst.q_hs + d] = out; + } else { + const int k = hd - h; + dst.k[r * dst.k_ld + k * dst.k_hs + d] = out; + if (kc != nullptr) kc[(at * hk + k) * dim + d] = out; + } +} + +// tf_qk_rows at dim 128, no or full rotation: a warp a head, lane l dims l + 32 j (same fold order, partner j ^ 2). +extern "C" __global__ void __launch_bounds__(128) tf_qk_rows_w(const QkSrc in, const QkW nw, + const float* cosv, const float* sinv, const float* inv, int rot, + const int* pos, const int* slot, const QkDst dst, + __nv_bfloat16* kc, __nv_bfloat16* vc, long long len, + int h, int hk, float eps) { + constexpr int D = 128, J = D / 32; + const int lane = threadIdx.x & 31; + const int64_t r = blockIdx.x; + const int hd = blockIdx.y * 4 + (threadIdx.x >> 5); + if (hd >= h + (in.v != nullptr ? 2 * hk : hk)) return; + const __nv_bfloat16* src = head_src(in, r, hd, h, hk); + const int p = pos[r]; + __nv_bfloat16 raw[J]; +#pragma unroll + for (int j = 0; j < J; ++j) raw[j] = src[lane + 32 * j]; + // every load up front (the rotation's row waits on pos, then on memory) + float c[J], sn[J], w[J]; + const bool qk = hd < h + hk; +#pragma unroll + for (int j = 0; j < J; ++j) { + c[j] = sn[j] = 0.f; + if (qk && rot != 0 && (cosv != nullptr || inv != nullptr)) rope_at(cosv, sinv, inv, p, D, lane + 32 * j, c[j], sn[j]); + w[j] = qk && nw.q != nullptr ? wv(nw, hd >= h, lane + 32 * j) : 0.f; + } + const int64_t pl = p < len ? p : p % len; + const int64_t at = slot != nullptr ? static_cast(slot[r]) * len + pl : pl; + if (hd >= h + hk) { // v: as is + const int k = hd - h - hk; +#pragma unroll + for (int j = 0; j < J; ++j) { + if (dst.v != nullptr) dst.v[r * dst.v_ld + k * dst.v_hs + lane + 32 * j] = raw[j]; + if (vc != nullptr) vc[(at * hk + k) * D + lane + 32 * j] = raw[j]; + } + return; + } + float x[J], y[J], yp[J]; +#pragma unroll + for (int j = 0; j < J; ++j) x[j] = __bfloat162float(raw[j]), y[j] = x[j], yp[j] = 0.f; + if (nw.q != nullptr) { + float part[J]; +#pragma unroll + for (int j = 0; j < J; ++j) part[j] = __fmul_rn(x[j], x[j]); + for (int m = 16; m > 0; m >>= 1) +#pragma unroll + for (int j = 0; j < J; ++j) part[j] = __fadd_rn(part[j], __shfl_xor_sync(0xFFFFFFFFu, part[j], m)); + float s = 0.f; +#pragma unroll + for (int j = 0; j < J; ++j) s = __fadd_rn(s, part[j]); + const float rstd = rsqrtf(__fadd_rn(__fdiv_rn(s, static_cast(D)), eps)); +#pragma unroll + for (int j = 0; j < J; ++j) { + y[j] = __fmul_rn(__fmul_rn(x[j], rstd), w[j]); + if (rot != 0) yp[j] = __fmul_rn(__fmul_rn(x[j ^ 2], rstd), w[j ^ 2]); + } + } else if (rot != 0) { +#pragma unroll + for (int j = 0; j < J; ++j) yp[j] = x[j ^ 2]; + } + if (rot != 0 && (cosv != nullptr || inv != nullptr)) { +#pragma unroll + for (int j = 0; j < J; ++j) y[j] = __fadd_rn(__fmul_rn(y[j], c[j]), __fmul_rn(j < 2 ? -yp[j] : yp[j], sn[j])); + } +#pragma unroll + for (int j = 0; j < J; ++j) { + const __nv_bfloat16 out = __float2bfloat16_rn(y[j]); + if (hd < h) { + dst.q[r * dst.q_ld + hd * dst.q_hs + lane + 32 * j] = out; + } else { + const int k = hd - h; + dst.k[r * dst.k_ld + k * dst.k_hs + lane + 32 * j] = out; + if (kc != nullptr) kc[(at * hk + k) * D + lane + 32 * j] = out; + } + } +} diff --git a/zig/kernels/cuda/tree_attention.cu b/zig/kernels/cuda/tree_attention.cu new file mode 100644 index 0000000000..310ab7d659 --- /dev/null +++ b/zig/kernels/cuda/tree_attention.cu @@ -0,0 +1,407 @@ +// Tree attention (float64 references: core/attention_ref.zig): committed keys, then the row's path, groups in order. + +#include +#include +#include +#include + +#include "qmm_frag.cuh" + +namespace tf_tree { + +using qmm_frag::commit; +using qmm_frag::cp16z; +using qmm_frag::ldmatrix4; +using qmm_frag::mma; +using qmm_frag::mma0; + +constexpr int SPAN = 512; // keys a group: one partial (4 slices of its tiles) +constexpr int MAXD = 128; // a path's most rows +constexpr float LOG2E = 1.4426950408889634f; + +template +__device__ __forceinline__ void wait_group() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); } + +__device__ __forceinline__ void ldmatrix4t(uint32_t (&r)[4], const void* p) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(qmm_frag::smem(p))); +} + +__device__ __forceinline__ float texp(float x) { + float y; + asm("ex2.approx.f32 %0, %1;\n" : "=f"(y) : "f"(__fmul_rn(x, LOG2E))); + return y; +} + +__device__ __forceinline__ uint32_t pack(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} + +template +__device__ __forceinline__ int sw(int r, int c) { return r * (D / 8) + (c ^ (r & 7)); } + +} // namespace tf_tree + +using namespace tf_tree; + +// Each row's root-to-row path (global window rows, root first) and its depth, from parents (-1 at a root). +extern "C" __global__ void tf_tree_paths(const int* __restrict__ parents, int* __restrict__ paths, + int* __restrict__ depths, int rows) { + const int node = blockIdx.x * blockDim.x + threadIdx.x; + if (node >= rows) return; + int depth = 0; + for (int cur = node; cur >= 0 && depth < MAXD; cur = parents[cur]) ++depth; + depths[node] = depth; + int cur = node; + for (int slot = depth - 1; slot >= 0; --slot, cur = parents[cur]) paths[node * MAXD + slot] = cur; +} + +// Group gi of key head kh for 16 (row, head) pairs from pair pf (pair m: row rq + m / G, head m % G), one mma tile. +template +__device__ __forceinline__ void tree_group( + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kn, const __nv_bfloat16* __restrict__ vn, + const __nv_bfloat16* __restrict__ origin, const long long* __restrict__ offs, const int* __restrict__ row_stream, + const int* __restrict__ streams, const int* __restrict__ paths, const int* __restrict__ depths, + float* __restrict__ po, float* __restrict__ pm, float* __restrict__ pl, int h, int hk, float scale, int gi, int ng, + int64_t rq, int pf, int npairs, int64_t row, int slice) { + constexpr int C = D / 8, K16 = D / 16, NB = D / 8, NT8 = HB / 8, PK = HB / 16; + constexpr bool QREG = D <= 128; + extern __shared__ uint4 smem[]; + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32, g8 = lane / 4, qd = lane % 4, mt = lane / 8; + const int wi = W == 4 ? warp : slice; // the group's slice: key tiles wi, wi + 4, ... + const int64_t r = row; // the row whose stream and depth set the keys + const int kh = blockIdx.y, G = h / hk; + const int s = row_stream[r]; + const int p = streams[4 * s + 2], depth = depths[r]; + const int total = p + depth, k0 = gi * SPAN, k1 = min(total, k0 + SPAN); + if (k0 >= total) return; // the merge reads only the row's own groups + const __nv_bfloat16* kc = origin + offs[2 * s]; + const __nv_bfloat16* vc = origin + offs[2 * s + 1]; + // ST tile stages: a one-warp block fetches ST - 1 tiles ahead while it computes one (stages never change bits) + uint4* const kv0 = smem + warp * ST * 2 * HB * C; + uint4* ks = kv0; + uint4* vs = ks + HB * C; + uint32_t qa[QREG ? K16 : 1][4]; + uint4* qs = smem + W * ST * 2 * HB * C; // !QREG: the row's heads [16][D], shared by the warps + if constexpr (QREG) { + const int a0 = pf + g8, a1 = pf + g8 + 8; + const bool v0 = a0 < npairs, v1 = a1 < npairs; + const uint32_t* q0 = reinterpret_cast(q + ((rq + (v0 ? a0 : pf) / G) * h + kh * G + (v0 ? a0 : pf) % G) * D); + const uint32_t* q1 = reinterpret_cast(q + ((rq + (v1 ? a1 : pf) / G) * h + kh * G + (v1 ? a1 : pf) % G) * D); +#pragma unroll + for (int k = 0; k < K16; ++k) { + qa[k][0] = v0 ? q0[8 * k + qd] : 0u; + qa[k][1] = v1 ? q1[8 * k + qd] : 0u; + qa[k][2] = v0 ? q0[8 * k + qd + 4] : 0u; + qa[k][3] = v1 ? q1[8 * k + qd + 4] : 0u; + } + } else { + for (int i = threadIdx.x; i < 16 * C; i += blockDim.x) { + const int a = pf + i / C; + const bool in = a < npairs; + const int64_t row = rq + (in ? a : pf) / G; + cp16z(qs + sw(i / C, i % C), q + (row * h + kh * G + (in ? a : pf) % G) * D + (i % C) * 8, in); + } + commit(); + wait_group<0>(); + __syncthreads(); + } + float o[NB][4]; +#pragma unroll + for (int i = 0; i < NB; ++i) o[i][0] = o[i][1] = o[i][2] = o[i][3] = 0.0f; + float m0 = -INFINITY, m1 = -INFINITY, l0 = 0.0f, l1 = 0.0f; + // tile t's keys and values into stage st (committed keys, then the row's path), one cp.async group + auto load = [&](int t, int st) { + uint4* const kd = kv0 + st * 2 * HB * C; + for (int i = lane; i < HB * C; i += 32) { + const int key = t * HB + i / C; + const bool in = key < k1; + const __nv_bfloat16* ksrc; + const __nv_bfloat16* vsrc; + if (key < p) { + ksrc = kc + ((int64_t)key * hk + kh) * D; + vsrc = vc + ((int64_t)key * hk + kh) * D; + } else { // the row's own path: key p + j is path row j + const int node = in ? paths[r * MAXD + (key - p)] : 0; + ksrc = kn + ((int64_t)node * hk + kh) * D; + vsrc = vn + ((int64_t)node * hk + kh) * D; + } + if (!in) ksrc = kn, vsrc = vn; + cp16z(kd + sw(i / C, i % C), ksrc + (i % C) * 8, in); + cp16z(kd + HB * C + sw(i / C, i % C), vsrc + (i % C) * 8, in); + } + commit(); + }; + // ST - 1 tiles ahead: one cp.async group a tile, an empty one past the end, so the wait count never changes + const int t0 = k0 / HB + wi; +#pragma unroll + for (int a = 0; a < ST - 1; ++a) { + if ((t0 + 4 * a) * HB < k1) load(t0 + 4 * a, a); + else commit(); + } + int st = 0; + for (int t = t0; t * HB < k1; t += 4) { + const int ahead = t + 4 * (ST - 1); + if (ahead * HB < k1) load(ahead, (st + ST - 1) % ST); + else commit(); + wait_group(); // tile t landed; the next ST - 1 are in flight + ks = kv0 + st * 2 * HB * C; + vs = ks + HB * C; + st = (st + 1) % ST; + + __syncwarp(); + float sc[NT8][4]; +#pragma unroll + for (int k = 0; k < K16; ++k) { + uint32_t qf[4]; + if constexpr (QREG) { + qf[0] = qa[k][0], qf[1] = qa[k][1], qf[2] = qa[k][2], qf[3] = qa[k][3]; + } else { + ldmatrix4(qf, qs + sw(lane % 16, 2 * k + lane / 16)); + } +#pragma unroll + for (int n = 0; n < NT8; n += 2) { + uint32_t b[4]; + ldmatrix4(b, ks + sw(8 * (n + mt / 2) + lane % 8, 2 * k + mt % 2)); + if (k == 0) { + mma0(sc[n], qf, b[0], b[1]); + mma0(sc[n + 1], qf, b[2], b[3]); + } else { + mma(sc[n], qf, b[0], b[1]); + mma(sc[n + 1], qf, b[2], b[3]); + } + } + } + const int kb = t * HB + 2 * qd; + float tm0 = -INFINITY, tm1 = -INFINITY; +#pragma unroll + for (int n = 0; n < NT8; ++n) { + const int c = kb + 8 * n; + const bool i0 = c < k1, i1 = c + 1 < k1; + sc[n][0] = i0 ? __fmul_rn(sc[n][0], scale) : -INFINITY; + sc[n][1] = i1 ? __fmul_rn(sc[n][1], scale) : -INFINITY; + sc[n][2] = i0 ? __fmul_rn(sc[n][2], scale) : -INFINITY; + sc[n][3] = i1 ? __fmul_rn(sc[n][3], scale) : -INFINITY; + tm0 = fmaxf(tm0, fmaxf(sc[n][0], sc[n][1])); + tm1 = fmaxf(tm1, fmaxf(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + tm0 = fmaxf(tm0, __shfl_xor_sync(0xffffffffu, tm0, x)); + tm1 = fmaxf(tm1, __shfl_xor_sync(0xffffffffu, tm1, x)); + } + const bool act = tm0 != -INFINITY; + const float n0 = act ? fmaxf(m0, tm0) : m0, n1 = act ? fmaxf(m1, tm1) : m1; + const float a0 = act ? (m0 == -INFINITY ? 0.0f : texp(__fsub_rn(m0, n0))) : 1.0f; + const float a1 = act ? (m1 == -INFINITY ? 0.0f : texp(__fsub_rn(m1, n1))) : 1.0f; + float c0 = 0.0f, c1 = 0.0f; +#pragma unroll + for (int n = 0; n < NT8; ++n) { + sc[n][0] = sc[n][0] != -INFINITY ? texp(__fsub_rn(sc[n][0], n0)) : 0.0f; + sc[n][1] = sc[n][1] != -INFINITY ? texp(__fsub_rn(sc[n][1], n0)) : 0.0f; + sc[n][2] = sc[n][2] != -INFINITY ? texp(__fsub_rn(sc[n][2], n1)) : 0.0f; + sc[n][3] = sc[n][3] != -INFINITY ? texp(__fsub_rn(sc[n][3], n1)) : 0.0f; + c0 = __fadd_rn(c0, __fadd_rn(sc[n][0], sc[n][1])); + c1 = __fadd_rn(c1, __fadd_rn(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + c0 = __fadd_rn(c0, __shfl_xor_sync(0xffffffffu, c0, x)); + c1 = __fadd_rn(c1, __shfl_xor_sync(0xffffffffu, c1, x)); + } + l0 = __fmaf_rn(l0, a0, c0); + l1 = __fmaf_rn(l1, a1, c1); + m0 = n0; + m1 = n1; +#pragma unroll + for (int i = 0; i < NB; ++i) { + o[i][0] = __fmul_rn(o[i][0], a0); + o[i][1] = __fmul_rn(o[i][1], a0); + o[i][2] = __fmul_rn(o[i][2], a1); + o[i][3] = __fmul_rn(o[i][3], a1); + } + uint32_t pa[PK][4]; +#pragma unroll + for (int j = 0; j < PK; ++j) { + pa[j][0] = pack(sc[2 * j][0], sc[2 * j][1]); + pa[j][1] = pack(sc[2 * j][2], sc[2 * j][3]); + pa[j][2] = pack(sc[2 * j + 1][0], sc[2 * j + 1][1]); + pa[j][3] = pack(sc[2 * j + 1][2], sc[2 * j + 1][3]); + } +#pragma unroll + for (int j = 0; j < PK; ++j) +#pragma unroll + for (int i = 0; i < NB; i += 2) { + uint32_t b[4]; + ldmatrix4t(b, vs + sw(16 * j + 8 * (mt % 2) + lane % 8, i + mt / 2)); + mma(o[i], pa[j], b[0], b[1]); + mma(o[i + 1], pa[j], b[2], b[3]); + } + __syncwarp(); + } + if constexpr (W == 1) { + // one warp a block: each slice's partial (o, max, sum) of its 16 pairs; the merge combines the 4 in order +#pragma unroll + for (int half = 0; half < 2; ++half) { + const int hd = g8 + 8 * half; + if (pf + hd >= npairs) continue; + const int64_t part = (((rq + (pf + hd) / G) * hk + kh) * ng + gi) * 4 + wi; + const int head = (pf + hd) % G; +#pragma unroll + for (int i = 0; i < NB; ++i) { + const int d = 8 * i + 2 * qd; + po[(part * G + head) * D + d] = o[i][2 * half]; + po[(part * G + head) * D + d + 1] = o[i][2 * half + 1]; + } + if (qd == 0) pm[part * G + head] = half ? m1 : m0, pl[part * G + head] = half ? l1 : l0; + } + return; + } else { + __syncthreads(); + float* os = reinterpret_cast(smem); // [4 warps][16 heads][D], then max and sum [4][16] + float* ms = os + 4 * 16 * D; + float* ls = ms + 4 * 16; +#pragma unroll + for (int i = 0; i < NB; ++i) { + const int d = 8 * i + 2 * qd; + os[(warp * 16 + g8) * D + d] = o[i][0]; + os[(warp * 16 + g8) * D + d + 1] = o[i][1]; + os[(warp * 16 + g8 + 8) * D + d] = o[i][2]; + os[(warp * 16 + g8 + 8) * D + d + 1] = o[i][3]; + } + if (qd == 0) { + ms[warp * 16 + g8] = m0, ls[warp * 16 + g8] = l0; + ms[warp * 16 + g8 + 8] = m1, ls[warp * 16 + g8 + 8] = l1; + } + __syncthreads(); + for (int idx = threadIdx.x; idx < 16 * D; idx += 128) { + const int hd = idx / D, e = idx % D; + if (pf + hd >= npairs) continue; + const int64_t part = ((rq + (pf + hd) / G) * hk + kh) * ng + gi; + const int head = (pf + hd) % G; + float mm = -INFINITY; + for (int w = 0; w < 4; ++w) mm = fmaxf(mm, ms[w * 16 + hd]); + float acc = 0.0f, l = 0.0f; + for (int w = 0; w < 4; ++w) { + const float mw = ms[w * 16 + hd]; + if (mw == -INFINITY) continue; + const float a = texp(__fsub_rn(mw, mm)); + acc = __fmaf_rn(a, os[(w * 16 + hd) * D + e], acc); + l = __fmaf_rn(a, ls[w * 16 + hd], l); + } + po[(part * G + head) * D + e] = acc; + if (e == 0) pm[part * G + head] = mm, pl[part * G + head] = l; + } + } +} + +// A stream's full committed groups once for its rows, 16 (row, head) pairs a block: each partial a row block's bits. +template +__device__ __forceinline__ void tree_shared( + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ origin, const long long* __restrict__ offs, + const int* __restrict__ row_stream, const int* __restrict__ streams, const int* __restrict__ paths, + const int* __restrict__ depths, float* __restrict__ po, float* __restrict__ pm, float* __restrict__ pl, int h, int hk, + float scale, int ng, int n) { + const int G = h / hk; + const int per = W == 4 ? 1 : 4; // slices an item takes a block + for (int item = blockIdx.x; item < n * ng * per; item += gridDim.x) { // (row, group[, slice]): a fixed grid + const int64_t r = item / per / ng; + const int gi = item / per % ng; + const int st = row_stream[r]; + const int64_t r0 = streams[4 * st]; + const int pairs = streams[4 * st + 1] * G, p = streams[4 * st + 2]; + const int tile = static_cast(r - r0); // a stream's j-th row hosts its j-th pair tile (tiles <= rows) + if (tile * 16 >= pairs || (gi + 1) * SPAN > p) continue; + tree_group(q, q, q, origin, offs, row_stream, streams, paths, depths, po, pm, pl, h, hk, scale, gi, ng, + r0, tile * 16, pairs, r, item % per); + __syncthreads(); + } +} + +#define TF_TREE_SHARED(NAME, D, HB, W, ST) \ + extern "C" __global__ void __launch_bounds__(32 * W) NAME( \ + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ origin, \ + const long long* __restrict__ offs, const int* __restrict__ row_stream, const int* __restrict__ streams, \ + const int* __restrict__ paths, const int* __restrict__ depths, float* __restrict__ po, \ + float* __restrict__ pm, float* __restrict__ pl, int h, int hk, float scale, int ng, int n) { \ + tree_shared(q, origin, offs, row_stream, streams, paths, depths, po, pm, pl, h, hk, scale, ng, n); \ + } + +TF_TREE_SHARED(tf_tree_shared, 128, 32, 4, 1) +TF_TREE_SHARED(tf_tree_shared_256, 256, 16, 4, 1) +TF_TREE_SHARED(tf_tree_shared_w1, 128, 32, 1, 2) +TF_TREE_SHARED(tf_tree_shared_256_w1, 256, 16, 1, 2) +TF_TREE_SHARED(tf_tree_shared_w1s3, 128, 32, 1, 3) +TF_TREE_SHARED(tf_tree_shared_256_w1s3, 256, 16, 1, 3) + +// Block (row, key head, group slot): the row's groups past its stream's full committed ones, slot, slot + slots, ... +template +__device__ __forceinline__ void tree_part( + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kn, const __nv_bfloat16* __restrict__ vn, + const __nv_bfloat16* __restrict__ origin, const long long* __restrict__ offs, const int* __restrict__ row_stream, + const int* __restrict__ streams, const int* __restrict__ paths, const int* __restrict__ depths, + float* __restrict__ po, float* __restrict__ pm, float* __restrict__ pl, int h, int hk, float scale, int ng) { + const int64_t r = blockIdx.x; + const int p = streams[4 * row_stream[r] + 2], total = p + depths[r]; + const int per = W == 4 ? 1 : 4, slot = blockIdx.z / per, slots = gridDim.z / per; + for (int gi = slot + p / SPAN; gi < ng && gi * SPAN < total; gi += slots) { // full committed: shared + tree_group(q, kn, vn, origin, offs, row_stream, streams, paths, depths, po, pm, pl, h, hk, scale, gi, + ng, r, 0, h / hk, r, blockIdx.z % per); + __syncthreads(); // the next group's tiles reuse the partials' shared memory + } +} + +#define TF_TREE_PART(NAME, D, HB, W, ST) \ + extern "C" __global__ void __launch_bounds__(32 * W) NAME( \ + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kn, \ + const __nv_bfloat16* __restrict__ vn, const __nv_bfloat16* __restrict__ origin, \ + const long long* __restrict__ offs, const int* __restrict__ row_stream, const int* __restrict__ streams, \ + const int* __restrict__ paths, const int* __restrict__ depths, float* __restrict__ po, \ + float* __restrict__ pm, float* __restrict__ pl, int h, int hk, float scale, int ng) { \ + tree_part(q, kn, vn, origin, offs, row_stream, streams, paths, depths, po, pm, pl, h, hk, scale, \ + ng); \ + } + +TF_TREE_PART(tf_tree_part, 128, 32, 4, 1) +TF_TREE_PART(tf_tree_part_256, 256, 16, 4, 1) +TF_TREE_PART(tf_tree_part_w1, 128, 32, 1, 2) +TF_TREE_PART(tf_tree_part_256_w1, 256, 16, 1, 2) +TF_TREE_PART(tf_tree_part_w1s3, 128, 32, 1, 3) +TF_TREE_PART(tf_tree_part_256_w1s3, 256, 16, 1, 3) + +// out [W, H, D] bf16: block (row, key head, head) folds the row's groups in order (sliced: a group's 4 slices first). +extern "C" __global__ void __launch_bounds__(256) tf_tree_merge( + const float* __restrict__ po, const float* __restrict__ pm, const float* __restrict__ pl, + const int* __restrict__ row_stream, const int* __restrict__ streams, const int* __restrict__ depths, + __nv_bfloat16* __restrict__ out, int h, int hk, int ng, int d, int sliced) { + const int64_t r = blockIdx.x; + const int kh = blockIdx.y, G = h / hk, hd = blockIdx.z; // block (row, key head, head): each element alone + const int total = streams[4 * row_stream[r] + 2] + depths[r]; + const int groups = (total + SPAN - 1) / SPAN; + for (int e = threadIdx.x; e < d; e += 256) { + float m = -INFINITY, l = 0.0f, acc = 0.0f; + for (int gi = 0; gi < groups; ++gi) { + const int64_t part = ((r * hk + kh) * ng + gi) * (sliced ? 4 : 1); + float cm = -INFINITY, co = 0.0f, cl = 0.0f; + if (!sliced) { // four-warp blocks combined the group's slices themselves + cm = pm[part * G + hd], co = po[(part * G + hd) * d + e], cl = pl[part * G + hd]; + } else { // one-warp blocks left the 4 slices: their combine, in the same order + for (int w = 0; w < 4; ++w) cm = fmaxf(cm, pm[(part + w) * G + hd]); + for (int w = 0; w < 4; ++w) { + const float mw = pm[(part + w) * G + hd]; + if (mw == -INFINITY) continue; + const float aw = texp(__fsub_rn(mw, cm)); + co = __fmaf_rn(aw, po[((part + w) * G + hd) * d + e], co); + cl = __fmaf_rn(aw, pl[(part + w) * G + hd], cl); + } + } + const bool active = cl > 0.0f; + const float next = active ? fmaxf(m, cm) : m; + const float a = active ? (m == -INFINITY ? 0.0f : texp(__fsub_rn(m, next))) : 1.0f; + const float b = active ? texp(__fsub_rn(cm, next)) : 0.0f; + acc = __fmaf_rn(acc, a, __fmul_rn(co, b)); + l = __fmaf_rn(l, a, __fmul_rn(cl, b)); + m = next; + } + out[(r * h + kh * G + hd) * d + e] = __float2bfloat16_rn(__fdiv_rn(acc, l)); + } +} diff --git a/zig/kernels/cuda/window_attention.cu b/zig/kernels/cuda/window_attention.cu new file mode 100644 index 0000000000..542e80a8ba --- /dev/null +++ b/zig/kernels/cuda/window_attention.cu @@ -0,0 +1,570 @@ +// Sliding-window attention for every CUDA family (float64 references: core/attention_ref.zig): decode and prompt rows. + +// Decode: a block a (row, key head); keys walk 64-key tiles by absolute position, so a row's bits are its own. + +#include +#include +#include +#include + +#include "qmm_frag.cuh" + +namespace tf_window { + +constexpr int NT = 256; +constexpr int BN = 64; // keys a tile +constexpr int GMAX = 16; // query heads a key head +constexpr int KT = BN + 2; // kt's row stride: a bank further each dim +constexpr int DMAX = 128; // head dim (static shared memory: q, a K and a V tile, scores within 48 KiB) + +} // namespace tf_window + +using namespace tf_window; + +// Decode split by key tile (block: row, key head, tile); tf_window_merge folds the partials in tile order. +extern "C" __global__ void __launch_bounds__(NT) tf_window_part(const __nv_bfloat16* __restrict__ q, + const __nv_bfloat16* __restrict__ kc, + const __nv_bfloat16* __restrict__ vc, + const int* __restrict__ pos, + const int* __restrict__ slot, float* __restrict__ po, + float* __restrict__ pm, float* __restrict__ pl, int h, + int hk, int d, int win, long long ring, float scale) { + __shared__ float qs[GMAX * DMAX]; + __shared__ __nv_bfloat16 kt[DMAX * KT]; // K transposed, [d][key], padded: a warp reads 32 keys of one dim + __shared__ __nv_bfloat16 vs[BN * DMAX]; + __shared__ float sc[GMAX * BN]; + __shared__ float mx[GMAX], ls[GMAX]; + const int64_t r = blockIdx.x; + const int kh = blockIdx.y, tiles = gridDim.z, j = blockIdx.z; + const int g = h / hk; + const int p = pos[r]; + const int64_t base = (slot != nullptr ? static_cast(slot[r]) : 0) * ring; + const int lo = win > 0 ? max(p - win + 1, 0) : 0; + const int t = lo / BN + j; + const int64_t part = (r * hk + kh) * tiles + j; // this block's partial, g heads of d values + if (t > p / BN) { // past the row's last tile: an empty partial + for (int i = threadIdx.x; i < g; i += NT) pm[part * g + i] = -INFINITY, pl[part * g + i] = 0.f; + return; + } + for (int i = threadIdx.x; i < g * d; i += NT) qs[i] = __bfloat162float(q[(r * h + kh * g + i / d) * d + i % d]); + for (int i = threadIdx.x; i < BN * d; i += NT) { + const int key = t * BN + i / d; + const bool in = key >= lo && key <= p; + const int64_t at = ((base + (in ? key : lo) % ring) * hk + kh) * d + i % d; + kt[(i % d) * KT + i / d] = kc[at]; + vs[i] = vc[at]; + } + __syncthreads(); + for (int i = threadIdx.x; i < g * BN; i += NT) { // score (head i / BN, key i % BN) + const int hd = i / BN, jj = i % BN, key = t * BN + jj; + float s = 0.f; +#pragma unroll 8 + for (int e = 0; e < d; ++e) s = __fmaf_rn(qs[hd * d + e], __bfloat162float(kt[e * KT + jj]), s); + sc[i] = (key >= lo && key <= p) ? __fmul_rn(s, scale) : -INFINITY; + } + __syncthreads(); + { // a warp a head: the tile's max, the weights exp(s - max) and their sum, lanes in a fixed tree + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32; + for (int hd = warp; hd < g; hd += NT / 32) { + float a = sc[hd * BN + lane], b = sc[hd * BN + 32 + lane]; + float m = fmaxf(a, b); + for (int k = 16; k > 0; k >>= 1) m = fmaxf(m, __shfl_xor_sync(0xFFFFFFFFu, m, k)); + a = a == -INFINITY ? 0.f : expf(__fsub_rn(a, m)); + b = b == -INFINITY ? 0.f : expf(__fsub_rn(b, m)); + sc[hd * BN + lane] = a; + sc[hd * BN + 32 + lane] = b; + float l = __fadd_rn(a, b); + for (int k = 16; k > 0; k >>= 1) l = __fadd_rn(l, __shfl_xor_sync(0xFFFFFFFFu, l, k)); + if (lane == 0) mx[hd] = m, ls[hd] = l; + } + } + __syncthreads(); + for (int o = threadIdx.x; o < g * d; o += NT) { + const int hd = o / d, e = o % d; + float a = 0.f; + for (int jj = 0; jj < BN; ++jj) a = __fmaf_rn(sc[hd * BN + jj], __bfloat162float(vs[jj * d + e]), a); + po[part * g * d + o] = a; + } + if (threadIdx.x < g) pm[part * g + threadIdx.x] = mx[threadIdx.x], pl[part * g + threadIdx.x] = ls[threadIdx.x]; +} + +// out [W, H, D] bf16: a row's partials folded in tile order, o = sum_j exp(m_j - M) o_j / sum_j exp(m_j - M) l_j. +extern "C" __global__ void __launch_bounds__(NT) tf_window_merge(const float* __restrict__ po, + const float* __restrict__ pm, + const float* __restrict__ pl, + __nv_bfloat16* __restrict__ out, int h, int hk, int d, + int tiles) { + const int64_t r = blockIdx.x; + const int kh = blockIdx.y, g = h / hk; + const int64_t part0 = (r * hk + kh) * tiles; + for (int o = threadIdx.x; o < g * d; o += NT) { + const int hd = o / d; + float m = -INFINITY; + for (int j = 0; j < tiles; ++j) m = fmaxf(m, pm[(part0 + j) * g + hd]); + float acc = 0.f, l = 0.f; + for (int j = 0; j < tiles; ++j) { + const float mj = pm[(part0 + j) * g + hd]; + if (mj == -INFINITY) continue; + const float a = expf(__fsub_rn(mj, m)); + acc = __fmaf_rn(a, po[(part0 + j) * g * d + o], acc); + l = __fmaf_rn(a, pl[(part0 + j) * g + hd], l); + } + out[(r * h + kh * g + hd) * d + o % d] = __float2bfloat16_rn(__fdiv_rn(acc, l)); + } +} + +// Prompt rows: prefill_attention.cu's tensor-core kernel with a window (keys (pos - win, pos]) and a ring of keys. +namespace tf_window_prompt { + +using qmm_frag::commit; +using qmm_frag::cp16z; +using qmm_frag::ldmatrix4; +using qmm_frag::mma; +using qmm_frag::mma0; + +constexpr int BN = 64, HALF = 32; // keys a tile (the fold's unit), keys a staging slot +constexpr float LOG2E = 1.4426950408889634f; + +template +__device__ __forceinline__ void wait_group() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); } + +__device__ __forceinline__ void ldmatrix4t(uint32_t (&r)[4], const void* p) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(qmm_frag::smem(p))); +} + +__device__ __forceinline__ float texp(float x) { + float y; + asm("ex2.approx.f32 %0, %1;\n" : "=f"(y) : "f"(__fmul_rn(x, LOG2E))); + return y; +} + +__device__ __forceinline__ float tdiv(float a, float b) { + float y; + asm("div.full.f32 %0, %1, %2;\n" : "=f"(y) : "f"(a), "f"(b)); + return y; +} + +__device__ __forceinline__ uint32_t pack(float lo, float hi) { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); +} + +template +__device__ __forceinline__ int sw(int r, int c) { return r * (D / 8) + (c ^ (r & 7)); } + +__device__ __forceinline__ float rowsum(const float (&c)[8]) { + float w[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float x = __fadd_rn(c[j], __shfl_xor_sync(0xffffffffu, c[j], 2)); + w[j] = __fadd_rn(x, __shfl_xor_sync(0xffffffffu, x, 1)); + } + return __fadd_rn(__fadd_rn(__fadd_rn(w[0], w[4]), __fadd_rn(w[2], w[6])), + __fadd_rn(__fadd_rn(w[1], w[5]), __fadd_rn(w[3], w[7]))); +} + +template +__global__ void __launch_bounds__(32 * WARPS, 1) +wattn_kernel(const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kc, + const __nv_bfloat16* __restrict__ vc, __nv_bfloat16* __restrict__ out, int p0, int W, int H, int HK, + int G, float scale, int win, int ring) { + constexpr int C = D / 8, K16 = D / 16, NB = D / 8, RB = WARPS / HPC, THREADS = 32 * WARPS; + constexpr bool QREG = D <= 128; // the queries' fragments stay in registers, else in shared memory + extern __shared__ uint4 smem[]; + uint4* slots = smem; + uint4* qs = smem + NS * HALF * C; + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32, g = lane / 4, qd = lane % 4, mt = lane / 8; + const int groups = G / HPC, hk = blockIdx.y / groups, hg = blockIdx.y % groups; + const int row0 = blockIdx.x * 16 * RB, last = min(row0 + 16 * RB, W) - 1; + const int head = hk * G + hg * HPC + warp % HPC, wrow = row0 + (warp / HPC) * 16; + // the first tile any row of the block sees: its first row's window start (win 0: key 0) + const int t0 = win > 0 ? max(p0 + row0 - win + 1, 0) / BN : 0; + const int tiles = (p0 + last) / BN + 1 - t0, items = 4 * tiles, limit = p0 + W; + auto stage = [&](int item) { + const __nv_bfloat16* src = item % 4 < 2 ? kc : vc; + const int key0 = (t0 + item / 4) * BN + (item % 2) * HALF; + uint4* dst = slots + (item % NS) * HALF * C; + for (int i = threadIdx.x; i < HALF * C; i += THREADS) { + const int key = key0 + i / C; + const bool in = key < limit; // past the cache's filled keys: zeros, never read + const int row = in ? (ring > 0 ? key % ring : key) : 0; // a ring keeps key p at row p % ring + cp16z(dst + sw(i / C, i % C), src + ((int64_t)row * HK + hk) * D + (i % C) * 8, in); + } + }; + uint32_t qa[QREG ? K16 : 1][4]; + if constexpr (QREG) { + const int r0 = wrow + g, r1 = r0 + 8; + const uint32_t* q0 = reinterpret_cast(q + ((int64_t)min(r0, W - 1) * H + head) * D); + const uint32_t* q1 = reinterpret_cast(q + ((int64_t)min(r1, W - 1) * H + head) * D); +#pragma unroll + for (int k = 0; k < K16; ++k) { + qa[k][0] = r0 < W ? q0[8 * k + qd] : 0u; + qa[k][1] = r1 < W ? q1[8 * k + qd] : 0u; + qa[k][2] = r0 < W ? q0[8 * k + qd + 4] : 0u; + qa[k][3] = r1 < W ? q1[8 * k + qd + 4] : 0u; + } + } else { + for (int i = threadIdx.x; i < WARPS * 16 * C; i += THREADS) { + const int w = i / (16 * C), r = (i / C) % 16, row = row0 + (w / HPC) * 16 + r; + const int hd = hk * G + hg * HPC + w % HPC; + cp16z(qs + sw(i / C, i % C), q + ((int64_t)min(row, W - 1) * H + hd) * D + (i % C) * 8, row < W); + } + commit(); + } +#pragma unroll + for (int i = 0; i < NS; ++i) { + if (i < items) stage(i); + commit(); + } + float o[NB][4]; +#pragma unroll + for (int i = 0; i < NB; ++i) o[i][0] = o[i][1] = o[i][2] = o[i][3] = 0.0f; + float sc[8][4]; + uint32_t pa[4][4]; + float m0 = -INFINITY, m1 = -INFINITY, l0 = 0.0f, l1 = 0.0f; + const int pos0 = p0 + wrow + g, pos1 = pos0 + 8; + auto take = [&](int it) -> const uint4* { // item it's slot, once every thread's copy has landed + wait_group(); + __syncthreads(); + return slots + (it % NS) * HALF * C; + }; + auto done = [&](int it) { // the slot is free: stage the item NS ahead into it + __syncthreads(); + if (it + NS < items) stage(it + NS); + commit(); + }; + auto scores = [&](float (&s)[4][4], const uint4* sl) { +#pragma unroll + for (int k = 0; k < K16; ++k) { + uint32_t a[4]; + if constexpr (QREG) { + a[0] = qa[k][0]; a[1] = qa[k][1]; a[2] = qa[k][2]; a[3] = qa[k][3]; + } else { + ldmatrix4(a, qs + sw(warp * 16 + lane % 16, 2 * k + lane / 16)); + } +#pragma unroll + for (int n = 0; n < 4; n += 2) { + uint32_t b[4]; + ldmatrix4(b, sl + sw(8 * (n + mt / 2) + lane % 8, 2 * k + mt % 2)); + if (k == 0) { + mma0(s[n], a, b[0], b[1]); + mma0(s[n + 1], a, b[2], b[3]); + } else { + mma(s[n], a, b[0], b[1]); + mma(s[n + 1], a, b[2], b[3]); + } + } + } + }; + auto values = [&](const uint32_t (&pa0)[4], const uint32_t (&pa1)[4], const uint4* sl) { +#pragma unroll + for (int i = 0; i < NB; i += 2) { + uint32_t b[4]; + ldmatrix4t(b, sl + sw(8 * (mt % 2) + lane % 8, i + mt / 2)); + mma(o[i], pa0, b[0], b[1]); + mma(o[i + 1], pa0, b[2], b[3]); + } +#pragma unroll + for (int i = 0; i < NB; i += 2) { + uint32_t b[4]; + ldmatrix4t(b, sl + sw(16 + 8 * (mt % 2) + lane % 8, i + mt / 2)); + mma(o[i], pa1, b[0], b[1]); + mma(o[i + 1], pa1, b[2], b[3]); + } + }; + for (int t = 0; t < tiles; ++t) { + const int it = 4 * t; + scores(*reinterpret_cast(&sc[0]), take(it)); + done(it); + scores(*reinterpret_cast(&sc[4]), take(it + 1)); + done(it + 1); + { // _tile's fold of the tile's 64 scores + const int kb = (t0 + t) * BN + 2 * qd; + const int lo0 = win > 0 ? pos0 - win + 1 : 0, lo1 = win > 0 ? pos1 - win + 1 : 0; + float tm0 = -INFINITY, tm1 = -INFINITY; +#pragma unroll + for (int n = 0; n < 8; ++n) { + const int c = kb + 8 * n; + sc[n][0] = c >= lo0 && c <= pos0 ? __fmul_rn(sc[n][0], scale) : -INFINITY; + sc[n][1] = c + 1 >= lo0 && c + 1 <= pos0 ? __fmul_rn(sc[n][1], scale) : -INFINITY; + sc[n][2] = c >= lo1 && c <= pos1 ? __fmul_rn(sc[n][2], scale) : -INFINITY; + sc[n][3] = c + 1 >= lo1 && c + 1 <= pos1 ? __fmul_rn(sc[n][3], scale) : -INFINITY; + tm0 = fmaxf(tm0, fmaxf(sc[n][0], sc[n][1])); + tm1 = fmaxf(tm1, fmaxf(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + tm0 = fmaxf(tm0, __shfl_xor_sync(0xffffffffu, tm0, x)); + tm1 = fmaxf(tm1, __shfl_xor_sync(0xffffffffu, tm1, x)); + } + const bool act0 = tm0 != -INFINITY, act1 = tm1 != -INFINITY; + const float n0 = act0 ? fmaxf(m0, tm0) : m0, n1 = act1 ? fmaxf(m1, tm1) : m1; + const float a0 = act0 ? (m0 == -INFINITY ? 0.0f : texp(__fsub_rn(m0, n0))) : 1.0f; + const float a1 = act1 ? (m1 == -INFINITY ? 0.0f : texp(__fsub_rn(m1, n1))) : 1.0f; + float c0[8], c1[8]; +#pragma unroll + for (int n = 0; n < 8; ++n) { + const int c = kb + 8 * n; + sc[n][0] = act0 && c >= lo0 && c <= pos0 ? texp(__fsub_rn(sc[n][0], n0)) : 0.0f; + sc[n][1] = act0 && c + 1 >= lo0 && c + 1 <= pos0 ? texp(__fsub_rn(sc[n][1], n0)) : 0.0f; + sc[n][2] = act1 && c >= lo1 && c <= pos1 ? texp(__fsub_rn(sc[n][2], n1)) : 0.0f; + sc[n][3] = act1 && c + 1 >= lo1 && c + 1 <= pos1 ? texp(__fsub_rn(sc[n][3], n1)) : 0.0f; + c0[n] = __fadd_rn(sc[n][0], sc[n][1]); + c1[n] = __fadd_rn(sc[n][2], sc[n][3]); + } + l0 = __fmaf_rn(l0, a0, rowsum(c0)); + l1 = __fmaf_rn(l1, a1, rowsum(c1)); + m0 = n0; + m1 = n1; +#pragma unroll + for (int i = 0; i < NB; ++i) { + o[i][0] = __fmul_rn(o[i][0], a0); + o[i][1] = __fmul_rn(o[i][1], a0); + o[i][2] = __fmul_rn(o[i][2], a1); + o[i][3] = __fmul_rn(o[i][3], a1); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + pa[j][0] = pack(sc[2 * j][0], sc[2 * j][1]); + pa[j][1] = pack(sc[2 * j][2], sc[2 * j][3]); + pa[j][2] = pack(sc[2 * j + 1][0], sc[2 * j + 1][1]); + pa[j][3] = pack(sc[2 * j + 1][2], sc[2 * j + 1][3]); + } + } + values(pa[0], pa[1], take(it + 2)); + done(it + 2); + values(pa[2], pa[3], take(it + 3)); + done(it + 3); + } + wait_group<0>(); + const int r0 = wrow + g, r1 = r0 + 8; +#pragma unroll + for (int i = 0; i < NB; ++i) { + const int d = 8 * i + 2 * qd; + if (r0 < W) + *reinterpret_cast(out + ((int64_t)r0 * H + head) * D + d) = + pack(tdiv(o[i][0], l0), tdiv(o[i][1], l0)); + if (r1 < W) + *reinterpret_cast(out + ((int64_t)r1 * H + head) * D + d) = + pack(tdiv(o[i][2], l1), tdiv(o[i][3], l1)); + } +} + + +// Decode on tensor cores (d 128): a row's heads one mma tile; warps take every 4th 32-key tile, fold in warp order. +extern "C" __global__ void __launch_bounds__(128) tf_window_decode_tc( + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ kc, const __nv_bfloat16* __restrict__ vc, + __nv_bfloat16* __restrict__ out, const int* __restrict__ pos, const int* __restrict__ slot, int h, int hk, int win, + long long ring, float scale) { + constexpr int D = 128, C = D / 8, K16 = D / 16, NB = D / 8, HB = 32; + extern __shared__ uint4 smem[]; + const int warp = threadIdx.x / 32, lane = threadIdx.x % 32, g8 = lane / 4, qd = lane % 4, mt = lane / 8; + const int64_t r = blockIdx.x; + const int kh = blockIdx.y, G = h / hk; + uint4* ks = smem + warp * 2 * HB * C; + uint4* vs = ks + HB * C; + const int p = pos[r]; + const int64_t base = (slot != nullptr ? static_cast(slot[r]) : 0) * ring; + const int lo = win > 0 ? max(p - win + 1, 0) : 0; + const int lor = static_cast(lo % ring); + uint32_t qa[K16][4]; + { + const int h0 = g8, h1 = g8 + 8; + const uint32_t* q0 = reinterpret_cast(q + ((int64_t)r * h + kh * G + min(h0, G - 1)) * D); + const uint32_t* q1 = reinterpret_cast(q + ((int64_t)r * h + kh * G + min(h1, G - 1)) * D); +#pragma unroll + for (int k = 0; k < K16; ++k) { + qa[k][0] = h0 < G ? q0[8 * k + qd] : 0u; + qa[k][1] = h1 < G ? q1[8 * k + qd] : 0u; + qa[k][2] = h0 < G ? q0[8 * k + qd + 4] : 0u; + qa[k][3] = h1 < G ? q1[8 * k + qd + 4] : 0u; + } + } + float o[NB][4]; +#pragma unroll + for (int i = 0; i < NB; ++i) o[i][0] = o[i][1] = o[i][2] = o[i][3] = 0.0f; + float m0 = -INFINITY, m1 = -INFINITY, l0 = 0.0f, l1 = 0.0f; + // K of tile t loads while tile t - 4's values fold in, V of tile t while its scores run: one commit group each + auto stage = [&](uint4* dst, const __nv_bfloat16* src, int t) { + if (t <= p / HB) { + const int r0 = static_cast((t * HB) % ring); +#pragma unroll 4 + for (int i = lane; i < HB * C; i += 32) { + const int j = i / C, key = t * HB + j; + const bool in = key >= lo && key <= p; + int row = r0 + j; + if (row >= ring) row -= static_cast(ring); + cp16z(dst + sw(j, i % C), src + ((base + (in ? row : lor)) * hk + kh) * D + (i % C) * 8, in); + } + } + commit(); + }; + stage(ks, kc, lo / HB + warp); + stage(vs, vc, lo / HB + warp); + for (int t = lo / HB + warp; t <= p / HB; t += 4) { + wait_group<1>(); // tile t's K (its V may still be in flight) + __syncwarp(); + float sc[4][4]; +#pragma unroll + for (int k = 0; k < K16; ++k) { +#pragma unroll + for (int n = 0; n < 4; n += 2) { + uint32_t b[4]; + ldmatrix4(b, ks + sw(8 * (n + mt / 2) + lane % 8, 2 * k + mt % 2)); + if (k == 0) { + mma0(sc[n], qa[k], b[0], b[1]); + mma0(sc[n + 1], qa[k], b[2], b[3]); + } else { + mma(sc[n], qa[k], b[0], b[1]); + mma(sc[n + 1], qa[k], b[2], b[3]); + } + } + } + __syncwarp(); + stage(ks, kc, t + 4); + const int kb = t * HB + 2 * qd; + float tm0 = -INFINITY, tm1 = -INFINITY; +#pragma unroll + for (int n = 0; n < 4; ++n) { + const int c = kb + 8 * n; + const bool i0 = c >= lo && c <= p, i1 = c + 1 >= lo && c + 1 <= p; + sc[n][0] = i0 ? __fmul_rn(sc[n][0], scale) : -INFINITY; + sc[n][1] = i1 ? __fmul_rn(sc[n][1], scale) : -INFINITY; + sc[n][2] = i0 ? __fmul_rn(sc[n][2], scale) : -INFINITY; + sc[n][3] = i1 ? __fmul_rn(sc[n][3], scale) : -INFINITY; + tm0 = fmaxf(tm0, fmaxf(sc[n][0], sc[n][1])); + tm1 = fmaxf(tm1, fmaxf(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + tm0 = fmaxf(tm0, __shfl_xor_sync(0xffffffffu, tm0, x)); + tm1 = fmaxf(tm1, __shfl_xor_sync(0xffffffffu, tm1, x)); + } + const bool act = tm0 != -INFINITY; // the keys' mask is the same for every head + const float n0 = act ? fmaxf(m0, tm0) : m0, n1 = act ? fmaxf(m1, tm1) : m1; + const float a0 = act ? (m0 == -INFINITY ? 0.0f : texp(__fsub_rn(m0, n0))) : 1.0f; + const float a1 = act ? (m1 == -INFINITY ? 0.0f : texp(__fsub_rn(m1, n1))) : 1.0f; + float c0 = 0.0f, c1 = 0.0f; +#pragma unroll + for (int n = 0; n < 4; ++n) { + sc[n][0] = sc[n][0] != -INFINITY ? texp(__fsub_rn(sc[n][0], n0)) : 0.0f; + sc[n][1] = sc[n][1] != -INFINITY ? texp(__fsub_rn(sc[n][1], n0)) : 0.0f; + sc[n][2] = sc[n][2] != -INFINITY ? texp(__fsub_rn(sc[n][2], n1)) : 0.0f; + sc[n][3] = sc[n][3] != -INFINITY ? texp(__fsub_rn(sc[n][3], n1)) : 0.0f; + c0 = __fadd_rn(c0, __fadd_rn(sc[n][0], sc[n][1])); + c1 = __fadd_rn(c1, __fadd_rn(sc[n][2], sc[n][3])); + } + for (int x = 1; x <= 2; x *= 2) { + c0 = __fadd_rn(c0, __shfl_xor_sync(0xffffffffu, c0, x)); + c1 = __fadd_rn(c1, __shfl_xor_sync(0xffffffffu, c1, x)); + } + l0 = __fmaf_rn(l0, a0, c0); + l1 = __fmaf_rn(l1, a1, c1); + m0 = n0; + m1 = n1; +#pragma unroll + for (int i = 0; i < NB; ++i) { + o[i][0] = __fmul_rn(o[i][0], a0); + o[i][1] = __fmul_rn(o[i][1], a0); + o[i][2] = __fmul_rn(o[i][2], a1); + o[i][3] = __fmul_rn(o[i][3], a1); + } + wait_group<1>(); // tile t's V (the next K may still be in flight) + __syncwarp(); + uint32_t pa[2][4]; +#pragma unroll + for (int j = 0; j < 2; ++j) { + pa[j][0] = pack(sc[2 * j][0], sc[2 * j][1]); + pa[j][1] = pack(sc[2 * j][2], sc[2 * j][3]); + pa[j][2] = pack(sc[2 * j + 1][0], sc[2 * j + 1][1]); + pa[j][3] = pack(sc[2 * j + 1][2], sc[2 * j + 1][3]); + } +#pragma unroll + for (int i = 0; i < NB; i += 2) { + uint32_t b[4]; + ldmatrix4t(b, vs + sw(8 * (mt % 2) + lane % 8, i + mt / 2)); + mma(o[i], pa[0], b[0], b[1]); + mma(o[i + 1], pa[0], b[2], b[3]); + } +#pragma unroll + for (int i = 0; i < NB; i += 2) { + uint32_t b[4]; + ldmatrix4t(b, vs + sw(16 + 8 * (mt % 2) + lane % 8, i + mt / 2)); + mma(o[i], pa[1], b[0], b[1]); + mma(o[i + 1], pa[1], b[2], b[3]); + } + __syncwarp(); + stage(vs, vc, t + 4); + } + wait_group<0>(); + const int na = min(4, p / HB - lo / HB + 1); // warps that saw keys + if (na == 1) { // one warp's fold is the result (the merge below would scale it by exp(0) = 1) + if (warp == 0) { +#pragma unroll + for (int i = 0; i < NB; ++i) { + const int d = 8 * i + 2 * qd; + if (g8 < G) + *reinterpret_cast(out + ((int64_t)r * h + kh * G + g8) * D + d) = + pack(tdiv(o[i][0], l0), tdiv(o[i][1], l0)); + if (g8 + 8 < G) + *reinterpret_cast(out + ((int64_t)r * h + kh * G + g8 + 8) * D + d) = + pack(tdiv(o[i][2], l1), tdiv(o[i][3], l1)); + } + } + return; + } + __syncthreads(); + float* os = reinterpret_cast(smem); // [4 warps][16 heads][D] + float* ms = os + 4 * 16 * D; // [4][16] + float* ls = ms + 4 * 16; +#pragma unroll + for (int i = 0; i < NB; ++i) { + const int d = 8 * i + 2 * qd; + os[(warp * 16 + g8) * D + d] = o[i][0]; + os[(warp * 16 + g8) * D + d + 1] = o[i][1]; + os[(warp * 16 + g8 + 8) * D + d] = o[i][2]; + os[(warp * 16 + g8 + 8) * D + d + 1] = o[i][3]; + } + if (qd == 0) { + ms[warp * 16 + g8] = m0, ls[warp * 16 + g8] = l0; + ms[warp * 16 + g8 + 8] = m1, ls[warp * 16 + g8 + 8] = l1; + } + __syncthreads(); + float* as = ls + 4 * 16; // [4][16]: warp w's factor for head hd + float* il = as + 4 * 16; // [16]: the folded l + if (threadIdx.x < 16) { + const int hd = threadIdx.x; + float mm = -INFINITY; + for (int w = 0; w < 4; ++w) mm = fmaxf(mm, ms[w * 16 + hd]); + float l = 0.0f; + for (int w = 0; w < 4; ++w) { + const float mw = ms[w * 16 + hd]; + const float a = mw == -INFINITY ? 0.0f : texp(__fsub_rn(mw, mm)); + as[w * 16 + hd] = a; + if (mw != -INFINITY) l = __fmaf_rn(a, ls[w * 16 + hd], l); + } + il[hd] = l; + } + __syncthreads(); + for (int idx = threadIdx.x; idx < G * D / 2; idx += 128) { + const int hd = idx / (D / 2), e = 2 * (idx % (D / 2)); + float acc0 = 0.0f, acc1 = 0.0f; +#pragma unroll + for (int w = 0; w < 4; ++w) { + if (ms[w * 16 + hd] == -INFINITY) continue; + const float a = as[w * 16 + hd]; + const float2 v = *reinterpret_cast(os + (w * 16 + hd) * D + e); + acc0 = __fmaf_rn(a, v.x, acc0); + acc1 = __fmaf_rn(a, v.y, acc1); + } + const float l = il[hd]; + *reinterpret_cast(out + ((int64_t)r * h + kh * G + hd) * D + e) = pack(tdiv(acc0, l), tdiv(acc1, l)); + } +} + +} // namespace tf_window_prompt + +// Head dim 128, eight warps, four or eight query heads a block, eight staging slots. +template __global__ void tf_window_prompt::wattn_kernel<128, 8, 4, 8>(const __nv_bfloat16*, const __nv_bfloat16*, + const __nv_bfloat16*, __nv_bfloat16*, int, int, int, int, int, float, int, int); +template __global__ void tf_window_prompt::wattn_kernel<128, 8, 8, 8>(const __nv_bfloat16*, const __nv_bfloat16*, + const __nv_bfloat16*, __nv_bfloat16*, int, int, int, int, int, float, int, int); diff --git a/zig/src/cli/qwen27_cuda.zig b/zig/src/cli/qwen27_cuda.zig index 84c8bda959..ebbe21c351 100644 --- a/zig/src/cli/qwen27_cuda.zig +++ b/zig/src/cli/qwen27_cuda.zig @@ -20,6 +20,8 @@ const usage = \\ --kv-cache fp8 against bf16, teacher-forced on bf16's greedy reply: top-1 agreement and KL a token. \\ tf-qwen27-cuda vision MODEL --fixtures DIR \\ The image tower on recorded 0.6.6 fixtures' patches: each image's rows against 0.6.6's (cosine, max error). + \\ tf-qwen27-cuda nll MODEL --prompts corpus.json [--max-tokens N] [--dump FILE] [--against FILE] + \\ Teacher-forced NLL and top-1; against another build's dump: KL, argmax agreement, flips, gated. \\ ; @@ -35,6 +37,7 @@ const Options = struct { cases: ?[]const u8 = null, // windows: only these cases, comma separated drafter: ?[]const u8 = null, // draft: the DFlash2 checkpoint kv8: bool = false, // --kv-cache fp8: e4m3 attention caches + against: ?[]const u8 = null, // nll: another build's --dump, compared and gated }; pub fn main(init: std.process.Init) !u8 { @@ -48,7 +51,8 @@ pub fn main(init: std.process.Init) !u8 { const kvcompare = args.len >= 2 and std.mem.eql(u8, args[1], "kvcompare"); if (args.len >= 5 and std.mem.eql(u8, args[1], "vision") and std.mem.eql(u8, args[3], "--fixtures")) return checkVision(gpa, arena, init.io, args[2], args[4]); - if (args.len < 3 or !(std.mem.eql(u8, args[1], "run") or windows or rounds or drafting or multi or kvcompare)) { + const nll = args.len >= 2 and std.mem.eql(u8, args[1], "nll"); + if (args.len < 3 or !(std.mem.eql(u8, args[1], "run") or windows or rounds or drafting or multi or kvcompare or nll)) { std.debug.print("{s}", .{usage}); return 2; } @@ -57,12 +61,12 @@ pub fn main(init: std.process.Init) !u8 { while (i + 1 < args.len) : (i += 2) { const k = args[i]; const v = args[i + 1]; - if (std.mem.eql(u8, k, "--kernels")) o.kernels = v else if (std.mem.eql(u8, k, "--prompts")) o.prompts = v else if (std.mem.eql(u8, k, "--expect")) o.expect = v else if (std.mem.eql(u8, k, "--max-tokens")) o.max_tokens = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--context")) o.context = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--device")) o.device = try std.fmt.parseInt(c_int, v, 10) else if (std.mem.eql(u8, k, "--dump")) o.dump = v else if (std.mem.eql(u8, k, "--cases")) o.cases = v else if (std.mem.eql(u8, k, "--drafter")) o.drafter = v else if (std.mem.eql(u8, k, "--kv-cache")) o.kv8 = std.mem.eql(u8, v, "fp8") else { + if (std.mem.eql(u8, k, "--kernels")) o.kernels = v else if (std.mem.eql(u8, k, "--prompts")) o.prompts = v else if (std.mem.eql(u8, k, "--expect")) o.expect = v else if (std.mem.eql(u8, k, "--max-tokens")) o.max_tokens = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--context")) o.context = try std.fmt.parseInt(usize, v, 10) else if (std.mem.eql(u8, k, "--device")) o.device = try std.fmt.parseInt(c_int, v, 10) else if (std.mem.eql(u8, k, "--dump")) o.dump = v else if (std.mem.eql(u8, k, "--cases")) o.cases = v else if (std.mem.eql(u8, k, "--drafter")) o.drafter = v else if (std.mem.eql(u8, k, "--against")) o.against = v else if (std.mem.eql(u8, k, "--kv-cache")) o.kv8 = std.mem.eql(u8, v, "fp8") else { std.debug.print("unknown option {s}\n{s}", .{ k, usage }); return 2; } } - if (o.kernels.len == 0 or o.prompts.len == 0) { + if (o.prompts.len == 0) { // BF16 native glue needs no captured set std.debug.print("{s}", .{usage}); return 2; } @@ -83,6 +87,7 @@ pub fn main(init: std.process.Init) !u8 { if (drafting) return runDrafted(gpa, arena, init.io, e, o, prompts, expect); if (multi) return runMulti(gpa, arena, init.io, e, o, prompts); if (kvcompare) return compareKv(arena, e, prompts, o.max_tokens); + if (nll) return runNll(gpa, arena, init.io, e, prompts, o.max_tokens, o.context, o.dump, o.against); var seq = try e.sequence(o.context); defer seq.deinit(); var mismatches: usize = 0; @@ -633,3 +638,140 @@ fn checkVision(gpa: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, mod fn bf(v: u16) f64 { return @as(f32, @bitCast(@as(u32, v) << 16)); } +/// Teacher-forced over a corpus ({"name": [ids], ...} or one id list): each row's next-token NLL and top-1; +/// `dump`: every row's bf16 logits; `against`: another build's dump, its KL, argmax agreement and flips gated. +fn runNll(gpa: std.mem.Allocator, a: std.mem.Allocator, io: std.Io, e: *qwen.Engine, doc: std.json.Value, most: usize, context: usize, dump: ?[]const u8, against: ?[]const u8) !u8 { + var names: std.ArrayList([]const u8) = .empty; + var lists: std.ArrayList([]u32) = .empty; + if (doc == .object) { + var it = doc.object.iterator(); + while (it.next()) |kv| { + try names.append(a, kv.key_ptr.*); + try lists.append(a, try tokenList(a, kv.value_ptr.*)); + } + } else { + try names.append(a, "text"); + try lists.append(a, try tokenList(a, doc)); + } + const vocab = qwen.config.vocab; + const host = try gpa.alloc(u16, vocab); + defer gpa.free(host); + const want_row = try gpa.alloc(u16, vocab); + defer gpa.free(want_row); + var file: ?std.Io.File = if (dump) |d| try std.Io.Dir.cwd().createFile(io, d, .{}) else null; + defer if (file) |*fl| fl.close(io); + var ref: ?std.Io.File = if (against) |p| try std.Io.Dir.cwd().openFile(io, p, .{}) else null; + defer if (ref) |*fl| fl.close(io); + var all: Quality = .{}; + var at: u64 = 0; + for (names.items, lists.items) |name, ids| { + const n = @min(ids.len, most); + var q: Quality = .{}; + var seq = try e.sequence(context); + defer seq.deinit(); + try e.prefill(&seq, ids[0..1]); + var f = e.forward(); + try f.head(&seq); + var i: usize = 1; + while (i < n) : (i += 1) { + if (i > 1) { + const part = [_]qwen.Part{.{ .seq = &seq, .tokens = ids[i - 1 .. i] }}; + try f.round(&part); + try f.commit(&part, &.{&.{0}}); + } + try f.ops.download(std.mem.sliceAsBytes(host), f.s.logits); + try f.ops.s.synchronize(); + if (file) |*fl| try fl.writeStreamingAll(io, std.mem.sliceAsBytes(host)); + const r: ?[]const u16 = if (ref) |*fl| blk: { + if (try fl.readPositionalAll(io, std.mem.sliceAsBytes(want_row), at) != vocab * 2) return error.ShortReference; + break :blk want_row; + } else null; + at += vocab * 2; + q.add(host, r, ids[i]); + } + q.print(name); + all.merge(q); + } + all.print("all"); + if (ref == null) return 0; + const ok = all.pass(); + std.debug.print("gate: |dNLL| <= {d}, argmax agreement >= {d} %, KL mean <= {d}, no flip with a reference margin above {d}: {s}\n", .{ Quality.max_dnll, Quality.min_agree, Quality.max_kl, Quality.max_flip, if (ok) "PASS" else "FAIL" }); + return if (ok) 0 else 1; +} + +/// One row's log-softmax normaliser and argmax. +fn rowStats(row: []const u16) struct { lse: f64, arg: usize } { + var mx: f64 = -std.math.inf(f64); + var arg: usize = 0; + for (row, 0..) |h, j| if (bf(h) > mx) { + mx = bf(h); + arg = j; + }; + var z: f64 = 0; + for (row) |h| z += @exp(bf(h) - mx); + return .{ .lse = mx + @log(z), .arg = arg }; +} + +/// Teacher-forced quality of one build, and against a reference build's logits when given. +const Quality = struct { + const max_dnll = 0.01; // nats a row, mean + const min_agree = 98.5; // % + const max_kl = 0.005; // nats a row, mean + const max_flip = 1.0; // reference logit gap between its argmax and ours + + rows: usize = 0, + nll: f64 = 0, + top1: usize = 0, + compared: usize = 0, + ref_nll: f64 = 0, + ref_top1: usize = 0, + agree: usize = 0, + kl: f64 = 0, + kl_max: f64 = 0, + flip_max: f64 = 0, + + fn add(q: *Quality, ours: []const u16, ref: ?[]const u16, want: u32) void { + const o = rowStats(ours); + q.rows += 1; + q.nll += o.lse - bf(ours[want]); + if (o.arg == want) q.top1 += 1; + const r = ref orelse return; + const s = rowStats(r); + q.compared += 1; + q.ref_nll += s.lse - bf(r[want]); + if (s.arg == want) q.ref_top1 += 1; + if (s.arg == o.arg) q.agree += 1 else q.flip_max = @max(q.flip_max, bf(r[s.arg]) - bf(r[o.arg])); + var kl: f64 = 0; + for (r, ours) |rh, oh| { + const lp = bf(rh) - s.lse; + kl += @exp(lp) * (lp - (bf(oh) - o.lse)); + } + q.kl += kl; + q.kl_max = @max(q.kl_max, kl); + } + + fn merge(q: *Quality, b: Quality) void { + inline for (.{ "rows", "nll", "top1", "compared", "ref_nll", "ref_top1", "agree", "kl" }) |f| @field(q, f) += @field(b, f); + q.kl_max = @max(q.kl_max, b.kl_max); + q.flip_max = @max(q.flip_max, b.flip_max); + } + + fn pct(k: usize, n: usize) f64 { + return 100 * @as(f64, @floatFromInt(k)) / @as(f64, @floatFromInt(@max(n, 1))); + } + + fn mean(x: f64, n: usize) f64 { + return x / @as(f64, @floatFromInt(@max(n, 1))); + } + + fn print(q: Quality, name: []const u8) void { + std.debug.print("{s}: {d} rows, NLL {d:.4} (ppl {d:.3}), top-1 {d:.2} %", .{ name, q.rows, mean(q.nll, q.rows), @exp(mean(q.nll, q.rows)), pct(q.top1, q.rows) }); + if (q.compared > 0) std.debug.print("; reference NLL {d:.4}, top-1 {d:.2} %; argmax agreement {d:.2} %, KL mean {d:.5} max {d:.4}, largest flip {d:.3}", .{ mean(q.ref_nll, q.compared), pct(q.ref_top1, q.compared), pct(q.agree, q.compared), mean(q.kl, q.compared), q.kl_max, q.flip_max }); + std.debug.print("\n", .{}); + } + + fn pass(q: Quality) bool { + return @abs(mean(q.nll - q.ref_nll, q.compared)) <= max_dnll and pct(q.agree, q.compared) >= min_agree and + mean(q.kl, q.compared) <= max_kl and q.flip_max <= max_flip; + } +}; diff --git a/zig/src/core/act_ref.zig b/zig/src/core/act_ref.zig index ccab2c254e..606a3efd8e 100644 --- a/zig/src/core/act_ref.zig +++ b/zig/src/core/act_ref.zig @@ -5,11 +5,21 @@ pub fn sigmoid(x: f64) f64 { return 1 / (1 + @exp(-x)); } +/// silu(gate) * up. +pub fn swiglu(gate: f64, up: f64) f64 { + return gate * sigmoid(gate) * up; +} + /// A 4-bit affine table's value: q * scale + bias. pub fn affine4(q: u4, scale: f64, bias: f64) f64 { return @as(f64, @floatFromInt(q)) * scale + bias; } +/// DFlash2's dynamic conv at one value: x k0 + prev k1 with the taps as rounded (k_t = bf16(base_t + dyn_t)). +pub fn dconv(x: f64, prev: f64, k0: f64, k1: f64) f64 { + return x * k0 + prev * k1; +} + /// A bf16 output of fp32 arithmetic through expf: half a bf16 ulp plus 2^-18 of |want|. pub fn tolerance(want: f64) f64 { return @abs(want) * (std.math.pow(f64, 2, -8) + std.math.pow(f64, 2, -18)) + 1e-30; @@ -18,5 +28,6 @@ pub fn tolerance(want: f64) f64 { test "activations against hand values" { try std.testing.expectEqual(@as(f64, 0.5), sigmoid(0)); try std.testing.expectEqual(@as(f64, 7 * 0.5 - 1), affine4(7, 0.5, -1)); + try std.testing.expectApproxEqRel(@as(f64, 2 * sigmoid(2) * 3), swiglu(2, 3), 1e-15); try std.testing.expect(tolerance(1) > tolerance(0.5)); } diff --git a/zig/src/core/attention_ref.zig b/zig/src/core/attention_ref.zig new file mode 100644 index 0000000000..b9f07beb2e --- /dev/null +++ b/zig/src/core/attention_ref.zig @@ -0,0 +1,38 @@ +//! Attention of one query head in float64, the reference for every backend's attention kernels. +const std = @import("std"); + +/// softmax(scale q . k_j) v over `keys`; tol 2^-7 of sum p_j |v_j| / sum p_j (bf16 weights) + a bf16 half-ulp. +pub fn head(q: []const f64, k: []const f64, v: []const f64, keys: []const usize, d: usize, scale: f64, out: []f64, tol: []f64) void { + var m: f64 = -std.math.inf(f64); + for (keys) |j| { + var s: f64 = 0; + for (0..d) |e| s += q[e] * k[j * d + e]; + m = @max(m, s * scale); + } + var l: f64 = 0; + @memset(out, 0); + @memset(tol, 0); + for (keys) |j| { + var s: f64 = 0; + for (0..d) |e| s += q[e] * k[j * d + e]; + const p = @exp(s * scale - m); + l += p; + for (0..d) |e| { + out[e] += p * v[j * d + e]; + tol[e] += p * @abs(v[j * d + e]); + } + } + for (out, tol) |*o, *t| { + o.* /= l; + t.* = t.* / l * std.math.pow(f64, 2, -7) + @abs(o.*) * std.math.pow(f64, 2, -8) + 1e-6; + } +} + +test "one key returns its value; equal scores average" { + var out: [2]f64 = undefined; + var tol: [2]f64 = undefined; + head(&.{ 1, 0 }, &.{ 1, 0, 1, 0 }, &.{ 2, 4, 6, 8 }, &.{1}, 2, 1, &out, &tol); + try std.testing.expectEqualSlices(f64, &.{ 6, 8 }, &out); + head(&.{ 0, 0 }, &.{ 1, 0, 1, 0 }, &.{ 2, 4, 6, 8 }, &.{ 0, 1 }, 2, 1, &out, &tol); + try std.testing.expectEqualSlices(f64, &.{ 4, 6 }, &out); +} diff --git a/zig/src/core/dense_ref.zig b/zig/src/core/dense_ref.zig new file mode 100644 index 0000000000..355de01daa --- /dev/null +++ b/zig/src/core/dense_ref.zig @@ -0,0 +1,24 @@ +//! bf16 rows times weight rows in float64, the reference for every backend's dense row kernels (cuda/dense_rows.zig). +const std = @import("std"); + +/// out[n] = x . w[n] for one row; tol: fp32 error of k / 32 sequential terms and a 5-step tree, plus the result's ulp. +pub fn row(x: []const f64, w: []const f64, k: usize, out: []f64, tol: []f64) void { + for (out, tol, 0..) |*o, *t, j| { + var s: f64 = 0; + var mag: f64 = 0; + for (x[0..k], w[j * k ..][0..k]) |a, b| { + s += a * b; + mag += @abs(a * b); + } + o.* = s; + t.* = (@as(f64, @floatFromInt(k / 32 + 6)) * mag + @abs(s)) * std.math.pow(f64, 2, -24) + 1e-30; + } +} + +test "a row against hand values" { + var out: [2]f64 = undefined; + var tol: [2]f64 = undefined; + row(&.{ 1, 2 }, &.{ 3, 4, -1, 0.5 }, 2, &out, &tol); + try std.testing.expectEqualSlices(f64, &.{ 11, 0 }, &out); + try std.testing.expect(tol[0] > 0 and tol[1] > 0); +} diff --git a/zig/src/core/gdn_prep_ref.zig b/zig/src/core/gdn_prep_ref.zig new file mode 100644 index 0000000000..59a812d6df --- /dev/null +++ b/zig/src/core/gdn_prep_ref.zig @@ -0,0 +1,45 @@ +//! Gated DeltaNet inputs in float64, the reference every backend's GDN prep kernels are checked against. +const std = @import("std"); + +/// A head's conv over `taps` rows (x [taps][dk], w [dk][taps]), then SiLU -> c; err: a channel's fp32 bound. +pub fn conv(x: []const []const f64, w: []const f64, taps: usize, c: []f64, err: []f64) void { + for (c, err, 0..) |*o, *e, i| { + var acc: f64 = 0; + var mag: f64 = 0; + for (0..taps) |t| { + acc += x[t][i] * w[i * taps + t]; + mag += @abs(x[t][i] * w[i * taps + t]); + } + o.* = acc / (1 + @exp(-acc)); + e.* = 1.1 * @as(f64, @floatFromInt(taps)) * std.math.pow(f64, 2, -24) * mag; // silu' is at most 1.1 + } +} + +/// q (is_q) or k from a head's c: c / sqrt(mean(c^2) + eps) * (1 / dk for q, 1 / sqrt(dk) for k). +pub fn norm(c: []const f64, is_q: bool, eps: f64, out: []f64) f64 { + var ss: f64 = 0; + for (c) |v| ss += v * v; + const dk: f64 = @floatFromInt(c.len); + const f = 1 / @sqrt(ss / dk + eps) * (if (is_q) 1 / dk else 1 / @sqrt(dk)); + for (out, c) |*o, v| o.* = v * f; + return f; +} + +/// g = exp(-exp(a_log) * softplus(a + dt_bias)). +pub fn gate(a: f64, dt_bias: f64, a_log: f64) f64 { + const x = a + dt_bias; + const sp = if (x > 20) x else std.math.log1p(@exp(x)); + return @exp(-@exp(a_log) * sp); +} + +test "conv, norms and the gate against hand values" { + var c: [2]f64 = undefined; + var e: [2]f64 = undefined; + conv(&.{ &.{ 1, 2 }, &.{ 3, 4 } }, &.{ 1, 0, 0, 1 }, 2, &c, &e); // channel 0 tap 0, channel 1 tap 1 + try std.testing.expectApproxEqRel(@as(f64, 1.0 / (1.0 + @exp(-1.0))), c[0], 1e-15); + try std.testing.expectApproxEqRel(@as(f64, 4.0 / (1.0 + @exp(-4.0))), c[1], 1e-15); + var o: [2]f64 = undefined; + _ = norm(&.{ 3, 4 }, false, 0, &o); + try std.testing.expectApproxEqRel(@as(f64, 0.6), o[0], 1e-15); + try std.testing.expectApproxEqRel(@as(f64, @exp(-@exp(0.5) * 25.0)), gate(25, 0, 0.5), 1e-15); +} diff --git a/zig/src/core/moe_ref.zig b/zig/src/core/moe_ref.zig new file mode 100644 index 0000000000..3789639ec5 --- /dev/null +++ b/zig/src/core/moe_ref.zig @@ -0,0 +1,55 @@ +//! MoE routing and combine in float64, the references every backend's MoE glue is checked against (cuda/moe_glue.zig). +const std = @import("std"); + +/// Sigmoid routing, as cuda/moe_glue.zig's Route. +pub const Route = struct { experts: usize, topk: usize, shared: bool, normalize: bool = false, scale: f64 = 1 }; + +/// A row's picks by the fp32 logit + bias (the kernels' ranking exactly; ties: lower id), weights in float64. +pub fn topk(logits: []const f32, bias: ?[]const f32, r: Route, pick: []u32, wts: []f64) void { + var taken: [4096]bool = @splat(false); + var sum: f64 = 0; + for (0..r.topk) |k| { + var best: ?usize = null; + var bv: f32 = -std.math.inf(f32); + for (logits[0..r.experts], 0..) |l, e| { + if (taken[e]) continue; + const v = l + (if (bias) |b| b[e] else 0); // one fp32 rounding, as the kernel's __fadd_rn + if (best == null or v > bv) { + best = e; + bv = v; + } + } + taken[best.?] = true; + pick[k] = @intCast(best.?); + wts[k] = 1 / (1 + @exp(-@as(f64, logits[best.?]))); + sum += wts[k]; + } + for (wts[0..r.topk]) |*w| w.* = (if (r.normalize) w.* / sum else w.*) * r.scale; + if (r.shared) { + pick[r.topk] = @intCast(r.experts); + wts[r.topk] = 1; + } +} + +/// One row: out[d] = sum over slots of w[k] * y[k][d]. +pub fn combine(y: []const f64, w: []const f64, dims: usize, out: []f64) void { + @memset(out, 0); + for (w, 0..) |wk, k| for (out, y[k * dims ..][0..dims]) |*o, v| { + o.* += wk * v; + }; +} + +test "picks rank by logit plus bias, weigh by the logit, and ties go to the lower id" { + var pick: [3]u32 = undefined; + var wts: [3]f64 = undefined; + topk(&.{ 0, 1, 1, -1 }, &.{ 0, 0, 0, 5 }, .{ .experts = 4, .topk = 2, .shared = true }, &pick, &wts); + try std.testing.expectEqualSlices(u32, &.{ 3, 1, 4 }, &pick); + try std.testing.expectApproxEqRel(@as(f64, 1 / (1 + @exp(@as(f64, 1)))), wts[0], 1e-15); + try std.testing.expectEqual(@as(f64, 1), wts[2]); + topk(&.{ 2, 2 }, null, .{ .experts = 2, .topk = 2, .shared = false, .normalize = true, .scale = 2 }, pick[0..2], wts[0..2]); + try std.testing.expectEqualSlices(u32, &.{ 0, 1 }, pick[0..2]); + try std.testing.expectApproxEqRel(@as(f64, 1), wts[0], 1e-15); + var out: [2]f64 = undefined; + combine(&.{ 1, 2, 10, 20 }, &.{ 0.5, 0.25 }, 2, &out); + try std.testing.expectEqualSlices(f64, &.{ 3, 6 }, &out); +} diff --git a/zig/src/core/qk_ref.zig b/zig/src/core/qk_ref.zig new file mode 100644 index 0000000000..b1763d9f34 --- /dev/null +++ b/zig/src/core/qk_ref.zig @@ -0,0 +1,30 @@ +//! QK norm and RoPE of one head in float64, the reference for every backend's attention inputs (cuda/qk_rows.zig). +const std = @import("std"); + +/// One head: RMS-normed by w (none: as is), then rotate-half RoPE on its first `rot` dims (no cos/sin: none). +pub fn head(x: []const f64, w: ?[]const f64, eps: f64, cos: ?[]const f64, sin: ?[]const f64, rot: usize, y: []f64) void { + const n = x.len; + var ss: f64 = 0; + for (x) |v| ss += v * v; + const inv = if (w != null) 1 / @sqrt(ss / @as(f64, @floatFromInt(n)) + eps) else 1; + var z: [256]f64 = undefined; + for (x, 0..) |v, i| z[i] = v * inv * (if (w) |ww| ww[i] else 1); + const half = rot / 2; + for (y, 0..) |*o, i| { + if (cos == null or i >= rot) { + o.* = z[i]; + continue; + } + const p = if (i < half) i + half else i - half; + const rotated = if (i < half) -z[p] else z[p]; + o.* = z[i] * cos.?[i] + rotated * sin.?[i]; + } +} + +test "rotate-half pairs (d, d + rot / 2) and leave the rest" { + var y: [4]f64 = undefined; + head(&.{ 1, 2, 3, 4 }, null, 0, &.{ 0, 0 }, &.{ 1, 1 }, 2, &y); // angle 90 degrees on the first pair + try std.testing.expectEqualSlices(f64, &.{ -2, 1, 3, 4 }, &y); + head(&.{ 2, 2, 2, 2 }, &.{ 1, 1, 1, 3 }, 0, null, null, 0, &y); + try std.testing.expectEqualSlices(f64, &.{ 1, 1, 1, 3 }, &y); +} diff --git a/zig/src/core/root.zig b/zig/src/core/root.zig index 736248a93d..a1e3472073 100644 --- a/zig/src/core/root.zig +++ b/zig/src/core/root.zig @@ -18,8 +18,20 @@ pub const registry = @import("registry.zig"); pub const tuning = @import("tuning.zig"); pub const norms_ref = @import("norms_ref.zig"); pub const act_ref = @import("act_ref.zig"); +pub const moe_ref = @import("moe_ref.zig"); +pub const dense_ref = @import("dense_ref.zig"); +pub const qk_ref = @import("qk_ref.zig"); +pub const gdn_prep_ref = @import("gdn_prep_ref.zig"); +pub const attention_ref = @import("attention_ref.zig"); test { + _ = norms_ref; + _ = moe_ref; + _ = dense_ref; + _ = qk_ref; + _ = act_ref; + _ = gdn_prep_ref; + _ = attention_ref; _ = safetensors; _ = affine4_host; _ = affine4_fold; diff --git a/zig/src/cuda/act_rows.zig b/zig/src/cuda/act_rows.zig new file mode 100644 index 0000000000..1d343f85f3 --- /dev/null +++ b/zig/src/cuda/act_rows.zig @@ -0,0 +1,97 @@ +//! act_rows.cu's activation rows for every CUDA family: SwiGLU, sigmoid output gates and 64-value group sums. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +const threads = 256; + +fn aligned(ptrs: []const u64, strides: []const usize) bool { + for (ptrs) |p| if (p % 16 != 0) return false; + for (strides) |v| if (v % 8 != 0) return false; + return true; +} + +fn grid(rows: usize, n: usize, t: u32) launch_.Dim3 { + return .{ .x = @intCast(rows), .y = @intCast((n / 8 + t - 1) / t), .z = 1 }; +} + +pub const ActRows = struct { + mod: Module, + swiglu_fn: Function, + gate_fn: Function, + sums_fn: Function, + dconv_fn: Function, + + pub fn load(d: *const Driver) !ActRows { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.act_rows); + errdefer mod.unload(); + return .{ .mod = mod, .swiglu_fn = try mod.function("tf_swiglu_rows"), .gate_fn = try mod.function("tf_gate_rows"), .sums_fn = try mod.function("tf_group_sums_rows"), .dconv_fn = try mod.function("tf_dconv_rows") }; + } + + pub fn unload(g: *ActRows) void { + g.mod.unload(); + } + + /// out [rows, n] = bf16(silu(gate) * up), rows at strides g_ld, u_ld; xs (or null): out's 64-value sums. + pub fn swiglu(g: *const ActRows, s: Stream, gate: u64, g_ld: usize, up: u64, u_ld: usize, out: u64, xs: ?u64, rows: usize, n: usize) !void { + if (n % 64 != 0 or !aligned(&.{ gate, up, out }, &.{ g_ld, u_ld })) return error.RowShape; + if (rows == 0) return; + var a: launch_.Args = .{}; + a.add(gate); + a.add(@as(i64, @intCast(g_ld))); + a.add(up); + a.add(@as(i64, @intCast(u_ld))); + a.add(out); + a.add(xs orelse 0); + a.add(@as(i32, @intCast(n))); + const t = launch_.rowThreads(rows, threads, 128); + try launch_.launch(g.swiglu_fn, .{ .grid = grid(rows, n, t), .block = .{ .x = t } }, s, &a); + } + + /// out = bf16(o * sigmoid(gate)), head h's gate at r * g_ld + h * g_hs ([q | gate]: q + d, 2 d); xs: sums. + pub fn outputGate(g: *const ActRows, s: Stream, o: u64, gate_: u64, g_ld: usize, g_hs: usize, out: u64, xs: ?u64, rows: usize, heads: usize, d: usize) !void { + if (d % 64 != 0 or !aligned(&.{ o, gate_, out }, &.{ g_ld, g_hs })) return error.RowShape; + if (rows == 0 or heads == 0) return; + var a: launch_.Args = .{}; + a.add(o); + a.add(gate_); + a.add(@as(i64, @intCast(g_ld))); + a.add(@as(i64, @intCast(g_hs))); + a.add(out); + a.add(xs orelse 0); + a.add(@as(i32, @intCast(heads))); + a.add(@as(i32, @intCast(d))); + const t = launch_.rowThreads(rows, threads, 64); + try launch_.launch(g.gate_fn, .{ .grid = grid(rows, heads * d, t), .block = .{ .x = t } }, s, &a); + } + + /// xs [rows, k / 64]: x's rows (stride ld) summed over each 64 values. k % 64 == 0. + pub fn groupSums(g: *const ActRows, s: Stream, x: u64, ld: usize, xs: u64, rows: usize, k: usize) !void { + if (k % 64 != 0 or !aligned(&.{x}, &.{ld})) return error.RowShape; + if (rows == 0) return; + var a: launch_.Args = .{}; + a.add(x); + a.add(@as(i64, @intCast(ld))); + a.add(xs); + a.add(@as(i32, @intCast(k))); + try launch_.launch(g.sums_fn, .{ .grid = grid(rows, k, threads), .block = .{ .x = threads } }, s, &a); + } + + /// DFlash2's two-tap conv over [rows, d] bf16: dyn [rows, 2, 2, d / gs], base [2, 2, d]; res (or null) added. + pub fn dconv(g: *const ActRows, s: Stream, x: u64, dyn: u64, base: u64, res: ?u64, out: u64, rows: usize, d: usize, gs: usize, branch: usize, seg: usize) !void { + if (d % 8 != 0 or gs % 8 != 0 or d % gs != 0 or branch > 1 or seg == 0 or !aligned(&.{ x, base, res orelse 0, out }, &.{})) return error.RowShape; + if (rows == 0) return; + var a: launch_.Args = .{}; + for ([_]u64{ x, dyn, base, res orelse 0, out }) |v| a.add(v); + for ([_]usize{ d, gs, branch, seg }) |v| a.add(@as(i32, @intCast(v))); + try launch_.launch(g.dconv_fn, .{ .grid = grid(rows, d, threads), .block = .{ .x = threads } }, s, &a); + } +}; diff --git a/zig/src/cuda/block_attention.zig b/zig/src/cuda/block_attention.zig new file mode 100644 index 0000000000..fd63c57d42 --- /dev/null +++ b/zig/src/cuda/block_attention.zig @@ -0,0 +1,48 @@ +//! block_attention.cu for every CUDA family: a DFlash-style drafter's block rows over their context and the block. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +/// A 32-key K and V tile (head dim 128), shared by the block's warps. +const smem: u32 = 2 * 32 * 16 * 16; + +/// q [h, rows, 128], kb and vb [hk, rows, 128] (head-major); stream j's block rows j L .. j L + L - 1. +pub const Shape = struct { h: usize, hk: usize, rows: usize, block: usize, window: usize, scale: f32, causal: bool = false }; + +pub const BlockAttention = struct { + mod: Module, + fn_: Function, + + pub fn load(d: *const Driver) !BlockAttention { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.block_attention); + errdefer mod.unload(); + const f = try mod.function("tf_block_attention"); + try f.allowDynamicShared(smem); + return .{ .mod = mod, .fn_ = f }; + } + + pub fn unload(g: *BlockAttention) void { + g.mod.unload(); + } + + /// out [rows, h, 128]; table i64 [streams, 2]: stream j's context K and V ([hk, lens[j], 128] bf16); lens i32. + pub fn run(g: *const BlockAttention, s: Stream, q: u64, kb: u64, vb: u64, table: u64, lens: u64, out: u64, sh: Shape) !void { + if (sh.hk == 0 or sh.h % sh.hk != 0 or sh.h / sh.hk > 8 or sh.block == 0 or sh.block > 16 or sh.rows % sh.block != 0) return error.BlockShape; + if (sh.rows == 0) return; + var a: launch_.Args = .{}; + for ([_]u64{ q, kb, vb, table, lens, out }) |v| a.add(v); + a.add(sh.scale); + for ([_]usize{ sh.window, sh.rows, sh.h, sh.hk, sh.block }) |v| a.add(@as(i32, @intCast(v))); + a.add(@as(i32, @intFromBool(sh.causal))); + const grid: launch_.Dim3 = .{ .x = @intCast(sh.rows / sh.block), .y = @intCast(sh.hk), .z = 1 }; + try launch_.launch(g.fn_, .{ .grid = grid, .block = .{ .x = @intCast(32 * (sh.h / sh.hk)) }, .shared = smem }, s, &a); + } +}; diff --git a/zig/src/cuda/dense_rows.zig b/zig/src/cuda/dense_rows.zig new file mode 100644 index 0000000000..9241a4596e --- /dev/null +++ b/zig/src/cuda/dense_rows.zig @@ -0,0 +1,51 @@ +//! dense_rows.cu's bf16 rows times bf16 weight rows into fp32 (routers, output heads): a row's bits at any row count. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +const warps = 8; +/// The launch tiles: rows a tile shares each weight row across. +const tiles = [_]usize{ 1, 4, 16, 32 }; + +pub const DenseRows = struct { + mod: Module, + fns: [tiles.len]Function, + + pub fn load(d: *const Driver) !DenseRows { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.dense_rows); + errdefer mod.unload(); + var fns: [tiles.len]Function = undefined; + inline for (tiles, 0..) |t, i| fns[i] = try mod.function(std.fmt.comptimePrint("tf_dense_rows_{d}", .{t})); + return .{ .mod = mod, .fns = fns }; + } + + pub fn unload(g: *DenseRows) void { + g.mod.unload(); + } + + /// out [m, n] fp32 = x [m, k] bf16 times w [n, k] bf16 transposed; strides in elements, k % 8 == 0, 16 B aligned. + pub fn rows(g: *const DenseRows, s: Stream, x: u64, x_stride: usize, w: u64, out: u64, out_stride: usize, m: usize, n: usize, k: usize) !void { + if (k % 8 != 0 or x % 16 != 0 or w % 16 != 0 or x_stride % 8 != 0) return error.Misaligned; + if (m == 0 or n == 0) return; + const i: usize = if (m == 1) 0 else if (m <= 4) 1 else if (m <= 16) 2 else 3; + var a: launch_.Args = .{}; + a.add(x); + a.add(@as(i64, @intCast(x_stride))); + a.add(w); + a.add(out); + a.add(@as(i64, @intCast(out_stride))); + a.add(@as(i32, @intCast(m))); + a.add(@as(i32, @intCast(n))); + a.add(@as(i32, @intCast(k))); + const grid: launch_.Dim3 = .{ .x = @intCast((n + warps - 1) / warps), .y = @intCast((m + tiles[i] - 1) / tiles[i]), .z = 1 }; + try launch_.launch(g.fns[i], .{ .grid = grid, .block = .{ .x = warps * 32 } }, s, &a); + } +}; diff --git a/zig/src/cuda/gdn_prep.zig b/zig/src/cuda/gdn_prep.zig new file mode 100644 index 0000000000..641a01cd8a --- /dev/null +++ b/zig/src/cuda/gdn_prep.zig @@ -0,0 +1,73 @@ +//! gdn_prep.cu's Gated DeltaNet inputs for every CUDA family: each row's conv window, SiLU, q/k norms, g and beta. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +/// The kernel's buffers (addresses; strides in elements), as gdn_prep.cu's Io. +pub const Io = extern struct { + qkv: u64, // [rows, qkv_ld] bf16: q (kh dk), k (kh dk), v (vh dk) channels + qkv_ld: i64, + state: u64, // [streams * keep, c] bf16: each stream's kept conv rows + conv: u64, // [c, keep + 1] bf16 taps a channel + win: u64, // [rows, keep + 1] i32: < keep a kept row of the row's stream, else qkv row (win - keep) + sid: u64 = 0, // [rows] i32 stream (0: every row stream 0) + a: u64, // [rows, a_ld] bf16 + a_ld: i64, + b: u64, // [rows, b_ld] bf16 + b_ld: i64, + a_log: u64, // [vh] fp32 + dt_bias: u64, // [vh] fp32 + q: u64, // [rows, kh, dk] bf16 + k: u64, + v: u64, // [rows, vh, dk] bf16 + g: u64, // [rows, vh] fp32 + beta: u64, +}; + +/// Channels a row (c = 2 kh dk + vh dk), heads and kept conv rows (taps - 1). +pub const Shape = struct { kh: usize, vh: usize, dk: usize, keep: usize, eps: f32 = 1e-6 }; + +pub const GdnPrep = struct { + mod: Module, + fns: [3]Function, // dk 64, 128, 256 + + pub fn load(d: *const Driver) !GdnPrep { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.gdn_prep); + errdefer mod.unload(); + return .{ .mod = mod, .fns = .{ try mod.function("tf_gdn_prep_64"), try mod.function("tf_gdn_prep_128"), try mod.function("tf_gdn_prep_256") } }; + } + + pub fn unload(g: *GdnPrep) void { + g.mod.unload(); + } + + /// Per row's conv window c = silu(conv): q = l2norm(c) / sqrt(dk), k = l2norm(c), v = c, g and beta. + pub fn run(g: *const GdnPrep, s: Stream, io: Io, rows: usize, sh: Shape) !void { + const which: usize = switch (sh.dk) { + 64 => 0, + 128 => 1, + 256 => 2, + else => return error.GdnShape, + }; + if (sh.keep + 1 > 8 or sh.keep == 0) return error.GdnShape; + if (rows == 0) return; + const heads = 2 * sh.kh + sh.vh; + var a: launch_.Args = .{}; + a.add(io); + a.add(@as(i32, @intCast(heads * sh.dk))); // c: a kept row's channels + a.add(@as(i32, @intCast(sh.kh))); + a.add(@as(i32, @intCast(sh.vh))); + a.add(@as(i32, @intCast(sh.keep))); + a.add(sh.eps); + const t = launch_.rowThreads(rows, 128, 64); // a warp a head either way + try launch_.launch(g.fns[which], .{ .grid = .{ .x = @intCast(rows), .y = @intCast((heads + t / 32 - 1) / (t / 32)), .z = 1 }, .block = .{ .x = t } }, s, &a); + } +}; diff --git a/zig/src/cuda/kernels.zig b/zig/src/cuda/kernels.zig index 8ff900952c..81f4d5ed5d 100644 --- a/zig/src/cuda/kernels.zig +++ b/zig/src/cuda/kernels.zig @@ -31,6 +31,16 @@ pub const exlr8_dense: []const u8 = if (available) &Blob("fatbin_exlr8_dense").b pub const nvfp4_experts: []const u8 = if (available) &Blob("fatbin_nvfp4_experts").bytes else &.{}; pub const qwen_ops: []const u8 = if (available) &Blob("fatbin_qwen_ops").bytes else &.{}; pub const qwen_vision: []const u8 = if (available) &Blob("fatbin_qwen_vision").bytes else &.{}; +pub const norms: []const u8 = if (available) &Blob("fatbin_norms").bytes else &.{}; +pub const moe_glue: []const u8 = if (available) &Blob("fatbin_moe_glue").bytes else &.{}; +pub const dense_rows: []const u8 = if (available) &Blob("fatbin_dense_rows").bytes else &.{}; +pub const qk_rows: []const u8 = if (available) &Blob("fatbin_qk_rows").bytes else &.{}; +pub const window_attention: []const u8 = if (available) &Blob("fatbin_window_attention").bytes else &.{}; +pub const tree_attention: []const u8 = if (available) &Blob("fatbin_tree_attention").bytes else &.{}; +pub const act_rows: []const u8 = if (available) &Blob("fatbin_act_rows").bytes else &.{}; +pub const gdn_prep: []const u8 = if (available) &Blob("fatbin_gdn_prep").bytes else &.{}; +pub const embed_rows: []const u8 = if (available) &Blob("fatbin_embed_rows").bytes else &.{}; +pub const block_attention: []const u8 = if (available) &Blob("fatbin_block_attention").bytes else &.{}; pub const sample: []const u8 = if (available) &Blob("fatbin_sample").bytes else &.{}; pub const train: []const u8 = if (available) &Blob("fatbin_train").bytes else &.{}; pub const qmmf: []const u8 = if (available) &Blob("fatbin_qmmf").bytes else &.{}; diff --git a/zig/src/cuda/launch.zig b/zig/src/cuda/launch.zig index 2472042638..56bba5272a 100644 --- a/zig/src/cuda/launch.zig +++ b/zig/src/cuda/launch.zig @@ -8,6 +8,14 @@ const Error = @import("driver.zig").Error; pub const Dim3 = abi.Dim3; +/// Rows a row kernel's call takes as decode-sized: one row spread over small blocks, as Triton's tuned glue does. +pub const few_rows = 16; + +/// Threads a block for a row kernel: `few` for decode-sized calls, `many` for prompt rows (bits never depend on it). +pub fn rowThreads(rows: usize, many: u32, few: u32) u32 { + return if (rows <= few_rows) few else many; +} + /// Argument values in kernel order; each must have exactly the C type of its kernel parameter. pub const Args = struct { pub const max_args = 48; diff --git a/zig/src/cuda/moe_glue.zig b/zig/src/cuda/moe_glue.zig new file mode 100644 index 0000000000..948f71634f --- /dev/null +++ b/zig/src/cuda/moe_glue.zig @@ -0,0 +1,76 @@ +//! moe_glue.cu's routing and combine for every CUDA family: sigmoid top-k with bias, the weighted sum of slots. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); +const norms = @import("norms.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; +const Kind = norms.Kind; + +const threads = 256; +/// The most experts a row ranks and the most it keeps (moe_glue.cu MOST, the picked array). +pub const most_experts = 4096; +pub const most_topk = 64; +/// Experts a lane ranks in the warp form (moe_glue.cu PER): up to 512 experts take it. +const per_lane = 16; + +/// Sigmoid routing: top `topk` by logit + bias, weights sigmoid(logit) [normalized] * scale; `shared` adds slot topk. +pub const Route = struct { experts: usize, topk: usize, shared: bool, normalize: bool = false, scale: f32 = 1 }; + +pub const MoeGlue = struct { + mod: Module, + topk_fn: Function, + topk_warp_fn: Function, + combine_fn: Function, + + pub fn load(d: *const Driver) !MoeGlue { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.moe_glue); + errdefer mod.unload(); + return .{ .mod = mod, .topk_fn = try mod.function("tf_moe_topk_sigmoid"), .topk_warp_fn = try mod.function("tf_moe_topk_sigmoid_warp"), .combine_fn = try mod.function("tf_moe_combine") }; + } + + pub fn unload(g: *MoeGlue) void { + g.mod.unload(); + } + + /// fp32 logits [rows, experts] (+ fp32 bias) into int32 picks and fp32 weights [rows, topk (+1 with shared)]. + pub fn topk(g: *const MoeGlue, s: Stream, logits: u64, bias: ?u64, r: Route, pick: u64, wts: u64, rows: usize) !void { + if (r.experts > most_experts or r.topk == 0 or r.topk > most_topk or r.topk > r.experts) return error.RouteOutOfRange; + if (rows == 0) return; + const warp = r.experts <= 32 * per_lane and r.topk <= 32; + var a: launch_.Args = .{}; + a.add(logits); + a.add(bias orelse 0); + a.add(pick); + a.add(wts); + if (warp) a.add(@as(i32, @intCast(rows))); + a.add(@as(i32, @intCast(r.experts))); + a.add(@as(i32, @intCast(r.topk))); + a.add(@as(i32, @intFromBool(r.shared))); + a.add(@as(i32, @intFromBool(r.normalize))); + a.add(r.scale); + if (warp) return launch_.launch(g.topk_warp_fn, .{ .grid = .{ .x = @intCast((rows + 7) / 8), .y = 1, .z = 1 }, .block = .{ .x = 256 } }, s, &a); + try launch_.launch(g.topk_fn, .{ .grid = .{ .x = @intCast(rows), .y = 1, .z = 1 }, .block = .{ .x = threads } }, s, &a); + } + + /// out [rows, dims] = the sum, in slot order, of w [rows, slots] (fp32) times y [rows, slots, dims]. + pub fn combine(g: *const MoeGlue, s: Stream, y: u64, y_kind: Kind, w: u64, out: u64, out_kind: Kind, rows: usize, slots: usize, dims: usize) !void { + if (rows == 0) return; + var a: launch_.Args = .{}; + a.add(y); + a.add(@as(i32, @backingInt(y_kind))); + a.add(w); + a.add(out); + a.add(@as(i32, @backingInt(out_kind))); + a.add(@as(i32, @intCast(slots))); + a.add(@as(i32, @intCast(dims))); + const cols: u32 = @intCast((dims + threads - 1) / threads); + try launch_.launch(g.combine_fn, .{ .grid = .{ .x = @intCast(rows), .y = cols, .z = 1 }, .block = .{ .x = threads } }, s, &a); + } +}; diff --git a/zig/src/cuda/norms.zig b/zig/src/cuda/norms.zig index e746777704..b9f827e0cd 100644 --- a/zig/src/cuda/norms.zig +++ b/zig/src/cuda/norms.zig @@ -135,9 +135,10 @@ pub const Norms = struct { for ([_]u64{ w, out, xs orelse 0 }) |v| a.add(v); a.add(@as(i32, @intCast(heads))); a.add(eps); - const per_block = 256 / (dv / 8); + const t = launch_.rowThreads(rows, 256, 32); // a head's lanes stay in one warp either way + const per_block = t / (dv / 8); const grid: launch_.Dim3 = .{ .x = @intCast(rows), .y = @intCast((heads + per_block - 1) / per_block), .z = 1 }; - try launch_.launch(n.gated_fns[which], .{ .grid = grid, .block = .{ .x = 256 } }, s, &a); + try launch_.launch(n.gated_fns[which], .{ .grid = grid, .block = .{ .x = t } }, s, &a); } }; diff --git a/zig/src/cuda/qk_rows.zig b/zig/src/cuda/qk_rows.zig new file mode 100644 index 0000000000..d1ccb1a4ed --- /dev/null +++ b/zig/src/cuda/qk_rows.zig @@ -0,0 +1,107 @@ +//! qk_rows.cu's attention inputs for every CUDA family: QK norm, RoPE (rotate-half, partial) and the cache write. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +/// A packed row's heads: h query, hk key and hk value heads of `dim` (a multiple of 32, at most 256). +pub const Heads = struct { h: usize, hk: usize, dim: usize }; + +/// Where a row's heads come from: q, k and v (0: none) at row stride *_ld and head stride *_hs (elements). +pub const Src = extern struct { q: u64, q_ld: i64, q_hs: i64, k: u64, k_ld: i64, k_hs: i64, v: u64 = 0, v_ld: i64 = 0, v_hs: i64 = 0 }; + +/// q/k norms [2, dim] (or q's with norm_k), fp32 or bf16; RoPE on `rot` dims: cos/sin tables or inv [rot / 2]. +pub const Shape = struct { norms: ?u64 = null, norm_k: ?u64 = null, norms_bf16: bool = false, cos: ?u64 = null, sin: ?u64 = null, inv: ?u64 = null, rot: usize = 0, eps: f32 = 1e-6 }; + +/// qk_rows.cu's QkW. +const W = extern struct { q: u64, k: u64, bf16: i32 }; + +/// Where the rows go: q [n, h, dim], k and v [n, hk, dim]; k and v also into kc/vc at (slot * len + pos % len). +pub const Out = struct { + q: u64, + k: u64, + v: u64, + kc: ?u64 = null, + vc: ?u64 = null, + len: usize = 1, + /// Row and head strides in elements (0: packed [n, heads, dim]); head-major [heads, n, dim]: ld dim, hs n dim. + q_ld: usize = 0, + q_hs: usize = 0, + kv_ld: usize = 0, + kv_hs: usize = 0, +}; + +/// qk_rows.cu's QkDst. +const Dst = extern struct { q: u64, q_ld: i64, q_hs: i64, k: u64, k_ld: i64, k_hs: i64, v: u64, v_ld: i64, v_hs: i64 }; + +pub const QkRows = struct { + mod: Module, + fn_: Function, + /// tf_qk_rows_w: dim 128 with no or full rotation, a warp a head. + warp_fn: Function, + + pub fn load(d: *const Driver) !QkRows { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.qk_rows); + errdefer mod.unload(); + return .{ .mod = mod, .fn_ = try mod.function("tf_qk_rows"), .warp_fn = try mod.function("tf_qk_rows_w") }; + } + + pub fn unload(g: *QkRows) void { + g.mod.unload(); + } + + /// n packed rows qkv [n, (h + 2 hk) dim] bf16 at positions pos (int32 [n]) in cache slots slot (int32 [n] or none). + pub fn rows(g: *const QkRows, s: Stream, qkv: u64, hs: Heads, shape: Shape, pos: u64, slot: ?u64, out: Out, n: usize) !void { + const ld: i64 = @intCast((hs.h + 2 * hs.hk) * hs.dim); + const dim: i64 = @intCast(hs.dim); + const src: Src = .{ .q = qkv, .q_ld = ld, .q_hs = dim, .k = qkv + hs.h * hs.dim * 2, .k_ld = ld, .k_hs = dim, .v = qkv + (hs.h + hs.hk) * hs.dim * 2, .v_ld = ld, .v_hs = dim }; + return g.rowsFrom(s, src, hs, shape, pos, slot, out, n); + } + + /// As `rows`, the heads read from `src` (e.g. [q | gate] rows: q_hs = 2 dim); no v source: no v out or vc. + pub fn rowsFrom(g: *const QkRows, s: Stream, src: Src, hs: Heads, shape: Shape, pos: u64, slot: ?u64, out: Out, n: usize) !void { + if (hs.dim % 32 != 0 or hs.dim > 256 or shape.rot % 2 != 0 or shape.rot > hs.dim) return error.HeadShape; + if ((shape.cos == null) != (shape.sin == null) or (shape.cos != null and shape.inv != null)) return error.HeadShape; + const roped = shape.cos != null or shape.inv != null; + if (roped and shape.rot == 0) return error.HeadShape; + if (src.v == 0 and out.vc != null) return error.HeadShape; + if (n == 0) return; + var a: launch_.Args = .{}; + a.add(src); + const wsize: usize = if (shape.norms_bf16) 2 else 4; + const nq = shape.norms orelse 0; + a.add(W{ .q = nq, .k = shape.norm_k orelse (if (nq != 0) nq + hs.dim * wsize else 0), .bf16 = @intFromBool(shape.norms_bf16) }); + a.add(shape.cos orelse 0); + a.add(shape.sin orelse 0); + a.add(shape.inv orelse 0); + a.add(@as(i32, @intCast(if (roped) shape.rot else 0))); + a.add(pos); + a.add(slot orelse 0); + const dim: i64 = @intCast(hs.dim); + const qld: i64 = if (out.q_ld != 0) @intCast(out.q_ld) else @as(i64, @intCast(hs.h)) * dim; + const qhs: i64 = if (out.q_hs != 0) @intCast(out.q_hs) else dim; + const kld: i64 = if (out.kv_ld != 0) @intCast(out.kv_ld) else @as(i64, @intCast(hs.hk)) * dim; + const khs: i64 = if (out.kv_hs != 0) @intCast(out.kv_hs) else dim; + a.add(Dst{ .q = out.q, .q_ld = qld, .q_hs = qhs, .k = out.k, .k_ld = kld, .k_hs = khs, .v = out.v, .v_ld = kld, .v_hs = khs }); + a.add(out.kc orelse 0); + a.add(out.vc orelse 0); + a.add(@as(i64, @intCast(out.len))); + a.add(@as(i32, @intCast(hs.h))); + a.add(@as(i32, @intCast(hs.hk))); + a.add(shape.eps); + const heads = hs.h + (if (src.v != 0) 2 * hs.hk else hs.hk); + if (hs.dim == 128 and (!roped or shape.rot == 128)) { + const grid: launch_.Dim3 = .{ .x = @intCast(n), .y = @intCast((heads + 3) / 4), .z = 1 }; + return launch_.launch(g.warp_fn, .{ .grid = grid, .block = .{ .x = 128 } }, s, &a); + } + const grid: launch_.Dim3 = .{ .x = @intCast(n), .y = @intCast(heads), .z = 1 }; + try launch_.launch(g.fn_, .{ .grid = grid, .block = .{ .x = @intCast(hs.dim) } }, s, &a); + } +}; diff --git a/zig/src/cuda/root.zig b/zig/src/cuda/root.zig index 3053c09a6b..1502797e42 100644 --- a/zig/src/cuda/root.zig +++ b/zig/src/cuda/root.zig @@ -36,6 +36,14 @@ pub const embed_rows = @import("embed_rows.zig"); pub const experts = @import("experts.zig"); pub const sample = @import("sample.zig"); pub const modelopt = @import("modelopt.zig"); +pub const moe_glue = @import("moe_glue.zig"); +pub const dense_rows = @import("dense_rows.zig"); +pub const qk_rows = @import("qk_rows.zig"); +pub const window_attention = @import("window_attention.zig"); +pub const tree_attention = @import("tree_attention.zig"); +pub const act_rows = @import("act_rows.zig"); +pub const gdn_prep = @import("gdn_prep.zig"); +pub const block_attention = @import("block_attention.zig"); test { _ = @import("memory.zig"); @@ -45,6 +53,16 @@ test { _ = @import("embed_rows.zig"); _ = @import("experts.zig"); _ = @import("sample.zig"); + _ = @import("norms.zig"); + _ = @import("moe_glue.zig"); + _ = @import("dense_rows.zig"); + _ = @import("qk_rows.zig"); + _ = @import("window_attention.zig"); + _ = @import("tree_attention.zig"); + _ = @import("act_rows.zig"); + _ = @import("gdn_prep.zig"); + _ = @import("embed_rows.zig"); + _ = @import("block_attention.zig"); _ = launch; _ = abi; _ = aot; diff --git a/zig/src/cuda/tree_attention.zig b/zig/src/cuda/tree_attention.zig new file mode 100644 index 0000000000..69b2ad598f --- /dev/null +++ b/zig/src/cuda/tree_attention.zig @@ -0,0 +1,129 @@ +//! tree_attention.cu for every CUDA family: drafted rows over committed keys and their paths, groups in order. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +/// Keys a group, a path's most rows, the part kernels' shared memory (four warps' K/V slots; at 256 also the queries). +pub const span = 512; +pub const max_depth = 128; +const part_smem = [2]u32{ 4 * 2 * 32 * 16 * 16, 4 * 2 * 16 * 32 * 16 + 16 * 32 * 16 }; +/// One-warp blocks' shared memory by tile stages (2: two blocks an SM; 3: one, deeper), at 256 also the queries. +fn smemW1(stages: u32, which: usize) u32 { + return if (which == 0) stages * 2 * 32 * 16 * 16 else stages * 2 * 16 * 32 * 16 + 16 * 32 * 16; +} +/// A group's slices: its key tiles dealt to 4 warps, combined in order (in the block, or by the merge). +const slices = 4; +/// (row, key head, group) items one-warp blocks may take: their slices' partials sit apart from the combined ones. +pub const slice_items = 1024; + +/// row_stream [n], streams [s, 4] (committed at [2]), offs [s, 2] (K, V from origin), paths [n, max_depth], depths. +pub const Rows = struct { row_stream: u64, streams: u64, offs: u64, origin: u64, paths: u64, depths: u64 }; + +pub const Scratch = struct { o: u64, m: u64, l: u64, so: u64, sm: u64, sl: u64 }; + +pub const TreeAttention = struct { + mod: Module, + paths_fn: Function, + part_fns: [2]Function, // head dim 128, 256 + shared_fns: [2]Function, + part_w1: [2][2]Function, // the same slices, one warp a block: [2 stages, 3 stages][head dim] + shared_w1: [2][2]Function, + merge_fn: Function, + /// The shared kernel's blocks a key head: a fixed grid walking every (row, group) (bits do not depend on it). + shared_blocks: usize = 96, + /// The device's SMs (0: four-warp blocks only): small grids take pipelined one-warp blocks (the same bits). + sms: usize = 0, + + pub fn load(d: *const Driver) !TreeAttention { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.tree_attention); + errdefer mod.unload(); + var parts: [2]Function = .{ try mod.function("tf_tree_part"), try mod.function("tf_tree_part_256") }; + for (&parts, part_smem) |*f, b| try f.allowDynamicShared(b); + var shared: [2]Function = .{ try mod.function("tf_tree_shared"), try mod.function("tf_tree_shared_256") }; + for (&shared, part_smem) |*f, b| try f.allowDynamicShared(b); + var parts1: [2][2]Function = .{ .{ try mod.function("tf_tree_part_w1"), try mod.function("tf_tree_part_256_w1") }, .{ try mod.function("tf_tree_part_w1s3"), try mod.function("tf_tree_part_256_w1s3") } }; + var shared1: [2][2]Function = .{ .{ try mod.function("tf_tree_shared_w1"), try mod.function("tf_tree_shared_256_w1") }, .{ try mod.function("tf_tree_shared_w1s3"), try mod.function("tf_tree_shared_256_w1s3") } }; + for (0..2) |st| for (0..2) |which| { + try parts1[st][which].allowDynamicShared(smemW1(@intCast(st + 2), which)); + try shared1[st][which].allowDynamicShared(smemW1(@intCast(st + 2), which)); + }; + return .{ .mod = mod, .paths_fn = try mod.function("tf_tree_paths"), .part_fns = parts, .shared_fns = shared, .part_w1 = parts1, .shared_w1 = shared1, .merge_fn = try mod.function("tf_tree_merge") }; + } + + pub fn unload(g: *TreeAttention) void { + g.mod.unload(); + } + + /// Groups a row with `keys` committed and path keys needs. + pub fn groups(keys: usize) usize { + return (keys + span - 1) / span; + } + + /// fp32 partials of `rows` rows' `h` heads (`hk` key heads) in `ng` groups: o [.., dim], max, sum; then slices. + pub fn scratchBytes(rows: usize, h: usize, hk: usize, ng: usize, dim: usize) usize { + return (rows * h * ng + slice_items * slices * (h / hk)) * (dim + 2) * 4; + } + + pub fn scratch(base: u64, rows: usize, h: usize, hk: usize, ng: usize, dim: usize) Scratch { + const parts = rows * h * ng; + const sb = base + parts * (dim + 2) * 4; + const sp = slice_items * slices * (h / hk); + return .{ .o = base, .m = base + parts * dim * 4, .l = base + parts * dim * 4 + parts * 4, .so = sb, .sm = sb + sp * dim * 4, .sl = sb + sp * dim * 4 + sp * 4 }; + } + + /// Each row's path (root first) and depth from `parents` int32 [n] (global window rows, -1 at a root). + pub fn paths(g: *const TreeAttention, s: Stream, parents: u64, out_paths: u64, depths: u64, n: usize) !void { + if (n == 0) return; + var a: launch_.Args = .{}; + for ([_]u64{ parents, out_paths, depths }) |v| a.add(v); + a.add(@as(i32, @intCast(n))); + try launch_.launch(g.paths_fn, .{ .grid = .{ .x = @intCast((n + 127) / 128), .y = 1, .z = 1 }, .block = .{ .x = 128 } }, s, &a); + } + + /// q [n, h, dim] (128 or 256) over committed keys and paths (kn, vn: window rows) -> out; ng >= a row's groups. + pub fn run(g: *const TreeAttention, s: Stream, q: u64, kn: u64, vn: u64, rows: Rows, sc: Scratch, out: u64, n: usize, h: usize, hk: usize, dim: usize, ng: usize, scale: f32) !void { + if (hk == 0 or h % hk != 0 or h / hk > 16 or ng == 0 or (dim != 128 and dim != 256)) return error.TreeShape; + const which: usize = if (dim == 256) 1 else 0; + if (n == 0) return; + // small grids (a few rows): each slice its own pipelined one-warp block and partial; else four-warp blocks + const one = g.sms > 0 and n * ng * hk <= slice_items; + const o = if (one) [3]u64{ sc.so, sc.sm, sc.sl } else [3]u64{ sc.o, sc.m, sc.l }; + var a: launch_.Args = .{}; + for ([_]u64{ q, kn, vn, rows.origin, rows.offs, rows.row_stream, rows.streams, rows.paths, rows.depths, o[0], o[1], o[2] }) |v| a.add(v); + a.add(@as(i32, @intCast(h))); + a.add(@as(i32, @intCast(hk))); + a.add(scale); + a.add(@as(i32, @intCast(ng))); + var c: launch_.Args = .{}; + for ([_]u64{ q, rows.origin, rows.offs, rows.row_stream, rows.streams, rows.paths, rows.depths, o[0], o[1], o[2] }) |v| c.add(v); + c.add(@as(i32, @intCast(h))); + c.add(@as(i32, @intCast(hk))); + c.add(scale); + c.add(@as(i32, @intCast(ng))); + c.add(@as(i32, @intCast(n))); + const per: usize = if (one) slices else 1; + const block: u32 = if (one) 32 else 128; + // three stages where the one-warp grid fits one wave of the SMs (a block each), else two (two blocks an SM) + const deep: usize = if (@min(n * ng, g.shared_blocks) * slices * hk <= g.sms) 1 else 0; + const smem = if (one) smemW1(@intCast(deep + 2), which) else part_smem[which]; + try launch_.launch(if (one) g.shared_w1[deep][which] else g.shared_fns[which], .{ .grid = .{ .x = @intCast(@min(n * ng, g.shared_blocks) * per), .y = @intCast(hk), .z = 1 }, .block = .{ .x = block }, .shared = smem }, s, &c); + // a row's own block: its boundary group and at most the next (a path is at most 128 keys) + try launch_.launch(if (one) g.part_w1[deep][which] else g.part_fns[which], .{ .grid = .{ .x = @intCast(n), .y = @intCast(hk), .z = @intCast(@min(ng, 2) * per) }, .block = .{ .x = block }, .shared = smem }, s, &a); + var b: launch_.Args = .{}; + for ([_]u64{ o[0], o[1], o[2], rows.row_stream, rows.streams, rows.depths, out }) |v| b.add(v); + b.add(@as(i32, @intCast(h))); + b.add(@as(i32, @intCast(hk))); + b.add(@as(i32, @intCast(ng))); + b.add(@as(i32, @intCast(dim))); + b.add(@as(i32, @intFromBool(one))); + try launch_.launch(g.merge_fn, .{ .grid = .{ .x = @intCast(n), .y = @intCast(hk), .z = @intCast(h / hk) }, .block = .{ .x = 256 } }, s, &b); + } +}; diff --git a/zig/src/cuda/window_attention.zig b/zig/src/cuda/window_attention.zig new file mode 100644 index 0000000000..40569b7ef9 --- /dev/null +++ b/zig/src/cuda/window_attention.zig @@ -0,0 +1,115 @@ +//! window_attention.cu's sliding-window attention for every CUDA family: decode rows over a ring of keys. +const std = @import("std"); +const kernels = @import("kernels.zig"); +const module = @import("module.zig"); +const launch_ = @import("launch.zig"); +const stream_ = @import("stream.zig"); + +const Driver = @import("driver.zig").Driver; +const Module = module.Module; +const Function = module.Function; +const Stream = stream_.Stream; + +/// The heads and the keys a row sees: (pos - win, pos] (win 0: all), cache row slot * ring + key % ring. +pub const Window = struct { h: usize, hk: usize, d: usize, win: usize, ring: usize, scale: f32 }; + +/// The prompt kernel's instances: head dim 128, 8 warps, 4 or 8 query heads a block (`hpc`), 8 staging slots. +const prompt_name = "_ZN16tf_window_prompt12wattn_kernelILi128ELi8ELi{d}ELi8EEEvPK13__nv_bfloat16S3_S3_PS1_iiiiifii"; +const prompt_smem: u32 = 8 * 32 * 16 * 16; // NS * HALF * (D / 8) uint4 +/// tf_window_decode_tc: four warps' K and V slots of 32 keys (64 KiB), reused for the warps' partials. +const tc_smem: u32 = 4 * 2 * 32 * 16 * 16; + +pub const WindowAttention = struct { + mod: Module, + part_fn: Function, + merge_fn: Function, + tc_fn: Function, + prompt_fns: [2]Function, // 4, then 8 query heads a block + + pub fn load(d: *const Driver) !WindowAttention { + if (!kernels.available) return error.BuiltWithoutKernels; + var mod = try Module.load(d, kernels.window_attention); + errdefer mod.unload(); + var p: [2]Function = undefined; + inline for (.{ 4, 8 }, 0..) |hpc, i| { + p[i] = try mod.function(std.fmt.comptimePrint(prompt_name, .{hpc})); + try p[i].allowDynamicShared(prompt_smem); + } + return .{ .mod = mod, .part_fn = try mod.function("tf_window_part"), .merge_fn = try mod.function("tf_window_merge"), .tc_fn = tc: { + const f = try mod.function("tf_window_decode_tc"); + try f.allowDynamicShared(tc_smem); + break :tc f; + }, .prompt_fns = p }; + } + + pub fn unload(g: *WindowAttention) void { + g.mod.unload(); + } + + /// Key tiles a decode row's window can touch (its partials a key head). + pub fn tiles(w: Window) usize { + return if (w.win > 0) (w.win + 62) / 64 + 1 else (w.ring + 63) / 64 + 1; + } + + /// The fp32 partials `decode` writes for up to `rows` rows: o, then m and l. + pub fn scratchBytes(w: Window, rows: usize) usize { + if (w.d == 128) return 16; // the tensor-core form keeps its partials in shared memory + const parts = rows * w.hk * tiles(w) * (w.h / w.hk); + return parts * w.d * 4 + 2 * parts * 4; + } + + /// n rows q [n, h, d] at pos/slot (int32 [n]) -> out [n, h, d]; `scratch`: scratchBytes(w, n) (d != 128). + pub fn decode(g: *const WindowAttention, s: Stream, q: u64, kc: u64, vc: u64, out: u64, pos: u64, slot: ?u64, w: Window, scratch: u64, n: usize) !void { + if (w.hk == 0 or w.h % w.hk != 0 or w.h / w.hk > 16 or w.d > 128 or w.ring == 0) return error.WindowShape; + if (w.win == 0 and w.ring > 4096) return error.WindowShape; // an unwindowed row's tiles come from the ring + if (n == 0) return; + if (w.d == 128) { // tensor cores; the heads of a row are one mma tile + var c: launch_.Args = .{}; + for ([_]u64{ q, kc, vc, out, pos, slot orelse 0 }) |v| c.add(v); + c.add(@as(i32, @intCast(w.h))); + c.add(@as(i32, @intCast(w.hk))); + c.add(@as(i32, @intCast(w.win))); + c.add(@as(i64, @intCast(w.ring))); + c.add(w.scale); + return launch_.launch(g.tc_fn, .{ .grid = .{ .x = @intCast(n), .y = @intCast(w.hk), .z = 1 }, .block = .{ .x = 128 }, .shared = tc_smem }, s, &c); + } + const t = tiles(w); + const parts = n * w.hk * t * (w.h / w.hk); + const po = scratch; + const pm = po + parts * w.d * 4; + const pl = pm + parts * 4; + var a: launch_.Args = .{}; + for ([_]u64{ q, kc, vc, pos, slot orelse 0, po, pm, pl }) |v| a.add(v); + a.add(@as(i32, @intCast(w.h))); + a.add(@as(i32, @intCast(w.hk))); + a.add(@as(i32, @intCast(w.d))); + a.add(@as(i32, @intCast(w.win))); + a.add(@as(i64, @intCast(w.ring))); + a.add(w.scale); + try launch_.launch(g.part_fn, .{ .grid = .{ .x = @intCast(n), .y = @intCast(w.hk), .z = @intCast(t) }, .block = .{ .x = 256 } }, s, &a); + var b: launch_.Args = .{}; + for ([_]u64{ po, pm, pl, out }) |v| b.add(v); + b.add(@as(i32, @intCast(w.h))); + b.add(@as(i32, @intCast(w.hk))); + b.add(@as(i32, @intCast(w.d))); + b.add(@as(i32, @intCast(t))); + try launch_.launch(g.merge_fn, .{ .grid = .{ .x = @intCast(n), .y = @intCast(w.hk), .z = 1 }, .block = .{ .x = 256 } }, s, &b); + } + + /// n prompt rows of one stream at p0, p0 + 1, ... (d 128); kc/vc that stream's ring, key p at row p % ring. + pub fn prompt(g: *const WindowAttention, s: Stream, q: u64, kc: u64, vc: u64, out: u64, p0: usize, n: usize, w: Window) !void { + const grp = w.h / w.hk; + if (w.d != 128 or w.hk == 0 or w.h % w.hk != 0 or grp % 4 != 0) return error.WindowShape; + if (n == 0) return; + const hpc: usize = if (grp % 8 == 0) 8 else 4; + var a: launch_.Args = .{}; + for ([_]u64{ q, kc, vc, out }) |v| a.add(v); + for ([_]usize{ p0, n, w.h, w.hk, grp }) |v| a.add(@as(i32, @intCast(v))); + a.add(w.scale); + a.add(@as(i32, @intCast(w.win))); + a.add(@as(i32, @intCast(w.ring))); + const rows = 16 * (8 / hpc); + const cfg: launch_.Config = .{ .grid = .{ .x = @intCast((n + rows - 1) / rows), .y = @intCast(w.hk * (grp / hpc)), .z = 1 }, .block = .{ .x = 256 }, .shared = prompt_smem }; + try launch_.launch(g.prompt_fns[if (hpc == 8) 1 else 0], cfg, s, &a); + } +}; diff --git a/zig/src/families/qwen27/cuda.zig b/zig/src/families/qwen27/cuda.zig index 3e10f18896..6a1a5dcfa1 100644 --- a/zig/src/families/qwen27/cuda.zig +++ b/zig/src/families/qwen27/cuda.zig @@ -28,6 +28,7 @@ test { _ = @import("cuda/forward.zig"); _ = @import("cuda/lanes.zig"); _ = @import("cuda/tree.zig"); + _ = @import("cuda/native_attention_plan.zig"); _ = @import("cuda/draft_load.zig"); _ = @import("cuda/draft_policy.zig"); _ = draft; diff --git a/zig/src/families/qwen27/cuda/draft.zig b/zig/src/families/qwen27/cuda/draft.zig index 587554ef05..7c631cdc15 100644 --- a/zig/src/families/qwen27/cuda/draft.zig +++ b/zig/src/families/qwen27/cuda/draft.zig @@ -240,21 +240,14 @@ pub const DFlash2 = struct { fn prep(x: *DFlash2, qkv: u64, layer: usize, rows: usize, with_q: bool, q: u64, k: u64, v: u64) !void { const l = x.w.layers[layer]; const h: usize = if (with_q) dl.heads else 0; - const stride = (h + 2 * dl.kv_heads) * D; - const p = aot.ptr; - try x.t.set.run(x.t.s, "_prep_kernel", .{ @intCast(rows), @intCast(h + 2 * dl.kv_heads), 1 }, &.{ - p("QKV", "*bf16", qkv), p("QN", "*bf16", l.q_norm), p("KN", "*bf16", l.k_norm), p("COS", "*fp32", x.cos), p("SIN", "*fp32", x.sin), - p("QO", "*bf16", if (with_q) q else qkv), p("KO", "*bf16", k), p("VO", "*bf16", v), aot.int("L", @intCast(rows)), aot.int("stride", @intCast(stride)), - aot.float("eps", 1e-6), - }, &.{ aot.ci("H", @intCast(h)), aot.ci("HKV", dl.kv_heads), aot.ci("HALF", D / 2) }); + const stride: i64 = @intCast((h + 2 * dl.kv_heads) * D); + const src: cuda.qk_rows.Src = .{ .q = qkv, .q_ld = stride, .q_hs = D, .k = qkv + h * D * bf, .k_ld = stride, .k_hs = D, .v = qkv + (h + dl.kv_heads) * D * bf, .v_ld = stride, .v_hs = D }; + try x.t.g.qk.rowsFrom(x.t.s, src, .{ .h = h, .hk = dl.kv_heads, .dim = D }, .{ .norms = l.q_norm, .norm_k = l.k_norm, .norms_bf16 = true, .inv = x.inv_freq, .rot = D, .eps = 1e-6 }, x.pos, null, .{ .q = q, .k = k, .v = v, .q_ld = D, .q_hs = rows * D, .kv_ld = D, .kv_hs = rows * D }, rows); } /// dflash2._dconv: the two-tap dynamic conv of each `seg`-row block, plus a residual. fn dconv(x: *DFlash2, in: u64, dyn: u64, base: u64, branch: usize, res: ?u64, out: u64, rows: usize, seg: usize) !void { - const p = aot.ptr; - try x.t.set.run(x.t.s, "_dconv_kernel", .{ @intCast(rows), (H + 1023) / 1024, 1 }, &.{ - p("X", "*bf16", in), p("DYN", "*bf16", dyn), p("BASE", "*bf16", base), p("RES", "*bf16", res orelse in), p("OUT", "*bf16", out), aot.int("SEG", @intCast(seg)), - }, &.{ aot.ci("D", H), aot.ci("G", H / dl.conv_group), aot.ci("GS", dl.conv_group), aot.ci("BRANCH", @intCast(branch)), aot.ci("HAS_RES", @intFromBool(res != null)), aot.ci("BLOCK", 1024) }); + try x.t.g.act.dconv(x.t.s, in, dyn, base, res, out, rows, H, dl.conv_group, branch, seg); } /// add_taps: the target's taps (rows of 5 x hidden bf16) projected into the context's layers. @@ -351,11 +344,7 @@ pub const DFlash2 = struct { try x.lin(x.conv, l.qkv, x.qkv, rows); try x.prep(x.qkv, i, rows, true, x.q, x.k, x.v); const tb = x.table + i * per; - const p = aot.ptr; - try x.t.set.run(x.t.s, "_block_attention", .{ @intCast(S), dl.kv_heads, 1 }, &.{ - p("Q", "*bf16", x.q), p("KB", "*bf16", x.k), p("VB", "*bf16", x.v), p("TABLE", "*i64", tb), p("LENS", "*i32", tb + S * 16), p("O", "*bf16", x.attn), - aot.float("scale", scale), aot.int("window", dl.window), aot.int("R", @intCast(rows)), - }, &.{ aot.ci("G", dl.heads / dl.kv_heads), aot.ci("HKV", dl.kv_heads), aot.ci("L", @intCast(L)), aot.ci("LP", @intCast(@max(16, std.math.ceilPowerOfTwo(usize, L) catch unreachable))), aot.ci("D", D), aot.ci("BN", 64), aot.ci("CAUSAL", 0) }); + try x.t.g.block.run(x.t.s, x.q, x.k, x.v, tb, tb + S * 16, x.attn, .{ .h = dl.heads, .hk = dl.kv_heads, .rows = rows, .block = L, .window = dl.window, .scale = scale }); try x.lin(x.attn, l.o, x.o, rows); try x.dconv(x.o, x.dyn, l.attn_base, 1, x.x, x.x, rows, L); try x.torchNorm(x.x, l.post_norm, x.normed, rows); diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig index af3c6594b4..632a806564 100644 --- a/zig/src/families/qwen27/cuda/engine.zig +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -73,6 +73,7 @@ pub const Engine = struct { grid: bool = false, // a prompt cache: prompts in zero-anchored chunks (Chunks), not even splits pub fn init(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: []const u8, o: Options) !*Engine { + if (o.kv8 and kernels.len == 0) return error.QwenNeedsCapturedKernels; const cfg = try c.Config.read(gpa, io, dir); const g = cfg.g; const e = try gpa.create(Engine); @@ -87,7 +88,7 @@ pub const Engine = struct { errdefer e.k.deinit(); e.w = try wts.load(gpa, io, e.ops(), dir, cfg); errdefer e.w.deinit(); - e.scratch = try st.Scratch.init(gpa, ctx.d, g, prompt_rows, o.context); + e.scratch = try st.Scratch.initForKv(gpa, ctx.d, g, prompt_rows, o.context, o.kv8); errdefer e.scratch.deinit(); try e.s.synchronize(); return e; @@ -106,7 +107,7 @@ pub const Engine = struct { } pub fn forward(e: *Engine) Forward { - return .{ .gpa = e.gpa, .w = &e.w, .ops = e.ops(), .t = .{ .set = &e.k.triton, .s = e.s }, .s = &e.scratch }; + return .{ .gpa = e.gpa, .w = &e.w, .ops = e.ops(), .t = .{ .g = &e.k.glue, .s = e.s, .set = if (e.k.glue.set) |*s| s else null }, .s = &e.scratch }; } /// A stream's state for `capacity` positions (its caches, recurrences and conv windows), empty. diff --git a/zig/src/families/qwen27/cuda/forward.zig b/zig/src/families/qwen27/cuda/forward.zig index e188647925..593c6e1f54 100644 --- a/zig/src/families/qwen27/cuda/forward.zig +++ b/zig/src/families/qwen27/cuda/forward.zig @@ -496,5 +496,28 @@ pub const Forward = struct { try f.ops.argmax(f.s.logits, c.vocab, c.vocab, f.s.picks, rows); try f.ops.download(std.mem.sliceAsBytes(out[0..rows]), f.s.picks); try f.ops.s.synchronize(); + if (std.c.getenv("TF_Q27_TOP2") != null and rows == 1) { // debug: the row's top-2 margin + const lg = try std.heap.page_allocator.alloc(u16, c.vocab); + defer std.heap.page_allocator.free(lg); + try f.ops.download(std.mem.sliceAsBytes(lg), f.s.logits); + try f.ops.s.synchronize(); + var b1: f32 = -std.math.inf(f32); + var b2: f32 = -std.math.inf(f32); + var ia: usize = 0; + var ib: usize = 0; + for (lg, 0..) |h, i| { + const v: f32 = @bitCast(@as(u32, h) << 16); + if (v > b1) { + b2 = b1; + ib = ia; + b1 = v; + ia = i; + } else if (v > b2) { + b2 = v; + ib = i; + } + } + std.debug.print("top2 {d} {d:.4} | {d} {d:.4}\n", .{ ia, b1, ib, b2 }); + } } }; diff --git a/zig/src/families/qwen27/cuda/kernels.zig b/zig/src/families/qwen27/cuda/kernels.zig index 964eb4536b..14c423c684 100644 --- a/zig/src/families/qwen27/cuda/kernels.zig +++ b/zig/src/families/qwen27/cuda/kernels.zig @@ -55,7 +55,7 @@ pub const QLinear = struct { w: u64, s: u64, b: u64, n: usize, k: usize, npad: u pub const Kernels = struct { d: *const cuda.Driver, mods: [15]cuda.Module, - triton: cuda.aot.Set, + glue: @import("triton.zig").Glue, group: cuda.Function, wide: [wide.len]cuda.Function, // tiles 3-5: rows <= 32, <= 64, more gemv: cuda.Function, @@ -120,8 +120,9 @@ pub const Kernels = struct { // gdn_prefill_cuda: a block takes 128 value rows, or 64 when there are fewer heads than SMs k.prompt_rows = if (heads >= sms) 128 else 64; k.prompt_chain = try k.mods[5].function(if (k.prompt_rows == 128) sym.chain128 else sym.chain64); - k.triton = try cuda.aot.Set.load(gpa, io, d, ctx.device, triton_dir); - errdefer k.triton.deinit(); + k.glue = try @import("triton.zig").Glue.load(gpa, io, d, ctx.device, triton_dir); + errdefer k.glue.unload(); + k.glue.tree.sms = if (std.c.getenv("TF_TREE_SMS")) |v| std.fmt.parseInt(usize, std.mem.span(v), 10) catch 0 else @intCast(try ctx.attribute(.multiprocessor_count)); try k.group.allowDynamicShared(group_smem); for (k.wide, wide) |f, t| try f.allowDynamicShared(t.smem); try k.gemv.allowDynamicShared(group_smem); @@ -140,7 +141,7 @@ pub const Kernels = struct { } pub fn deinit(k: *Kernels) void { - k.triton.deinit(); + k.glue.unload(); for (&k.mods) |*m| m.unload(); } }; diff --git a/zig/src/families/qwen27/cuda/native.zig b/zig/src/families/qwen27/cuda/native.zig index cbe9817214..00ad3ecfea 100644 --- a/zig/src/families/qwen27/cuda/native.zig +++ b/zig/src/families/qwen27/cuda/native.zig @@ -100,8 +100,9 @@ fn drafterDir(a: std.mem.Allocator, io: std.Io, given: ?[]const u8) !?[]const u8 const most_streams = 32; pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: []const u8, kernels: ?[]const u8, o: Options) !Loaded { - // the 27B's glue runs the Python engine's Triton kernels, captured on this chip (no own glue kernels yet) - const e = try engine.Engine.init(gpa, io, ctx, dir, kernels orelse return error.QwenNeedsCapturedKernels, .{ .context = o.context, .kv8 = o.kv8 }); + // BF16 target and drafter glue use native CUDA; FP8 attention retains its captured set. + if (o.kv8 and (kernels == null or kernels.?.len == 0)) return error.QwenNeedsCapturedKernels; + const e = try engine.Engine.init(gpa, io, ctx, dir, kernels orelse "", .{ .context = o.context, .kv8 = o.kv8 }); errdefer e.deinit(); const own = try gpa.create(Owned); errdefer gpa.destroy(own); @@ -130,7 +131,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [ .facts = own.lanes.facts(), .rows = st.max_rows, // each stream's caches, its drafter context, and its share of a round's attention partials (`fit`) - .stream_bytes = st.Seq.bytes(e.w.g, o.context, o.kv8) + st.Scratch.partialBytes(e.w.g, st.Scratch.streamPartRows(o.context)) + if (own.draft != null) dr.Context.bytes() else 0, + .stream_bytes = st.Seq.bytes(e.w.g, o.context, o.kv8) + st.Scratch.partialBytesFor(e.w.g, st.Scratch.streamPartRows(o.context), !o.kv8) + if (own.draft != null) dr.Context.bytes() else 0, .vision_bytes = vision_bytes, .ctx = own, .deinit = release, diff --git a/zig/src/families/qwen27/cuda/native_attention_plan.zig b/zig/src/families/qwen27/cuda/native_attention_plan.zig new file mode 100644 index 0000000000..c63f82ac71 --- /dev/null +++ b/zig/src/families/qwen27/cuda/native_attention_plan.zig @@ -0,0 +1,51 @@ +//! Bounds for native 512-key groups and slice storage within PR726 dynamic partial allocations. +const std = @import("std"); +pub const span: usize = 512; +pub const slice_items: usize = 1024; +const slices: usize = 4; + +/// Each piece's committed keys plus its window rows bound every tree path in that piece. +pub fn groups(keys: usize) usize { + return (keys + span - 1) / span; +} + +/// Fixed one-warp slice storage follows combined partials even when a launch uses four-warp blocks. +pub fn sliceBytes(heads: usize, key_heads: usize, dim: usize) usize { + return slice_items * slices * (heads / key_heads) * (dim + 2) * 4; +} + +pub fn required(rows: usize, heads: usize, key_heads: usize, group_count: usize, dim: usize) usize { + return rows * heads * group_count * (dim + 2) * 4 + sliceBytes(heads, key_heads, dim); +} + +fn available(slot_rows: usize, heads: usize, key_heads: usize, dim: usize) usize { + const aligned = std.mem.alignForward; + return aligned(usize, slot_rows * heads * dim * 4, 256) + + 2 * aligned(usize, slot_rows * heads * 4, 256) + sliceBytes(heads, key_heads, dim); +} + +test "512-key group bound includes longest piece and old 2048/4096/6144 boundaries" { + for ([_]usize{ 1, 512, 513, 2048, 2049, 4096, 4097, 6144, 6145 }, [_]usize{ 1, 1, 2, 4, 5, 8, 9, 12, 13 }) |keys, want| + try std.testing.expectEqual(want, groups(keys)); + try std.testing.expectEqual(@as(usize, 13), @max(groups(2048), groups(6145))); +} + +test "initial calibration needs slice storage and fits all 14 measured widths" { + const old = 1024 * 24 * (256 + 2) * 4; + try std.testing.expect(required(256, 24, 4, 1, 256) > old); + for ([_]usize{ 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256 }) |rows| + try std.testing.expect(required(rows, 24, 4, 1, 256) <= available(1024, 24, 4, 256)); +} + +test "n1 and n16 fitted capacities retain enough combined and slice storage" { + for ([_]usize{ 1, 512, 2047, 2048, 2049, 4095, 4096, 4097, 6144, 32768 }) |capacity| { + for ([_]usize{ 1, 16 }) |streams| { + const slot_rows = @max(1024, streams * ((capacity + 16 + 511) / 512 + 1) * 16); + try std.testing.expect(required(streams * 16, 24, 4, groups(capacity + 16), 256) <= available(slot_rows, 24, 4, 256)); + } + } +} + +test "wide unfitted CLI cannot assume contiguous native scratch fits" { + try std.testing.expect(required(128, 24, 4, groups(6144 + 128), 256) > available(1024, 24, 4, 256)); +} diff --git a/zig/src/families/qwen27/cuda/round.zig b/zig/src/families/qwen27/cuda/round.zig index 7fd0b2f770..8ff75f6837 100644 --- a/zig/src/families/qwen27/cuda/round.zig +++ b/zig/src/families/qwen27/cuda/round.zig @@ -6,6 +6,7 @@ const kern = @import("kernels.zig"); const tri = @import("triton.zig"); const st = @import("state.zig"); const tree = @import("tree.zig"); +const native_plan = @import("native_attention_plan.zig"); const bf = 2; @@ -160,8 +161,10 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, if (W > st.round_rows) return error.WindowTooWide; // attention's partials: the most slots any stream takes, for every row of the round var most_slots: usize = 0; + var native_groups: usize = 0; for (parts) |p| { most_slots = @max(most_slots, tri.slots(p.seq.pos, p.tokens.len)); + native_groups = @max(native_groups, native_plan.groups(p.seq.pos + p.tokens.len)); if (p.seq.kv8 != parts[0].seq.kv8) return error.InvalidQwenWindows; } if (most_slots * W > s.part_rows) return error.WindowTooWide; @@ -212,7 +215,7 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, r.starts = s.plan + 3 * W * 4; r.tables = 0; r.offs = s.offs; - r.attn = .{ .rows = s.attn_plan, .streams = s.attn_plan + ap.streams * 4, .items = s.attn_plan + ap.items * 4, .n_items = ap.n_items, .paths = s.paths, .depths = s.depths, .width = W }; + r.attn = .{ .rows = s.attn_plan, .streams = s.attn_plan + ap.streams * 4, .items = s.attn_plan + ap.items * 4, .n_items = ap.n_items, .paths = s.paths, .depths = s.depths, .width = W, .native_groups = native_groups, .scratch_bytes = s.parts.?.len }; try t.paths(s.attn_plan + ap.parents * 4, s.paths, s.depths, W); return r; } @@ -236,7 +239,7 @@ pub fn stage(gpa: std.mem.Allocator, ops: kern.Ops, t: tri.Tri, s: *st.Scratch, r.ids = base + (5 * W + S + 1) * 4; r.windows = base + (6 * W + S + 1) * 4; const ab = base + at_attn * 4; - r.attn = .{ .rows = ab, .streams = ab + ap.streams * 4, .items = ab + ap.items * 4, .n_items = ap.n_items, .paths = s.paths, .depths = s.depths, .width = W }; + r.attn = .{ .rows = ab, .streams = ab + ap.streams * 4, .items = ab + ap.items * 4, .n_items = ap.n_items, .paths = s.paths, .depths = s.depths, .width = W, .native_groups = native_groups, .scratch_bytes = s.parts.?.len }; // the state pointer table and cache offsets, one copy each; every stream's conv rows stacked by layer var ptrs: std.ArrayList(u64) = .empty; defer ptrs.deinit(gpa); diff --git a/zig/src/families/qwen27/cuda/state.zig b/zig/src/families/qwen27/cuda/state.zig index e85f68a5b5..8204da8fcd 100644 --- a/zig/src/families/qwen27/cuda/state.zig +++ b/zig/src/families/qwen27/cuda/state.zig @@ -214,9 +214,17 @@ pub const Scratch = struct { attn_records: []AttnRecord, // by layer: the round's rotated keys and values gpa: std.mem.Allocator, + native_partials: bool, + /// Scratch for prompt chunks of `prompt_rows` and windows of `max_rows`, over caches of `capacity` positions. pub fn init(gpa: std.mem.Allocator, d: *const cuda.Driver, g: c.Geometry, prompt_rows: usize, capacity: usize) !Scratch { + return initForKv(gpa, d, g, prompt_rows, capacity, false); + } + + /// FP8 retains captured partial sizing; BF16 reserves native attention slice storage. + pub fn initForKv(gpa: std.mem.Allocator, d: *const cuda.Driver, g: c.Geometry, prompt_rows: usize, capacity: usize, kv8: bool) !Scratch { var s: Scratch = undefined; + s.native_partials = !kv8; s.gpa = gpa; s.records = try gpa.alloc(GdnRecord, g.layers); errdefer gpa.free(s.records); @@ -241,6 +249,12 @@ pub const Scratch = struct { return a(usize, slot_rows * g.query_heads * c.head_dim * 4, 256) + 2 * a(usize, slot_rows * g.query_heads * 4, 256); } + /// Native slice storage is shared by a launch; per-stream admission conservatively budgets it in full. + pub fn partialBytesFor(g: c.Geometry, slot_rows: usize, native: bool) usize { + const extra: usize = if (native) @import("native_attention_plan.zig").sliceBytes(g.query_heads, g.kv_heads, c.head_dim) else 0; + return partialBytes(g, slot_rows) + extra; + } + /// The slot-rows a stream of `capacity` positions may add to a round: its most slots (attention.slots of any /// keys and window it can hold) times its widest window. pub fn streamPartRows(capacity: usize) usize { @@ -252,7 +266,7 @@ pub const Scratch = struct { const rows = @max(slot_rows, initial_part_rows); if (s.parts) |*b| b.free(); s.parts = null; - var b = try cuda.DeviceBuffer.alloc(d, partialBytes(g, rows)); + var b = try cuda.DeviceBuffer.alloc(d, partialBytesFor(g, rows, s.native_partials)); errdefer b.free(); var A: Arena = .{ .buf = b }; s.part_o = try A.take(rows * g.query_heads * c.head_dim * 4); diff --git a/zig/src/families/qwen27/cuda/triton.zig b/zig/src/families/qwen27/cuda/triton.zig index bc737a39ec..68e8b9c5ee 100644 --- a/zig/src/families/qwen27/cuda/triton.zig +++ b/zig/src/families/qwen27/cuda/triton.zig @@ -1,19 +1,17 @@ -//! The 27B's Triton kernels from the captured cubins: each launch has the Python wrapper's grid and constexprs. +//! The 27B's glue on the shared CUDA modules (norms, act_rows, gdn_prep, qk_rows, embed_rows, tree_attention). const std = @import("std"); const cuda = @import("cuda"); const aot = cuda.aot; - const p = aot.ptr; -fn int(name: []const u8, v: usize) aot.Arg { - return aot.int(name, @intCast(v)); +fn int(name: []const u8, value: usize) aot.Arg { + return aot.int(name, @intCast(value)); } -fn ci(name: []const u8, v: usize) aot.Const { - return aot.ci(name, @intCast(v)); +fn ci(name: []const u8, value: usize) aot.Const { + return aot.ci(name, @intCast(value)); } - fn u(x: usize) u32 { return @intCast(x); } @@ -38,85 +36,114 @@ pub const Gdn = struct { c: usize, kh: usize, vh: usize, dk: usize, keep: usize /// The attention shapes: query and key heads, head dim, rotary half. pub const Attn = struct { heads: usize, kv_heads: usize, dim: usize, half: usize }; +/// The shared CUDA glue modules (zig/kernels/cuda) that replace the captured Triton set. +pub const Glue = struct { + norms: cuda.norms.Norms, + act: cuda.act_rows.ActRows, + gdn: cuda.gdn_prep.GdnPrep, + qk: cuda.qk_rows.QkRows, + embed: cuda.embed_rows.EmbedRows, + tree: cuda.tree_attention.TreeAttention, + block: cuda.block_attention.BlockAttention, + /// Only FP8 target attention uses a captured set; BF16 target and DFlash2 use native CUDA. + set: ?cuda.aot.Set, + + pub fn load(gpa: std.mem.Allocator, io: std.Io, d: *const cuda.Driver, device: anytype, triton_dir: []const u8) !Glue { + var g: Glue = undefined; + g.set = if (triton_dir.len > 0) try cuda.aot.Set.load(gpa, io, d, device, triton_dir) else null; + errdefer if (g.set) |*s| s.deinit(); + g.norms = try cuda.norms.Norms.load(d); + errdefer g.norms.unload(); + g.act = try cuda.act_rows.ActRows.load(d); + errdefer g.act.unload(); + g.gdn = try cuda.gdn_prep.GdnPrep.load(d); + errdefer g.gdn.unload(); + g.qk = try cuda.qk_rows.QkRows.load(d); + errdefer g.qk.unload(); + g.embed = try cuda.embed_rows.EmbedRows.load(d); + errdefer g.embed.unload(); + g.tree = try cuda.tree_attention.TreeAttention.load(d); + errdefer g.tree.unload(); + g.block = try cuda.block_attention.BlockAttention.load(d); + return g; + } + + pub fn unload(g: *Glue) void { + if (g.set) |*s| s.deinit(); + g.norms.unload(); + g.act.unload(); + g.gdn.unload(); + g.qk.unload(); + g.embed.unload(); + g.tree.unload(); + g.block.unload(); + } +}; + +/// The glue calls the 27B makes, on the shared modules (same signatures as the Triton set's wrappers had). pub const Tri = struct { - set: *const aot.Set, + g: *const Glue, s: cuda.Stream, - + /// The optional captured FP8 attention kernels. + set: ?*const cuda.aot.Set, fn run(t: Tri, name: []const u8, grid: [3]u32, args: []const aot.Arg, consts: []const aot.Const) !void { - try t.set.run(t.s, name, grid, args, consts); + const set = t.set orelse return error.QwenNeedsCapturedKernels; + try set.run(t.s, name, grid, args, consts); } - /// glue.embed: a 4-bit table row dequantized, one program a row and 64-group. pub fn embed(t: Tri, ids: u64, w: u64, s: u64, b: u64, out: u64, rows: usize, d: usize) !void { - try t.run("_embed", .{ u(rows), u(d / 64), 1 }, &.{ p("IDS", "*i32", ids), p("Wt", "*i32", w), p("S", "*bf16", s), p("B", "*bf16", b), p("OUT", "*bf16", out) }, &.{ci("D", d)}); + try t.g.embed.affine4(t.s, ids, w, s, b, out, rows, d); } - /// glue.add_rmsnorm: h = x + r (or x itself without r), y = rmsnorm(h) * w, xs = y's 64-group sums. pub fn addRmsnorm(t: Tri, x: u64, r: ?u64, w: u64, h: u64, y: u64, xs: u64, rows: usize, d: usize, eps: f32) !void { - try t.run("_add_rmsnorm", .{ u(rows), 1, 1 }, &.{ p("X", "*bf16", x), p("R", "*bf16", r orelse x), p("W", "*bf16", w), p("H", "*bf16", if (r != null) h else x), p("Y", "*bf16", y), p("XS", "*fp32", xs), aot.float("eps", eps) }, &.{ ci("D", d), ci("BLOCK", pow2(d)), ci("HAS_R", @intFromBool(r != null)) }); + try t.g.norms.addRmsH(t.s, x, r, w, .bf16, h, y, xs, rows, d, eps); } - /// glue.gdn_pre: conv over each row's window of [state; qkv], q/k norms, g and beta; `sid`: several streams. pub fn gdnPre(t: Tri, qkv: u64, cs: u64, cw: u64, win: u64, sid: ?u64, a: u64, b: u64, alog: u64, dtb: u64, q: u64, k: u64, v: u64, g: u64, beta: u64, rows: usize, m: Gdn) !void { - try t.run("_gdn_pre", .{ u(rows), u(2 * m.kh + m.vh), 1 }, &.{ - p("QKV", "*bf16", qkv), p("CS", "*bf16", cs), p("CW", "*bf16", cw), p("WIN", "*i32", win), p("A", "*bf16", a), p("B", "*bf16", b), p("ALOG", "*fp32", alog), - p("DTB", "*fp32", dtb), p("Q", "*bf16", q), p("K", "*bf16", k), p("V", "*bf16", v), p("G", "*fp32", g), p("BETA", "*fp32", beta), p("SID", "*i32", sid orelse win), - }, &.{ ci("C", m.c), ci("KH", m.kh), ci("VH", m.vh), ci("DK", m.dk), ci("NKEEP", m.keep), ci("MULTI", @intFromBool(sid != null)) }); + if (m.c != (2 * m.kh + m.vh) * m.dk) return error.GdnShape; + const ld: i64 = @intCast(m.c); + const vh: i64 = @intCast(m.vh); + try t.g.gdn.run(t.s, .{ .qkv = qkv, .qkv_ld = ld, .state = cs, .conv = cw, .win = win, .sid = sid orelse 0, .a = a, .a_ld = vh, .b = b, .b_ld = vh, .a_log = alog, .dt_bias = dtb, .q = q, .k = k, .v = v, .g = g, .beta = beta }, rows, .{ .kh = m.kh, .vh = m.vh, .dk = m.dk, .keep = m.keep }); } - /// glue.gated_norm: silu(z) * rmsnorm(y) * w per value head, and its group sums. pub fn gatedNorm(t: Tri, y: u64, z: u64, w: u64, out: u64, xs: u64, rows: usize, vh: usize, dv: usize, eps: f32) !void { - try t.run("_gated_norm", .{ u(rows), u(vh), 1 }, &.{ p("Yr", "*bf16", y), p("Z", "*bf16", z), p("W", "*bf16", w), p("OUT", "*bf16", out), p("XS", "*fp32", xs), aot.float("eps", eps) }, &.{ ci("VH", vh), ci("DV", dv) }); + try t.g.norms.gatedRms(t.s, y, vh * dv, z, vh * dv, w, out, xs, rows, vh, dv, eps); } - /// glue.swiglu: silu(gate) * up in 1024-column blocks, and its group sums. pub fn swiglu(t: Tri, gate: u64, up: u64, out: u64, xs: u64, rows: usize, n: usize) !void { - try t.run("_swiglu", .{ u(rows), cdiv(n, 1024), 1 }, &.{ p("GATE", "*bf16", gate), p("UP", "*bf16", up), p("OUT", "*bf16", out), p("XS", "*fp32", xs) }, &.{ ci("N", n), ci("BLOCK", 1024) }); + try t.g.act.swiglu(t.s, gate, n, up, n, out, xs, rows, n); } - /// glue.attn_prep: q (from the [q | gate] rows) and k normed, then rotated at each row's position. pub fn attnPrep(t: Tri, qg: u64, kv: u64, qn: u64, kn: u64, pos: u64, inv: u64, qout: u64, kout: u64, rows: usize, a: Attn, eps: f32) !void { - try t.run("_attn_prep", .{ u(rows), u(a.heads + a.kv_heads), 1 }, &.{ p("QG", "*bf16", qg), p("KV", "*bf16", kv), p("QN", "*bf16", qn), p("KN", "*bf16", kn), p("POS", "*i32", pos), p("INV", "*fp32", inv), p("QOUT", "*bf16", qout), p("KOUT", "*bf16", kout), aot.float("eps", eps) }, &.{ - ci("H", a.heads), ci("HKV", a.kv_heads), ci("D", a.dim), ci("HALF", a.half), ci("MROPE", 0), ci("ROWS", 0), ci("HSEC", 11), ci("WSEC", 10), - }); + const d: i64 = @intCast(a.dim); + const src: cuda.qk_rows.Src = .{ .q = qg, .q_ld = @as(i64, @intCast(a.heads)) * 2 * d, .q_hs = 2 * d, .k = kv, .k_ld = @as(i64, @intCast(a.kv_heads)) * d, .k_hs = d }; + try t.g.qk.rowsFrom(t.s, src, .{ .h = a.heads, .hk = a.kv_heads, .dim = a.dim }, .{ .norms = qn, .norm_k = kn, .norms_bf16 = true, .inv = inv, .rot = 2 * a.half, .eps = eps }, pos, null, .{ .q = qout, .k = kout, .v = 0 }, rows); } - /// glue.gate_mul: the attention output times sigmoid(gate) from the [q | gate] rows, and its group sums. pub fn gateMul(t: Tri, o: u64, qg: u64, out: u64, xs: u64, rows: usize, heads: usize, dim: usize) !void { - try t.run("_gate_mul", .{ u(rows), u(heads), 1 }, &.{ p("O", "*bf16", o), p("QG", "*bf16", qg), p("OUT", "*bf16", out), p("XS", "*fp32", xs) }, &.{ ci("H", heads), ci("D", dim) }); + try t.g.act.outputGate(t.s, o, qg + dim * 2, heads * 2 * dim, 2 * dim, out, xs, rows, heads, dim); } - /// qmm.group_sums: each row's 64-group input sums in fp32 (rows may be strided by `ld`). pub fn groupSums(t: Tri, x: u64, xs: u64, ld: usize, rows: usize, k: usize) !void { - try t.run("_group_sums", .{ u(rows), cdiv(k / 64, 16), 1 }, &.{ p("X", "*bf16", x), p("XS", "*fp32", xs), int("ldx", ld) }, &.{ ci("KG", k / 64), ci("GS", 64), ci("GB", 16) }); + try t.g.act.groupSums(t.s, x, ld, xs, rows, k); } - /// attention.from_packed's _paths: each row's root-to-row window rows and its depth. pub fn paths(t: Tri, parents: u64, out: u64, depths: u64, rows: usize) !void { - try t.run("_paths", .{ u(rows), 1, 1 }, &.{ p("PARENTS", "*i32", parents), p("PATHS", "*i32", out), p("DEPTHS", "*i32", depths) }, &.{ci("MAXD", max_nodes)}); + try t.g.tree.paths(t.s, parents, out, depths, rows); } /// The layout attention.attention reads: the plan's tables and the partial buffers it writes. - pub const Plan = struct { streams: u64, items: u64, n_items: usize, rows: u64, paths: u64, depths: u64, width: usize }; + pub const Plan = struct { streams: u64, items: u64, n_items: usize, rows: u64, paths: u64, depths: u64, width: usize, native_groups: usize, scratch_bytes: usize }; pub const Partials = struct { o: u64, m: u64, l: u64 }; - /// attention.attention: committed keys in 512-key chunks, the row's own path in its tail, merged in order. + /// Committed keys then each row's path in native 512-key groups (cuda/tree_attention), scratch in part.o. pub fn attention(t: Tri, q: u64, kn: u64, vn: u64, base: u64, offs: u64, plan: Plan, part: Partials, out: u64, a: Attn) !void { - const g = a.heads / a.kv_heads; + const TA = cuda.tree_attention.TreeAttention; + const ng = plan.native_groups; + if (ng == 0 or TA.scratchBytes(plan.width, a.heads, a.kv_heads, ng, a.dim) > plan.scratch_bytes) return error.TreeScratch; const scale: f32 = @floatCast(std.math.pow(f64, @floatFromInt(a.dim), -0.5)); - const w = plan.width; - if (plan.n_items > 0) try t.run("_shared", .{ u(plan.n_items * a.kv_heads), 1, 1 }, &.{ - p("Q", "*bf16", q), p("KC", "*bf16", base), p("VC", "*bf16", base), p("OFF", "*i64", offs), p("STREAM", "*i32", plan.streams), - p("ITEMS", "*i32", plan.items), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), - }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), aot.cf("SCALE", scale), ci("GR", group_chunks) }); - const tails = 1 + (max_nodes + chunk - 1) / chunk; - try t.run("_tail", .{ u(w), u(a.kv_heads), u(tails) }, &.{ - p("Q", "*bf16", q), p("KN", "*bf16", kn), p("VN", "*bf16", vn), p("KC", "*bf16", base), p("VC", "*bf16", base), p("OFF", "*i64", offs), p("STREAM", "*i32", plan.streams), - p("ROWS", "*i32", plan.rows), p("PATHS", "*i32", plan.paths), p("DEPTHS", "*i32", plan.depths), p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), int("W", w), - }, &.{ ci("H", a.heads), ci("HK", a.kv_heads), ci("D", a.dim), ci("G", g), ci("CH", chunk), ci("MAXD", max_nodes), aot.cf("SCALE", scale), ci("GR", group_chunks) }); - try t.run("_merge", .{ u(w), u(a.kv_heads), u(a.dim / merge_columns) }, &.{ - p("PO", "*fp32", part.o), p("PM", "*fp32", part.m), p("PL", "*fp32", part.l), p("OUT", "*bf16", out), p("STREAM", "*i32", plan.streams), p("ROWS", "*i32", plan.rows), int("W", w), - }, &.{ ci("H", a.heads), ci("D", a.dim), ci("G", g), ci("DS", merge_columns), ci("CH", chunk), ci("GR", group_chunks) }); + const rows: cuda.tree_attention.Rows = .{ .row_stream = plan.rows, .streams = plan.streams, .offs = offs, .origin = base, .paths = plan.paths, .depths = plan.depths }; + try t.g.tree.run(t.s, q, kn, vn, rows, TA.scratch(part.o, plan.width, a.heads, a.kv_heads, ng, a.dim), out, plan.width, a.heads, a.kv_heads, a.dim, ng, scale); } /// attention8 (attention_kv8.py, captured outside the tree like the rest of the set): attention over e4m3 @@ -160,3 +187,18 @@ test "attention slots follow the Python plan" { try std.testing.expectEqual(@as(usize, 13), slots(6144, 16)); // 3 groups x 16 rows < 64: every chunk its own try std.testing.expectEqual(@as(usize, 4), slots(6144, 32)); // 3 folded groups, then the window's chunk } + +test "native scratch formula and contiguous slice pointers bind the pinned CUDA wrapper" { + const TA = cuda.tree_attention.TreeAttention; + const plan = @import("native_attention_plan.zig"); + try std.testing.expectEqual(plan.span, cuda.tree_attention.span); + try std.testing.expectEqual(plan.slice_items, cuda.tree_attention.slice_items); + for ([_]usize{ 128, 256 }) |dim| for ([_]usize{ 1, 16, 256 }) |rows| for ([_]usize{ 1, 4, 8, 13, 65 }) |ng| { + const bytes = TA.scratchBytes(rows, 24, 4, ng, dim); + try std.testing.expectEqual(plan.required(rows, 24, 4, ng, dim), bytes); + const base: u64 = 4096; + const sc = TA.scratch(base, rows, 24, 4, ng, dim); + try std.testing.expect(sc.o == base and sc.m < sc.l and sc.l < sc.so and sc.so < sc.sm and sc.sm < sc.sl); + try std.testing.expectEqual(base + bytes, sc.sl + cuda.tree_attention.slice_items * 4 * 6 * 4); + }; +} diff --git a/zig/tests/cuda/act_rows_tests.zig b/zig/tests/cuda/act_rows_tests.zig new file mode 100644 index 0000000000..f525e7da63 --- /dev/null +++ b/zig/tests/cuda/act_rows_tests.zig @@ -0,0 +1,330 @@ +//! act_rows.cu and norms.cu's bf16-residual and gated rows against float64, group sums, bits at every row count. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const nref = core.norms_ref; +const aref = core.act_ref; +const counts = [_]usize{ 1, 3, 17, 64 }; +const most = counts[counts.len - 1]; +const eps: f32 = 1e-6; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn bf(h: u16) f64 { + return @as(f32, @bitCast(@as(u32, h) << 16)); +} + +const Rig = struct { + gpu: check.Gpu, + a: std.mem.Allocator, + s: cuda.Stream, + rng: std.Random, + bufs: std.ArrayList(cuda.DeviceBuffer) = .empty, + worst: f64 = 0, + + fn dev(r: *Rig, bytes: []const u8) !u64 { + const b = try cuda.DeviceBuffer.fromHost(r.gpu.d, bytes); + try r.bufs.append(r.gpu.gpa, b); + return b.ptr; + } + + fn out(r: *Rig, bytes: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(r.gpu.d, bytes); + try r.bufs.append(r.gpu.gpa, b); + try b.fill8(0xff, r.s.handle); // a value the kernel misses reads NaN + return b.ptr; + } + + fn back(r: *Rig, comptime T: type, ptr: u64, n: usize) ![]T { + const o = try r.a.alloc(T, n); + try r.s.synchronize(); + const b: cuda.DeviceBuffer = .{ .d = r.gpu.d, .ptr = ptr, .len = n * @sizeOf(T) }; + try b.download(0, std.mem.sliceAsBytes(o)); + return o; + } + + /// bf16 values: mostly within [-scale, scale), one in 64 twenty times larger. + fn bfs(r: *Rig, n: usize, scale: f32) []u16 { + const v = r.a.alloc(u16, n) catch unreachable; + for (v, 0..) |*x, i| x.* = bf16((r.rng.float(f32) * 2 - 1) * scale * (if (i % 64 == 7) @as(f32, 20) else 1)); + return v; + } + + /// Weights near 1. + fn weights(r: *Rig, n: usize) []f32 { + const v = r.a.alloc(f32, n) catch unreachable; + for (v) |*x| x.* = 0.5 + r.rng.float(f32); + return v; + } + + fn within(r: *Rig, what: []const u8, got: f64, want: f64, tol: f64) !void { + const q = @abs(got - want) / tol; + if (!(q <= 1)) { + try check.expect(false, "{s}: within float64 tolerance (got {d}, want {d}, {d:.3} tolerances)", .{ what, got, want, q }); + } + r.worst = @max(r.worst, q); + } + + /// Group sums `xs` of the stored bf16 rows `y` ([rows, k]). + fn sums(r: *Rig, what: []const u8, y: []const u16, xs: []const f32, rows: usize, k: usize) !void { + var row: [64]f64 = undefined; + for (0..rows) |i| for (0..k / 64) |j| { + for (&row, 0..) |*v, e| v.* = bf(y[i * k + 64 * j + e]); + var want: [1]f64 = undefined; + nref.groupSums(&row, &want); + try r.within(what, xs[i * (k / 64) + j], want[0], nref.groupSumTolerance(&row)); + }; + } +}; + +fn addRmsH(r: *Rig, n: *const cuda.norms.Norms, width: usize, residual: bool, wk: cuda.norms.Kind) !void { + const x = r.bfs(most * width, 1); + const rv = r.bfs(most * width, 1); + const wf = r.weights(width); + const wb = try r.a.alloc(u16, width); + for (wb, wf) |*o, v| o.* = bf16(v); + const wvals = try r.a.alloc(f64, width); + for (wvals, wf, wb) |*o, f, b| o.* = if (wk == .bf16) bf(b) else f; + const dx = try r.dev(std.mem.sliceAsBytes(x)); + const dr = try r.dev(std.mem.sliceAsBytes(rv)); + const dw = try r.dev(if (wk == .bf16) std.mem.sliceAsBytes(wb) else std.mem.sliceAsBytes(wf)); + var hs: [counts.len]u64 = undefined; + var ys: [counts.len]u64 = undefined; + var xss: [counts.len]u64 = undefined; + for (counts, 0..) |m, c| { + hs[c] = try r.out(m * width * 2); + ys[c] = try r.out(m * width * 2); + xss[c] = try r.out(m * width / 64 * 4); + try n.addRmsH(r.s, dx, if (residual) dr else null, dw, wk, hs[c], ys[c], xss[c], m, width, eps); + } + var name: [96]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "add_rms_h width {d}{s}, {t} weights", .{ width, if (residual) "" else ", no residual", wk }); + const h = try r.back(u16, hs[counts.len - 1], most * width); + const y = try r.back(u16, ys[counts.len - 1], most * width); + const xs = try r.back(f32, xss[counts.len - 1], most * width / 64); + const hv = try r.a.alloc(f64, width); + const want = try r.a.alloc(f64, width); + for (0..most) |row| { + for (hv, 0..) |*v, i| { + const at = row * width + i; + if (residual) { + const sum = bf(x[at]) + bf(rv[at]); + try r.within(label, bf(h[at]), sum, nref.tolerance(sum, .bf16)); + v.* = bf(h[at]); + } else v.* = bf(x[at]); + } + nref.rms(hv, wvals, eps, want); + for (want, 0..) |w, i| try r.within(label, bf(y[row * width + i]), w, nref.tolerance(w, .bf16)); + } + try r.sums(label, y, xs, most, width); + for (counts[0 .. counts.len - 1], 0..) |m, c| { + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(u16, ys[c], m * width)), std.mem.sliceAsBytes(y[0 .. m * width])); + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(f32, xss[c], m * width / 64)), std.mem.sliceAsBytes(xs[0 .. m * width / 64])); + if (residual) try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(u16, hs[c], m * width)), std.mem.sliceAsBytes(h[0 .. m * width])); + } +} + +fn gatedRms(r: *Rig, n: *const cuda.norms.Norms, heads: usize, dv: usize) !void { + const width = heads * dv; + const y_ld = width + 64; // a stacked row with something after the heads + const z_ld = 2 * width; + const y = r.bfs(most * y_ld, 1); + const z = r.bfs(most * z_ld, 3); + const wb = try r.a.alloc(u16, dv); + for (wb, r.weights(dv)) |*o, v| o.* = bf16(v); + const wvals = try r.a.alloc(f64, dv); + for (wvals, wb) |*o, b| o.* = bf(b); + const dy = try r.dev(std.mem.sliceAsBytes(y)); + const dz = try r.dev(std.mem.sliceAsBytes(z)); + const dw = try r.dev(std.mem.sliceAsBytes(wb)); + var os: [counts.len]u64 = undefined; + var xss: [counts.len]u64 = undefined; + for (counts, 0..) |m, c| { + os[c] = try r.out(m * width * 2); + xss[c] = try r.out(m * width / 64 * 4); + try n.gatedRms(r.s, dy, y_ld, dz, z_ld, dw, os[c], xss[c], m, heads, dv, eps); + } + var name: [96]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "gated rms {d} heads of {d}", .{ heads, dv }); + const o = try r.back(u16, os[counts.len - 1], most * width); + const xs = try r.back(f32, xss[counts.len - 1], most * width / 64); + const yv = try r.a.alloc(f64, dv); + const zv = try r.a.alloc(f64, dv); + const want = try r.a.alloc(f64, dv); + for (0..most) |row| for (0..heads) |hd| { + for (yv, zv, 0..) |*a, *b, e| { + a.* = bf(y[row * y_ld + hd * dv + e]); + b.* = bf(z[row * z_ld + hd * dv + e]); + } + nref.gatedRms(yv, zv, wvals, eps, want); + for (want, 0..) |w, e| try r.within(label, bf(o[(row * heads + hd) * dv + e]), w, aref.tolerance(w)); + }; + try r.sums(label, o, xs, most, width); + for (counts[0 .. counts.len - 1], 0..) |m, c| { + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(u16, os[c], m * width)), std.mem.sliceAsBytes(o[0 .. m * width])); + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(f32, xss[c], m * width / 64)), std.mem.sliceAsBytes(xs[0 .. m * width / 64])); + } +} + +fn swiglu(r: *Rig, g: *const cuda.act_rows.ActRows, n: usize) !void { + const gu = r.bfs(most * 2 * n, 3); // [gate | up] rows + const d = try r.dev(std.mem.sliceAsBytes(gu)); + var os: [counts.len]u64 = undefined; + var xss: [counts.len]u64 = undefined; + for (counts, 0..) |m, c| { + os[c] = try r.out(m * n * 2); + xss[c] = try r.out(m * n / 64 * 4); + try g.swiglu(r.s, d, 2 * n, d + 2 * n, 2 * n, os[c], xss[c], m, n); + } + var name: [64]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "swiglu {d}", .{n}); + const o = try r.back(u16, os[counts.len - 1], most * n); + const xs = try r.back(f32, xss[counts.len - 1], most * n / 64); + for (0..most) |row| for (0..n) |i| { + const w = aref.swiglu(bf(gu[row * 2 * n + i]), bf(gu[row * 2 * n + n + i])); + try r.within(label, bf(o[row * n + i]), w, aref.tolerance(w)); + }; + try r.sums(label, o, xs, most, n); + for (counts[0 .. counts.len - 1], 0..) |m, c| + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(u16, os[c], m * n)), std.mem.sliceAsBytes(o[0 .. m * n])); +} + +fn outputGate(r: *Rig, g: *const cuda.act_rows.ActRows, heads: usize, d: usize) !void { + const n = heads * d; + const o = r.bfs(most * n, 1); + const qg = r.bfs(most * 2 * n, 3); // [q_h | gate_h] a head + const do_ = try r.dev(std.mem.sliceAsBytes(o)); + const dq = try r.dev(std.mem.sliceAsBytes(qg)); + var os: [counts.len]u64 = undefined; + var xss: [counts.len]u64 = undefined; + for (counts, 0..) |m, c| { + os[c] = try r.out(m * n * 2); + xss[c] = try r.out(m * n / 64 * 4); + try g.outputGate(r.s, do_, dq + 2 * d, 2 * n, 2 * d, os[c], xss[c], m, heads, d); + } + var name: [64]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "output gate {d} heads of {d}", .{ heads, d }); + const got = try r.back(u16, os[counts.len - 1], most * n); + const xs = try r.back(f32, xss[counts.len - 1], most * n / 64); + for (0..most) |row| for (0..heads) |h| for (0..d) |e| { + const w = bf(o[row * n + h * d + e]) * aref.sigmoid(bf(qg[row * 2 * n + h * 2 * d + d + e])); + try r.within(label, bf(got[row * n + h * d + e]), w, aref.tolerance(w)); + }; + try r.sums(label, got, xs, most, n); + for (counts[0 .. counts.len - 1], 0..) |m, c| + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(u16, os[c], m * n)), std.mem.sliceAsBytes(got[0 .. m * n])); +} + +fn groupSums(r: *Rig, g: *const cuda.act_rows.ActRows, k: usize) !void { + const ld = k + 64; + const x = r.bfs(most * ld, 2); + const d = try r.dev(std.mem.sliceAsBytes(x)); + const xsd = try r.out(most * k / 64 * 4); + try g.groupSums(r.s, d, ld, xsd, most, k); + const xs = try r.back(f32, xsd, most * k / 64); + const dense = try r.a.alloc(u16, most * k); + for (0..most) |row| @memcpy(dense[row * k ..][0..k], x[row * ld ..][0..k]); + try r.sums("group sums", dense, xs, most, k); +} + +fn embed(r: *Rig, d: usize) !void { + var e = try cuda.embed_rows.EmbedRows.load(r.gpu.d); + defer e.unload(); + const vocab = 300; + const w = try r.a.alloc(u32, vocab * d / 8); + for (w) |*x| x.* = r.rng.int(u32); + const sc = r.bfs(vocab * d / 64, 0.1); + const bi = r.bfs(vocab * d / 64, 0.5); + const tab = r.bfs(vocab * d, 1); + const ids = try r.a.alloc(i32, most); + for (ids) |*x| x.* = @intCast(r.rng.uintLessThan(usize, vocab)); + const did = try r.dev(std.mem.sliceAsBytes(ids)); + const o4 = try r.out(most * d * 2); + const o16 = try r.out(most * d * 2); + try e.affine4(r.s, did, try r.dev(std.mem.sliceAsBytes(w)), try r.dev(std.mem.sliceAsBytes(sc)), try r.dev(std.mem.sliceAsBytes(bi)), o4, most, d); + try e.bf16(r.s, did, try r.dev(std.mem.sliceAsBytes(tab)), o16, most, d); + const g4 = try r.back(u16, o4, most * d); + const g16 = try r.back(u16, o16, most * d); + for (0..most) |row| { + const t: usize = @intCast(ids[row]); + for (0..d) |i| { + const q: u4 = @truncate(w[t * d / 8 + i / 8] >> @intCast(4 * (i % 8))); + const want = aref.affine4(q, bf(sc[t * d / 64 + i / 64]), bf(bi[t * d / 64 + i / 64])); + try r.within("embed affine4", bf(g4[row * d + i]), want, nref.tolerance(want, .bf16)); + } + try check.sameBytes("embed bf16", std.mem.sliceAsBytes(g16[row * d ..][0..d]), std.mem.sliceAsBytes(tab[t * d ..][0..d])); + } +} + +fn dconv(r: *Rig, g: *const cuda.act_rows.ActRows, d: usize, gs: usize, seg: usize, branch: usize, residual: bool) !void { + const x = r.bfs(most * d, 1); + const dyn = r.bfs(most * 4 * (d / gs), 0.3); + const base = r.bfs(4 * d, 0.7); + const res = r.bfs(most * d, 1); + const dx = try r.dev(std.mem.sliceAsBytes(x)); + const ddyn = try r.dev(std.mem.sliceAsBytes(dyn)); + const dbase = try r.dev(std.mem.sliceAsBytes(base)); + const dres = try r.dev(std.mem.sliceAsBytes(res)); + var os: [counts.len]u64 = undefined; + for (counts, 0..) |m, c| { + os[c] = try r.out(m * d * 2); + try g.dconv(r.s, dx, ddyn, dbase, if (residual) dres else null, os[c], m, d, gs, branch, seg); + } + var name: [64]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "dconv d {d} gs {d} seg {d} branch {d}{s}", .{ d, gs, seg, branch, if (residual) " +res" else "" }); + const got = try r.back(u16, os[counts.len - 1], most * d); + const half = std.math.pow(f64, 2, -8); + for (0..most) |row| for (0..d) |i| { + const grp = i / gs; + const k0 = bf(bf16(@as(f32, @floatCast(bf(base[(branch * 2) * d + i]))) + @as(f32, @floatCast(bf(dyn[((row * 2 + branch) * 2) * (d / gs) + grp]))))); + const k1 = bf(bf16(@as(f32, @floatCast(bf(base[(branch * 2 + 1) * d + i]))) + @as(f32, @floatCast(bf(dyn[((row * 2 + branch) * 2 + 1) * (d / gs) + grp]))))); + const xv = bf(x[row * d + i]); + const pv = if (row % seg > 0) bf(x[(row - 1) * d + i]) else 0; + const y = aref.dconv(xv, pv, k0, k1); + const yt = (@abs(xv * k0) + @abs(pv * k1)) * std.math.pow(f64, 2, -22); + if (residual) { + const rv = bf(res[row * d + i]); + try r.within(label, bf(got[row * d + i]), rv + y, @abs(rv + y) * half + @abs(y) * half + yt + 1e-30); + } else try r.within(label, bf(got[row * d + i]), y, @abs(y) * half + yt + 1e-30); + }; + for (counts[0 .. counts.len - 1], 0..) |m, c| + try check.sameBytes(label, std.mem.sliceAsBytes(try r.back(u16, os[c], m * d)), std.mem.sliceAsBytes(got[0 .. m * d])); +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var prng = std.Random.DefaultPrng.init(27); + var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = try cuda.Stream.init(gpu.d, false), .rng = prng.random() }; + defer { + for (r.bufs.items) |*b| b.free(); + r.bufs.deinit(gpu.gpa); + r.s.deinit(); + } + var n = try cuda.norms.Norms.load(gpu.d); + defer n.unload(); + var g = try cuda.act_rows.ActRows.load(gpu.d); + defer g.unload(); + try addRmsH(&r, &n, 5120, true, .bf16); // Qwen3.8-27B's residual + try addRmsH(&r, &n, 5120, false, .bf16); // the first layer + try addRmsH(&r, &n, 2048, true, .f32); + try gatedRms(&r, &n, 48, 128); // the 27B's GDN output + try gatedRms(&r, &n, 5, 64); + try gatedRms(&r, &n, 3, 256); + try swiglu(&r, &g, 17408); // the 27B's MLP + try swiglu(&r, &g, 192); + try outputGate(&r, &g, 24, 256); // the 27B's attention output gate + try outputGate(&r, &g, 3, 64); + try groupSums(&r, &g, 5120); + try embed(&r, 5120); // the 27B's hidden + try embed(&r, 192); + try dconv(&r, &g, 5120, 16, 16, 0, false); // DFlash2's block conv (groups of 16) + try dconv(&r, &g, 5120, 16, 16, 1, true); + try dconv(&r, &g, 256, 64, 7, 0, true); + check.pass("act rows: add_rms_h, gated rms, swiglu, output gate, embed rows, dconv within float64 tolerance (worst {d:.3}), group sums of the stored outputs, every row count's bits equal", .{r.worst}); +} diff --git a/zig/tests/cuda/block_tests.zig b/zig/tests/cuda/block_tests.zig new file mode 100644 index 0000000000..48d07a021c --- /dev/null +++ b/zig/tests/cuda/block_tests.zig @@ -0,0 +1,152 @@ +//! block_attention.cu against core/attention_ref.zig: streams' blocks over windowed contexts, a stream's bits alone. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const D = 128; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn bf(h: u16) f64 { + return @as(f32, @bitCast(@as(u32, h) << 16)); +} + +const Bufs = struct { + gpu: check.Gpu, + list: std.ArrayList(cuda.DeviceBuffer) = .empty, + + fn up(b: *Bufs, bytes: []const u8) !u64 { + const d = try cuda.DeviceBuffer.fromHost(b.gpu.d, bytes); + try b.list.append(b.gpu.gpa, d); + return d.ptr; + } + + fn out(b: *Bufs, s: cuda.Stream, n: usize) !u64 { + const d = try cuda.DeviceBuffer.alloc(b.gpu.d, n); + try b.list.append(b.gpu.gpa, d); + try d.fill8(0xff, s.handle); + return d.ptr; + } + + fn back(b: *Bufs, a: std.mem.Allocator, ptr: u64, n: usize) ![]u16 { + const o = try a.alloc(u16, n); + const d: cuda.DeviceBuffer = .{ .d = b.gpu.d, .ptr = ptr, .len = n * 2 }; + try d.download(0, std.mem.sliceAsBytes(o)); + return o; + } +}; + +/// Streams `which` of the case: their blocks packed head-major, their contexts; the output rows. +fn call(b: *Bufs, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.block_attention.BlockAttention, which: []const usize, q: []const u16, kb: []const u16, vb: []const u16, ctx_k: []const u64, ctx_v: []const u64, lens: []const i32, h: usize, hk: usize, L: usize, all_rows: usize, window: usize, causal: bool) ![]u16 { + const rows = which.len * L; + const pq = try a.alloc(u16, h * rows * D); + const pk = try a.alloc(u16, hk * rows * D); + const pv = try a.alloc(u16, hk * rows * D); + const table = try a.alloc(i64, 2 * which.len); + const ls = try a.alloc(i32, which.len); + for (which, 0..) |j, i| { + for (0..h) |hd| @memcpy(pq[(hd * rows + i * L) * D ..][0 .. L * D], q[(hd * all_rows + j * L) * D ..][0 .. L * D]); + for (0..hk) |hd| { + @memcpy(pk[(hd * rows + i * L) * D ..][0 .. L * D], kb[(hd * all_rows + j * L) * D ..][0 .. L * D]); + @memcpy(pv[(hd * rows + i * L) * D ..][0 .. L * D], vb[(hd * all_rows + j * L) * D ..][0 .. L * D]); + } + table[2 * i] = @intCast(ctx_k[j]); + table[2 * i + 1] = @intCast(ctx_v[j]); + ls[i] = lens[j]; + } + const o = try b.out(s, rows * h * D * 2); + const sc: f32 = @floatCast(1 / @sqrt(@as(f64, D))); + try g.run(s, try b.up(std.mem.sliceAsBytes(pq)), try b.up(std.mem.sliceAsBytes(pk)), try b.up(std.mem.sliceAsBytes(pv)), try b.up(std.mem.sliceAsBytes(table)), try b.up(std.mem.sliceAsBytes(ls)), o, .{ .h = h, .hk = hk, .rows = rows, .block = L, .window = window, .scale = sc, .causal = causal }); + try s.synchronize(); + return b.back(a, o, rows * h * D); +} + +fn case(gpu: check.Gpu, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.block_attention.BlockAttention, rng: std.Random, L: usize, causal: bool, worst: *f64) !void { + var b: Bufs = .{ .gpu = gpu }; + defer { + for (b.list.items) |*d| d.free(); + b.list.deinit(gpu.gpa); + } + const h = 32; + const hk = 8; + const window = 2048; + const lens = [_]i32{ 5, 700, 2100 }; + const S = lens.len; + const rows = S * L; + const q = try a.alloc(u16, h * rows * D); + const kb = try a.alloc(u16, hk * rows * D); + const vb = try a.alloc(u16, hk * rows * D); + for (q) |*x| x.* = bf16(rng.floatNorm(f32)); + for (kb) |*x| x.* = bf16(rng.floatNorm(f32)); + for (vb) |*x| x.* = bf16(rng.floatNorm(f32)); + var ck: [S][]u16 = undefined; + var cv: [S][]u16 = undefined; + var pk: [S]u64 = undefined; + var pv: [S]u64 = undefined; + for (lens, 0..) |n, j| { + const m: usize = @intCast(n); + ck[j] = try a.alloc(u16, hk * m * D); + cv[j] = try a.alloc(u16, hk * m * D); + for (ck[j]) |*x| x.* = bf16(rng.floatNorm(f32)); + for (cv[j]) |*x| x.* = bf16(rng.floatNorm(f32)); + pk[j] = try b.up(std.mem.sliceAsBytes(ck[j])); + pv[j] = try b.up(std.mem.sliceAsBytes(cv[j])); + } + const all = [_]usize{ 0, 1, 2 }; + const got = try call(&b, a, s, g, &all, q, kb, vb, &pk, &pv, &lens, h, hk, L, rows, window, causal); + const scale = 1 / @sqrt(@as(f64, D)); + const kd = try a.alloc(f64, (window + 1 + L) * D); + const vd = try a.alloc(f64, kd.len); + const keys = try a.alloc(usize, window + 1 + L); + const qv = try a.alloc(f64, D); + const want = try a.alloc(f64, D); + const tol = try a.alloc(f64, D); + var most: f64 = 0; + for (0..S) |j| for (0..L) |r| for (0..h) |hd| { + const kh = hd / (h / hk); + const n: usize = @intCast(lens[j]); + var nk: usize = 0; + for (0..n) |k| if (n + r - k <= window) { + for (0..D) |e| { + kd[nk * D + e] = bf(ck[j][(kh * n + k) * D + e]); + vd[nk * D + e] = bf(cv[j][(kh * n + k) * D + e]); + } + keys[nk] = nk; + nk += 1; + }; + for (0..L) |c| if (!causal or c <= r) { + for (0..D) |e| { + kd[nk * D + e] = bf(kb[(kh * rows + j * L + c) * D + e]); + vd[nk * D + e] = bf(vb[(kh * rows + j * L + c) * D + e]); + } + keys[nk] = nk; + nk += 1; + }; + for (qv, 0..) |*x, e| x.* = bf(q[(hd * rows + j * L + r) * D + e]); + core.attention_ref.head(qv, kd, vd, keys[0..nk], D, scale, want, tol); + for (want, tol, 0..) |w, t, e| most = @max(most, @abs(bf(got[((j * L + r) * h + hd) * D + e]) - w) / t); + }; + worst.* = @max(worst.*, most); + try check.expect(most <= 1, "block attention L {d}{s}: within float64 tolerance (worst {d:.3})", .{ L, if (causal) " causal" else "", most }); + const one = [_]usize{2}; + const alone = try call(&b, a, s, g, &one, q, kb, vb, &pk, &pv, &lens, h, hk, L, rows, window, causal); + try check.sameBytes("block attention: one stream alone", std.mem.sliceAsBytes(alone), std.mem.sliceAsBytes(got[2 * L * h * D ..][0 .. L * h * D])); +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try cuda.block_attention.BlockAttention.load(gpu.d); + defer g.unload(); + var prng = std.Random.DefaultPrng.init(16); + var worst: f64 = 0; + try case(gpu, arena.allocator(), s, &g, prng.random(), 16, false, &worst); // Qwen3.8-27B's DFlash2: 32/8 heads, a 2,048 window + try case(gpu, arena.allocator(), s, &g, prng.random(), 7, true, &worst); + check.pass("block attention: windowed contexts then the block, within float64 tolerance (worst {d:.3}), a stream's bits alone", .{worst}); +} diff --git a/zig/tests/cuda/dense_rows_tests.zig b/zig/tests/cuda/dense_rows_tests.zig new file mode 100644 index 0000000000..7ec56a424b --- /dev/null +++ b/zig/tests/cuda/dense_rows_tests.zig @@ -0,0 +1,79 @@ +//! dense_rows.cu against core/dense_ref.zig's float64 rows, and every row's bits at every row count. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const counts = [_]usize{ 1, 3, 17, 40, 300 }; +const most = counts[counts.len - 1]; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn case(gpu: check.Gpu, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.dense_rows.DenseRows, rng: std.Random, n: usize, k: usize, worst: *f64) !void { + const xs = try a.alloc(u16, most * k); + const xv = try a.alloc(f64, most * k); + for (xs, xv, 0..) |*h, *v, i| { + h.* = bf16(rng.floatNorm(f32) * (if (i % 61 == 0) @as(f32, 12) else 1)); + v.* = @as(f32, @bitCast(@as(u32, h.*) << 16)); + } + const ws = try a.alloc(u16, n * k); + const wv = try a.alloc(f64, n * k); + for (ws, wv) |*h, *v| { + h.* = bf16(rng.floatNorm(f32) * 0.02); + v.* = @as(f32, @bitCast(@as(u32, h.*) << 16)); + } + var bufs: std.ArrayList(cuda.DeviceBuffer) = .empty; + defer { + for (bufs.items) |*b| b.free(); + bufs.deinit(gpu.gpa); + } + const dx = try cuda.DeviceBuffer.fromHost(gpu.d, std.mem.sliceAsBytes(xs)); + try bufs.append(gpu.gpa, dx); + const dw = try cuda.DeviceBuffer.fromHost(gpu.d, std.mem.sliceAsBytes(ws)); + try bufs.append(gpu.gpa, dw); + const outs = try a.alloc(cuda.DeviceBuffer, counts.len); + for (counts, outs) |m, *o| { + o.* = try cuda.DeviceBuffer.alloc(gpu.d, m * n * 4); + try bufs.append(gpu.gpa, o.*); + try o.fill8(0xff, s.handle); + try g.rows(s, dx.ptr, k, dw.ptr, o.ptr, n, m, n, k); + } + try s.synchronize(); + const wide = try a.alloc(u8, most * n * 4); + try outs[counts.len - 1].download(0, wide); + const want = try a.alloc(f64, n); + const tol = try a.alloc(f64, n); + var q_most: f64 = 0; + for (0..most) |r| { + core.dense_ref.row(xv[r * k ..][0..k], wv, k, want, tol); + for (want, tol, 0..) |w, t, j| { + const got: f32 = @bitCast(std.mem.readInt(u32, wide[4 * (r * n + j) ..][0..4], .little)); + q_most = @max(q_most, @abs(got - w) / t); + } + } + worst.* = @max(worst.*, q_most); + try check.expect(q_most <= 1, "dense rows [{d}, {d}]: within float64 tolerance (worst {d:.3})", .{ n, k, q_most }); + for (counts[0 .. counts.len - 1], outs[0 .. counts.len - 1]) |m, o| { + const got = try a.alloc(u8, m * n * 4); + try o.download(0, got); + try check.sameBytes("dense rows: a row count's bits equal the widest call's", got, wide[0 .. m * n * 4]); + } +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try cuda.dense_rows.DenseRows.load(gpu.d); + defer g.unload(); + var prng = std.Random.DefaultPrng.init(29); + var worst: f64 = 0; + try case(gpu, arena.allocator(), s, &g, prng.random(), 384, 2560, &worst); // Kolibri 1's router + try case(gpu, arena.allocator(), s, &g, prng.random(), 4103, 2560, &worst); // a head's shape, odd columns + try case(gpu, arena.allocator(), s, &g, prng.random(), 257, 7168, &worst); + check.pass("dense rows: within float64 tolerance (worst {d:.3}), every row count's bits equal", .{worst}); +} diff --git a/zig/tests/cuda/gdn_prep_tests.zig b/zig/tests/cuda/gdn_prep_tests.zig new file mode 100644 index 0000000000..1ad46ec614 --- /dev/null +++ b/zig/tests/cuda/gdn_prep_tests.zig @@ -0,0 +1,156 @@ +//! gdn_prep.cu against core/gdn_prep_ref.zig: random conv windows over two streams, bits at every row count. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const ref = core.gdn_prep_ref; +const counts = [_]usize{ 1, 3, 17, 40 }; +const most = counts[counts.len - 1]; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn bf(h: u16) f64 { + return @as(f32, @bitCast(@as(u32, h) << 16)); +} + +fn bfs(a: std.mem.Allocator, rng: std.Random, n: usize, scale: f32) ![]u16 { + const v = try a.alloc(u16, n); + for (v) |*x| x.* = bf16(rng.floatNorm(f32) * scale); + return v; +} + +const Bufs = struct { + gpu: check.Gpu, + list: std.ArrayList(cuda.DeviceBuffer) = .empty, + + fn up(b: *Bufs, bytes: []const u8) !u64 { + const d = try cuda.DeviceBuffer.fromHost(b.gpu.d, bytes); + try b.list.append(b.gpu.gpa, d); + return d.ptr; + } + + fn out(b: *Bufs, s: cuda.Stream, n: usize) !u64 { + const d = try cuda.DeviceBuffer.alloc(b.gpu.d, @max(n, 16)); + try b.list.append(b.gpu.gpa, d); + try d.fill8(0xff, s.handle); + return d.ptr; + } + + fn back(b: *Bufs, a: std.mem.Allocator, comptime T: type, ptr: u64, n: usize) ![]T { + const o = try a.alloc(T, n); + const d: cuda.DeviceBuffer = .{ .d = b.gpu.d, .ptr = ptr, .len = n * @sizeOf(T) }; + try d.download(0, std.mem.sliceAsBytes(o)); + return o; + } +}; + +fn case(gpu: check.Gpu, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.gdn_prep.GdnPrep, rng: std.Random, sh: cuda.gdn_prep.Shape, worst: *f64) !void { + var b: Bufs = .{ .gpu = gpu }; + defer { + for (b.list.items) |*d| d.free(); + b.list.deinit(gpu.gpa); + } + const c = (2 * sh.kh + sh.vh) * sh.dk; + const taps = sh.keep + 1; + const ld = c + 64; // stacked rows: something after the channels + const ab_ld = sh.vh + 3; + const qkv = try bfs(a, rng, most * ld, 1); + const state = try bfs(a, rng, 2 * sh.keep * c, 1); + const conv = try bfs(a, rng, c * taps, 0.4); + const av = try bfs(a, rng, most * ab_ld, 2); + const bv = try bfs(a, rng, most * ab_ld, 2); + const a_log = try a.alloc(f32, sh.vh); + const dtb = try a.alloc(f32, sh.vh); + for (a_log, dtb) |*x, *y| { + x.* = rng.float(f32) * 2 - 1; + y.* = rng.floatNorm(f32); + } + const sid = try a.alloc(i32, most); + const win = try a.alloc(i32, most * taps); + for (sid, 0..) |*x, i| x.* = @intCast(i % 2); + for (win, 0..) |*w, i| { // kept rows or any qkv row (a tree's window is any rows) + w.* = if (rng.uintLessThan(u32, 3) == 0) @intCast(rng.uintLessThan(usize, sh.keep)) else @intCast(sh.keep + rng.uintLessThan(usize, most)); + if (i % taps == taps - 1) w.* = @intCast(sh.keep + i / taps); // the row itself last + } + const base: cuda.gdn_prep.Io = .{ .qkv = try b.up(std.mem.sliceAsBytes(qkv)), .qkv_ld = @intCast(ld), .state = try b.up(std.mem.sliceAsBytes(state)), .conv = try b.up(std.mem.sliceAsBytes(conv)), .win = try b.up(std.mem.sliceAsBytes(win)), .sid = try b.up(std.mem.sliceAsBytes(sid)), .a = try b.up(std.mem.sliceAsBytes(av)), .a_ld = @intCast(ab_ld), .b = try b.up(std.mem.sliceAsBytes(bv)), .b_ld = @intCast(ab_ld), .a_log = try b.up(std.mem.sliceAsBytes(a_log)), .dt_bias = try b.up(std.mem.sliceAsBytes(dtb)), .q = 0, .k = 0, .v = 0, .g = 0, .beta = 0 }; + var ios: [counts.len]cuda.gdn_prep.Io = undefined; + for (counts, &ios) |m, *io| { + io.* = base; + io.q = try b.out(s, m * sh.kh * sh.dk * 2); + io.k = try b.out(s, m * sh.kh * sh.dk * 2); + io.v = try b.out(s, m * sh.vh * sh.dk * 2); + io.g = try b.out(s, m * sh.vh * 4); + io.beta = try b.out(s, m * sh.vh * 4); + try g.run(s, io.*, m, sh); + } + try s.synchronize(); + const io = ios[counts.len - 1]; + const q = try b.back(a, u16, io.q, most * sh.kh * sh.dk); + const k = try b.back(a, u16, io.k, most * sh.kh * sh.dk); + const v = try b.back(a, u16, io.v, most * sh.vh * sh.dk); + const gg = try b.back(a, f32, io.g, most * sh.vh); + const be = try b.back(a, f32, io.beta, most * sh.vh); + var name: [64]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "gdn prep kh {d} vh {d} dk {d} taps {d}", .{ sh.kh, sh.vh, sh.dk, taps }); + const rows = try a.alloc([]f64, taps); + for (rows) |*x| x.* = try a.alloc(f64, sh.dk); + const xs = try a.alloc([]const f64, taps); + const w = try a.alloc(f64, sh.dk * taps); + const cv = try a.alloc(f64, sh.dk); + const err = try a.alloc(f64, sh.dk); + const want = try a.alloc(f64, sh.dk); + var most_q: f64 = 0; + const rel = 3 * std.math.pow(f64, 2, -8) + std.math.pow(f64, 2, -18); // c rounded, the norm through it, the output + for (0..most) |r| for (0..2 * sh.kh + sh.vh) |hd| { + for (rows, xs, 0..) |x, *p, t| { + const src: usize = @intCast(win[r * taps + t]); + for (x, 0..) |*o, i| o.* = if (src < sh.keep) bf(state[(@as(usize, @intCast(sid[r])) * sh.keep + src) * c + hd * sh.dk + i]) else bf(qkv[(src - sh.keep) * ld + hd * sh.dk + i]); + p.* = x; + } + for (0..sh.dk) |i| for (0..taps) |t| { + w[i * taps + t] = bf(conv[(hd * sh.dk + i) * taps + t]); + }; + ref.conv(xs, w, taps, cv, err); + if (hd < 2 * sh.kh) { + const is_q = hd < sh.kh; + const f = ref.norm(cv, is_q, sh.eps, want); + const got = if (is_q) q[(r * sh.kh + hd) * sh.dk ..] else k[(r * sh.kh + hd - sh.kh) * sh.dk ..]; + for (want, err, 0..) |x, e, i| most_q = @max(most_q, @abs(bf(got[i]) - x) / (@abs(x) * rel + f * e + 1e-30)); + } else { + const hv = hd - 2 * sh.kh; + for (cv, err, 0..) |x, e, i| most_q = @max(most_q, @abs(bf(v[(r * sh.vh + hv) * sh.dk + i]) - x) / (@abs(x) * (std.math.pow(f64, 2, -8) + std.math.pow(f64, 2, -20)) + e + 1e-30)); + const x = bf(av[r * ab_ld + hv]) + dtb[hv]; + const sp = if (x > 20) x else std.math.log1p(@exp(x)); + const wg = ref.gate(bf(av[r * ab_ld + hv]), dtb[hv], a_log[hv]); + most_q = @max(most_q, @abs(gg[r * sh.vh + hv] - wg) / (@abs(wg) * std.math.pow(f64, 2, -20) * (4 + 4 * @exp(@as(f64, a_log[hv])) * sp) + 1e-30)); + const wb = 1 / (1 + @exp(-bf(bv[r * ab_ld + hv]))); + most_q = @max(most_q, @abs(be[r * sh.vh + hv] - wb) / (wb * std.math.pow(f64, 2, -19) + 1e-30)); + } + }; + worst.* = @max(worst.*, most_q); + try check.expect(most_q <= 1, "{s}: within float64 tolerance (worst {d:.3})", .{ label, most_q }); + for (counts[0 .. counts.len - 1], ios[0 .. counts.len - 1]) |m, o| { + try check.sameBytes(label, std.mem.sliceAsBytes(try b.back(a, u16, o.q, m * sh.kh * sh.dk)), std.mem.sliceAsBytes(q[0 .. m * sh.kh * sh.dk])); + try check.sameBytes(label, std.mem.sliceAsBytes(try b.back(a, u16, o.v, m * sh.vh * sh.dk)), std.mem.sliceAsBytes(v[0 .. m * sh.vh * sh.dk])); + try check.sameBytes(label, std.mem.sliceAsBytes(try b.back(a, f32, o.g, m * sh.vh)), std.mem.sliceAsBytes(gg[0 .. m * sh.vh])); + } +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try cuda.gdn_prep.GdnPrep.load(gpu.d); + defer g.unload(); + var prng = std.Random.DefaultPrng.init(48); + var worst: f64 = 0; + try case(gpu, arena.allocator(), s, &g, prng.random(), .{ .kh = 16, .vh = 48, .dk = 128, .keep = 3 }, &worst); // Qwen3.8-27B + try case(gpu, arena.allocator(), s, &g, prng.random(), .{ .kh = 2, .vh = 3, .dk = 64, .keep = 3 }, &worst); + try case(gpu, arena.allocator(), s, &g, prng.random(), .{ .kh = 1, .vh = 2, .dk = 256, .keep = 1 }, &worst); + check.pass("gdn prep: conv windows, q/k norms, v, g and beta within float64 tolerance (worst {d:.3}), bits at every row count", .{worst}); +} diff --git a/zig/tests/cuda/main.zig b/zig/tests/cuda/main.zig index 7b8a089947..915ae7c3ed 100644 --- a/zig/tests/cuda/main.zig +++ b/zig/tests/cuda/main.zig @@ -27,6 +27,14 @@ const exlr8_tests = @import("exlr8_tests.zig"); const exlr8_dense_tests = @import("exlr8_dense_tests.zig"); const norms_tests = @import("norms_tests.zig"); const norm_rows_tests = @import("norm_rows_tests.zig"); +const moe_glue_tests = @import("moe_glue_tests.zig"); +const dense_rows_tests = @import("dense_rows_tests.zig"); +const qk_rows_tests = @import("qk_rows_tests.zig"); +const window_tests = @import("window_tests.zig"); +const tree_tests = @import("tree_tests.zig"); +const act_rows_tests = @import("act_rows_tests.zig"); +const gdn_prep_tests = @import("gdn_prep_tests.zig"); +const block_tests = @import("block_tests.zig"); const usage = \\usage: tf-cuda-test @@ -101,6 +109,15 @@ fn run(gpu: check.Gpu, cmd: []const u8, rest: []const [:0]const u8) !void { if (std.mem.eql(u8, cmd, "info")) return info(gpu); if (std.mem.eql(u8, cmd, "smoke")) return runtime_tests.smoke(gpu); if (std.mem.eql(u8, cmd, "graph")) return runtime_tests.graphs(gpu); + if (std.mem.eql(u8, cmd, "norms")) return norms_tests.run(gpu); + if (std.mem.eql(u8, cmd, "moe-glue")) return moe_glue_tests.run(gpu); + if (std.mem.eql(u8, cmd, "dense-rows")) return dense_rows_tests.run(gpu); + if (std.mem.eql(u8, cmd, "qk-rows")) return qk_rows_tests.run(gpu); + if (std.mem.eql(u8, cmd, "window")) return window_tests.run(gpu); + if (std.mem.eql(u8, cmd, "tree")) return tree_tests.run(gpu); + if (std.mem.eql(u8, cmd, "act-rows")) return act_rows_tests.run(gpu); + if (std.mem.eql(u8, cmd, "gdn-prep")) return gdn_prep_tests.run(gpu); + if (std.mem.eql(u8, cmd, "block")) return block_tests.run(gpu); if (std.mem.eql(u8, cmd, "overhead") or std.mem.eql(u8, cmd, "overhead-pdl")) { const n = if (rest.len > 0) try std.fmt.parseInt(usize, rest[0], 10) else 1000; const reps = if (rest.len > 1) try std.fmt.parseInt(usize, rest[1], 10) else 20; diff --git a/zig/tests/cuda/moe_glue_tests.zig b/zig/tests/cuda/moe_glue_tests.zig new file mode 100644 index 0000000000..2445e6dedf --- /dev/null +++ b/zig/tests/cuda/moe_glue_tests.zig @@ -0,0 +1,161 @@ +//! moe_glue.cu against core/moe_ref.zig: picks exact, weights and sums in float64 tolerance, bits at any row count. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const ref = core.moe_ref; +const counts = [_]usize{ 1, 3, 17, 300 }; +const most = counts[counts.len - 1]; + +const Rig = struct { + gpu: check.Gpu, + a: std.mem.Allocator, + s: cuda.Stream, + g: *const cuda.moe_glue.MoeGlue, + rng: std.Random, + bufs: std.ArrayList(cuda.DeviceBuffer) = .empty, + worst: f64 = 0, + + fn dev(r: *Rig, bytes: []const u8) !u64 { + const b = try cuda.DeviceBuffer.fromHost(r.gpu.d, bytes); + try r.bufs.append(r.gpu.gpa, b); + return b.ptr; + } + + fn out(r: *Rig, bytes: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(r.gpu.d, @max(bytes, 4)); + try r.bufs.append(r.gpu.gpa, b); + try b.fill8(0xff, r.s.handle); + return b.ptr; + } + + fn back(r: *Rig, ptr: u64, len: usize) ![]u8 { + const o = try r.a.alloc(u8, len); + try r.s.synchronize(); + const b: cuda.DeviceBuffer = .{ .d = r.gpu.d, .ptr = ptr, .len = len }; + try b.download(0, o); + return o; + } +}; + +fn f32At(raw: []const u8, i: usize) f32 { + return @bitCast(std.mem.readInt(u32, raw[4 * i ..][0..4], .little)); +} + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn topkCase(r: *Rig, route: cuda.moe_glue.Route, with_bias: bool) !void { + const ne = route.experts; + const logits = try r.a.alloc(f32, most * ne); + for (logits, 0..) |*l, i| { + l.* = r.rng.floatNorm(f32) * 2; + if (i % 7 == 3) l.* = logits[i - 1]; // equal neighbours: the tie rule decides + } + const bias = try r.a.alloc(f32, ne); + for (bias, 0..) |*b, i| b.* = if (i % 5 == 0) 0.25 else 0; + const slots = route.topk + @intFromBool(route.shared); + const dl = try r.dev(std.mem.sliceAsBytes(logits)); + const db = try r.dev(std.mem.sliceAsBytes(bias)); + const want_p = try r.a.alloc(u32, most * slots); + const want_w = try r.a.alloc(f64, most * slots); + const rr: ref.Route = .{ .experts = ne, .topk = route.topk, .shared = route.shared, .normalize = route.normalize, .scale = route.scale }; + for (0..most) |row| ref.topk(logits[row * ne ..][0..ne], if (with_bias) bias else null, rr, want_p[row * slots ..][0..slots], want_w[row * slots ..][0..slots]); + var name: [96]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "topk {d} of {d}{s}{s}{s}", .{ route.topk, ne, if (route.shared) " + shared" else "", if (with_bias) ", bias" else "", if (route.normalize) ", normalized" else "" }); + const ps = try r.a.alloc(u64, counts.len); + const ws = try r.a.alloc(u64, counts.len); + for (counts, ps, ws) |m, *p, *w| { + p.* = try r.out(m * slots * 4); + w.* = try r.out(m * slots * 4); + try r.g.topk(r.s, dl, if (with_bias) db else null, route, p.*, w.*, m); + } + const gp = try r.back(ps[counts.len - 1], most * slots * 4); + const gw = try r.back(ws[counts.len - 1], most * slots * 4); + const picks = std.mem.bytesAsSlice(u32, @as([]align(1) u8, gp)); + var q_most: f64 = 0; + for (want_p, 0..) |p, i| try check.expect(picks[i] == p, "{s}: pick {d} is {d}, want {d}", .{ label, i, picks[i], p }); + for (want_w, 0..) |w, i| q_most = @max(q_most, @abs(f32At(gw, i) - w) / (@abs(w) * std.math.pow(f64, 2, -20) + 1e-30)); + r.worst = @max(r.worst, q_most); + try check.expect(q_most <= 1, "{s}: weights within 2^-20 (worst {d:.3})", .{ label, q_most }); + for (counts[0 .. counts.len - 1], ps[0 .. counts.len - 1], ws[0 .. counts.len - 1]) |m, p, w| { + try check.sameBytes(label, try r.back(p, m * slots * 4), gp[0 .. m * slots * 4]); + try check.sameBytes(label, try r.back(w, m * slots * 4), gw[0 .. m * slots * 4]); + } +} + +fn combineCase(r: *Rig, slots: usize, dims: usize, y_bf16: bool) !void { + const n = most * slots * dims; + const yv = try r.a.alloc(f64, n); + const ybytes = try r.a.alloc(u8, n * @as(usize, if (y_bf16) 2 else 4)); + for (yv, 0..) |*v, i| { + const x = r.rng.floatNorm(f32) * (if (i % 97 == 0) @as(f32, 30) else 1); + if (y_bf16) { + const h = bf16(x); + std.mem.writeInt(u16, ybytes[2 * i ..][0..2], h, .little); + v.* = @as(f32, @bitCast(@as(u32, h) << 16)); + } else { + std.mem.writeInt(u32, ybytes[4 * i ..][0..4], @bitCast(x), .little); + v.* = x; + } + } + const w = try r.a.alloc(f32, most * slots); + for (w) |*x| x.* = r.rng.float(f32); + const wv = try r.a.alloc(f64, w.len); + for (w, wv) |x, *v| v.* = x; + const dy = try r.dev(ybytes); + const dw = try r.dev(std.mem.sliceAsBytes(w)); + const want = try r.a.alloc(f64, most * dims); + const tol = try r.a.alloc(f64, most * dims); + for (0..most) |row| { + ref.combine(yv[row * slots * dims ..][0 .. slots * dims], wv[row * slots ..][0..slots], dims, want[row * dims ..][0..dims]); + for (0..dims) |d| { // bf16 half-ulp of the sum, plus fp32 fma error over the slots + var mag: f64 = 0; + for (0..slots) |k| mag += @abs(wv[row * slots + k] * yv[(row * slots + k) * dims + d]); + tol[row * dims + d] = @abs(want[row * dims + d]) * std.math.pow(f64, 2, -8) + mag * @as(f64, @floatFromInt(slots)) * std.math.pow(f64, 2, -24) + 1e-30; + } + } + var name: [96]u8 = undefined; + const label = try std.fmt.bufPrint(&name, "combine {d} slots x {d} ({s} -> bf16)", .{ slots, dims, if (y_bf16) "bf16" else "fp32" }); + const outs = try r.a.alloc(u64, counts.len); + for (counts, outs) |m, *o| { + o.* = try r.out(m * dims * 2); + try r.g.combine(r.s, dy, if (y_bf16) .bf16 else .f32, dw, o.*, .bf16, m, slots, dims); + } + const wide = try r.back(outs[counts.len - 1], most * dims * 2); + var q_most: f64 = 0; + for (want, tol, 0..) |x, t, i| { + const got: f64 = @as(f32, @bitCast(@as(u32, std.mem.readInt(u16, wide[2 * i ..][0..2], .little)) << 16)); + q_most = @max(q_most, @abs(got - x) / t); + } + r.worst = @max(r.worst, q_most); + try check.expect(q_most <= 1, "{s}: within its float64 tolerance (worst {d:.3})", .{ label, q_most }); + for (counts[0 .. counts.len - 1], outs[0 .. counts.len - 1]) |m, o| try check.sameBytes(label, try r.back(o, m * dims * 2), wide[0 .. m * dims * 2]); +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try cuda.moe_glue.MoeGlue.load(gpu.d); + defer g.unload(); + var prng = std.Random.DefaultPrng.init(23); + var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = s, .g = &g, .rng = prng.random() }; + defer { + for (r.bufs.items) |*b| b.free(); + r.bufs.deinit(gpu.gpa); + } + try topkCase(&r, .{ .experts = 384, .topk = 6, .shared = true }, true); // Kolibri 1 + try topkCase(&r, .{ .experts = 256, .topk = 8, .shared = false, .normalize = true, .scale = 2.5 }, true); // DeepSeek-style + try topkCase(&r, .{ .experts = 512, .topk = 32, .shared = true }, true); // the warp form's widest + try topkCase(&r, .{ .experts = 513, .topk = 6, .shared = true }, true); // the block form's narrowest + try topkCase(&r, .{ .experts = 4096, .topk = 64, .shared = false }, false); + try combineCase(&r, 7, 2560, false); + try combineCase(&r, 7, 2560, true); + try combineCase(&r, 9, 7168, false); + check.pass("moe glue: picks exact, weights and sums within float64 tolerance (worst {d:.3}), every row count's bits equal", .{r.worst}); +} diff --git a/zig/tests/cuda/qk_rows_tests.zig b/zig/tests/cuda/qk_rows_tests.zig new file mode 100644 index 0000000000..cdb5105ef3 --- /dev/null +++ b/zig/tests/cuda/qk_rows_tests.zig @@ -0,0 +1,226 @@ +//! qk_rows.cu against core/qk_ref.zig: q, k within float64 tolerance, v and the caches exact, bits at any row count. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const counts = [_]usize{ 1, 3, 17, 300 }; +const most = counts[counts.len - 1]; +const positions = 4096; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn bf(h: u16) f64 { + return @as(f32, @bitCast(@as(u32, h) << 16)); +} + +const Case = struct { h: usize, hk: usize, dim: usize, rot: usize, norm: bool }; + +fn run1(gpu: check.Gpu, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.qk_rows.QkRows, rng: std.Random, c: Case, worst: *f64) !void { + const heads = c.h + 2 * c.hk; + const width = heads * c.dim; + const xs = try a.alloc(u16, most * width); + for (xs, 0..) |*h, i| h.* = bf16(rng.floatNorm(f32) * (if (i % 37 == 0) @as(f32, 8) else 1)); + const nw = try a.alloc(f32, 2 * c.dim); + for (nw) |*w| w.* = 0.5 + rng.float(f32); + const tc = try a.alloc(f32, positions * @max(c.rot, 1)); + const ts = try a.alloc(f32, positions * @max(c.rot, 1)); + for (0..positions) |p| for (0..c.rot) |i| { + const j = i % (c.rot / 2); + const ang = @as(f64, @floatFromInt(p)) / std.math.pow(f64, 10000, @as(f64, @floatFromInt(2 * j)) / @as(f64, @floatFromInt(c.rot))); + tc[p * c.rot + i] = @floatCast(@cos(ang)); + ts[p * c.rot + i] = @floatCast(@sin(ang)); + }; + const len = 600; + const pos = try a.alloc(i32, most); + const slot = try a.alloc(i32, most); + for (pos, slot, 0..) |*p, *sl, i| { + p.* = @intCast((i * 7919 + 13) % positions); + sl.* = @intCast(i % 2); + } + // distinct (slot, pos % len) a row, so no row's cache entry overwrites another's + for (0..most) |i| pos[i] = @intCast((i / 2) % len + len * (i % 5)); + var bufs: std.ArrayList(cuda.DeviceBuffer) = .empty; + defer { + for (bufs.items) |*b| b.free(); + bufs.deinit(gpu.gpa); + } + const up = struct { + fn f(gp: check.Gpu, l: *std.ArrayList(cuda.DeviceBuffer), bytes: []const u8) !u64 { + const b = try cuda.DeviceBuffer.fromHost(gp.d, bytes); + try l.append(gp.gpa, b); + return b.ptr; + } + fn z(gp: check.Gpu, l: *std.ArrayList(cuda.DeviceBuffer), st: cuda.Stream, n: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(gp.d, n); + try l.append(gp.gpa, b); + try b.fill8(0xff, st.handle); + return b.ptr; + } + }; + const dx = try up.f(gpu, &bufs, std.mem.sliceAsBytes(xs)); + const dnw = try up.f(gpu, &bufs, std.mem.sliceAsBytes(nw)); + const dc = try up.f(gpu, &bufs, std.mem.sliceAsBytes(tc)); + const ds = try up.f(gpu, &bufs, std.mem.sliceAsBytes(ts)); + const dp = try up.f(gpu, &bufs, std.mem.sliceAsBytes(pos)); + const dsl = try up.f(gpu, &bufs, std.mem.sliceAsBytes(slot)); + const shape: cuda.qk_rows.Shape = .{ .norms = if (c.norm) dnw else null, .cos = if (c.rot > 0) dc else null, .sin = if (c.rot > 0) ds else null, .rot = c.rot }; + const cache_bytes = 2 * len * c.hk * c.dim * 2; + const outs = try a.alloc(cuda.qk_rows.Out, counts.len); + for (counts, outs) |m, *o| { + o.* = .{ .q = try up.z(gpu, &bufs, s, m * c.h * c.dim * 2), .k = try up.z(gpu, &bufs, s, m * c.hk * c.dim * 2), .v = try up.z(gpu, &bufs, s, m * c.hk * c.dim * 2), .kc = try up.z(gpu, &bufs, s, cache_bytes), .vc = try up.z(gpu, &bufs, s, cache_bytes), .len = len }; + try g.rows(s, dx, .{ .h = c.h, .hk = c.hk, .dim = c.dim }, shape, dp, dsl, o.*, m); + } + try s.synchronize(); + const o = outs[counts.len - 1]; + const back = struct { + fn f(gp: check.Gpu, al: std.mem.Allocator, ptr: u64, n: usize) ![]u16 { + const raw = try al.alloc(u16, n); + const b: cuda.DeviceBuffer = .{ .d = gp.d, .ptr = ptr, .len = n * 2 }; + try b.download(0, std.mem.sliceAsBytes(raw)); + return raw; + } + }; + const gq = try back.f(gpu, a, o.q, most * c.h * c.dim); + const gk = try back.f(gpu, a, o.k, most * c.hk * c.dim); + const gv = try back.f(gpu, a, o.v, most * c.hk * c.dim); + const gkc = try back.f(gpu, a, o.kc.?, cache_bytes / 2); + const gvc = try back.f(gpu, a, o.vc.?, cache_bytes / 2); + const xv = try a.alloc(f64, c.dim); + const wq = try a.alloc(f64, c.dim); + const wk = try a.alloc(f64, c.dim); + for (wq, wk, 0..) |*p, *q, i| { + p.* = nw[i]; + q.* = nw[c.dim + i]; + } + const cv = try a.alloc(f64, @max(c.rot, 1)); + const sv = try a.alloc(f64, @max(c.rot, 1)); + const y = try a.alloc(f64, c.dim); + var q_most: f64 = 0; + for (0..most) |r| { + const p: usize = @intCast(pos[r]); + for (0..c.rot) |i| { + cv[i] = tc[p * c.rot + i]; + sv[i] = ts[p * c.rot + i]; + } + const at = @as(usize, @intCast(slot[r])) * len + p % len; + for (0..heads) |hd| { + for (xv, 0..) |*v, i| v.* = bf(xs[r * width + hd * c.dim + i]); + if (hd >= c.h + c.hk) { // v: exact copies, in the output and the cache + const k = hd - c.h - c.hk; + for (0..c.dim) |i| { + try check.expect(gv[(r * c.hk + k) * c.dim + i] == xs[r * width + hd * c.dim + i], "qk rows: v copied", .{}); + try check.expect(gvc[(at * c.hk + k) * c.dim + i] == xs[r * width + hd * c.dim + i], "qk rows: v cached", .{}); + } + continue; + } + const is_q = hd < c.h; + core.qk_ref.head(xv, if (c.norm) (if (is_q) wq else wk) else null, 1e-6, if (c.rot > 0) cv else null, if (c.rot > 0) sv else null, c.rot, y); + for (y, 0..) |want, i| { + const got = if (is_q) bf(gq[(r * c.h + hd) * c.dim + i]) else bf(gk[(r * c.hk + hd - c.h) * c.dim + i]); + const tol = (@abs(want) + 2 * @abs(xv[i]) + 1e-3) * std.math.pow(f64, 2, -8); + q_most = @max(q_most, @abs(got - want) / tol); + if (!is_q) try check.expect(gkc[(at * c.hk + hd - c.h) * c.dim + i] == gk[(r * c.hk + hd - c.h) * c.dim + i], "qk rows: k cached as output", .{}); + } + } + } + worst.* = @max(worst.*, q_most); + // the same heads from [q | gate] rows and a padded k buffer, no v: the same q, k and k cache bits + const qg = try a.alloc(u16, most * c.h * 2 * c.dim); + const kb = try a.alloc(u16, most * (c.hk * c.dim + 64)); + @memset(qg, 0x7fc0); // the gate halves: NaN, never read + @memset(kb, 0x7fc0); + for (0..most) |r| { + for (0..c.h) |hd| @memcpy(qg[(r * c.h + hd) * 2 * c.dim ..][0..c.dim], xs[r * width + hd * c.dim ..][0..c.dim]); + @memcpy(kb[r * (c.hk * c.dim + 64) ..][0 .. c.hk * c.dim], xs[r * width + c.h * c.dim ..][0 .. c.hk * c.dim]); + } + const src: cuda.qk_rows.Src = .{ .q = try up.f(gpu, &bufs, std.mem.sliceAsBytes(qg)), .q_ld = @intCast(c.h * 2 * c.dim), .q_hs = @intCast(2 * c.dim), .k = try up.f(gpu, &bufs, std.mem.sliceAsBytes(kb)), .k_ld = @intCast(c.hk * c.dim + 64), .k_hs = @intCast(c.dim) }; + const o2: cuda.qk_rows.Out = .{ .q = try up.z(gpu, &bufs, s, most * c.h * c.dim * 2), .k = try up.z(gpu, &bufs, s, most * c.hk * c.dim * 2), .v = 0, .kc = try up.z(gpu, &bufs, s, cache_bytes), .len = len }; + try g.rowsFrom(s, src, .{ .h = c.h, .hk = c.hk, .dim = c.dim }, shape, dp, dsl, o2, most); + try s.synchronize(); + try check.sameBytes("qk rows from strided heads: q", std.mem.sliceAsBytes(try back.f(gpu, a, o2.q, most * c.h * c.dim)), std.mem.sliceAsBytes(gq)); + try check.sameBytes("qk rows from strided heads: k", std.mem.sliceAsBytes(try back.f(gpu, a, o2.k, most * c.hk * c.dim)), std.mem.sliceAsBytes(gk)); + try check.sameBytes("qk rows from strided heads: k cache", std.mem.sliceAsBytes(try back.f(gpu, a, o2.kc.?, cache_bytes / 2)), std.mem.sliceAsBytes(gkc)); + // head-major outputs [heads, rows, dim] (strides): the packed call's bits, rearranged + { + const o4: cuda.qk_rows.Out = .{ .q = try up.z(gpu, &bufs, s, most * c.h * c.dim * 2), .k = try up.z(gpu, &bufs, s, most * c.hk * c.dim * 2), .v = try up.z(gpu, &bufs, s, most * c.hk * c.dim * 2), .len = len, .q_ld = c.dim, .q_hs = most * c.dim, .kv_ld = c.dim, .kv_hs = most * c.dim }; + try g.rows(s, dx, .{ .h = c.h, .hk = c.hk, .dim = c.dim }, shape, dp, dsl, o4, most); + try s.synchronize(); + const hq = try back.f(gpu, a, o4.q, most * c.h * c.dim); + const hk_ = try back.f(gpu, a, o4.k, most * c.hk * c.dim); + const hv = try back.f(gpu, a, o4.v, most * c.hk * c.dim); + for (0..most) |r| { + for (0..c.h) |hd| try check.sameBytes("qk rows head-major: q", std.mem.sliceAsBytes(hq[(hd * most + r) * c.dim ..][0..c.dim]), std.mem.sliceAsBytes(gq[(r * c.h + hd) * c.dim ..][0..c.dim])); + for (0..c.hk) |hd| { + try check.sameBytes("qk rows head-major: k", std.mem.sliceAsBytes(hk_[(hd * most + r) * c.dim ..][0..c.dim]), std.mem.sliceAsBytes(gk[(r * c.hk + hd) * c.dim ..][0..c.dim])); + try check.sameBytes("qk rows head-major: v", std.mem.sliceAsBytes(hv[(hd * most + r) * c.dim ..][0..c.dim]), std.mem.sliceAsBytes(gv[(r * c.hk + hd) * c.dim ..][0..c.dim])); + } + } + } + // bf16 norm weights at their own addresses and RoPE from inv (angle = pos * inv in fp32): float64 tolerance + if (c.norm) { + const wqb = try a.alloc(u16, c.dim); + const wkb = try a.alloc(u16, c.dim); + for (wqb, wkb, 0..) |*p, *q, i| { + p.* = bf16(nw[i]); + q.* = bf16(nw[c.dim + i]); + } + const half = @max(c.rot / 2, 1); + const inv = try a.alloc(f32, half); + for (inv, 0..) |*v, j| v.* = @floatCast(1 / std.math.pow(f64, 10000, @as(f64, @floatFromInt(2 * j)) / @as(f64, @floatFromInt(@max(c.rot, 1))))); + const sh3: cuda.qk_rows.Shape = .{ .norms = try up.f(gpu, &bufs, std.mem.sliceAsBytes(wqb)), .norm_k = try up.f(gpu, &bufs, std.mem.sliceAsBytes(wkb)), .norms_bf16 = true, .inv = if (c.rot > 0) try up.f(gpu, &bufs, std.mem.sliceAsBytes(inv)) else null, .rot = c.rot }; + const o3: cuda.qk_rows.Out = .{ .q = try up.z(gpu, &bufs, s, most * c.h * c.dim * 2), .k = try up.z(gpu, &bufs, s, most * c.hk * c.dim * 2), .v = try up.z(gpu, &bufs, s, most * c.hk * c.dim * 2), .len = len }; + try g.rows(s, dx, .{ .h = c.h, .hk = c.hk, .dim = c.dim }, sh3, dp, dsl, o3, most); + try s.synchronize(); + const q3 = try back.f(gpu, a, o3.q, most * c.h * c.dim); + const k3 = try back.f(gpu, a, o3.k, most * c.hk * c.dim); + for (wq, wk, 0..) |*p, *q, i| { + p.* = bf(wqb[i]); + q.* = bf(wkb[i]); + } + for (0..most) |r| { + const p: usize = @intCast(pos[r]); + for (0..c.rot) |i| { + const ang: f64 = @as(f32, @floatFromInt(p)) * inv[i % half]; + cv[i] = @cos(ang); + sv[i] = @sin(ang); + } + for (0..c.h + c.hk) |hd| { + for (xv, 0..) |*v, i| v.* = bf(xs[r * width + hd * c.dim + i]); + const is_q = hd < c.h; + core.qk_ref.head(xv, if (is_q) wq else wk, 1e-6, if (c.rot > 0) cv else null, if (c.rot > 0) sv else null, c.rot, y); + for (y, 0..) |want, i| { + const got = if (is_q) bf(q3[(r * c.h + hd) * c.dim + i]) else bf(k3[(r * c.hk + hd - c.h) * c.dim + i]); + q_most = @max(q_most, @abs(got - want) / ((@abs(want) + 2 * @abs(xv[i]) + 1e-3) * std.math.pow(f64, 2, -8))); + } + } + } + worst.* = @max(worst.*, q_most); + } + try check.expect(q_most <= 1, "qk rows {d}/{d}x{d}, rot {d}{s}: within float64 tolerance (worst {d:.3})", .{ c.h, c.hk, c.dim, c.rot, if (c.norm) ", normed" else "", q_most }); + for (counts[0 .. counts.len - 1], outs[0 .. counts.len - 1]) |m, oo| { + try check.sameBytes("qk rows: q at a smaller row count", std.mem.sliceAsBytes(try back.f(gpu, a, oo.q, m * c.h * c.dim)), std.mem.sliceAsBytes(gq[0 .. m * c.h * c.dim])); + try check.sameBytes("qk rows: k at a smaller row count", std.mem.sliceAsBytes(try back.f(gpu, a, oo.k, m * c.hk * c.dim)), std.mem.sliceAsBytes(gk[0 .. m * c.hk * c.dim])); + } +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try cuda.qk_rows.QkRows.load(gpu.d); + defer g.unload(); + var prng = std.Random.DefaultPrng.init(31); + var worst: f64 = 0; + const a = arena.allocator(); + try run1(gpu, a, s, &g, prng.random(), .{ .h = 48, .hk = 4, .dim = 128, .rot = 128, .norm = true }, &worst); // Kolibri, sliding + try run1(gpu, a, s, &g, prng.random(), .{ .h = 48, .hk = 4, .dim = 128, .rot = 0, .norm = true }, &worst); // Kolibri, full + try run1(gpu, a, s, &g, prng.random(), .{ .h = 8, .hk = 2, .dim = 64, .rot = 32, .norm = false }, &worst); // partial rotary + try run1(gpu, a, s, &g, prng.random(), .{ .h = 16, .hk = 2, .dim = 128, .rot = 128, .norm = false }, &worst); // a warp a head, no norm + try run1(gpu, a, s, &g, prng.random(), .{ .h = 24, .hk = 4, .dim = 256, .rot = 64, .norm = true }, &worst); // Qwen3.8-27B + check.pass("qk rows: q and k within float64 tolerance (worst {d:.3}), v and the caches exact, bits at any row count", .{worst}); +} diff --git a/zig/tests/cuda/tree_tests.zig b/zig/tests/cuda/tree_tests.zig new file mode 100644 index 0000000000..fedfdb134f --- /dev/null +++ b/zig/tests/cuda/tree_tests.zig @@ -0,0 +1,217 @@ +//! tree_attention.cu against core/attention_ref.zig, and one stream's rows alone giving the same bits. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const tree = cuda.tree_attention; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn bf(h: u16) f64 { + return @as(f32, @bitCast(@as(u32, h) << 16)); +} + +const Set = struct { + gpu: check.Gpu, + a: std.mem.Allocator, + bufs: std.ArrayList(cuda.DeviceBuffer) = .empty, + + fn up(x: *Set, bytes: []const u8) !u64 { + const b = try cuda.DeviceBuffer.fromHost(x.gpu.d, bytes); + try x.bufs.append(x.gpu.gpa, b); + return b.ptr; + } + + fn zero(x: *Set, s: cuda.Stream, n: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(x.gpu.d, @max(n, 16)); + try x.bufs.append(x.gpu.gpa, b); + try b.fill8(0xff, s.handle); + return b.ptr; + } + + fn down(x: *Set, comptime T: type, ptr: u64, n: usize) ![]T { + const o = try x.a.alloc(T, n); + const b: cuda.DeviceBuffer = .{ .d = x.gpu.d, .ptr = ptr, .len = n * @sizeOf(T) }; + try b.download(0, std.mem.sliceAsBytes(o)); + return o; + } +}; + +/// One call over `rows` (global window rows of the given streams, in order): their outputs. +fn call(x: *Set, s: cuda.Stream, g: *const tree.TreeAttention, d: usize, rows: []const usize, row_stream_all: []const i32, parents_all: []const i32, qs: []const u16, kns: []const u16, vns: []const u16, h: usize, hk: usize, streams_all: []const i32, offs: u64, origin: u64, ng: usize, scale: f32) ![]u16 { + const n = rows.len; + var local: std.AutoHashMap(usize, i32) = .init(x.a); + for (rows, 0..) |r, i| try local.put(r, @intCast(i)); + const rs = try x.a.alloc(i32, n); + const par = try x.a.alloc(i32, n); + const q = try x.a.alloc(u16, n * h * d); + const kn = try x.a.alloc(u16, n * hk * d); + const vn = try x.a.alloc(u16, n * hk * d); + for (rows, 0..) |r, i| { + rs[i] = row_stream_all[r]; + par[i] = if (parents_all[r] < 0) -1 else local.get(@intCast(parents_all[r])).?; + @memcpy(q[i * h * d ..][0 .. h * d], qs[r * h * d ..][0 .. h * d]); + @memcpy(kn[i * hk * d ..][0 .. hk * d], kns[r * hk * d ..][0 .. hk * d]); + @memcpy(vn[i * hk * d ..][0 .. hk * d], vns[r * hk * d ..][0 .. hk * d]); + } + // the streams' table for these rows: each stream's first row and rows among them, its committed keys + const st = try x.a.dupe(i32, streams_all); + for (0..st.len / 4) |k| { + st[4 * k] = 0; + st[4 * k + 1] = 0; + } + for (rows, 0..) |r, i| { + const k: usize = @intCast(row_stream_all[r]); + if (st[4 * k + 1] == 0) st[4 * k] = @intCast(i); + st[4 * k + 1] += 1; + } + const streams_dev = try x.up(std.mem.sliceAsBytes(st)); + const paths = try x.zero(s, n * tree.max_depth * 4); + const depths = try x.zero(s, n * 4); + try g.paths(s, try x.up(std.mem.sliceAsBytes(par)), paths, depths, n); + const sc_base = try x.zero(s, tree.TreeAttention.scratchBytes(n, h, hk, ng, d)); + const out = try x.zero(s, n * h * d * 2); + const rws: tree.Rows = .{ .row_stream = try x.up(std.mem.sliceAsBytes(rs)), .streams = streams_dev, .offs = offs, .origin = origin, .paths = paths, .depths = depths }; + try g.run(s, try x.up(std.mem.sliceAsBytes(q)), try x.up(std.mem.sliceAsBytes(kn)), try x.up(std.mem.sliceAsBytes(vn)), rws, tree.TreeAttention.scratch(sc_base, n, h, hk, ng, d), out, n, h, hk, d, ng, scale); + try s.synchronize(); + return x.down(u16, out, n * h * d); +} + +fn case(gpu: check.Gpu, d: usize, h: usize, hk: usize, seed: u64, worst_all: *f64) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + const a = arena.allocator(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try tree.TreeAttention.load(gpu.d); + defer g.unload(); + var x: Set = .{ .gpu = gpu, .a = a }; + defer { + for (x.bufs.items) |*b| b.free(); + x.bufs.deinit(gpu.gpa); + } + var prng = std.Random.DefaultPrng.init(seed); + const rng = prng.random(); + const committed = [_]usize{ 5, 1500, 4200 }; + const widths = [_]usize{ 9, 33, 17 }; + // the caches: stream s's rows at element offset cache_at[s] of one buffer (keys, then values) + var cache_rows: usize = 0; + for (committed) |p| cache_rows += p; + const kcache = try a.alloc(u16, 2 * cache_rows * hk * d); + for (kcache) |*v| v.* = bf16(rng.floatNorm(f32)); + var offs: [2 * committed.len]i64 = undefined; + var at: usize = 0; + for (committed, 0..) |p, si| { + offs[2 * si] = @intCast(at * hk * d); + offs[2 * si + 1] = @intCast((cache_rows + at) * hk * d); + at += p; + } + var total: usize = 0; + for (widths) |w| total += w; + const row_stream = try a.alloc(i32, total); + const parents = try a.alloc(i32, total); + var streams: [4 * committed.len]i32 = undefined; + var start: usize = 0; + for (widths, committed, 0..) |w, p, si| { + for (0..w) |j| { + row_stream[start + j] = @intCast(si); + parents[start + j] = if (j == 0 or rng.uintLessThan(u32, 6) == 0) -1 else @intCast(start + rng.uintLessThan(usize, j)); + } + for ([_]usize{ start, w, p, 0 }, 0..) |v, k| streams[4 * si + k] = @intCast(v); + start += w; + } + const qs = try a.alloc(u16, total * h * d); + const kns = try a.alloc(u16, total * hk * d); + const vns = try a.alloc(u16, total * hk * d); + for (qs) |*v| v.* = bf16(rng.floatNorm(f32)); + for (kns) |*v| v.* = bf16(rng.floatNorm(f32)); + for (vns) |*v| v.* = bf16(rng.floatNorm(f32)); + const origin = try x.up(std.mem.sliceAsBytes(kcache)); + const offs_dev = try x.up(std.mem.sliceAsBytes(&offs)); + var ng: usize = 0; + for (committed, widths) |p, w| ng = @max(ng, tree.TreeAttention.groups(p + w)); + const scale: f32 = @floatCast(1 / @sqrt(@as(f64, @floatFromInt(d)))); + const all = try a.alloc(usize, total); + for (all, 0..) |*r, i| r.* = i; + const got = try call(&x, s, &g, d, all, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, ng, scale); + // the reference: committed keys, then the path root first + var worst: f64 = 0; + const kd = try a.alloc(f64, (4200 + 128) * d); + const vd = try a.alloc(f64, kd.len); + const keys = try a.alloc(usize, 4200 + 128); + const q = try a.alloc(f64, d); + const want = try a.alloc(f64, d); + const tol = try a.alloc(f64, d); + var path: [tree.max_depth]usize = undefined; + for (0..total) |r| { + const si: usize = @intCast(row_stream[r]); + const p = committed[si]; + var depth: usize = 0; + var cur: i32 = @intCast(r); + while (cur >= 0) : (cur = parents[@intCast(cur)]) { + path[depth] = @intCast(cur); + depth += 1; + } + std.mem.reverse(usize, path[0..depth]); + for (0..hk) |kh| { + for (0..p) |j| for (0..d) |e| { + const ko: usize = @intCast(offs[2 * si]); + const vo: usize = @intCast(offs[2 * si + 1]); + kd[j * d + e] = bf(kcache[ko + (j * hk + kh) * d + e]); + vd[j * d + e] = bf(kcache[vo + (j * hk + kh) * d + e]); + }; + for (0..depth) |j| for (0..d) |e| { + kd[(p + j) * d + e] = bf(kns[(path[j] * hk + kh) * d + e]); + vd[(p + j) * d + e] = bf(vns[(path[j] * hk + kh) * d + e]); + }; + for (0..p + depth) |j| keys[j] = j; + for (0..h / hk) |hh| { + const hd = kh * (h / hk) + hh; + for (q, 0..) |*v, e| v.* = bf(qs[(r * h + hd) * d + e]); + core.attention_ref.head(q, kd, vd, keys[0 .. p + depth], d, scale, want, tol); + for (want, tol, 0..) |w, t, e| worst = @max(worst, @abs(bf(got[(r * h + hd) * d + e]) - w) / t); + } + } + } + try check.expect(worst <= 1, "tree attention dim {d}: within float64 tolerance (worst {d:.3})", .{ d, worst }); + worst_all.* = @max(worst_all.*, worst); + // stream 1's rows alone (re-indexed): the same bits + const only = try a.alloc(usize, widths[1]); + for (only, 0..) |*r, i| r.* = widths[0] + i; + const alone = try call(&x, s, &g, d, only, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, ng, scale); + try check.sameBytes("tree attention: one stream's rows alone", std.mem.sliceAsBytes(alone), std.mem.sliceAsBytes(got[widths[0] * h * d ..][0 .. widths[1] * h * d])); + // one shared block walking every (row, group), and a grid sized for a long context: the same bits + g.shared_blocks = 1; + const strided = try call(&x, s, &g, d, all, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, ng, scale); + try check.sameBytes("tree attention: one shared block", std.mem.sliceAsBytes(strided), std.mem.sliceAsBytes(got)); + g.shared_blocks = 96; + const wide = try call(&x, s, &g, d, all, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, 513, scale); + try check.sameBytes("tree attention: groups for a 262k context", std.mem.sliceAsBytes(wide), std.mem.sliceAsBytes(got)); + // one warp a block (a device with SMs to spare): stream 2's rows alone, its full groups too, the batch's bits + g.sms = 1 << 20; + defer g.sms = 0; + const third = try a.alloc(usize, widths[2]); + for (third, 0..) |*r, i| r.* = widths[0] + widths[1] + i; + try check.expect(widths[2] * ng * hk <= tree.slice_items, "tree attention: the one-warp case fits the slices", .{}); + const one = try call(&x, s, &g, d, third, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, ng, scale); + try check.sameBytes("tree attention: one warp a block", std.mem.sliceAsBytes(one), std.mem.sliceAsBytes(got[(widths[0] + widths[1]) * h * d ..][0 .. widths[2] * h * d])); + g.shared_blocks = 1; + const one_strided = try call(&x, s, &g, d, third, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, ng, scale); + try check.sameBytes("tree attention: one warp a block, one shared block", std.mem.sliceAsBytes(one_strided), std.mem.sliceAsBytes(one)); + g.shared_blocks = 96; + g.sms = 1; // more one-warp blocks than SMs: two tile stages instead of three + const two = try call(&x, s, &g, d, third, row_stream, parents, qs, kns, vns, h, hk, &streams, offs_dev, origin, ng, scale); + try check.sameBytes("tree attention: one warp a block, two stages", std.mem.sliceAsBytes(two), std.mem.sliceAsBytes(one)); + g.shared_blocks = 96; +} + +pub fn run(gpu: check.Gpu) !void { + var worst: f64 = 0; + try case(gpu, 128, 48, 4, 41, &worst); // Kolibri + try case(gpu, 256, 24, 4, 42, &worst); // Qwen3.8-27B + check.pass("tree attention: dim 128 and 256, committed keys and paths within float64 tolerance (worst {d:.3}), a stream's bits alone, any group slots, four-warp or one-warp blocks", .{worst}); +} diff --git a/zig/tests/cuda/window_tests.zig b/zig/tests/cuda/window_tests.zig new file mode 100644 index 0000000000..c3d8c68899 --- /dev/null +++ b/zig/tests/cuda/window_tests.zig @@ -0,0 +1,217 @@ +//! window_attention.cu against core/attention_ref.zig: decode rows over a ring of keys, bits at any row count. +const std = @import("std"); +const cuda = @import("cuda"); +const core = @import("core"); +const check = @import("check.zig"); + +const counts = [_]usize{ 1, 3, 17, 64 }; +const most = counts[counts.len - 1]; + +fn bf16(x: f32) u16 { + const u: u32 = @bitCast(x); + return @truncate((u + 0x7fff + ((u >> 16) & 1)) >> 16); +} + +fn bf(h: u16) f64 { + return @as(f32, @bitCast(@as(u32, h) << 16)); +} + +const Case = struct { h: usize, hk: usize, d: usize, win: usize, ring: usize, streams: usize, top: usize }; + +fn run1(gpu: check.Gpu, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.window_attention.WindowAttention, rng: std.Random, c: Case, worst: *f64) !void { + const gq = c.h / c.hk; + const qn = most * c.h * c.d; + const cn = c.streams * c.ring * c.hk * c.d; + const qs = try a.alloc(u16, qn); + const ks = try a.alloc(u16, cn); + const vs = try a.alloc(u16, cn); + for (qs) |*x| x.* = bf16(rng.floatNorm(f32)); + for (ks) |*x| x.* = bf16(rng.floatNorm(f32)); + for (vs) |*x| x.* = bf16(rng.floatNorm(f32)); + const pos = try a.alloc(i32, most); + const slot = try a.alloc(i32, most); + for (pos, slot, 0..) |*p, *sl, i| { + p.* = @intCast(if (i % 4 == 0) i % 40 else (i * 7919) % c.top); // short histories and wrapped rings both + sl.* = @intCast(i % c.streams); + } + var bufs: std.ArrayList(cuda.DeviceBuffer) = .empty; + defer { + for (bufs.items) |*b| b.free(); + bufs.deinit(gpu.gpa); + } + const up = struct { + fn f(gp: check.Gpu, l: *std.ArrayList(cuda.DeviceBuffer), bytes: []const u8) !u64 { + const b = try cuda.DeviceBuffer.fromHost(gp.d, bytes); + try l.append(gp.gpa, b); + return b.ptr; + } + }; + const dq = try up.f(gpu, &bufs, std.mem.sliceAsBytes(qs)); + const dk = try up.f(gpu, &bufs, std.mem.sliceAsBytes(ks)); + const dv = try up.f(gpu, &bufs, std.mem.sliceAsBytes(vs)); + const dp = try up.f(gpu, &bufs, std.mem.sliceAsBytes(pos)); + const dsl = try up.f(gpu, &bufs, std.mem.sliceAsBytes(slot)); + const scale: f32 = @floatCast(1 / @sqrt(@as(f64, @floatFromInt(c.d)))); + const w: cuda.window_attention.Window = .{ .h = c.h, .hk = c.hk, .d = c.d, .win = c.win, .ring = c.ring, .scale = scale }; + const scratch = try cuda.DeviceBuffer.alloc(gpu.d, cuda.window_attention.WindowAttention.scratchBytes(w, most)); + try bufs.append(gpu.gpa, scratch); + const outs = try a.alloc(cuda.DeviceBuffer, counts.len); + for (counts, outs) |m, *o| { + o.* = try cuda.DeviceBuffer.alloc(gpu.d, m * c.h * c.d * 2); + try bufs.append(gpu.gpa, o.*); + try o.fill8(0xff, s.handle); + try g.decode(s, dq, dk, dv, o.ptr, dp, dsl, w, scratch.ptr, m); + } + try s.synchronize(); + const wide = try a.alloc(u16, qn); + try outs[counts.len - 1].download(0, std.mem.sliceAsBytes(wide)); + // per (row, key head): the window's keys gathered from the ring into dense rows, then each query head + const kd = try a.alloc(f64, c.top * c.d); + const vd = try a.alloc(f64, c.top * c.d); + const keys = try a.alloc(usize, c.top); + const q = try a.alloc(f64, c.d); + const want = try a.alloc(f64, c.d); + const tol = try a.alloc(f64, c.d); + var q_most: f64 = 0; + for (0..most) |r| { + const p: usize = @intCast(pos[r]); + const lo = if (c.win > 0 and p + 1 > c.win) p + 1 - c.win else 0; + const nk = p + 1 - lo; + for (0..c.hk) |kh| { + for (0..nk) |j| { + const row = @as(usize, @intCast(slot[r])) * c.ring + (lo + j) % c.ring; + for (0..c.d) |e| { + kd[j * c.d + e] = bf(ks[(row * c.hk + kh) * c.d + e]); + vd[j * c.d + e] = bf(vs[(row * c.hk + kh) * c.d + e]); + } + keys[j] = j; + } + for (0..gq) |hh| { + const hd = kh * gq + hh; + for (q, 0..) |*x, e| x.* = bf(qs[(r * c.h + hd) * c.d + e]); + core.attention_ref.head(q, kd, vd, keys[0..nk], c.d, scale, want, tol); + for (want, tol, 0..) |x, t, e| q_most = @max(q_most, @abs(bf(wide[(r * c.h + hd) * c.d + e]) - x) / t); + } + } + } + worst.* = @max(worst.*, q_most); + try check.expect(q_most <= 1, "window decode h {d}/{d} d {d} win {d}: within float64 tolerance (worst {d:.3})", .{ c.h, c.hk, c.d, c.win, q_most }); + for (counts[0 .. counts.len - 1], outs[0 .. counts.len - 1]) |m, o| { + const got = try a.alloc(u16, m * c.h * c.d); + try o.download(0, std.mem.sliceAsBytes(got)); + try check.sameBytes("window decode: a row count's bits equal the widest call's", std.mem.sliceAsBytes(got), std.mem.sliceAsBytes(wide[0 .. m * c.h * c.d])); + } +} + +const PCase = struct { h: usize, hk: usize, win: usize, ring: usize, p0: usize, n: usize }; + +/// Prompt rows p0 .. p0 + n: float64 tolerance, a shorter call's bits, a later chunk's bits (from p0 + n / 2). +fn prompt1(gpu: check.Gpu, a: std.mem.Allocator, s: cuda.Stream, g: *const cuda.window_attention.WindowAttention, rng: std.Random, c: PCase, worst: *f64) !void { + const d = 128; + const qn = c.n * c.h * d; + const crow = if (c.ring > 0) c.ring else c.p0 + c.n; // ring 0: a row a position + const cn = crow * c.hk * d; + const qs = try a.alloc(u16, qn); + const ks = try a.alloc(u16, cn); + const vs = try a.alloc(u16, cn); + for (qs) |*x| x.* = bf16(rng.floatNorm(f32)); + for (ks) |*x| x.* = bf16(rng.floatNorm(f32)); + for (vs) |*x| x.* = bf16(rng.floatNorm(f32)); + var bufs: std.ArrayList(cuda.DeviceBuffer) = .empty; + defer { + for (bufs.items) |*b| b.free(); + bufs.deinit(gpu.gpa); + } + const up = struct { + fn f(gp: check.Gpu, l: *std.ArrayList(cuda.DeviceBuffer), bytes: []const u8) !u64 { + const b = try cuda.DeviceBuffer.fromHost(gp.d, bytes); + try l.append(gp.gpa, b); + return b.ptr; + } + fn z(gp: check.Gpu, l: *std.ArrayList(cuda.DeviceBuffer), st: cuda.Stream, n: usize) !u64 { + const b = try cuda.DeviceBuffer.alloc(gp.d, n); + try l.append(gp.gpa, b); + try b.fill8(0xff, st.handle); + return b.ptr; + } + }; + const dq = try up.f(gpu, &bufs, std.mem.sliceAsBytes(qs)); + const dk = try up.f(gpu, &bufs, std.mem.sliceAsBytes(ks)); + const dv = try up.f(gpu, &bufs, std.mem.sliceAsBytes(vs)); + const scale: f32 = @floatCast(1 / @sqrt(@as(f64, d))); + const w: cuda.window_attention.Window = .{ .h = c.h, .hk = c.hk, .d = d, .win = c.win, .ring = c.ring, .scale = scale }; + const full = try up.z(gpu, &bufs, s, qn * 2); + try g.prompt(s, dq, dk, dv, full, c.p0, c.n, w); + const pre_n = 17; + const pre = try up.z(gpu, &bufs, s, pre_n * c.h * d * 2); + try g.prompt(s, dq, dk, dv, pre, c.p0, pre_n, w); + const half = c.n / 2; + const late = try up.z(gpu, &bufs, s, (c.n - half) * c.h * d * 2); + try g.prompt(s, dq + half * c.h * d * 2, dk, dv, late, c.p0 + half, c.n - half, w); + try s.synchronize(); + const got = try a.alloc(u16, qn); + const fb: cuda.DeviceBuffer = .{ .d = gpu.d, .ptr = full, .len = qn * 2 }; + try fb.download(0, std.mem.sliceAsBytes(got)); + const gq = c.h / c.hk; + const top = c.p0 + c.n; + const kd = try a.alloc(f64, @min(top, c.win + 0) * d + top * d); + const vd = try a.alloc(f64, kd.len); + const keys = try a.alloc(usize, top); + const q = try a.alloc(f64, d); + const want = try a.alloc(f64, d); + const tol = try a.alloc(f64, d); + var q_most: f64 = 0; + var r: usize = 0; + while (r < c.n) : (r += 7) { // every seventh row, and the last + const p = c.p0 + r; + const lo = if (c.win > 0 and p + 1 > c.win) p + 1 - c.win else 0; + const nk = p + 1 - lo; + for (0..c.hk) |kh| { + for (0..nk) |j| { + const row = if (c.ring > 0) (lo + j) % c.ring else lo + j; + for (0..d) |e| { + kd[j * d + e] = bf(ks[(row * c.hk + kh) * d + e]); + vd[j * d + e] = bf(vs[(row * c.hk + kh) * d + e]); + } + keys[j] = j; + } + for (0..gq) |hh| { + const hd = kh * gq + hh; + for (q, 0..) |*x, e| x.* = bf(qs[(r * c.h + hd) * d + e]); + core.attention_ref.head(q, kd, vd, keys[0..nk], d, scale, want, tol); + for (want, tol, 0..) |x, t, e| q_most = @max(q_most, @abs(bf(got[(r * c.h + hd) * d + e]) - x) / t); + } + } + if (r + 7 >= c.n and r != c.n - 1) r = c.n - 1 - 7; + } + worst.* = @max(worst.*, q_most); + try check.expect(q_most <= 1, "window prompt h {d}/{d} win {d} ring {d} p0 {d} n {d}: within float64 tolerance (worst {d:.3})", .{ c.h, c.hk, c.win, c.ring, c.p0, c.n, q_most }); + const pb: cuda.DeviceBuffer = .{ .d = gpu.d, .ptr = pre, .len = pre_n * c.h * d * 2 }; + const pg = try a.alloc(u16, pre_n * c.h * d); + try pb.download(0, std.mem.sliceAsBytes(pg)); + try check.sameBytes("window prompt: a shorter call's rows", std.mem.sliceAsBytes(pg), std.mem.sliceAsBytes(got[0 .. pre_n * c.h * d])); + const lb: cuda.DeviceBuffer = .{ .d = gpu.d, .ptr = late, .len = (c.n - half) * c.h * d * 2 }; + const lg = try a.alloc(u16, (c.n - half) * c.h * d); + try lb.download(0, std.mem.sliceAsBytes(lg)); + try check.sameBytes("window prompt: a later chunk's rows", std.mem.sliceAsBytes(lg), std.mem.sliceAsBytes(got[half * c.h * d ..])); +} + +pub fn run(gpu: check.Gpu) !void { + var arena: std.heap.ArenaAllocator = .init(gpu.gpa); + defer arena.deinit(); + var s = try cuda.Stream.init(gpu.d, false); + defer s.deinit(); + var g = try cuda.window_attention.WindowAttention.load(gpu.d); + defer g.unload(); + var prng = std.Random.DefaultPrng.init(37); + var worst: f64 = 0; + const a = arena.allocator(); + try run1(gpu, a, s, &g, prng.random(), .{ .h = 48, .hk = 4, .d = 128, .win = 513, .ring = 640, .streams = 2, .top = 3000 }, &worst); // Kolibri + try run1(gpu, a, s, &g, prng.random(), .{ .h = 48, .hk = 4, .d = 128, .win = 513, .ring = 600, .streams = 2, .top = 3000 }, &worst); // the ring's end inside a key tile + try run1(gpu, a, s, &g, prng.random(), .{ .h = 8, .hk = 2, .d = 64, .win = 100, .ring = 128, .streams = 3, .top = 1000 }, &worst); + try run1(gpu, a, s, &g, prng.random(), .{ .h = 16, .hk = 1, .d = 128, .win = 0, .ring = 300, .streams = 1, .top = 300 }, &worst); // no window + try prompt1(gpu, a, s, &g, prng.random(), .{ .h = 48, .hk = 4, .win = 513, .ring = 2624, .p0 = 0, .n = 600 }, &worst); // Kolibri, first chunk + try prompt1(gpu, a, s, &g, prng.random(), .{ .h = 48, .hk = 4, .win = 513, .ring = 2624, .p0 = 2300, .n = 900 }, &worst); // the ring wraps + try prompt1(gpu, a, s, &g, prng.random(), .{ .h = 16, .hk = 2, .win = 0, .ring = 0, .p0 = 100, .n = 333 }, &worst); // no window, 8 heads a block + check.pass("window attention: decode and prompt rows within float64 tolerance (worst {d:.3}), bits at any row count and chunk", .{worst}); +} From 6b07f775a96a47271a75d4b29a63dd5a149aa176 Mon Sep 17 00:00:00 2001 From: Jon Regan-Beasley Date: Wed, 14 Oct 2026 08:47:43 +1100 Subject: [PATCH 15/19] test(qwen27): apply PR747 TTFT increment to current native backend --- zig/src/core/engine_api.zig | 1 + zig/src/core/lane_host.zig | 12 +- zig/src/core/lane_host_first_test.zig | 119 +++++ zig/src/core/lanes/backend.zig | 3 + zig/src/core/lanes/engine.zig | 2 + zig/src/core/lanes/lanes.zig | 1 + zig/src/core/lanes/open.zig | 17 +- zig/src/core/lanes/open_first.zig | 64 +++ zig/src/core/lanes/open_first_test.zig | 163 ++++++ zig/src/families/qwen27/cuda/draft_pool.zig | 147 ++++++ .../families/qwen27/cuda/draft_pool_test.zig | 105 ++++ zig/src/families/qwen27/cuda/drafts.zig | 15 + zig/src/families/qwen27/cuda/engine.zig | 6 + .../qwen27/cuda/exact_cache_policy.zig | 58 +++ .../families/qwen27/cuda/exact_cache_test.zig | 491 ++++++++++++++++++ .../qwen27/cuda/exact_draft_reuse.zig | 173 ++++++ .../families/qwen27/cuda/exact_eligible.zig | 40 ++ .../qwen27/cuda/exact_first_logits.zig | 31 ++ zig/src/families/qwen27/cuda/exact_init.zig | 30 ++ .../qwen27/cuda/exact_logits_policy.zig | 75 +++ .../qwen27/cuda/exact_prompt_cache.zig | 339 ++++++++++++ .../families/qwen27/cuda/exact_seq_reuse.zig | 189 +++++++ zig/src/families/qwen27/cuda/lanes.zig | 142 ++--- zig/src/families/qwen27/cuda/reset_skip.zig | 58 +++ .../qwen27/cuda/reset_skip_factory_test.zig | 179 +++++++ .../families/qwen27/cuda/reset_skip_test.zig | 105 ++++ .../families/qwen27/cuda/restore_proof.zig | 76 +++ zig/src/families/qwen27/cuda/seq_pool.zig | 120 +++++ .../families/qwen27/cuda/seq_pool_test.zig | 172 ++++++ zig/src/families/qwen27/cuda/state.zig | 15 +- zig/src/native/cuda.zig | 2 +- 31 files changed, 2880 insertions(+), 70 deletions(-) create mode 100644 zig/src/core/lane_host_first_test.zig create mode 100644 zig/src/core/lanes/open_first.zig create mode 100644 zig/src/core/lanes/open_first_test.zig create mode 100644 zig/src/families/qwen27/cuda/draft_pool.zig create mode 100644 zig/src/families/qwen27/cuda/draft_pool_test.zig create mode 100644 zig/src/families/qwen27/cuda/exact_cache_policy.zig create mode 100644 zig/src/families/qwen27/cuda/exact_cache_test.zig create mode 100644 zig/src/families/qwen27/cuda/exact_draft_reuse.zig create mode 100644 zig/src/families/qwen27/cuda/exact_eligible.zig create mode 100644 zig/src/families/qwen27/cuda/exact_first_logits.zig create mode 100644 zig/src/families/qwen27/cuda/exact_init.zig create mode 100644 zig/src/families/qwen27/cuda/exact_logits_policy.zig create mode 100644 zig/src/families/qwen27/cuda/exact_prompt_cache.zig create mode 100644 zig/src/families/qwen27/cuda/exact_seq_reuse.zig create mode 100644 zig/src/families/qwen27/cuda/reset_skip.zig create mode 100644 zig/src/families/qwen27/cuda/reset_skip_factory_test.zig create mode 100644 zig/src/families/qwen27/cuda/reset_skip_test.zig create mode 100644 zig/src/families/qwen27/cuda/restore_proof.zig create mode 100644 zig/src/families/qwen27/cuda/seq_pool.zig create mode 100644 zig/src/families/qwen27/cuda/seq_pool_test.zig diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 3cbfb4b454..3cd1d42a2b 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -463,6 +463,7 @@ test { _ = @import("lane_host_reuse_test.zig"); _ = @import("lane_host_warm_test.zig"); _ = @import("lane_host_fill_test.zig"); + _ = @import("lane_host_first_test.zig"); _ = @import("prompt_cache.zig"); _ = @import("prompt_radix.zig"); _ = @import("prompt_imprint.zig"); diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index 4d791bcad2..f45e29b994 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -358,7 +358,7 @@ pub const LaneHost = struct { h.filling.append(h.gpa, job) catch return h.drop(job, "out of memory"); return true; } - h.core.addStream(&job.stream) catch |e| return switch (e) { + h.core.addStreamCommitted(&job.stream, .{ .ptr = job, .call = firstCommitted }) catch |e| return switch (e) { error.Cancelled => h.cancel(job), error.LaneMemory => h.requeue(job), else => h.drop(job, h.words(e)), @@ -375,7 +375,7 @@ pub const LaneHost = struct { const job = h.filling.items[at]; const first = !job.fill_began; job.fill_began = true; - const done = h.core.fillStream(&job.stream, first) catch |e| { + const done = h.core.fillStreamCommitted(&job.stream, first, .{ .ptr = job, .call = firstCommitted }) catch |e| { _ = switch (e) { error.Cancelled => h.cancel(job), error.LaneMemory => blk: { @@ -398,7 +398,14 @@ pub const LaneHost = struct { return c.constraint(h.gpa, h.io, @fromBackingInt(@intCast(@backingInt(st.kind))), st.text, st.after); } + fn firstCommitted(ptr: *anyopaque) void { + const job: *Job = @ptrCast(@alignCast(ptr)); + job.host.prefilled(job, job.began); + job.host.send(job); + } + pub fn prefilled(h: *LaneHost, job: *Job, began: i96) void { + if (job.prefilled != null) return; const done = std.Io.Clock.awake.now(h.io).toNanoseconds(); h.lock(); if (done > began) h.prefill_rate = @as(f64, @floatFromInt(job.request.prompt.len - job.stream.cached)) / (@as(f64, @floatFromInt(done - began)) / 1e9); @@ -446,6 +453,7 @@ pub const LaneHost = struct { /// No memory for another lane yet: the job waits at the queue's head, and admission holds until a job ends. fn requeue(h: *LaneHost, job: *Job) bool { + if (job.stream.pending != null or job.delivered != 0) return h.drop(job, "memory failure after first-token commit"); if (job.stream.grammar) |g| g.deinit(h.gpa); // the next admission compiles its own job.stream.deinit(h.gpa); job.proposer.deinit(); diff --git a/zig/src/core/lane_host_first_test.zig b/zig/src/core/lane_host_first_test.zig new file mode 100644 index 0000000000..04b71df7b5 --- /dev/null +++ b/zig/src/core/lane_host_first_test.zig @@ -0,0 +1,119 @@ +//! Actual lane-host delivery keeps terminal jobs alive through the callback and never requeues an emitted first token. +const std = @import("std"); +const lanes = @import("lanes"); +const api = @import("engine_api.zig"); +const LaneHost = @import("lane_host.zig").LaneHost; +const a = std.testing.allocator; + +const Box = struct { + mutex: std.Io.Mutex = .init, + tokens: usize = 0, + logprobs: usize = 0, + prefilled: usize = 0, + finishes: usize = 0, + reason: ?api.Reason = null, + fn event(ptr: *anyopaque, _: api.Id, ev: *const api.Event) void { + const b: *Box = @ptrCast(@alignCast(ptr)); + b.mutex.lockUncancelable(std.testing.io); + defer b.mutex.unlock(std.testing.io); + switch (ev.*) { + .tokens => |t| b.tokens += t.len, + .logprobs => |r| b.logprobs += r.len, + .prefilled => b.prefilled += 1, + .finished => |f| { + b.reason = f.reason; + b.finishes += 1; + }, + else => {}, + } + } + fn wait(b: *Box) !void { + for (0..1000) |_| { + b.mutex.lockUncancelable(std.testing.io); + const done = b.reason != null; + b.mutex.unlock(std.testing.io); + if (done) return; + try std.Io.sleep(std.testing.io, .fromMilliseconds(1), .awake); + } + return error.TestTimedOut; + } +}; + +const Target = struct { + fake: lanes.fake.Fake = .{ .gpa = a }, + table: lanes.backend.Backend.VTable = undefined, + drafts: usize = 0, + releases: usize = 0, + lane_memory: bool, + observed: *Box, + delivered_before_draft: bool = false, + fn backend(t: *Target) lanes.backend.Backend { + t.table = t.fake.backend().vtable.*; + t.table.draft = draft; + t.table.release = release; + return .{ .ptr = &t.fake, .vtable = &t.table, .first_before_draft = true }; + } + fn self(ptr: *anyopaque) *Target { + const fake: *lanes.fake.Fake = @ptrCast(@alignCast(ptr)); + return @fieldParentPtr("fake", fake); + } + fn draft(ptr: *anyopaque, _: []const lanes.backend.DraftRequest) !void { + const t = self(ptr); + t.drafts += 1; + t.observed.mutex.lockUncancelable(std.testing.io); + t.delivered_before_draft = t.observed.tokens == 1 and t.observed.logprobs == 1; + t.observed.mutex.unlock(std.testing.io); + return if (t.lane_memory) error.LaneMemory else error.InitialDraftFailed; + } + fn release(ptr: *anyopaque, s: *lanes.Stream) void { + const t = self(ptr); + t.releases += 1; + t.fake.backend().release(s); + } +}; + +fn check(lane_memory: bool) !void { + var one: Box = .{}; + var other: Box = .{}; + var t: Target = .{ .lane_memory = lane_memory, .observed = &other }; + defer t.fake.deinit(); + var cfg = try lanes.Config.init(a, .{ .exact_width = 8, .hidden_rows = true, .mtp = true, .speculate = true, .drafts = 7 }, 8, 7); + defer cfg.deinit(a); + var clock: lanes.fake.FixedClock = .{}; + var core = lanes.Engine.init(a, &cfg, t.backend(), clock.clock()); + defer core.deinit(); + var h = LaneHost.init(a, std.testing.io, &core, .{ .lanes = 2 }); + const e = h.engine(); + const terminal: api.Request = .{ .prompt = &.{ 3, 1, 4 }, .max_tokens = 1, .logprobs = 2 }; + const failing: api.Request = .{ .prompt = &.{ 3, 1, 4 }, .max_tokens = 8, .logprobs = 2 }; + try e.submit(1, &terminal, .{ .ctx = &one, .event = Box.event }); + try e.submit(2, &failing, .{ .ctx = &other, .event = Box.event }); + try h.start(); + var stopped = false; + defer if (!stopped) h.stop(); + try one.wait(); + try other.wait(); + h.stop(); + stopped = true; + try std.testing.expectEqual(api.Reason.length, one.reason.?); + try std.testing.expectEqual(api.Reason.failed, other.reason.?); + try std.testing.expectEqual(@as(usize, 1), one.tokens); + try std.testing.expectEqual(@as(usize, 1), other.tokens); + try std.testing.expectEqual(@as(usize, 1), one.logprobs); + try std.testing.expectEqual(@as(usize, 1), other.logprobs); + try std.testing.expectEqual(@as(usize, 1), one.prefilled); + try std.testing.expectEqual(@as(usize, 1), other.prefilled); + try std.testing.expectEqual(@as(usize, 1), one.finishes); + try std.testing.expectEqual(@as(usize, 1), other.finishes); + try std.testing.expectEqual(@as(usize, 1), t.drafts); + try std.testing.expectEqual(@as(usize, 2), t.releases); + try std.testing.expect(t.delivered_before_draft); +} + +test "first token and logprob sent once; terminal job survives another job's draft error" { + try check(false); +} + +test "LaneMemory after the committed handoff fails instead of resetting and requeueing the reply" { + try check(true); +} diff --git a/zig/src/core/lanes/backend.zig b/zig/src/core/lanes/backend.zig index 7d2eb1fb08..8bfef2869b 100644 --- a/zig/src/core/lanes/backend.zig +++ b/zig/src/core/lanes/backend.zig @@ -59,9 +59,12 @@ pub const DraftRequest = struct { ranks: bool = false, // also keep each chained level's best tokens (alternatives(), for grafts) }; +pub const Committed = struct { ptr: *anyopaque, call: *const fn (*anyopaque) void }; + pub const Backend = struct { ptr: *anyopaque, vtable: *const VTable, + first_before_draft: bool = false, pub const VTable = struct { /// Prefill the stream's prompt into its caches (and its draft head's). diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index cf0a97ea4a..95c878fbca 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -79,9 +79,11 @@ pub const Engine = struct { } pub const addStream = open.add; + pub const addStreamCommitted = open.addCommitted; pub const opened = open.opened; pub const fills = open.fills; pub const fillStream = open.fillStream; + pub const fillStreamCommitted = open.fillStreamCommitted; /// Take over a stream another driver decoded so far: its cache settled, `pending` and `cache_len` set, no drafts held. pub fn adopt(e: *Engine, s: *Stream) !void { diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index 5d50a16ef3..e961a2aed3 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -38,6 +38,7 @@ pub const LogprobRow = logprob.Row; test { std.testing.refAllDecls(@This()); _ = @import("engine_test.zig"); + _ = @import("open_first_test.zig"); _ = @import("gpu_full_test.zig"); _ = @import("drafted_test.zig"); _ = @import("pricing_test.zig"); diff --git a/zig/src/core/lanes/open.zig b/zig/src/core/lanes/open.zig index 7cddf9a064..fcb4a52196 100644 --- a/zig/src/core/lanes/open.zig +++ b/zig/src/core/lanes/open.zig @@ -14,17 +14,26 @@ const int = trail.int; /// Prefill a stream, draw its first token and ask for its first drafts; it takes part from the next round. pub fn add(e: *Engine, s: *Stream) !void { + return addCommitted(e, s, null); +} + +pub fn addCommitted(e: *Engine, s: *Stream, hook: ?be.Committed) !void { _ = e.arena.reset(.retain_capacity); try trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }); e.backend.prefill(s) catch |err| { if (err == error.Cancelled) e.backend.release(s); // the host finishes a cancelled stream without the core return err; }; - try opened(e, s); + try openedCommitted(e, s, hook); } /// After the prompt pass: the first token drawn and committed, the first drafts asked; rounds from the next. pub fn opened(e: *Engine, s: *Stream) !void { + return openedCommitted(e, s, null); +} + +fn openedCommitted(e: *Engine, s: *Stream, hook: ?be.Committed) !void { + if (e.backend.first_before_draft) return @import("open_first.zig").opened(e, s, hook); if (s.isCancelled()) { // cancelled in its last chunk: no first token e.backend.release(s); return error.Cancelled; @@ -77,6 +86,10 @@ pub fn fills(e: *const Engine) bool { /// One chunk of a stream's prompt pass; true once the stream is opened and joins the next round. pub fn fillStream(e: *Engine, s: *Stream, first: bool) !bool { + return fillStreamCommitted(e, s, first, null); +} + +pub fn fillStreamCommitted(e: *Engine, s: *Stream, first: bool, hook: ?be.Committed) !bool { _ = e.arena.reset(.retain_capacity); if (first) try trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }); const chunk = e.backend.vtable.prefill_step orelse return error.NoPrefillSteps; @@ -85,6 +98,6 @@ pub fn fillStream(e: *Engine, s: *Stream, first: bool) !bool { return err; }; if (!done) return false; - try opened(e, s); + try openedCommitted(e, s, hook); return true; } diff --git a/zig/src/core/lanes/open_first.zig b/zig/src/core/lanes/open_first.zig new file mode 100644 index 0000000000..8638695786 --- /dev/null +++ b/zig/src/core/lanes/open_first.zig @@ -0,0 +1,64 @@ +//! An opt-in single-stream admission commits and hands off its first token before preparing initial drafts. +const Engine = @import("engine.zig").Engine; +const be = @import("backend.zig"); +const Stream = @import("stream.zig").Stream; +const ev = @import("events.zig"); +const win = @import("windows.zig"); +const trail = @import("trail.zig"); +const LogRow = @import("logprob.zig").Row; +const f = ev.f; +const str = trail.str; +const int = trail.int; + +pub fn opened(e: *Engine, s: *Stream, hook: ?be.Committed) !void { + if (s.isCancelled()) { + e.backend.release(s); + return error.Cancelled; + } + try e.live.ensureUnusedCapacity(e.gpa, 1); + s.context.shrinkRetainingCapacity(s.prompt_len); + s.rows.clearRetainingCapacity(); + s.pending = null; + s.cache_len = s.prompt_len; + const position: u64 = s.prompt_len; + const drawn = try e.drawFirst(s, position); + var feed: be.Feed = .{ .handle = drawn }; + if (try e.forcedNext(s)) |t| feed = .{ .value = t }; + const depth: ?u32 = if (e.cfg.family_mtp and s.drafts) @intCast(try e.rule.depth(win.who(s))) else null; + const value = try e.readFeed(feed); + if (e.log != null) { + const actual = if (feed == .handle) value else try e.backend.read(drawn); + try trail.event(e, &.{ f("ev", str("first")), f("stream", str(s.id)), f("position", int(position)), f("drawn", int(actual)), f("token", int(value)) }); + } + var row: [1]LogRow = undefined; + if (s.logprobs != null) row[0] = (try e.backend.firstRow(s)).forToken(value); + _ = try s.commit(e.gpa, &.{value}, if (s.logprobs != null) &row else &.{}); + s.pending = value; + try trail.resolve(e); + if (s.finished) { + try trail.finish(e, s); + e.release(s); + if (hook) |h| h.call(h.ptr); + return; + } + if (hook) |h| h.call(h.ptr); + if (s.isCancelled()) { + e.release(s); + return error.Cancelled; + } + if (depth) |d| { + e.backend.draft(&.{.{ .stream = s, .follow = &.{}, .first = .{ .value = value }, .rows = null, .start = s.prompt_len, .position = position + 1, .depth = d }}) catch |err| { + if (err == error.Cancelled) e.release(s); + return err; + }; + s.dropHeld(e.gpa); + s.next = .{ .count = d }; + if (e.backend.vtable.tree) |tree| if (try tree(e.backend.ptr, s, e.gpa)) |held| { + s.next = held; + }; + try trail.event(e, &.{ f("ev", str("draft")), f("stream", str(s.id)), f("depth", int(d)), f("position", int(position + 1)), f("follow", .{ .u32s = &.{value} }), f("rows", .null) }); + } else if (e.cfg.pipelined and s.logprobs == null and s.grammar == null) { + try e.queueNext(s, feed); + } + e.live.appendAssumeCapacity(s); +} diff --git a/zig/src/core/lanes/open_first_test.zig b/zig/src/core/lanes/open_first_test.zig new file mode 100644 index 0000000000..274994dd10 --- /dev/null +++ b/zig/src/core/lanes/open_first_test.zig @@ -0,0 +1,163 @@ +//! Single-stream opt-in admission preserves committed tokens and default ordering across terminal and failure paths. +const std = @import("std"); +const Engine = @import("engine.zig").Engine; +const Config = @import("config.zig").Config; +const be = @import("backend.zig"); +const sm = @import("stream.zig"); +const fake = @import("fake.zig"); +const a = std.testing.allocator; +const ids = [_]u32{ 3, 1, 4 }; + +const Probe = struct { + target: fake.Fake = .{ .gpa = a }, + table: be.Backend.VTable = undefined, + hooks: usize = 0, + drafts: usize = 0, + releases: usize = 0, + committed_in_draft: bool = false, + hook_in_draft: bool = false, + fail_draft: bool = false, + cancel: bool = false, + selected: u32 = 0, + + fn backend(p: *Probe, early: bool, stepped: bool) be.Backend { + p.table = (if (stepped) p.target.stepped() else p.target.backend()).vtable.*; + p.table.draft = draft; + p.table.release = release; + return .{ .ptr = &p.target, .vtable = &p.table, .first_before_draft = early }; + } + fn self(ptr: *anyopaque) *Probe { + const t: *fake.Fake = @ptrCast(@alignCast(ptr)); + return @fieldParentPtr("target", t); + } + fn draft(ptr: *anyopaque, req: []const be.DraftRequest) !void { + const p = self(ptr); + p.drafts += 1; + p.committed_in_draft = req[0].stream.emitted().len == 1; + p.hook_in_draft = p.hooks == 1; + if (req[0].first) |f| p.selected = switch (f) { + .value => |v| v, + .handle => |h| try p.target.backend().read(h), + }; + if (p.fail_draft) return error.InitialDraftFailed; + try p.target.backend().draft(req); + } + fn release(ptr: *anyopaque, s: *sm.Stream) void { + const p = self(ptr); + p.releases += 1; + p.target.backend().release(s); + } + fn hook(ptr: *anyopaque) void { + const p: *Probe = @ptrCast(@alignCast(ptr)); + p.hooks += 1; + } + fn cancelled(ptr: *anyopaque) bool { + const p: *Probe = @ptrCast(@alignCast(ptr)); + return p.cancel and p.hooks > 0; + } +}; + +fn config() !Config { + return Config.init(a, .{ .exact_width = 8, .hidden_rows = true, .mtp = true, .speculate = true, .drafts = 7 }, 8, 7); +} + +fn runOrder(early: bool) !void { + var p: Probe = .{}; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(early, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "order", .prompt = &ids, .max_new = 8 }); + defer s.deinit(a); + try e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook }); + try std.testing.expectEqual(early, p.committed_in_draft); + try std.testing.expectEqual(early, p.hook_in_draft); + try std.testing.expectEqual(@as(usize, @intFromBool(early)), p.hooks); + try std.testing.expectEqual(@as(usize, 1), s.emitted().len); + e.discard(&s); + try std.testing.expectEqual(@as(usize, 1), p.releases); +} + +test "opt-in commits and hands off before draft; default retains old ordering" { + try runOrder(true); + try runOrder(false); +} + +test "max-one and EOS terminate successfully without preparing any drafts" { + for ([_]bool{ false, true }) |eos| { + var p: Probe = .{ .fail_draft = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + const token = fake.next(&ids, null, ids.len); + var s = try sm.Stream.init(a, .{ .id = "terminal", .prompt = &ids, .max_new = if (eos) 8 else 1, .eos = if (eos) &.{token} else &.{} }); + defer s.deinit(a); + try e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook }); + try std.testing.expect(s.finished); + try std.testing.expectEqual(@as(usize, 0), p.drafts); + try std.testing.expectEqual(@as(usize, 1), p.hooks); + try std.testing.expectEqual(@as(usize, 1), p.releases); + try std.testing.expectEqual(@as(usize, 0), e.activeCount()); + } +} + +test "forced first and its logprob are committed exactly once before a later draft failure" { + var p: Probe = .{ .fail_draft = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "forced", .prompt = &ids, .max_new = 8, .logprobs = 2 }); + defer s.deinit(a); + try s.force.append(a, 42); + try std.testing.expectError(error.InitialDraftFailed, e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expectEqualSlices(u32, &.{42}, s.emitted()); + try std.testing.expectEqual(@as(u32, 42), p.selected); + try std.testing.expectEqual(@as(usize, 1), s.rows.items.len); + try std.testing.expectEqual(@as(u32, 42), s.rows.items[0].token); + try std.testing.expectEqual(@as(usize, 1), p.hooks); + e.discard(&s); + try std.testing.expectEqual(@as(usize, 1), p.releases); +} + +test "cancellation at handoff releases once and skips initial drafts" { + var p: Probe = .{ .cancel = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "cancel", .prompt = &ids, .max_new = 8, .cancel_check = .{ .ptr = &p, .check = Probe.cancelled } }); + defer s.deinit(a); + try std.testing.expectError(error.Cancelled, e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expectEqual(@as(usize, 1), p.hooks); + try std.testing.expectEqual(@as(usize, 1), p.releases); + try std.testing.expectEqual(@as(usize, 0), p.drafts); +} + +test "stepped prefill uses the same committed handoff only after its final chunk" { + var p: Probe = .{}; + p.target.prefill_chunks = 2; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, true), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "step", .prompt = &ids, .max_new = 8 }); + defer s.deinit(a); + try std.testing.expect(!try e.fillStreamCommitted(&s, true, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expectEqual(@as(usize, 0), p.hooks); + while (!try e.fillStreamCommitted(&s, false, .{ .ptr = &p, .call = Probe.hook })) {} + try std.testing.expectEqual(@as(usize, 1), p.hooks); + try std.testing.expect(p.committed_in_draft and p.hook_in_draft); + e.discard(&s); +} diff --git a/zig/src/families/qwen27/cuda/draft_pool.zig b/zig/src/families/qwen27/cuda/draft_pool.zig new file mode 100644 index 0000000000..7b2da57e8d --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_pool.zig @@ -0,0 +1,147 @@ +//! Detached fixed-class draft Context ownership; no CUDA allocation, copy or fence occurs in this policy. +const std = @import("std"); +pub const max_slots: usize = 16; +pub const max_capacity: usize = 512; +pub const max_bytes: usize = 1536 * 1024 * 1024; + +pub const Stats = struct { + enabled: bool = false, + failed: bool = false, + driver_creations: u64 = 0, + reuse_hits: u64 = 0, + evictions: u64 = 0, + discarded: u64 = 0, + initialized: u64 = 0, + retained_slots: usize = 0, + retained_bytes: usize = 0, + quarantined_slots: usize = 0, + quarantined_bytes: usize = 0, + fence_failures: u64 = 0, +}; + +pub fn enabled(raw: ?[]const u8, cache_enabled: bool) !bool { + const text = raw orelse return false; + if (std.mem.eql(u8, text, "0")) return false; + if (!std.mem.eql(u8, text, "1")) return error.InvalidDraftReuseFlag; + if (!cache_enabled) return error.DraftReuseRequiresExactCache; + return true; +} + +pub fn configured(cache_enabled: bool) !bool { + const env = std.c.getenv("TF_PR608_EXACT_CACHE_DRAFT_REUSE"); + return enabled(if (env) |v| std.mem.span(v) else null, cache_enabled); +} + +pub fn eligible(drafted: bool, drafter: bool, tokens: usize, capacity: usize, fresh: bool, cancelled: bool) bool { + return drafted and drafter and tokens > 0 and tokens <= max_capacity and capacity <= max_capacity and + tokens < capacity and fresh and !cancelled; +} + +/// A completed request authorizes storage recycling only when every independent success gate holds. +pub fn completed(initialized: bool, terminal: bool, uncancelled: bool, settled: bool, cache_ok: bool, cache_failed: bool) bool { + return initialized and terminal and uncancelled and settled and cache_ok and !cache_failed; +} + +pub const Proof = struct { + stream: *const anyopaque, + seq: *const anyopaque, + wrapper: *const anyopaque, + arena: u64, + tokens: usize, + capacity: usize, +}; + +pub const Proofs = struct { + items: [max_slots]?Proof = @splat(null), + pub fn mark(p: *Proofs, proof: Proof) !void { + if (proof.arena == 0 or proof.tokens == 0 or proof.tokens >= proof.capacity or proof.capacity > max_capacity) + return error.InvalidDraftInitialization; + for (p.items) |slot| if (slot) |old| { + if (old.stream == proof.stream) return error.DuplicateDraftInitialization; + }; + for (&p.items) |*slot| if (slot.* == null) { + slot.* = proof; + return; + }; + return error.DraftInitializationLimit; + } + pub fn remove(p: *Proofs, stream: *const anyopaque) ?Proof { + for (&p.items) |*slot| if (slot.*) |proof| { + if (proof.stream != stream) continue; + slot.* = null; + return proof; + }; + return null; + } +}; + +/// Idle entries were fenced; quarantined entries must not be freed or taken until a new successful fence. +pub fn Pool(comptime T: type) type { + return struct { + const Self = @This(); + entries: [max_slots]*T = undefined, + quarantine: [max_slots]*T = undefined, + limit: usize, + item_bytes: usize, + stats: Stats, + + pub fn init(on: bool, slots: usize, item_bytes: usize) Self { + return .{ .limit = @min(slots, max_slots), .item_bytes = item_bytes, .stats = .{ .enabled = on } }; + } + pub fn active(p: *const Self) bool { + return p.stats.enabled and !p.stats.failed; + } + pub fn hasRoom(p: *const Self, live: usize) bool { + return live < p.limit and p.stats.retained_slots + p.stats.quarantined_slots < p.limit - live; + } + fn duplicate(p: *const Self, ptr: *T) bool { + for (p.entries[0..p.stats.retained_slots]) |old| if (old == ptr) return true; + for (p.quarantine[0..p.stats.quarantined_slots]) |old| if (old == ptr) return true; + return false; + } + pub fn take(p: *Self) ?*T { + if (!p.active() or p.stats.retained_slots == 0) return null; + p.stats.retained_slots -= 1; + p.stats.retained_bytes -= p.item_bytes; + p.stats.reuse_hits += 1; + return p.entries[p.stats.retained_slots]; + } + pub fn evict(p: *Self) ?*T { + if (p.stats.retained_slots == 0) return null; + p.stats.retained_slots -= 1; + p.stats.retained_bytes -= p.item_bytes; + p.stats.evictions += 1; + return p.entries[p.stats.retained_slots]; + } + pub fn retain(p: *Self, ptr: *T, live: usize) bool { + if (!p.active() or !p.hasRoom(live) or p.item_bytes > max_bytes or + p.stats.retained_bytes > max_bytes - p.item_bytes) return false; + if (p.duplicate(ptr)) @panic("duplicate draft pool owner"); + p.entries[p.stats.retained_slots] = ptr; + p.stats.retained_slots += 1; + p.stats.retained_bytes += p.item_bytes; + return true; + } + pub fn fail(p: *Self) void { + if (p.stats.enabled) p.stats.failed = true; + } + pub fn park(p: *Self, ptr: *T, live: usize) void { + p.fail(); + if (!p.hasRoom(live) or p.duplicate(ptr)) @panic("draft quarantine ownership limit"); + p.quarantine[p.stats.quarantined_slots] = ptr; + p.stats.quarantined_slots += 1; + p.stats.quarantined_bytes += p.item_bytes; + p.stats.fence_failures += 1; + } + pub fn reap(p: *Self, fenced: bool) ?*T { + if (!fenced or p.stats.quarantined_slots == 0) return null; + p.stats.quarantined_slots -= 1; + p.stats.quarantined_bytes -= p.item_bytes; + return p.quarantine[p.stats.quarantined_slots]; + } + }; +} + +test { + _ = @import("draft_pool_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/draft_pool_test.zig b/zig/src/families/qwen27/cuda/draft_pool_test.zig new file mode 100644 index 0000000000..e1fc92575d --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_pool_test.zig @@ -0,0 +1,105 @@ +//! Host-only Context storage ownership and initialization provenance regressions. +const std = @import("std"); +const p = @import("draft_pool.zig"); +const expect = std.testing.expect; +const equal = std.testing.expectEqual; + +test "strict flag is default off and requires exact cache" { + try expect(!try p.enabled(null, false)); + try expect(!try p.enabled("0", false)); + try expect(try p.enabled("1", true)); + try std.testing.expectError(error.DraftReuseRequiresExactCache, p.enabled("1", false)); + try std.testing.expectError(error.InvalidDraftReuseFlag, p.enabled("yes", true)); +} + +test "fresh small drafted requests include cross-key and capacity changes" { + try expect(p.eligible(true, true, 14, 286, true, false)); + try expect(p.eligible(true, true, 31, 303, true, false)); + try expect(p.eligible(true, true, 1, 512, true, false)); + try expect(!p.eligible(false, true, 14, 286, true, false)); + try expect(!p.eligible(true, false, 14, 286, true, false)); + try expect(!p.eligible(true, true, 31, 513, true, false)); + try expect(!p.eligible(true, true, 0, 286, true, false)); + try expect(!p.eligible(true, true, 31, 31, true, false)); + try expect(!p.eligible(true, true, 31, 303, false, false)); + try expect(!p.eligible(true, true, 31, 303, true, true)); +} + +test "detached storage transfers once and total ownership is bounded" { + var owners = [_]u8{ 1, 2, 3 }; + var pool = p.Pool(u8).init(true, 2, 80); + try expect(pool.retain(&owners[0], 1)); + try expect(!pool.retain(&owners[1], 1)); + try expect(!pool.hasRoom(1)); + try expect(pool.take().? == &owners[0]); + try expect(pool.take() == null); + try equal(@as(u64, 1), pool.stats.reuse_hits); + try expect(pool.retain(&owners[0], 0)); + try expect(pool.retain(&owners[1], 0)); + try expect(!pool.retain(&owners[2], 0)); + try equal(@as(usize, 160), pool.stats.retained_bytes); + try expect(pool.evict().? == &owners[1]); + try expect(pool.evict().? == &owners[0]); + try equal(@as(usize, 0), pool.stats.retained_bytes); + try equal(@as(u64, 2), pool.stats.evictions); +} + +test "retained bytes and backend configured ownership cap are enforced" { + var owners = [_]u8{ 1, 2, 3 }; + var pool = p.Pool(u8).init(true, 32, p.max_bytes / 2); + try equal(@as(usize, 16), pool.limit); + try expect(pool.retain(&owners[0], 0)); + try expect(pool.retain(&owners[1], 0)); + try expect(!pool.retain(&owners[2], 0)); + try equal(p.max_bytes, pool.stats.retained_bytes); + var disabled = p.Pool(u8).init(false, 16, 1); + try expect(!disabled.retain(&owners[0], 0)); + var zero = p.Pool(u8).init(true, 0, 1); + try expect(!zero.hasRoom(0)); +} + +test "failed fence quarantines without reuse or free until successful fence" { + var owners = [_]u8{ 1, 2 }; + var pool = p.Pool(u8).init(true, 2, 80); + pool.park(&owners[0], 1); + try expect(pool.stats.failed); + try expect(!pool.hasRoom(1)); + try expect(pool.take() == null); + try expect(pool.reap(false) == null); + try equal(@as(usize, 1), pool.stats.quarantined_slots); + try equal(@as(usize, 80), pool.stats.quarantined_bytes); + try expect(pool.reap(true).? == &owners[0]); + try expect(pool.reap(true) == null); + try equal(@as(usize, 0), pool.stats.quarantined_bytes); + try expect(!pool.retain(&owners[1], 0)); + try equal(@as(u64, 1), pool.stats.fence_failures); +} + +test "initialization proof binds independent request target wrapper and arena" { + var owners = [_]u8{ 1, 2, 3, 4 }; + const proof: p.Proof = .{ .stream = &owners[0], .seq = &owners[1], .wrapper = &owners[2], .arena = 4096, .tokens = 31, .capacity = 303 }; + var proofs: p.Proofs = .{}; + try proofs.mark(proof); + try std.testing.expectError(error.DuplicateDraftInitialization, proofs.mark(proof)); + try expect(proofs.remove(&owners[3]) == null); + const taken = proofs.remove(&owners[0]).?; + try expect(taken.seq == @as(*const anyopaque, &owners[1]) and taken.wrapper == @as(*const anyopaque, &owners[2])); + try equal(@as(u64, 4096), taken.arena); + try expect(proofs.remove(&owners[0]) == null); + var invalid = proof; + invalid.capacity = 513; + try std.testing.expectError(error.InvalidDraftInitialization, proofs.mark(invalid)); + invalid = proof; + invalid.arena = 0; + try std.testing.expectError(error.InvalidDraftInitialization, proofs.mark(invalid)); +} + +test "retention requires every independent clean completion gate" { + try expect(p.completed(true, true, true, true, true, false)); + try expect(!p.completed(false, true, true, true, true, false)); + try expect(!p.completed(true, false, true, true, true, false)); + try expect(!p.completed(true, true, false, true, true, false)); + try expect(!p.completed(true, true, true, false, true, false)); + try expect(!p.completed(true, true, true, true, false, false)); + try expect(!p.completed(true, true, true, true, true, true)); +} diff --git a/zig/src/families/qwen27/cuda/drafts.zig b/zig/src/families/qwen27/cuda/drafts.zig index 02e3659524..99303e9832 100644 --- a/zig/src/families/qwen27/cuda/drafts.zig +++ b/zig/src/families/qwen27/cuda/drafts.zig @@ -31,12 +31,27 @@ pub const Stream = struct { pub fn deinit(x: *Stream) void { x.ctx.deinit(); + x.deinitHost(); + } + + fn deinitHost(x: *Stream) void { x.copies.deinit(); x.tokens.deinit(x.gpa); x.parents.deinit(x.gpa); x.scores.deinit(x.gpa); } + /// Reconstruct every request-specific host field around exclusively owned, already-fenced storage. + pub fn resetStorage(x: *Stream) void { + const gpa = x.gpa; + var ctx = x.ctx; + x.deinitHost(); + ctx.cur = 0; + ctx.len = 0; + ctx.end = 0; + x.* = init(gpa, ctx); + } + fn clear(x: *Stream) void { x.kind = .none; x.tokens.clearRetainingCapacity(); diff --git a/zig/src/families/qwen27/cuda/engine.zig b/zig/src/families/qwen27/cuda/engine.zig index 632a806564..4708a5d7e0 100644 --- a/zig/src/families/qwen27/cuda/engine.zig +++ b/zig/src/families/qwen27/cuda/engine.zig @@ -115,6 +115,12 @@ pub const Engine = struct { return st.Seq.init(e.gpa, e.ops(), e.w.g, capacity, e.kv8); } + /// Private BF16 admission only; no target may read this destination before restore or reset. + pub fn sequenceForRestore(e: *Engine, capacity: usize) !st.Seq { + if (e.kv8) return error.ExactCacheRequiresBf16; + return st.Seq.initRestoreDestination(e.gpa, e.ops(), e.w.g, capacity); + } + /// Attention's partials for `streams` streams of the engine's context in one round. pub fn fit(e: *Engine, streams: usize) !void { try e.s.synchronize(); diff --git a/zig/src/families/qwen27/cuda/exact_cache_policy.zig b/zig/src/families/qwen27/cuda/exact_cache_policy.zig new file mode 100644 index 0000000000..3aa664485b --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_cache_policy.zig @@ -0,0 +1,58 @@ +//! Private full-prompt cache eligibility and bounded ownership decisions. +const std = @import("std"); +pub const max_tokens = 512; +pub const max_bytes = 256 * 1024 * 1024; + +pub fn enabled(value: ?[]const u8) !bool { + const text = value orelse return false; + if (std.mem.eql(u8, text, "0")) return false; + if (std.mem.eql(u8, text, "1")) return true; + return error.InvalidExactPromptCacheFlag; +} + +pub fn eligible(tokens: usize, chunks: usize, fresh: bool, cancelled: bool) bool { + return tokens > 0 and tokens <= max_tokens and chunks <= max_tokens and fresh and !cancelled; +} + +pub fn clean(finished: bool, normal_reason: bool, cancelled: bool, settled: bool) bool { + return finished and normal_reason and !cancelled and settled; +} + +pub const Decision = enum { refuse, busy, retain, evict }; +pub fn admission(active_bytes: usize, pending_bytes: usize, requested: usize) Decision { + if (requested == 0 or requested > max_bytes) return .refuse; + if (pending_bytes != 0) return .busy; + return if (active_bytes > max_bytes - requested) .evict else .retain; +} + +test "flag is strict and disabled by default" { + try std.testing.expect(!try enabled(null)); + try std.testing.expect(!try enabled("0")); + try std.testing.expect(try enabled("1")); + try std.testing.expectError(error.InvalidExactPromptCacheFlag, enabled("true")); +} + +test "only bounded fresh uncancelled prompts can enter" { + try std.testing.expect(eligible(1, 0, true, false)); + try std.testing.expect(eligible(512, 512, true, false)); + try std.testing.expect(!eligible(0, 0, true, false)); + try std.testing.expect(!eligible(513, 0, true, false)); + try std.testing.expect(!eligible(14, 0, false, false)); + try std.testing.expect(!eligible(14, 0, true, true)); +} + +test "clean completion never includes uncertain settle or cancellation" { + try std.testing.expect(clean(true, true, false, true)); + try std.testing.expect(!clean(true, true, false, false)); + try std.testing.expect(!clean(true, true, true, true)); + try std.testing.expect(!clean(true, false, false, true)); + try std.testing.expect(!clean(false, true, false, true)); +} + +test "one pending owner and total byte bound include replacement with room" { + try std.testing.expectEqual(.busy, admission(0, 1, 100)); + try std.testing.expectEqual(.refuse, admission(0, 0, max_bytes + 1)); + try std.testing.expectEqual(.retain, admission(100, 0, 100)); + try std.testing.expectEqual(.evict, admission(197995008, 0, 197995008)); + try std.testing.expectEqual(.retain, admission(0, 0, 197995008)); +} diff --git a/zig/src/families/qwen27/cuda/exact_cache_test.zig b/zig/src/families/qwen27/cuda/exact_cache_test.zig new file mode 100644 index 0000000000..db3166b367 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_cache_test.zig @@ -0,0 +1,491 @@ +//! Host-only CUDA-driver mocks exercise snapshot ownership and failure paths. +const std = @import("std"); +const cuda = @import("cuda"); +const lanes = @import("lanes"); +const cache_mod = @import("exact_prompt_cache.zig"); +const st = @import("state.zig"); +const shape = @import("shape.zig"); +const kern = @import("kernels.zig"); +const dr = @import("draft.zig"); +const g = shape.the_27b; +const a = std.testing.allocator; + +const Mock = struct { + var live: usize = 0; + var freed: usize = 0; + const Copy = struct { dst: u64, src: u64, bytes: usize }; + var copy_rows: [512]Copy = undefined; + var copies: usize = 0; + var fences: usize = 0; + var fail_copy = false; + var fail_fence = false; + var next: u64 = 0x100000000; + fn alloc(ptr: *cuda.abi.DevicePtr, _: usize) callconv(.c) cuda.abi.Result { + ptr.* = next; + next += 0x100000000; + live += 1; + return cuda.abi.success; + } + fn free(_: cuda.abi.DevicePtr) callconv(.c) cuda.abi.Result { + live -= 1; + freed += 1; + return cuda.abi.success; + } + fn copy(dst: cuda.abi.DevicePtr, src: cuda.abi.DevicePtr, bytes: usize, _: cuda.abi.Stream) callconv(.c) cuda.abi.Result { + if (fail_copy) return cuda.abi.error_not_ready; + std.debug.assert(copies < copy_rows.len); + copy_rows[copies] = .{ .dst = dst, .src = src, .bytes = bytes }; + copies += 1; + return cuda.abi.success; + } + fn fence(_: cuda.abi.Stream) callconv(.c) cuda.abi.Result { + if (fail_fence) return cuda.abi.error_not_ready; + fences += 1; + return cuda.abi.success; + } + fn reset() void { + live = 0; + freed = 0; + copies = 0; + fences = 0; + fail_copy = false; + fail_fence = false; + } +}; + +const Fixture = struct { + driver: cuda.Driver = undefined, + kernels: kern.Kernels = undefined, + rec: [64]u64 = @splat(0x1000), + conv: [64]u64 = @splat(0x2000), + keys: [64]u64 = @splat(0x3000), + values: [64]u64 = @splat(0x4000), + fn ops(f: *Fixture) kern.Ops { + f.driver.api.cuMemAlloc_v2 = Mock.alloc; + f.driver.api.cuMemFree_v2 = Mock.free; + f.driver.api.cuMemcpyDtoDAsync_v2 = Mock.copy; + f.driver.api.cuStreamSynchronize = Mock.fence; + f.kernels.d = &f.driver; + return .{ .k = &f.kernels, .s = .{ .d = &f.driver, .handle = null } }; + } + fn seq(f: *Fixture, n: usize) st.Seq { + return .{ .gpa = a, .arena = .{ .buf = null }, .rec = &f.rec, .conv = &f.conv, .keys = &f.keys, .values = &f.values, .kscale = &.{}, .vscale = &.{}, .kv8 = false, .head_in = 0x5000, .head_xs = 0x6000, .pos = n, .capacity = n + 256 }; + } +}; + +fn stream(ids: []const u32, drafted: bool) !lanes.Stream { + return lanes.Stream.init(a, .{ .id = "cache-test", .prompt = ids, .max_new = 256, .drafts = drafted }); +} +fn finish(s: *lanes.Stream) void { + s.finished = true; + s.reason = .length; +} + +test "actual snapshot owner replacement and hit copy never alias a live destination" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + finish(&seed); + try cache.release(&seed, true, ops); + const saved = cache.active.?; + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + var dest = f.seq(0); + try std.testing.expect(try cache.lookup(&hit, &dest, null, ops, g)); + try std.testing.expectEqual(@as(u32, 3), hit.cached); + try std.testing.expectEqual(@as(usize, 3), dest.pos); + try std.testing.expect(saved.head_in != dest.head_in); + var miss = try stream(&.{ 4, 5, 6 }, false); + defer miss.deinit(a); + try cache.capture(&miss, &seq, null, ops, g); + try std.testing.expectEqual(@as(u64, 1), cache.stats.evictions); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + try std.testing.expect(Mock.fences >= 2); + finish(&miss); + try cache.release(&miss, true, ops); + try std.testing.expectEqual(@as(u64, 2), cache.stats.promotions); +} + +test "pending is never hit and error or cancellation cannot promote" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var s = try stream(&.{ 1, 2 }, false); + defer s.deinit(a); + var seq = f.seq(2); + try cache.capture(&s, &seq, null, ops, g); + var other = try stream(&.{ 1, 2 }, false); + defer other.deinit(a); + try std.testing.expect(!try cache.lookup(&other, &seq, null, ops, g)); + try cache.capture(&other, &seq, null, ops, g); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + s.finished = true; + s.reason = .cancelled; + try cache.release(&s, true, ops); + try std.testing.expect(cache.active == null and cache.pending == null); + s.finished = false; + s.reason = .none; + try cache.capture(&s, &seq, null, ops, g); + finish(&s); + try cache.release(&other, false, ops); + try std.testing.expect(cache.failed and cache.active == null and Mock.live == 0); + try cache.release(&s, true, ops); + try std.testing.expectEqual(@as(u64, 0), cache.stats.promotions); +} + +test "copy failure purges and failed fence defers freeing in-flight source" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var s = try stream(&.{1}, false); + defer s.deinit(a); + var seq = f.seq(1); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, cache.capture(&s, &seq, null, ops, g)); + Mock.fail_copy = false; + Mock.fail_fence = true; + try std.testing.expectError(error.NotReady, cache.abandon(&s, ops)); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + Mock.fail_fence = false; + try cache.abandon(&s, ops); + try std.testing.expectEqual(@as(usize, 0), Mock.live); +} + +test "allocation refusal and disabled path perform no CUDA copy or fence" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var s = try stream(&.{1}, false); + defer s.deinit(a); + var seq = f.seq(1); + var off = cache_mod.Cache.init(a, false); + defer off.deinit(ops) catch unreachable; + try off.capture(&s, &seq, null, ops, g); + try std.testing.expect(!try off.lookup(&s, &seq, null, ops, g)); + var failing = std.testing.FailingAllocator.init(a, .{ .fail_index = 0 }); + var refused = cache_mod.Cache.init(failing.allocator(), true); + defer refused.deinit(ops) catch unreachable; + try refused.capture(&s, &seq, null, ops, g); + try std.testing.expectEqual(@as(u64, 1), refused.stats.refusals); + try std.testing.expect(Mock.copies == 0 and Mock.fences == 0 and Mock.live == 0); +} + +test "draft active-bank compact copies preserve metadata and bound maximum snapshot" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var s = try stream(&.{ 1, 2 }, true); + defer s.deinit(a); + var seq = f.seq(2); + var ctx: dr.Context = .{ .arena = .{ .buf = null }, .keys = @splat(@splat(0x10000)), .values = @splat(@splat(0x20000)), .cur = 1, .len = 2, .end = 2 }; + try cache.capture(&s, &seq, &ctx, ops, g); + // absorbMany and _block_attention pack each head at len*128 BF16 values, not the allocation window. + const head_bytes = 2 * 128 * 2; + const target_copies = 2 + 48 * 2 + 16 * 2; + const saved = cache.pending.?; + for (0..5) |layer| for (0..8) |head| { + const pair = Mock.copy_rows[target_copies + layer * 16 + head * 2 ..][0..2]; + try std.testing.expectEqual(0x10000 + head * head_bytes, pair[0].src); + try std.testing.expectEqual(0x20000 + head * head_bytes, pair[1].src); + try std.testing.expectEqual(saved.draft_keys[layer] + head * head_bytes, pair[0].dst); + try std.testing.expectEqual(saved.draft_values[layer] + head * head_bytes, pair[1].dst); + try std.testing.expectEqual(head_bytes, pair[0].bytes); + try std.testing.expectEqual(head_bytes, pair[1].bytes); + }; + const capture_copies = Mock.copies; + finish(&s); + try cache.release(&s, true, ops); + var hit = try stream(&.{ 1, 2 }, true); + defer hit.deinit(a); + ctx.cur = 0; + ctx.len = 0; + ctx.end = 0; + try std.testing.expect(try cache.lookup(&hit, &seq, &ctx, ops, g)); + try std.testing.expect(ctx.cur == 1 and ctx.len == 2 and ctx.end == 2); + for (0..5) |layer| for (0..8) |head| { + const pair = Mock.copy_rows[capture_copies + target_copies + layer * 16 + head * 2 ..][0..2]; + try std.testing.expectEqual(0x10000 + head * head_bytes, pair[0].dst); + try std.testing.expectEqual(0x20000 + head * head_bytes, pair[1].dst); + try std.testing.expectEqual(saved.draft_keys[layer] + head * head_bytes, pair[0].src); + try std.testing.expectEqual(saved.draft_values[layer] + head * head_bytes, pair[1].src); + }; + try std.testing.expectEqual(@as(usize, 197995008), cache_mod.Snapshot.bytes(g, 512, true)); + try std.testing.expect(cache_mod.Snapshot.bytes(g, 512, true) <= cache_mod.policy.max_bytes); +} + +test "acquisition hint shares exact eligibility but never copies or reports a hit" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + const before = cache.stats; + const copies = Mock.copies; + const fences = Mock.fences; + const active = cache.active; + try std.testing.expect(cache.canRestore(&hit, false)); + try std.testing.expect(!cache.canRestore(&hit, true)); + hit.cached = 1; + try std.testing.expect(!cache.canRestore(&hit, false)); + hit.cached = 0; + cache.failed = true; + try std.testing.expect(!cache.canRestore(&hit, false)); + cache.failed = false; + var miss = try stream(&.{ 1, 2, 4 }, false); + defer miss.deinit(a); + try std.testing.expect(!cache.canRestore(&miss, false)); + try std.testing.expect(std.meta.eql(before, cache.stats)); + try std.testing.expectEqual(copies, Mock.copies); + try std.testing.expectEqual(fences, Mock.fences); + try std.testing.expect(cache.active == active and hit.cached == 0); +} + +test "enabled first logits add one bounded row and absent marker prevents promotion" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + cache.logits_enabled = true; + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + const snap = cache.pending.?; + try std.testing.expect(!snap.logits_valid and snap.first_logits != 0); + try std.testing.expectEqual(cache_mod.Snapshot.bytes(g, 3, false) + 496640, snap.device_bytes); + try std.testing.expectEqual(@as(usize, 198491648), cache_mod.Snapshot.bytesWithLogits(g, 512, true, true)); + try std.testing.expect(cache_mod.Snapshot.bytesWithLogits(g, 512, true, true) < cache_mod.policy.max_bytes); + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expect(cache.active == null and cache.pending == null and Mock.live == 0); + try std.testing.expectEqual(@as(u64, 1), cache.stats.purges); +} + +test "actual raw capture and proven current-active copy preserve direction and eviction fallback" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + cache.logits_enabled = true; + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + const saved_logits = cache.pending.?.first_logits; + const before = Mock.copies; + try std.testing.expect(try cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + try std.testing.expectEqual(before + 1, Mock.copies); + try std.testing.expectEqual(saved_logits, Mock.copy_rows[before].dst); + try std.testing.expectEqual(@as(u64, 0x7770), Mock.copy_rows[before].src); + try std.testing.expectEqual(@as(usize, 496640), Mock.copy_rows[before].bytes); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + var dest = f.seq(0); + try std.testing.expect(try cache.lookup(&hit, &dest, null, ops, g)); + const copied = Mock.copies; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, false)); + try std.testing.expectEqual(copied, Mock.copies); + dest.pos = 4; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + dest.pos = 3; + try std.testing.expect(try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + try std.testing.expectEqual(saved_logits, Mock.copy_rows[copied].src); + try std.testing.expectEqual(@as(u64, 0x8880), Mock.copy_rows[copied].dst); + try std.testing.expectEqual(@as(usize, 496640), Mock.copy_rows[copied].bytes); + try std.testing.expect(cache.active.?.logits_valid); + try std.testing.expectEqual(@as(u64, 1), cache.stats.first_logits_hits); + Mock.copies = 0; + var miss = try stream(&.{ 4, 5, 6 }, false); + defer miss.deinit(a); + try cache.capture(&miss, &seq, null, ops, g); + try std.testing.expect(cache.active == null and cache.pending != null); + const after_eviction = Mock.copies; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + try std.testing.expectEqual(after_eviction, Mock.copies); + try std.testing.expect(Mock.fences > 0); +} + +test "raw capture error clears validity and hit copy error disables future use" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + cache.logits_enabled = true; + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{1}, false); + defer seed.deinit(a); + var seq = f.seq(1); + try cache.capture(&seed, &seq, null, ops, g); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + try std.testing.expect(cache.failed and !cache.pending.?.logits_valid); + Mock.fail_copy = false; + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expect(cache.active == null and cache.pending == null); + cache.failed = false; + seed.finished = false; + seed.reason = .none; + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expect(try cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{1}, false); + defer hit.deinit(a); + var dest = f.seq(0); + try std.testing.expect(try cache.lookup(&hit, &dest, null, ops, g)); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); + try std.testing.expect(cache.failed); + Mock.fail_copy = false; + try std.testing.expect(!try cache.restoreFirstLogits(&hit, &dest, ops, 0x8880, true)); +} + +test "logits-off capture and first-logit methods add no row copy or fence" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{1}, false); + defer seed.deinit(a); + var seq = f.seq(1); + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expectEqual(cache_mod.Snapshot.bytes(g, 1, false), cache.pending.?.device_bytes); + try std.testing.expectEqual(@as(u64, 0), cache.pending.?.first_logits); + const copied = Mock.copies; + const fenced = Mock.fences; + try std.testing.expect(!try cache.captureFirstLogits(&seed, &seq, ops, 0x7770)); + try std.testing.expect(!try cache.restoreFirstLogits(&seed, &seq, ops, 0x8880, true)); + try std.testing.expectEqual(copied, Mock.copies); + try std.testing.expectEqual(fenced, Mock.fences); + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expect(cache.active != null); +} + +test "unsupported Seq formats decline active lookup and new capture before any device operation" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2 }, false); + defer seed.deinit(a); + var seq = f.seq(2); + try cache.capture(&seed, &seq, null, ops, g); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2 }, false); + defer hit.deinit(a); + var miss = try stream(&.{ 3, 4 }, false); + defer miss.deinit(a); + const raw = try cache_mod.Snapshot.init(a, ops, g, &hit); + defer raw.deinit(); + var media: @import("vision.zig").Media = undefined; + const before = .{ Mock.live, Mock.copies, Mock.fences }; + for (0..3) |kind| { + seq.kv8 = kind == 0; + seq.media = if (kind == 1) &media else null; + seq.rope_delta = if (kind == 2) 7 else 0; + try cache.capture(&miss, &seq, null, ops, g); + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(@as(u32, 0), hit.cached); + try std.testing.expectError(error.UnsupportedExactCacheState, raw.save(ops, g, &seq, null)); + try std.testing.expectError(error.UnsupportedExactCacheState, raw.restore(ops, g, &seq, null)); + try std.testing.expectEqual(before[0], Mock.live); + try std.testing.expectEqual(before[1], Mock.copies); + try std.testing.expectEqual(before[2], Mock.fences); + } + seq.kv8 = false; + seq.media = null; + seq.rope_delta = 0; + hit.images = &.{.{ .pixels = &.{}, .gh = 1, .gw = 1, .at = 0, .tokens = 1 }}; + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(before[1], Mock.copies); + hit.images = &.{}; + try std.testing.expect(try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(@as(u32, 2), hit.cached); +} + +test "partial host-prefix metadata cannot become full-N exact restoration proof" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + var seed = try stream(&.{ 1, 2, 3 }, false); + defer seed.deinit(a); + var seq = f.seq(3); + try cache.capture(&seed, &seq, null, ops, g); + finish(&seed); + try cache.release(&seed, true, ops); + var hit = try stream(&.{ 1, 2, 3 }, false); + defer hit.deinit(a); + const before = Mock.copies; + hit.reuse.at = 2; + try std.testing.expect(!cache.canRestore(&hit, false)); + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + hit.reuse.at = 0; + hit.reuse.marks = &.{2}; + try std.testing.expect(!cache.canRestore(&hit, false)); + hit.reuse.marks = &.{}; + hit.cached = 2; + try std.testing.expect(!cache.canRestore(&hit, false)); + try std.testing.expect(!try cache.lookup(&hit, &seq, null, ops, g)); + try std.testing.expectEqual(before, Mock.copies); + try std.testing.expectEqual(@as(u32, 2), hit.cached); + hit.cached = 0; + try std.testing.expect(cache.canRestore(&hit, false)); +} + +test "normal chat assistant-header marker permits full-N snapshot promotion and repeat hit" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var cache = cache_mod.Cache.init(a, true); + defer cache.deinit(ops) catch unreachable; + const ids: [31]u32 = @splat(1); + var seed = try lanes.Stream.init(a, .{ .id = "chat-cold", .prompt = &ids, .max_new = 256, .drafts = false, .decide = 29 }); + defer seed.deinit(a); + var seq = f.seq(ids.len); + try std.testing.expect(!cache.canRestore(&seed, false)); + try cache.capture(&seed, &seq, null, ops, g); + try std.testing.expectEqual(@as(u64, 1), cache.stats.captures); + finish(&seed); + try cache.release(&seed, true, ops); + try std.testing.expectEqual(@as(u64, 1), cache.stats.promotions); + var repeat = try lanes.Stream.init(a, .{ .id = "chat-repeat", .prompt = &ids, .max_new = 256, .drafts = false, .decide = 29 }); + defer repeat.deinit(a); + var dest = f.seq(0); + try std.testing.expect(cache.canRestore(&repeat, false)); + try std.testing.expect(try cache.lookup(&repeat, &dest, null, ops, g)); + try std.testing.expectEqual(@as(u32, 31), repeat.cached); + try std.testing.expectEqual(@as(usize, 31), dest.pos); + try std.testing.expectEqual(@as(u64, 1), cache.stats.hits); +} diff --git a/zig/src/families/qwen27/cuda/exact_draft_reuse.zig b/zig/src/families/qwen27/cuda/exact_draft_reuse.zig new file mode 100644 index 0000000000..94a27b05d2 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_draft_reuse.zig @@ -0,0 +1,173 @@ +//! Private cold-capable Context storage recycling; uncertain in-flight ownership is quarantined, never reused. +const std = @import("std"); +const Cuda = @import("lanes.zig").Cuda; +const lanes = @import("lanes"); +const drafts = @import("drafts.zig"); +const dr = @import("draft.zig"); +const seq_reuse = @import("exact_seq_reuse.zig"); +pub const policy = @import("draft_pool.zig"); +pub const Storage = policy.Pool(drafts.Stream); + +fn destroy(x: *Cuda, ds: *drafts.Stream) void { + ds.deinit(); + x.gpa.destroy(ds); +} + +fn clearIdle(x: *Cuda) void { + while (x.draft_pool.evict()) |ds| destroy(x, ds); +} + +/// Only already-fenced idle storage can be freed without another fence. +pub fn fail(x: *Cuda) void { + if (!x.draft_pool.stats.enabled) return; + x.draft_pool.fail(); + clearIdle(x); +} + +fn fresh(s: *lanes.Stream) bool { + return s.chunks.len == 0 and !s.reuse_failed and s.reuse.saved == null and s.reuse.at == 0 and + s.reuse.marks.len == 0 and s.reuse.hook == null; +} + +fn eligible(x: *Cuda, s: *lanes.Stream, capacity: usize) bool { + return @import("exact_eligible.zig").text(x.e.kv8, s) and policy.eligible(s.drafts, x.drafter != null, s.prompt().len, capacity, fresh(s), s.isCancelled()); +} + +fn reap(x: *Cuda) !void { + if (x.draft_pool.stats.quarantined_slots == 0) return; + try x.e.s.synchronize(); + while (x.draft_pool.reap(true)) |ds| destroy(x, ds); +} + +/// Misses within the fixed small class may take storage; unsupported admissions drain before target allocation. +pub fn beforeOpen(x: *Cuda, s: *lanes.Stream, capacity: usize) !void { + if (!x.draft_pool.stats.enabled) return; + try reap(x); + if (x.seqs.count() >= x.draft_pool.limit) return error.DraftPoolOwnershipLimit; + if (!x.draft_pool.active() or !eligible(x, s, capacity)) clearIdle(x); +} + +pub fn obtain(x: *Cuda, s: *lanes.Stream) !*drafts.Stream { + errdefer seq_reuse.fail(x); + if (x.drafting.get(s)) |ds| return ds; + var ds: *drafts.Stream = undefined; + if (x.draft_pool.stats.enabled) { + try reap(x); + const seq = x.seqs.get(s) orelse return error.UnknownQwenStream; + if (x.draft_pool.active() and eligible(x, s, seq.capacity)) { + if (x.draft_pool.take()) |saved| { + ds = saved; + errdefer destroy(x, ds); + ds.resetStorage(); + try x.drafting.put(x.gpa, s, ds); + return ds; + } + } else clearIdle(x); + if (!x.draft_pool.hasRoom(x.drafting.count())) return error.DraftPoolOwnershipLimit; + } + ds = try x.gpa.create(drafts.Stream); + errdefer x.gpa.destroy(ds); + const ctx = dr.Context.init(x.e.ctx.d) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + if (x.draft_pool.stats.enabled) x.draft_pool.stats.driver_creations += 1; + ds.* = drafts.Stream.init(x.gpa, ctx); + errdefer ds.deinit(); + try x.drafting.put(x.gpa, s, ds); + return ds; +} + +/// The tag authorizes recycling empty storage later, not reuse of prior request state now. +pub fn initialized(x: *Cuda, s: *lanes.Stream) !void { + if (!x.draft_pool.active()) return; + errdefer fail(x); + const seq = x.seqs.get(s) orelse return error.UnknownQwenStream; + if (!eligible(x, s, seq.capacity)) return; + const ds = x.drafting.get(s) orelse return error.UnknownDraftStream; + const buf = ds.ctx.arena.buf orelse return error.InvalidDraftInitialization; + if (seq.pos != s.prompt_len or ds.ctx.len != s.prompt_len or ds.ctx.end != s.prompt_len) + return error.InvalidDraftInitialization; + try x.draft_initialized.mark(.{ .stream = s, .seq = seq, .wrapper = ds, .arena = buf.ptr, .tokens = s.prompt_len, .capacity = seq.capacity }); + x.draft_pool.stats.initialized += 1; +} + +fn detach(x: *Cuda, s: *lanes.Stream) ?*drafts.Stream { + if (x.drafting.fetchRemove(s)) |entry| return entry.value; + return null; +} + +/// Error ownership never enters the reusable pool, even when a later fence succeeds. +pub fn drop(x: *Cuda, s: *lanes.Stream) void { + _ = x.draft_initialized.remove(s); + const ds = detach(x, s) orelse return; + if (!x.draft_pool.stats.enabled) return destroy(x, ds); + fail(x); + x.e.s.synchronize() catch |err| { + std.log.err("quarantining failed draft Context: {s}", .{@errorName(err)}); + x.draft_pool.park(ds, x.drafting.count()); + return; + }; + x.draft_pool.stats.discarded += 1; + destroy(x, ds); +} + +/// Target release has already happened; pointer comparison does not dereference that former Seq owner. +pub fn release(x: *Cuda, s: *lanes.Stream, seq: ?*const anyopaque, settled: bool, cache_ok: bool) void { + const proof = x.draft_initialized.remove(s); + const ds = detach(x, s) orelse return; + if (!x.draft_pool.stats.enabled) return destroy(x, ds); + const normal = s.finished and (s.reason == .stop or s.reason == .length); + const valid = if (proof) |p| p.seq == seq and p.wrapper == @as(*const anyopaque, ds) and + p.arena == (if (ds.ctx.arena.buf) |buf| buf.ptr else 0) and p.tokens == s.prompt_len and + eligible(x, s, p.capacity) else false; + const clean = policy.completed(valid, normal, !s.isCancelled(), settled, cache_ok, x.exact_cache.failed); + if (!settled or !cache_ok or x.exact_cache.failed) fail(x); + x.e.s.synchronize() catch |err| { + std.log.err("quarantining released draft Context: {s}", .{@errorName(err)}); + fail(x); + x.draft_pool.park(ds, x.drafting.count()); + return; + }; + if (clean and !s.isCancelled() and x.draft_pool.active()) { + ds.resetStorage(); + if (x.draft_pool.retain(ds, x.drafting.count())) return; + } + x.draft_pool.stats.discarded += 1; + destroy(x, ds); +} + +fn shutdownFence(stream: anytype) !void { + stream.synchronize() catch |first_err| { + std.log.warn("draft teardown fence, retrying once: {s}", .{@errorName(first_err)}); + try stream.synchronize(); + }; +} + +/// One bounded retry precedes a fatal caller exit; no uncertain Context is freed on either failed fence. +pub fn shutdown(x: *Cuda) !void { + if (!x.draft_pool.stats.enabled) return; + if (x.draft_pool.stats.quarantined_slots > 0 or x.drafting.count() > 0) { + try shutdownFence(&x.e.s); + } + while (x.draft_pool.reap(true)) |ds| destroy(x, ds); + clearIdle(x); + std.debug.print("PR608_EXACT_CACHE_DRAFT_POOL {f}\n", .{std.json.fmt(x.draft_pool.stats, .{})}); +} + +test "shutdown fence has one bounded retry and propagates persistent failure" { + const Mock = struct { + calls: usize = 0, + failures: usize, + pub fn synchronize(x: *@This()) !void { + x.calls += 1; + if (x.calls <= x.failures) return error.MockFenceFailed; + } + }; + var good: Mock = .{ .failures = 0 }; + try shutdownFence(&good); + try std.testing.expectEqual(@as(usize, 1), good.calls); + var recovered: Mock = .{ .failures = 1 }; + try shutdownFence(&recovered); + try std.testing.expectEqual(@as(usize, 2), recovered.calls); + var failed: Mock = .{ .failures = 3 }; + try std.testing.expectError(error.MockFenceFailed, shutdownFence(&failed)); + try std.testing.expectEqual(@as(usize, 2), failed.calls); +} diff --git a/zig/src/families/qwen27/cuda/exact_eligible.zig b/zig/src/families/qwen27/cuda/exact_eligible.zig new file mode 100644 index 0000000000..7f9703b016 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_eligible.zig @@ -0,0 +1,40 @@ +//! The exact-cache helpers support only the original BF16 fresh text arithmetic and state layout. +const lanes = @import("lanes"); + +pub fn text(kv8: bool, s: *const lanes.Stream) bool { + // The server's assistant-header decide marker is inert in Qwen's nonlearning CUDA backend. + return !kv8 and s.images.len == 0 and s.prompt_logprobs == null and s.decode_spans.len == 0 and + !s.untaught and !s.reuse_failed; +} + +test "FP8, images and changed prompt arithmetic decline every exact-cache treatment" { + const std = @import("std"); + var s = try lanes.Stream.init(std.testing.allocator, .{ .id = "eligible", .prompt = &.{1}, .max_new = 4 }); + defer s.deinit(std.testing.allocator); + try std.testing.expect(text(false, &s)); + try std.testing.expect(!text(true, &s)); + s.images = &.{.{ .pixels = &.{}, .gh = 1, .gw = 1, .at = 0, .tokens = 1 }}; + try std.testing.expect(!text(false, &s)); + s.images = &.{}; + s.prompt_logprobs = 1; + try std.testing.expect(!text(false, &s)); + s.prompt_logprobs = null; + s.decode_spans = &.{.{ 0, 1 }}; + try std.testing.expect(!text(false, &s)); + s.decode_spans = &.{}; + s.untaught = true; + try std.testing.expect(!text(false, &s)); + s.untaught = false; + s.reuse_failed = true; + try std.testing.expect(!text(false, &s)); +} + +test "ordinary chat assistant-header metadata does not change Qwen text eligibility" { + const std = @import("std"); + const ids: [31]u32 = @splat(1); + var s = try lanes.Stream.init(std.testing.allocator, .{ .id = "chat-marker", .prompt = &ids, .max_new = 256, .decide = 29 }); + defer s.deinit(std.testing.allocator); + try std.testing.expectEqual(@as(?u32, 29), s.decide); + try std.testing.expect(text(false, &s)); + try std.testing.expect(!text(true, &s)); +} diff --git a/zig/src/families/qwen27/cuda/exact_first_logits.zig b/zig/src/families/qwen27/cuda/exact_first_logits.zig new file mode 100644 index 0000000000..5b709a7bfa --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_first_logits.zig @@ -0,0 +1,31 @@ +//! First-head substitution only; the core forced-token selection and every sampling operation remain unchanged. +const std = @import("std"); +const lanes = @import("lanes"); +const Cuda = @import("lanes.zig").Cuda; +const reuse = @import("exact_seq_reuse.zig"); + +pub fn first(p: *anyopaque, s: *lanes.Stream, position: u64) !u64 { + const x: *Cuda = @ptrCast(@alignCast(p)); + errdefer { + reuse.fail(x); + if (x.exact_cache.logits_enabled) { + _ = x.restored.remove(s); + x.exact_cache.failed = true; + x.exact_cache.abandon(s, x.e.ops()) catch |err| std.log.err("first-logits purge: {s}", .{@errorName(err)}); + } + } + const seq = try x.settled(s); + if (position != seq.pos) return error.InvalidQwenPosition; + var f = x.e.forward(); + if (x.exact_cache.logits_enabled) { + const proven = x.seqs.get(s) == seq and x.restored.matches(s, seq, s.prompt().len); + if (!try x.exact_cache.restoreFirstLogits(s, seq, f.ops, f.s.logits, proven)) { + x.exact_cache.stats.first_logits_fallbacks += 1; + try f.head(seq); + _ = try x.exact_cache.captureFirstLogits(s, seq, f.ops, f.s.logits); + } + } else try f.head(seq); + var out: [1]u32 = undefined; + try x.drawRows(&.{s.sampling}, &.{position}, &out); + return out[0]; +} diff --git a/zig/src/families/qwen27/cuda/exact_init.zig b/zig/src/families/qwen27/cuda/exact_init.zig new file mode 100644 index 0000000000..b4657aec4c --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_init.zig @@ -0,0 +1,30 @@ +//! The default-off exact-cache owners initialized beside the unchanged Qwen CUDA host buffers. +const std = @import("std"); +const Cuda = @import("lanes.zig").Cuda; +const Engine = @import("engine.zig").Engine; +const c = @import("shape.zig"); +const st = @import("state.zig"); +const exact = @import("exact_prompt_cache.zig"); +const exact_reuse = @import("exact_seq_reuse.zig"); +const draft_reuse = @import("exact_draft_reuse.zig"); +const dr = @import("draft.zig"); + +pub fn init(gpa: std.mem.Allocator, e: *Engine, streams: u32) !Cuda { + const env = std.c.getenv("TF_PR608_EXACT_PROMPT_CACHE"); + const on = try exact.policy.enabled(if (env) |v| std.mem.span(v) else null); + var cache = exact.Cache.init(gpa, on); + cache.logits_enabled = try exact.logits_policy.configured(on); + return .{ + .gpa = gpa, + .e = e, + .streams = streams, + .exact_cache = cache, + .reset_skip = .{ .enabled = try exact_reuse.reset_skip.configured(on) }, + .seq_pool = exact_reuse.Storage.init(try exact_reuse.policy.configured(on), streams), + .draft_pool = draft_reuse.Storage.init(try draft_reuse.policy.configured(on), streams, dr.Context.bytes()), + .logits = try gpa.alloc(u16, c.vocab), + .cand_values = try gpa.alloc(f32, st.round_rows * st.max_candidates), + .cand_ids = try gpa.alloc(i64, st.round_rows * st.max_candidates), + .values = try gpa.alloc(f64, st.max_candidates), + }; +} diff --git a/zig/src/families/qwen27/cuda/exact_logits_policy.zig b/zig/src/families/qwen27/cuda/exact_logits_policy.zig new file mode 100644 index 0000000000..f4dc1f6d3f --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_logits_policy.zig @@ -0,0 +1,75 @@ +//! Bounded raw first-logit cache policy; decisions never cache sampling or a selected token. +const std = @import("std"); +pub const row_bytes: usize = 248320 * 2; // shape.zig's fixed BF16 vocabulary row, already 256-byte aligned + +pub fn enabled(text: ?[]const u8, cache_enabled: bool) !bool { + const value = text orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (!std.mem.eql(u8, value, "1")) return error.InvalidExactCacheLogitsFlag; + if (!cache_enabled) return error.FirstLogitsRequireExactCache; + return true; +} + +pub fn configured(cache_enabled: bool) !bool { + const env = std.c.getenv("TF_PR608_EXACT_CACHE_LOGITS"); + return enabled(if (env) |v| std.mem.span(v) else null, cache_enabled); +} + +pub fn canUse(healthy: bool, proven: bool, key_matches: bool, valid: bool, position_matches: bool, cancelled: bool) bool { + return healthy and proven and key_matches and valid and position_matches and !cancelled; +} + +pub fn promotable(on: bool, valid: bool) bool { + return !on or valid; +} + +/// Only successful submission grants validity; the existing promotion fence supplies completion. +pub fn capture(valid: *bool, copier: anytype, dst: u64, src: u64) !void { + valid.* = false; + try copier.copy(dst, src, row_bytes); + valid.* = true; +} + +test "raw logits are off by default and strictly require the full-prompt cache" { + try std.testing.expect(!try enabled(null, false)); + try std.testing.expect(!try enabled("0", false)); + try std.testing.expect(try enabled("1", true)); + try std.testing.expectError(error.FirstLogitsRequireExactCache, enabled("1", false)); + try std.testing.expectError(error.InvalidExactCacheLogitsFlag, enabled("true", true)); + try std.testing.expectError(error.InvalidExactCacheLogitsFlag, enabled("", true)); +} + +test "current active key and actual stream restore proof are both required" { + try std.testing.expect(canUse(true, true, true, true, true, false)); + try std.testing.expect(!canUse(false, true, true, true, true, false)); + try std.testing.expect(!canUse(true, false, true, true, true, false)); + try std.testing.expect(!canUse(true, true, false, true, true, false)); + try std.testing.expect(!canUse(true, true, true, false, true, false)); + try std.testing.expect(!canUse(true, true, true, true, false, false)); + try std.testing.expect(!canUse(true, true, true, true, true, true)); + try std.testing.expect(!promotable(true, false)); + try std.testing.expect(promotable(true, true)); + try std.testing.expect(promotable(false, false)); +} + +test "copy submission failure cannot create a valid raw row or promotion" { + const Copier = struct { + fail: bool, + calls: usize = 0, + fn copy(self: *@This(), dst: u64, src: u64, n: usize) !void { + self.calls += 1; + try std.testing.expectEqual(@as(u64, 256), dst); + try std.testing.expectEqual(@as(u64, 512), src); + try std.testing.expectEqual(@as(usize, 496640), n); + if (self.fail) return error.InjectedCopyFailure; + } + }; + var valid = true; + var copier: Copier = .{ .fail = true }; + try std.testing.expectError(error.InjectedCopyFailure, capture(&valid, &copier, 256, 512)); + try std.testing.expect(!valid and !promotable(true, valid)); + copier.fail = false; + try capture(&valid, &copier, 256, 512); + try std.testing.expect(valid and promotable(true, valid)); + try std.testing.expectEqual(@as(usize, 2), copier.calls); +} diff --git a/zig/src/families/qwen27/cuda/exact_prompt_cache.zig b/zig/src/families/qwen27/cuda/exact_prompt_cache.zig new file mode 100644 index 0000000000..41f5662879 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_prompt_cache.zig @@ -0,0 +1,339 @@ +//! Default-off full-prompt snapshots into independent normal Qwen allocations. +const std = @import("std"); +const lanes = @import("lanes"); +const st = @import("state.zig"); +const c = @import("shape.zig"); +const dr = @import("draft.zig"); +const dl = @import("draft_load.zig"); +const kern = @import("kernels.zig"); +pub const policy = @import("exact_cache_policy.zig"); +pub const logits_policy = @import("exact_logits_policy.zig"); +const roundedBytes = struct { + fn bytes(n: usize) usize { + return std.mem.alignForward(usize, n, 256); + } +}.bytes; + +pub const Snapshot = struct { + gpa: std.mem.Allocator, + arena: st.Arena, + prompt: []u32, + chunks: []u32, + drafted: bool, + pos: usize, + cur: u1 = 0, + len: usize = 0, + end: usize = 0, + rec: [64]u64 = @splat(0), + conv: [64]u64 = @splat(0), + keys: [64]u64 = @splat(0), + values: [64]u64 = @splat(0), + head_in: u64, + head_xs: u64, + draft_keys: [dl.layers]u64 = @splat(0), + draft_values: [dl.layers]u64 = @splat(0), + device_bytes: usize, + first_logits: u64 = 0, + logits_valid: bool = false, + + pub fn bytes(g: c.Geometry, n: usize, drafted: bool) usize { + return bytesWithLogits(g, n, drafted, false); + } + + pub fn bytesWithLogits(g: c.Geometry, n: usize, drafted: bool, logits: bool) usize { + return st.Seq.bytes(g, n, false) + (if (drafted) 2 * dl.layers * roundedBytes(dl.kv_heads * n * dl.head_dim * 2) else 0) + + (if (logits) logits_policy.row_bytes else 0); + } + + pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, s: *lanes.Stream) !*Snapshot { + return initWithLogits(gpa, ops, g, s, false); + } + + pub fn initWithLogits(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, s: *lanes.Stream, logits: bool) !*Snapshot { + if (g.layers != 64) return error.UnsupportedExactCacheGeometry; + const n = s.prompt().len; + const total = bytesWithLogits(g, n, s.drafts, logits); + if (total > policy.max_bytes) return error.ExactCacheBudget; + const out = try gpa.create(Snapshot); + errdefer gpa.destroy(out); + const prompt = try gpa.dupe(u32, s.prompt()); + errdefer gpa.free(prompt); + const chunks = try gpa.dupe(u32, s.chunks); + errdefer gpa.free(chunks); + out.* = .{ .gpa = gpa, .arena = try st.Arena.init(ops.k.d, total), .prompt = prompt, .chunks = chunks, .drafted = s.drafts, .pos = n, .head_in = 0, .head_xs = 0, .device_bytes = total }; + errdefer out.arena.deinit(); + out.head_in = try out.arena.take(g.hidden * 2); + out.head_xs = try out.arena.take(g.hidden / 64 * 4); + for (0..g.layers) |i| { + if (c.linear(i)) { + out.rec[i] = try out.arena.take(g.deltaBytes()); + out.conv[i] = try out.arena.take(g.convBytes()); + } else { + out.keys[i] = try out.arena.take(n * g.kvInner() * 2); + out.values[i] = try out.arena.take(n * g.kvInner() * 2); + } + } + if (s.drafts) for (0..dl.layers) |i| { + out.draft_keys[i] = try out.arena.take(dl.kv_heads * n * dl.head_dim * 2); + out.draft_values[i] = try out.arena.take(dl.kv_heads * n * dl.head_dim * 2); + }; + if (logits) out.first_logits = try out.arena.take(logits_policy.row_bytes); + return out; + } + + fn target(snap: *const Snapshot, ops: kern.Ops, g: c.Geometry, seq: *st.Seq, restoring: bool) !void { + if (seq.capacity < snap.pos) return error.ExactCacheCapacity; + const pair = struct { + fn run(o: kern.Ops, saved: u64, live: u64, n: usize, restored: bool) !void { + try o.copy(if (restored) live else saved, if (restored) saved else live, n); + } + }.run; + try pair(ops, snap.head_in, seq.head_in, g.hidden * 2, restoring); + try pair(ops, snap.head_xs, seq.head_xs, g.hidden / 64 * 4, restoring); + for (0..g.layers) |i| if (c.linear(i)) { + try pair(ops, snap.rec[i], seq.rec[i], g.deltaBytes(), restoring); + try pair(ops, snap.conv[i], seq.conv[i], g.convBytes(), restoring); + } else { + try pair(ops, snap.keys[i], seq.keys[i], snap.pos * g.kvInner() * 2, restoring); + try pair(ops, snap.values[i], seq.values[i], snap.pos * g.kvInner() * 2, restoring); + }; + } + + fn context(snap: *const Snapshot, ops: kern.Ops, ctx: *dr.Context, restoring: bool) !void { + const n = snap.len * dl.head_dim * 2; + const live_stride = n; + for (0..dl.layers) |i| for (0..dl.kv_heads) |h| { + const k = ctx.keys[i][snap.cur] + h * live_stride; + const v = ctx.values[i][snap.cur] + h * live_stride; + const sk = snap.draft_keys[i] + h * n; + const sv = snap.draft_values[i] + h * n; + try ops.copy(if (restoring) k else sk, if (restoring) sk else k, n); + try ops.copy(if (restoring) v else sv, if (restoring) sv else v, n); + }; + } + + pub fn save(snap: *Snapshot, ops: kern.Ops, g: c.Geometry, seq: *st.Seq, ctx: ?*dr.Context) !void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0) return error.UnsupportedExactCacheState; + if (seq.pos != snap.pos or (snap.drafted != (ctx != null))) return error.InvalidExactCacheState; + if (ctx) |x| { + if (x.len != snap.pos or x.end != snap.pos) return error.InvalidExactCacheContext; + snap.cur = x.cur; + snap.len = x.len; + snap.end = x.end; + } + try snap.target(ops, g, seq, false); + if (ctx) |x| try snap.context(ops, x, false); + } + + pub fn restore(snap: *const Snapshot, ops: kern.Ops, g: c.Geometry, seq: *st.Seq, ctx: ?*dr.Context) !void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0) return error.UnsupportedExactCacheState; + if (snap.drafted != (ctx != null)) return error.InvalidExactCacheState; + try snap.target(ops, g, seq, true); + if (ctx) |x| { + try snap.context(ops, x, true); + x.cur = snap.cur; + x.len = snap.len; + x.end = snap.end; + } + seq.pos = snap.pos; + } + + pub fn matches(snap: *const Snapshot, s: *lanes.Stream) bool { + return snap.drafted == s.drafts and std.mem.eql(u32, snap.prompt, s.prompt()) and std.mem.eql(u32, snap.chunks, s.chunks); + } + + pub fn deinit(snap: *Snapshot) void { + snap.arena.deinit(); + snap.gpa.free(snap.prompt); + snap.gpa.free(snap.chunks); + snap.gpa.destroy(snap); + } +}; + +pub const Stats = struct { + lookups: u64 = 0, + hits: u64 = 0, + misses: u64 = 0, + captures: u64 = 0, + promotions: u64 = 0, + purges: u64 = 0, + evictions: u64 = 0, + refusals: u64 = 0, + active_bytes: usize = 0, + pending_bytes: usize = 0, + first_logits_captures: u64 = 0, + first_logits_hits: u64 = 0, + first_logits_fallbacks: u64 = 0, +}; + +pub const Cache = struct { + gpa: std.mem.Allocator, + enabled: bool, + logits_enabled: bool = false, + failed: bool = false, + active: ?*Snapshot = null, + pending: ?*Snapshot = null, + owner: ?*lanes.Stream = null, + stats: Stats = .{}, + + pub fn init(gpa: std.mem.Allocator, on: bool) Cache { + return .{ .gpa = gpa, .enabled = on }; + } + + pub fn eligible(cache: *const Cache, s: *lanes.Stream) bool { + const fresh = s.reuse.saved == null and s.reuse.at == 0 and s.reuse.marks.len == 0 and s.cached == 0; + return cache.enabled and !cache.failed and @import("exact_eligible.zig").text(false, s) and policy.eligible(s.prompt().len, s.chunks.len, fresh, s.isCancelled()); + } + + /// Pure acquisition hint; successful lookup, not this hint, proves restoration. + pub fn canRestore(cache: *const Cache, s: *lanes.Stream, drafted: bool) bool { + if (!cache.eligible(s) or s.drafts != drafted) return false; + const snap = cache.active orelse return false; + return snap.matches(s); + } + + fn fence(cache: *Cache, ops: kern.Ops) !void { + ops.s.synchronize() catch |err| { + cache.failed = true; + return err; + }; + } + + fn freeActive(cache: *Cache, ops: kern.Ops) !void { + if (cache.active) |snap| { + try cache.fence(ops); + snap.deinit(); + cache.active = null; + cache.stats.active_bytes = 0; + } + } + + fn freePending(cache: *Cache, ops: kern.Ops) !void { + if (cache.pending) |snap| { + try cache.fence(ops); + snap.deinit(); + cache.pending = null; + cache.owner = null; + cache.stats.pending_bytes = 0; + } + } + + pub fn lookup(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry) !bool { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0 or !cache.eligible(s) or s.drafts != (ctx != null)) return false; + cache.stats.lookups += 1; + if (cache.active) |snap| if (snap.matches(s)) { + snap.restore(ops, g, seq, ctx) catch |err| { + cache.failed = true; + return err; + }; + s.cached = @intCast(snap.pos); + cache.stats.hits += 1; + return true; + }; + cache.stats.misses += 1; + return false; + } + + pub fn capture(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry) !void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0 or !cache.eligible(s) or s.drafts != (ctx != null) or (cache.active != null and cache.active.?.matches(s))) return; + const size = Snapshot.bytesWithLogits(g, s.prompt().len, s.drafts, cache.logits_enabled); + switch (policy.admission(cache.stats.active_bytes, cache.stats.pending_bytes, size)) { + .refuse, .busy => { + cache.stats.refusals += 1; + return; + }, + .evict => { + try cache.freeActive(ops); + cache.stats.evictions += 1; + }, + .retain => {}, + } + const snap = Snapshot.initWithLogits(cache.gpa, ops, g, s, cache.logits_enabled) catch { + cache.stats.refusals += 1; + return; + }; + cache.pending = snap; + cache.owner = s; + cache.stats.pending_bytes = snap.device_bytes; + snap.save(ops, g, seq, ctx) catch |err| { + cache.failed = true; + return err; + }; + cache.stats.captures += 1; + } + + /// Copy only from the current matching immutable entry; eviction means an ordinary head fallback. + pub fn restoreFirstLogits(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ops: kern.Ops, raw: u64, proven: bool) !bool { + if (!cache.logits_enabled or seq.kv8 or !@import("exact_eligible.zig").text(false, s)) return false; + const snap = cache.active orelse return false; + if (!logits_policy.canUse(cache.enabled and !cache.failed, proven, snap.matches(s), snap.logits_valid, seq.pos == snap.pos and s.prompt().len == snap.pos and s.cached == snap.pos, s.isCancelled())) return false; + if (snap.first_logits == 0) return error.InvalidFirstLogitsStorage; + ops.copy(raw, snap.first_logits, logits_policy.row_bytes) catch |err| { + cache.failed = true; + return err; + }; + cache.stats.first_logits_hits += 1; + return true; + } + + /// Preserve the cold owner's raw row before any draw; the normal promotion fence completes this copy. + pub fn captureFirstLogits(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ops: kern.Ops, raw: u64) !bool { + if (seq.kv8 or !@import("exact_eligible.zig").text(false, s) or !cache.logits_enabled or cache.failed or cache.owner != s) return false; + const snap = cache.pending orelse return false; + if (snap.logits_valid) return false; + if (!snap.matches(s) or seq.pos != snap.pos or snap.first_logits == 0) return error.InvalidFirstLogitsCapture; + logits_policy.capture(&snap.logits_valid, ops, snap.first_logits, raw) catch |err| { + cache.failed = true; + snap.logits_valid = false; + return err; + }; + cache.stats.first_logits_captures += 1; + return true; + } + + pub fn release(cache: *Cache, s: *lanes.Stream, settled: bool, ops: kern.Ops) !void { + if (!settled) { + cache.failed = true; + if (cache.pending != null) { + try cache.freePending(ops); + cache.stats.purges += 1; + } + return; + } + if (cache.owner != s) return; + const normal = s.reason == .stop or s.reason == .length; + const logits_ready = if (cache.pending) |snap| logits_policy.promotable(cache.logits_enabled, snap.logits_valid) else false; + if (cache.failed or !logits_ready or !policy.clean(s.finished, normal, s.isCancelled(), settled)) { + try cache.freePending(ops); + cache.stats.purges += 1; + return; + } + try cache.fence(ops); + try cache.freeActive(ops); + cache.active = cache.pending; + cache.pending = null; + cache.owner = null; + cache.stats.active_bytes = cache.stats.pending_bytes; + cache.stats.pending_bytes = 0; + cache.stats.promotions += 1; + } + + pub fn abandon(cache: *Cache, s: *lanes.Stream, ops: kern.Ops) !void { + if (cache.owner == s) { + try cache.freePending(ops); + cache.stats.purges += 1; + } + } + + pub fn deinit(cache: *Cache, ops: kern.Ops) !void { + if (cache.enabled) std.debug.print("PR608_EXACT_CACHE {{\"hits\":{d},\"misses\":{d},\"captures\":{d},\"promotions\":{d},\"purges\":{d},\"evictions\":{d},\"refusals\":{d},\"active_bytes\":{d},\"pending_bytes\":{d},\"failed\":{}}}\n", .{ cache.stats.hits, cache.stats.misses, cache.stats.captures, cache.stats.promotions, cache.stats.purges, cache.stats.evictions, cache.stats.refusals, cache.stats.active_bytes, cache.stats.pending_bytes, cache.failed }); + if (cache.logits_enabled) std.debug.print("PR608_EXACT_CACHE_LOGITS {f}\n", .{std.json.fmt(.{ + .captures = cache.stats.first_logits_captures, + .hits = cache.stats.first_logits_hits, + .fallbacks = cache.stats.first_logits_fallbacks, + .failed = cache.failed, + }, .{})}); + try cache.freePending(ops); + try cache.freeActive(ops); + } +}; diff --git a/zig/src/families/qwen27/cuda/exact_seq_reuse.zig b/zig/src/families/qwen27/cuda/exact_seq_reuse.zig new file mode 100644 index 0000000000..a101769248 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_seq_reuse.zig @@ -0,0 +1,189 @@ +//! Private target-only allocation reuse after proven full-N cache restore; reset and draft lifecycle are unchanged. +const std = @import("std"); +const st = @import("state.zig"); +const dr = @import("draft.zig"); +const lanes = @import("lanes"); +const Cuda = @import("lanes.zig").Cuda; +const draft_reuse = @import("exact_draft_reuse.zig"); +pub const policy = @import("seq_pool.zig"); +pub const Proofs = @import("restore_proof.zig").Proofs; +pub const Storage = policy.Pool(st.Seq); +pub const reset_skip = @import("reset_skip.zig"); +pub const Opened = struct { seq: *st.Seq, deferred_reset: bool = false, pooled: bool = false }; + +fn destroy(x: *Cuda, seq: *st.Seq) void { + seq.deinit(); + x.gpa.destroy(seq); +} + +pub fn clear(x: *Cuda) void { + while (x.seq_pool.evict()) |seq| destroy(x, seq); +} + +pub fn fail(x: *Cuda) void { + draft_reuse.fail(x); + if (!x.seq_pool.stats.enabled) return; + x.seq_pool.fail(); + clear(x); +} + +pub fn shutdown(x: *Cuda) void { + if (x.reset_skip.enabled) std.debug.print("PR608_EXACT_CACHE_RESET_SKIP {f}\n", .{std.json.fmt(x.reset_skip, .{})}); + if (x.seq_pool.stats.enabled) std.debug.print("PR608_EXACT_CACHE_SEQ_POOL {f}\n", .{std.json.fmt(x.seq_pool.stats, .{})}); + clear(x); +} + +/// A new stream's state, sized for its prompt and reply. +pub fn open(x: *Cuda, s: *lanes.Stream) !*st.Seq { + errdefer fail(x); + try x.settle(); + const ids = s.prompt(); + if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; + const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); + if (ids.len >= capacity) return error.PromptTooLong; + try draft_reuse.beforeOpen(x, s, capacity); + return obtain(x, s, capacity); +} + +/// A peek permits acquisition only; a separate successful lookup records restore provenance. +pub fn obtain(x: *Cuda, s: *lanes.Stream, capacity: usize) !*st.Seq { + return (try obtainState(x, s, capacity, false)).seq; +} + +fn obtainState(x: *Cuda, s: *lanes.Stream, capacity: usize, defer_reset: bool) !Opened { + errdefer fail(x); + if (x.seq_pool.active()) { + if (x.seqs.count() >= x.seq_pool.limit) return error.SeqReuseOwnershipLimit; + if (@import("exact_eligible.zig").text(x.e.kv8, s) and s.reuse.hook == null and + x.exact_cache.canRestore(s, x.drafter != null and s.drafts) and capacity <= x.e.context) + { + if (x.seq_pool.take(capacity)) |seq| { + errdefer destroy(x, seq); + if (seq.capacity != capacity) return error.SeqReuseCapacityMismatch; + if (defer_reset) { + seq.pos = 0; + seq.rope_delta = 0; + seq.media = null; + } else try seq.reset(x.e.ops(), x.e.w.g); + try x.seqs.put(x.gpa, s, seq); + if (defer_reset) x.reset_skip.deferred_pool += 1; + return .{ .seq = seq, .deferred_reset = defer_reset, .pooled = true }; + } + } + // Every fresh creation drains retained ownership, including a prompt-key miss with room. + x.seq_pool.beforeCreate(x, destroy); + if (!x.seq_pool.hasRoom(x.seqs.count())) return error.SeqReuseOwnershipLimit; + } + const seq = try x.gpa.create(st.Seq); + errdefer x.gpa.destroy(seq); + x.seq_pool.stats.create_attempts += 1; + seq.* = (if (defer_reset) x.e.sequenceForRestore(capacity) else x.e.sequence(capacity)) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; + x.seq_pool.stats.driver_creations += 1; + errdefer seq.deinit(); + try x.seqs.put(x.gpa, s, seq); + if (defer_reset) x.reset_skip.deferred_fresh += 1; + return .{ .seq = seq, .deferred_reset = defer_reset }; +} + +/// Private admission only; public Cuda.open always initializes the target. +pub fn openPrefill(x: *Cuda, s: *lanes.Stream) !Opened { + if (!x.reset_skip.enabled) return .{ .seq = try x.open(s) }; + errdefer fail(x); + try x.settle(); + const ids = s.prompt(); + if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; + const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); + if (ids.len >= capacity) return error.PromptTooLong; + try draft_reuse.beforeOpen(x, s, capacity); + const hint = capacity <= reset_skip.max_capacity and @import("exact_eligible.zig").text(x.e.kv8, s) and s.reuse.hook == null and + x.exact_cache.canRestore(s, x.drafter != null and s.drafts); + return obtainState(x, s, capacity, hint); +} + +const Resolve = struct { + x: *Cuda, + stream: *lanes.Stream, + seq: *st.Seq, + ctx: ?*dr.Context, + pub fn cancelled(a: Resolve) bool { + return a.stream.isCancelled(); + } + pub fn lookup(a: Resolve) !bool { + return @import("exact_seq_reuse.zig").lookup(a.x, a.stream, a.seq, a.ctx); + } + pub fn reset(a: Resolve) !void { + try a.seq.reset(a.x.e.ops(), a.x.e.w.g); + } +}; + +/// No target call may precede resolution; restore errors abort, never fall back from a partial copy. +pub fn lookupPrefill(x: *Cuda, s: *lanes.Stream, opened: *Opened, ctx: ?*dr.Context) !bool { + errdefer fail(x); + if (!opened.deferred_reset) return lookup(x, s, opened.seq, ctx); + const action: Resolve = .{ .x = x, .stream = s, .seq = opened.seq, .ctx = ctx }; + return reset_skip.resolve(action, &opened.deferred_reset, opened.pooled, &x.reset_skip, reset_skip.resetWork(x.e.w.g)); +} + +pub fn lookup(x: *Cuda, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context) !bool { + errdefer fail(x); + if (!@import("exact_eligible.zig").text(x.e.kv8, s)) return false; + const hit = try x.exact_cache.lookup(s, seq, ctx, x.e.ops(), x.e.w.g); + if (hit and (x.seq_pool.active() or x.exact_cache.logits_enabled)) { + if (x.seqs.get(s) != seq or seq.pos != s.prompt().len or s.cached != s.prompt().len) + return error.InvalidSeqRestoreProof; + try x.restored.mark(s, seq, s.prompt().len); + } + if (hit) try draft_reuse.initialized(x, s); + return hit; +} + +fn retainOrDestroy(x: *Cuda, s: *lanes.Stream, seq: *st.Seq, restored: bool, settled: bool, cache_ok: bool) void { + const bytes = st.Seq.bytes(x.e.w.g, seq.capacity, seq.kv8); + const safe = @import("exact_eligible.zig").text(x.e.kv8, s) and !seq.kv8 and cache_ok and !x.exact_cache.failed and seq.capacity <= x.e.context and + seq.pos >= s.prompt_len and seq.pos <= seq.capacity; + if (safe and policy.completed(s, settled, restored, cache_ok, x.exact_cache.failed) and x.seq_pool.canRetain(x.seqs.count(), seq.capacity, bytes)) { + x.e.s.synchronize() catch |err| { + std.log.err("synchronizing before exact-cache Seq retention: {s}", .{@errorName(err)}); + fail(x); + x.seq_pool.stats.discarded += 1; + destroy(x, seq); + return; + }; + if (policy.completed(s, settled, restored, cache_ok, x.exact_cache.failed) and !x.exact_cache.failed and + x.seq_pool.retain(seq, x.seqs.count(), seq.capacity, bytes)) return; + } + x.seq_pool.stats.discarded += 1; + destroy(x, seq); +} + +/// Release retains only detached, synchronized ownership after actual settle and cache-release success. +pub fn release(ptr: *anyopaque, s: *lanes.Stream) void { + const x: *Cuda = @ptrCast(@alignCast(ptr)); + const settled_ok = blk: { + x.settle() catch |err| { + std.log.err("committing the last round before a release: {s}", .{@errorName(err)}); + break :blk false; + }; + break :blk true; + }; + const cache_ok = blk: { + x.exact_cache.release(s, settled_ok, x.e.ops()) catch |err| { + std.log.err("exact cache release: {s}", .{@errorName(err)}); + fail(x); + break :blk false; + }; + break :blk true; + }; + const draft_seq: ?*const anyopaque = if (x.seqs.get(s)) |seq| seq else null; + const proof = x.restored.remove(s); + if (x.seqs.fetchRemove(s)) |entry| { + const restored = if (proof) |p| p.seq == @as(*const anyopaque, entry.value) and p.tokens == s.prompt_len else false; + retainOrDestroy(x, s, entry.value, restored, settled_ok, cache_ok); + } + draft_reuse.release(x, s, draft_seq, settled_ok, cache_ok); +} + +test { + _ = policy; + _ = @import("restore_proof.zig"); +} diff --git a/zig/src/families/qwen27/cuda/lanes.zig b/zig/src/families/qwen27/cuda/lanes.zig index adae5d4913..2008211bb6 100644 --- a/zig/src/families/qwen27/cuda/lanes.zig +++ b/zig/src/families/qwen27/cuda/lanes.zig @@ -10,6 +10,9 @@ const Forward = @import("forward.zig").Forward; const eng = @import("engine.zig"); const prompt_rows = eng.prompt_rows; const reuse = @import("reuse.zig"); +const exact = @import("exact_prompt_cache.zig"); +const exact_reuse = @import("exact_seq_reuse.zig"); +const draft_reuse = @import("exact_draft_reuse.zig"); const dl = @import("draft_load.zig"); const dr = @import("draft.zig"); const drafts = @import("drafts.zig"); @@ -48,20 +51,27 @@ pub const Cuda = struct { drafting: std.AutoHashMapUnmanaged(*lanes.Stream, *drafts.Stream) = .empty, costs: [calibration.len + 1]lanes.config.Cost = undefined, // the forward's ms by a round's rows (calibrate) cost_count: usize = 0, + exact_cache: exact.Cache, + seq_pool: exact_reuse.Storage, + restored: exact_reuse.Proofs = .{}, + draft_pool: draft_reuse.Storage, + draft_initialized: draft_reuse.policy.Proofs = .{}, + reset_skip: exact_reuse.reset_skip.Stats, + + pub const openPrefill = exact_reuse.openPrefill; + pub const lookupPrefill = exact_reuse.lookupPrefill; pub fn init(gpa: std.mem.Allocator, e: *Engine, streams: u32) !Cuda { - return .{ - .gpa = gpa, - .e = e, - .streams = streams, - .logits = try gpa.alloc(u16, c.vocab), - .cand_values = try gpa.alloc(f32, st.round_rows * st.max_candidates), - .cand_ids = try gpa.alloc(i64, st.round_rows * st.max_candidates), - .values = try gpa.alloc(f64, st.max_candidates), - }; + return @import("exact_init.zig").init(gpa, e, streams); } pub fn deinit(x: *Cuda) void { + draft_reuse.shutdown(x) catch |err| { + std.log.err("fatal unresolved draft Context fence; terminating before device frees: {s}", .{@errorName(err)}); + std.process.exit(1); + }; + exact_reuse.shutdown(x); + x.exact_cache.deinit(x.e.ops()) catch |err| std.log.err("exact cache shutdown: {s}", .{@errorName(err)}); var it = x.seqs.valueIterator(); while (it.next()) |seq| { seq.*.deinit(); @@ -134,19 +144,12 @@ pub const Cuda = struct { } pub fn backend(x: *Cuda) be.Backend { - return .{ .ptr = x, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany } }; + return .{ .ptr = x, .first_before_draft = true, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany } }; } /// The stream's drafting (its DFlash2 context and copy index), made on first use. pub fn draftingOf(x: *Cuda, s: *lanes.Stream) !*drafts.Stream { - if (x.drafting.get(s)) |ds| return ds; - const ds = try x.gpa.create(drafts.Stream); - errdefer x.gpa.destroy(ds); - const ctx = dr.Context.init(x.e.ctx.d) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; - ds.* = drafts.Stream.init(x.gpa, ctx); - errdefer ds.deinit(); - try x.drafting.put(x.gpa, s, ds); - return ds; + return draft_reuse.obtain(x, s); } fn cast(p: *anyopaque) *Cuda { @@ -158,7 +161,7 @@ pub const Cuda = struct { } /// Commits the last verified round: every stream its kept rows (its whole window unless keep cut it). - fn settle(x: *Cuda) !void { + pub fn settle(x: *Cuda) !void { const p = &x.pending; if (p.n == 0) return; const n = p.n; @@ -168,7 +171,7 @@ pub const Cuda = struct { } /// The stream's state, the last round committed first. - fn settled(x: *Cuda, s: *lanes.Stream) !*st.Seq { + pub fn settled(x: *Cuda, s: *lanes.Stream) !*st.Seq { try x.settle(); return x.seqOf(s); } @@ -208,31 +211,24 @@ pub const Cuda = struct { } } - /// A new stream's state, sized for its prompt and reply. - pub fn open(x: *Cuda, s: *lanes.Stream) !*st.Seq { - try x.settle(); - const ids = s.prompt(); - if (ids.len == 0 or x.seqs.contains(s)) return error.InvalidQwenPrompt; - const capacity = @min(x.e.context, ids.len + s.max_new + st.max_rows); - if (ids.len >= capacity) return error.PromptTooLong; - const seq = try x.gpa.create(st.Seq); - errdefer x.gpa.destroy(seq); - seq.* = x.e.sequence(capacity) catch |err| return if (err == error.OutOfMemory) err else error.OutOfDeviceMemory; - errdefer seq.deinit(); - try x.seqs.put(x.gpa, s, seq); - return seq; - } + pub const open = exact_reuse.open; fn prefill(p: *anyopaque, s: *lanes.Stream) !void { const x = cast(p); - const seq = try x.open(s); + var opened = try x.openPrefill(s); errdefer x.dropSeq(s); - try x.prefillOpened(s, seq); + try x.prefillOpened(s, &opened); } /// prefill_state into an opened stream: the prompt in chunks, a drafting stream's last window of taps absorbed. A /// kept state the lane host found is restored first, and the pass stops at its marks for the cache to copy. - fn prefillOpened(x: *Cuda, s: *lanes.Stream, seq: *st.Seq) !void { + fn prefillOpened(x: *Cuda, s: *lanes.Stream, opened: *exact_reuse.Opened) !void { + errdefer exact_reuse.fail(x); + const seq = opened.seq; + if (x.exact_cache.eligible(s) and !x.e.kv8) { + const ctx = if (x.drafter != null and s.drafts) &(try x.draftingOf(s)).ctx else null; + if (try exact_reuse.lookupPrefill(x, s, opened, ctx)) return; + } // an image prompt: the tower's rows and the prompt's three positions for its pass (EncodedVision) const media: ?*vision.Media = if (s.images.len == 0) null else blk: { const tower = x.tower orelse return error.VisionDisabled; @@ -245,9 +241,12 @@ pub const Cuda = struct { const marks: ?eng.Keep = if (s.reuse.marks.len == 0 or media != null) null else .{ .at = s.reuse.marks, .ctx = s, .call = keptAt }; if (x.drafter) |d| if (s.drafts) { const ds = try x.draftingOf(s); - return lone.prefillFrom(x.e, seq, &ds.ctx, d, s.prompt(), from, media, marks); + try lone.prefillFrom(x.e, seq, &ds.ctx, d, s.prompt(), from, media, marks); + try x.exact_cache.capture(s, seq, &ds.ctx, x.e.ops(), x.e.w.g); + return draft_reuse.initialized(x, s); }; try x.e.prefillFrom(seq, s.prompt(), from, media, marks); + try x.exact_cache.capture(s, seq, null, x.e.ops(), x.e.w.g); } /// The stream's kept state restored on its sequence: where its prompt pass starts (0: none, off the grid, failed). @@ -280,13 +279,27 @@ pub const Cuda = struct { var n: usize = 0; var rows: usize = 0; for (streams, errs, 0..) |s, *err, i| { - const seq = x.open(s) catch |e| { + var opened = x.openPrefill(s) catch |e| { err.* = e; continue; }; + const seq = opened.seq; const ids = s.prompt(); + if (x.exact_cache.eligible(s) and !x.e.kv8) { + const ctx = if (x.drafter != null and s.drafts) &(x.draftingOf(s) catch |e| { + x.dropSeq(s); + err.* = e; + continue; + }).ctx else null; + const hit = exact_reuse.lookupPrefill(x, s, &opened, ctx) catch |e| { + x.dropSeq(s); + err.* = e; + continue; + }; + if (hit) continue; + } if (ids.len > prompt_rows) { - x.prefillOpened(s, seq) catch |e| { + x.prefillOpened(s, &opened) catch |e| { x.dropSeq(s); err.* = e; }; @@ -315,7 +328,13 @@ pub const Cuda = struct { fn passPieces(x: *Cuda, streams: []const *lanes.Stream, pieces: []const Forward.Piece, which: []const usize) !void { var f = x.e.forward(); - const d = x.drafter orelse return f.chunks(pieces); + const d = x.drafter orelse { + try f.chunks(pieces); + if (x.exact_cache.enabled and !x.e.kv8) for (pieces, which) |pc, i| { + try x.exact_cache.capture(streams[i], pc.seq, null, x.e.ops(), x.e.w.g); + }; + return; + }; f.taps = x.e.scratch.taps; try f.chunks(pieces); // each drafting stream's last window of taps, consecutive whole prompts absorbed together @@ -348,25 +367,26 @@ pub const Cuda = struct { g_rows += taken; } if (g > 0) try d.absorbMany(group[0..g], x.e.scratch.taps + g_from * row); + for (pieces, which) |pc, i| { + const s = streams[i]; + const ctx = if (s.drafts) &(try x.draftingOf(s)).ctx else null; + if (!x.e.kv8) try x.exact_cache.capture(s, pc.seq, ctx, x.e.ops(), x.e.w.g); + if (ctx != null) try draft_reuse.initialized(x, s); + } } fn dropSeq(x: *Cuda, s: *lanes.Stream) void { + if (x.draft_pool.stats.enabled) draft_reuse.drop(x, s); + _ = x.restored.remove(s); + exact_reuse.fail(x); + x.exact_cache.abandon(s, x.e.ops()) catch |err| std.log.err("exact cache abandon: {s}", .{@errorName(err)}); if (x.seqs.fetchRemove(s)) |entry| { entry.value.deinit(); x.gpa.destroy(entry.value); } } - fn first(p: *anyopaque, s: *lanes.Stream, position: u64) !u64 { - const x = cast(p); - const seq = try x.settled(s); - if (position != seq.pos) return error.InvalidQwenPosition; - var f = x.e.forward(); - try f.head(seq); - var out: [1]u32 = undefined; - try x.drawRows(&.{s.sampling}, &.{position}, &out); - return out[0]; - } + const first = @import("exact_first_logits.zig").first; fn queue(p: *anyopaque, s: *lanes.Stream, feed: be.Feed, position: u64) !u64 { const x = cast(p); @@ -500,7 +520,7 @@ pub const Cuda = struct { .value => |v| v, .handle => |h| try read(p, h), }; - try firsts[na].appendSlice(x.gpa, context); + try firsts[na].appendSlice(x.gpa, r.stream.prompt()); try firsts[na].append(x.gpa, token); context = firsts[na].items; } @@ -531,18 +551,7 @@ pub const Cuda = struct { return true; } - fn release(p: *anyopaque, s: *lanes.Stream) void { - const x = cast(p); - x.settle() catch |err| std.log.err("committing the last round before a release: {s}", .{@errorName(err)}); - if (x.seqs.fetchRemove(s)) |entry| { - entry.value.deinit(); - x.gpa.destroy(entry.value); - } - if (x.drafting.fetchRemove(s)) |entry| { - entry.value.deinit(); - x.gpa.destroy(entry.value); - } - } + const release = exact_reuse.release; }; fn value(word: u16) f64 { @@ -584,3 +593,8 @@ pub fn draw(gpa: std.mem.Allocator, logits: []const u16, settings: ?lanes.Sampli } return @intCast(try lanes.sampling.choose(gpa, values, ids, position, sampling)); } + +test { + _ = @import("exact_cache_test.zig"); + _ = @import("exact_eligible.zig"); +} diff --git a/zig/src/families/qwen27/cuda/reset_skip.zig b/zig/src/families/qwen27/cuda/reset_skip.zig new file mode 100644 index 0000000000..fc625e7103 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reset_skip.zig @@ -0,0 +1,58 @@ +//! Private prefill reset transaction: only actual full-state restoration resolves uninitialized target state. +const std = @import("std"); +pub const max_capacity: usize = 512; +pub const Stats = struct { + enabled: bool = false, + deferred_fresh: u64 = 0, + deferred_pool: u64 = 0, + restored_fresh: u64 = 0, + restored_pool: u64 = 0, + avoided_fill_calls: u64 = 0, + avoided_fill_bytes: u64 = 0, + fallback_resets: u64 = 0, + failed_deferred_transactions: u64 = 0, +}; + +pub fn compatible(raw: ?[]const u8, cache_enabled: bool) !bool { + const value = raw orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (!std.mem.eql(u8, value, "1")) return error.InvalidResetSkip; + if (!cache_enabled) return error.ResetSkipNeedsExactCache; + return true; +} + +pub fn configured(cache_enabled: bool) !bool { + const raw = std.c.getenv("TF_PR608_EXACT_CACHE_SKIP_RESET"); + return compatible(if (raw) |value| std.mem.span(value) else null, cache_enabled); +} + +/// Counts the actual reset domains, excluding attention KV, heads and alignment padding. +pub fn resetWork(g: anytype) struct { calls: u64, bytes: u64 } { + var linear_layers: u64 = 0; + for (0..g.layers) |i| if (i % 4 != 3) { + linear_layers += 1; + }; + return .{ .calls = linear_layers * 2, .bytes = linear_layers * @as(u64, @intCast(g.deltaBytes() + g.convBytes())) }; +} + +/// A hint never resolves deferred state: actual hit or original reset must happen before target work. +pub fn resolve(action: anytype, deferred: *bool, pooled: bool, stats: *Stats, work: anytype) !bool { + if (!deferred.*) return action.lookup(); + errdefer stats.failed_deferred_transactions += 1; + if (action.cancelled()) return error.Cancelled; + const hit = try action.lookup(); + if (hit) { + if (pooled) stats.restored_pool += 1 else stats.restored_fresh += 1; + stats.avoided_fill_calls += work.calls; + stats.avoided_fill_bytes += work.bytes; + } else { + try action.reset(); + stats.fallback_resets += 1; + } + deferred.* = false; + return hit; +} + +test { + _ = @import("reset_skip_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/reset_skip_factory_test.zig b/zig/src/families/qwen27/cuda/reset_skip_factory_test.zig new file mode 100644 index 0000000000..31aaa4fe15 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reset_skip_factory_test.zig @@ -0,0 +1,179 @@ +//! Real Seq factories and Ops.fill32 wired to synthetic CUDA driver calls; no device or model is used. +const std = @import("std"); +const cuda = @import("cuda"); +const st = @import("state.zig"); +const shape = @import("shape.zig"); +const kern = @import("kernels.zig"); +const policy = @import("reset_skip.zig"); +const Call = struct { dst: u64, value: c_uint, words: usize }; +var calls: [96]Call = undefined; +var call_count: usize = 0; +var allocations: usize = 0; +var frees: usize = 0; +var fail_alloc: bool = false; +var fail_fill: bool = false; + +fn allocate(ptr: *u64, bytes: usize) callconv(.c) c_int { + if (fail_alloc) return cuda.abi.error_not_ready; + if (bytes == 0) return 1; + allocations += 1; + ptr.* = 0x100000000; + return 0; +} +fn freeDevice(ptr: u64) callconv(.c) c_int { + if (ptr == 0x100000000) frees += 1; + return 0; +} +fn fill(dst: u64, value: c_uint, words: usize, stream: cuda.abi.Stream) callconv(.c) c_int { + _ = stream; + if (fail_fill) return cuda.abi.error_not_ready; + if (call_count >= calls.len) return 1; + calls[call_count] = .{ .dst = dst, .value = value, .words = words }; + call_count += 1; + return 0; +} +fn errorText(code: c_int, out: *?[*:0]const u8) callconv(.c) c_int { + _ = code; + out.* = "synthetic reset failure"; + return 0; +} +fn setup(driver: *cuda.Driver, kernels: *kern.Kernels) kern.Ops { + call_count = 0; + allocations = 0; + frees = 0; + fail_alloc = false; + fail_fill = false; + driver.* = undefined; + driver.api.cuMemAlloc_v2 = allocate; + driver.api.cuMemFree_v2 = freeDevice; + driver.api.cuMemsetD32Async = fill; + driver.api.cuGetErrorName = errorText; + driver.api.cuGetErrorString = errorText; + kernels.* = undefined; + kernels.d = driver; + return .{ .k = kernels, .s = .{ .d = driver, .handle = null } }; +} +fn checkReset(seq: *const st.Seq) !void { + const g = shape.the_27b; + var index: usize = 0; + var bytes: u64 = 0; + for (0..g.layers) |layer| if (shape.linear(layer)) { + try std.testing.expectEqual(seq.rec[layer], calls[index].dst); + try std.testing.expectEqual(g.deltaBytes() / 4, calls[index].words); + try std.testing.expectEqual(seq.conv[layer], calls[index + 1].dst); + try std.testing.expectEqual(g.convBytes() / 4, calls[index + 1].words); + try std.testing.expectEqual(@as(c_uint, 0), calls[index].value | calls[index + 1].value); + bytes += (calls[index].words + calls[index + 1].words) * 4; + index += 2; + }; + try std.testing.expectEqual(@as(usize, 96), index); + try std.testing.expectEqual(index, call_count); + try std.testing.expectEqual(@as(u64, 153944064), bytes); + const actual = policy.resetWork(g); + try std.testing.expectEqual(@as(u64, @intCast(index)), actual.calls); + try std.testing.expectEqual(bytes, actual.bytes); +} + +test "public Seq factory calls all original driver fills and owns one arena" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + var seq = try st.Seq.init(std.testing.allocator, ops, shape.the_27b, 303, false); + try checkReset(&seq); + try std.testing.expectEqual(@as(usize, 303), seq.capacity); + try std.testing.expectEqual(@as(usize, 0), seq.pos); + seq.deinit(); + try std.testing.expectEqual(@as(usize, 1), allocations); + try std.testing.expectEqual(allocations, frees); +} + +test "restore destination preserves capacity and layout while omitting only initial fills" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + var seq = try st.Seq.initRestoreDestination(std.testing.allocator, ops, shape.the_27b, 303); + defer seq.deinit(); + try std.testing.expectEqual(@as(usize, 0), call_count); + try std.testing.expectEqual(@as(usize, 303), seq.capacity); + try std.testing.expectEqual(st.Seq.bytes(shape.the_27b, 303, false), seq.arena.buf.?.len); + var layout: st.Arena = .{ .buf = null }; + _ = try layout.take(shape.the_27b.hidden * 2); + _ = try layout.take(shape.the_27b.hidden / 64 * 4); + try std.testing.expectEqual(@as(u64, 0x100000000), seq.head_in); + try std.testing.expectEqual(@as(u64, 0x100000000 + 10240), seq.head_xs); + for (0..shape.the_27b.layers) |layer| { + if (shape.linear(layer)) { + const rec_at = std.mem.alignForward(usize, layout.used, 256); + _ = try layout.take(shape.the_27b.deltaBytes()); + const conv_at = std.mem.alignForward(usize, layout.used, 256); + _ = try layout.take(shape.the_27b.convBytes()); + try std.testing.expectEqual(0x100000000 + @as(u64, @intCast(rec_at)), seq.rec[layer]); + try std.testing.expectEqual(0x100000000 + @as(u64, @intCast(conv_at)), seq.conv[layer]); + } else { + _ = try layout.take(303 * shape.the_27b.kvInner() * 2); + _ = try layout.take(303 * shape.the_27b.kvInner() * 2); + } + } + try seq.reset(ops, shape.the_27b); + try checkReset(&seq); +} + +test "restore factory allocation failure owns no device buffer" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + fail_alloc = true; + try std.testing.expectError(error.NotReady, st.Seq.initRestoreDestination(std.testing.allocator, ops, shape.the_27b, 303)); + try std.testing.expectEqual(@as(usize, 0), allocations + frees + call_count); +} + +test "public factory reset error frees the allocated arena and host arrays" { + var d: cuda.Driver = undefined; + var k: kern.Kernels = undefined; + const ops = setup(&d, &k); + fail_fill = true; + try std.testing.expectError(error.NotReady, st.Seq.init(std.testing.allocator, ops, shape.the_27b, 303, false)); + try std.testing.expectEqual(@as(usize, 1), allocations); + try std.testing.expectEqual(allocations, frees); +} + +test "public Engine sequence stays initialized and private restore factory omits fills" { + var d: cuda.Driver = undefined; + var engine: @import("engine.zig").Engine = undefined; + const ops = setup(&d, &engine.k); + engine.gpa = std.testing.allocator; + engine.s = ops.s; + engine.w.g = shape.the_27b; + engine.kv8 = false; + var safe = try engine.sequence(303); + try checkReset(&safe); + safe.deinit(); + call_count = 0; + var deferred = try engine.sequenceForRestore(303); + try std.testing.expectEqual(@as(usize, 0), call_count); + try std.testing.expectEqual(@as(usize, 303), deferred.capacity); + deferred.deinit(); + try std.testing.expectEqual(@as(usize, 2), allocations); + try std.testing.expectEqual(allocations, frees); +} + +test "FP8 public factory keeps scales and fills; private BF16 factory rejects FP8 before allocation" { + var d: cuda.Driver = undefined; + var engine: @import("engine.zig").Engine = undefined; + const ops = setup(&d, &engine.k); + engine.gpa = std.testing.allocator; + engine.s = ops.s; + engine.w.g = shape.the_27b; + engine.kv8 = true; + try std.testing.expectError(error.ExactCacheRequiresBf16, engine.sequenceForRestore(303)); + try std.testing.expectEqual(@as(usize, 0), allocations + call_count); + var safe = try engine.sequence(303); + defer safe.deinit(); + try checkReset(&safe); + try std.testing.expect(safe.kv8); + try std.testing.expectEqual(st.Seq.bytes(shape.the_27b, 303, true), safe.arena.buf.?.len); + for (0..shape.the_27b.layers) |layer| { + if (shape.linear(layer)) continue; + try std.testing.expect(safe.kscale[layer] != 0 and safe.vscale[layer] != 0); + } +} diff --git a/zig/src/families/qwen27/cuda/reset_skip_test.zig b/zig/src/families/qwen27/cuda/reset_skip_test.zig new file mode 100644 index 0000000000..732303ae87 --- /dev/null +++ b/zig/src/families/qwen27/cuda/reset_skip_test.zig @@ -0,0 +1,105 @@ +//! Host transaction checks prove deferred target state never reaches work after a miss or failed restore. +const std = @import("std"); +const policy = @import("reset_skip.zig"); +const work = .{ .calls = @as(u64, 96), .bytes = @as(u64, 153944064) }; +const Mock = struct { + hit: bool = true, + cancel: bool = false, + copy_error: bool = false, + reset_error: bool = false, + copies: usize = 0, + resets: usize = 0, + initialized: bool = false, + pub fn cancelled(a: *Mock) bool { + return a.cancel; + } + pub fn lookup(a: *Mock) !bool { + a.copies += 1; + if (a.copy_error) return error.PartialCopy; + if (a.hit) a.initialized = true; + return a.hit; + } + pub fn reset(a: *Mock) !void { + a.resets += 1; + if (a.reset_error) return error.ResetFailed; + a.initialized = true; + } + fn target(a: *Mock) !void { + if (!a.initialized) return error.UninitializedTarget; + } +}; + +test "strict opt-in requires exact cache and rejects malformed flags" { + try std.testing.expect(!try policy.compatible(null, true)); + try std.testing.expect(!try policy.compatible("0", false)); + try std.testing.expect(try policy.compatible("1", true)); + try std.testing.expectError(error.ResetSkipNeedsExactCache, policy.compatible("1", false)); + try std.testing.expectError(error.InvalidResetSkip, policy.compatible("true", true)); +} + +test "actual hit resolves fresh and retained state without reset" { + for ([_]bool{ false, true }) |pooled| { + var m: Mock = .{}; + var deferred = true; + var stats: policy.Stats = .{ .enabled = true }; + try std.testing.expect(try policy.resolve(&m, &deferred, pooled, &stats, work)); + try m.target(); + try std.testing.expect(!deferred); + try std.testing.expectEqual(@as(usize, 0), m.resets); + try std.testing.expectEqual(@as(u64, 96), stats.avoided_fill_calls); + try std.testing.expectEqual(work.bytes, stats.avoided_fill_bytes); + try std.testing.expectEqual(@as(u64, 1), stats.restored_fresh + stats.restored_pool); + } +} + +test "positive hint then actual miss resets before target work" { + var m: Mock = .{ .hit = false }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.UninitializedTarget, m.target()); + try std.testing.expect(!try policy.resolve(&m, &deferred, false, &stats, work)); + try m.target(); + try std.testing.expectEqual(@as(usize, 1), m.copies); + try std.testing.expectEqual(@as(usize, 1), m.resets); + try std.testing.expectEqual(@as(u64, 1), stats.fallback_resets); + try std.testing.expectEqual(@as(u64, 0), stats.avoided_fill_calls); +} + +test "partial copy aborts without reset fallback or resolved admission" { + var m: Mock = .{ .copy_error = true }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.PartialCopy, policy.resolve(&m, &deferred, false, &stats, work)); + try std.testing.expect(deferred); + try std.testing.expectEqual(@as(usize, 0), m.resets); + try std.testing.expectEqual(@as(u64, 1), stats.failed_deferred_transactions); +} + +test "cancelled deferred destination aborts before copying or resetting" { + var m: Mock = .{ .cancel = true }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.Cancelled, policy.resolve(&m, &deferred, true, &stats, work)); + try std.testing.expect(deferred); + try std.testing.expectEqual(@as(usize, 0), m.copies + m.resets); +} + +test "reset failure on actual miss remains unresolved and propagates" { + var m: Mock = .{ .hit = false, .reset_error = true }; + var deferred = true; + var stats: policy.Stats = .{}; + try std.testing.expectError(error.ResetFailed, policy.resolve(&m, &deferred, false, &stats, work)); + try std.testing.expect(deferred); + try std.testing.expectEqual(@as(u64, 0), stats.fallback_resets + stats.avoided_fill_calls); +} + +test "ordinary initialized path retains original lookup with no extra reset or cancel check" { + var m: Mock = .{ .hit = false, .cancel = true, .initialized = true }; + var deferred = false; + var stats: policy.Stats = .{}; + try std.testing.expect(!try policy.resolve(&m, &deferred, false, &stats, work)); + try m.target(); + try std.testing.expectEqual(@as(usize, 1), m.copies); + try std.testing.expectEqual(@as(usize, 0), m.resets); + try std.testing.expectEqual(@as(u64, 0), stats.failed_deferred_transactions); +} diff --git a/zig/src/families/qwen27/cuda/restore_proof.zig b/zig/src/families/qwen27/cuda/restore_proof.zig new file mode 100644 index 0000000000..455e1ed9b9 --- /dev/null +++ b/zig/src/families/qwen27/cuda/restore_proof.zig @@ -0,0 +1,76 @@ +//! Bounded provenance binds a successful full-N restore to one stream and its exclusively owned Seq. +const std = @import("std"); +pub const Proof = struct { stream: *const anyopaque, seq: *const anyopaque, tokens: usize }; +pub const Proofs = struct { + items: [16]?Proof = @splat(null), + + pub fn mark(p: *Proofs, stream: *const anyopaque, seq: *const anyopaque, tokens: usize) !void { + if (tokens == 0 or tokens > 512) return error.InvalidSeqRestoreProof; + for (&p.items) |*slot| if (slot.*) |existing| { + if (existing.stream == stream) return error.DuplicateSeqRestoreProof; + }; + for (&p.items) |*slot| if (slot.* == null) { + slot.* = .{ .stream = stream, .seq = seq, .tokens = tokens }; + return; + }; + return error.SeqRestoreProofCapacity; + } + + /// A read-only binding check leaves the release-time pool proof intact. + pub fn matches(p: *const Proofs, stream: *const anyopaque, seq: *const anyopaque, tokens: usize) bool { + for (p.items) |slot| if (slot) |proof| { + if (proof.stream == stream) return proof.seq == seq and proof.tokens == tokens; + }; + return false; + } + + pub fn remove(p: *Proofs, stream: *const anyopaque) ?Proof { + for (&p.items) |*slot| if (slot.*) |existing| { + if (existing.stream != stream) continue; + slot.* = null; + return existing; + }; + return null; + } +}; + +test "provenance binds stream and Seq once and is cleared before pointer reuse" { + var p: Proofs = .{}; + var streams: [2]u8 = @splat(0); + var seqs: [2]u8 = @splat(0); + try p.mark(&streams[0], &seqs[0], 14); + try std.testing.expect(p.remove(&streams[1]) == null); + try std.testing.expectError(error.DuplicateSeqRestoreProof, p.mark(&streams[0], &seqs[1], 14)); + const proof = p.remove(&streams[0]).?; + try std.testing.expect(proof.seq == @as(*const anyopaque, &seqs[0]) and proof.seq != @as(*const anyopaque, &seqs[1]) and proof.tokens == 14); + try std.testing.expect(p.remove(&streams[0]) == null); + try p.mark(&streams[0], &seqs[1], 31); + const next = p.remove(&streams[0]).?; + try std.testing.expect(next.seq == @as(*const anyopaque, &seqs[1]) and next.tokens == 31); +} + +test "fixed proof ownership refuses overflow and unsupported restoration lengths" { + var p: Proofs = .{}; + var streams: [17]u8 = @splat(0); + var seqs: [17]u8 = @splat(0); + try std.testing.expectError(error.InvalidSeqRestoreProof, p.mark(&streams[0], &seqs[0], 0)); + try std.testing.expectError(error.InvalidSeqRestoreProof, p.mark(&streams[0], &seqs[0], 513)); + for (0..16) |i| try p.mark(&streams[i], &seqs[i], 14); + try std.testing.expectError(error.SeqRestoreProofCapacity, p.mark(&streams[16], &seqs[16], 14)); + _ = p.remove(&streams[0]); + try p.mark(&streams[16], &seqs[16], 31); +} + +test "read-only first-logit proof rejects another owner, Seq and position without consuming release authority" { + var proofs: Proofs = .{}; + var streams: [2]u8 = @splat(0); + var seqs: [2]u8 = @splat(0); + try proofs.mark(&streams[0], &seqs[0], 14); + try std.testing.expect(proofs.matches(&streams[0], &seqs[0], 14)); + try std.testing.expect(!proofs.matches(&streams[1], &seqs[0], 14)); + try std.testing.expect(!proofs.matches(&streams[0], &seqs[1], 14)); + try std.testing.expect(!proofs.matches(&streams[0], &seqs[0], 31)); + try std.testing.expect(proofs.matches(&streams[0], &seqs[0], 14)); + _ = proofs.remove(&streams[0]); + try std.testing.expect(!proofs.matches(&streams[0], &seqs[0], 14)); +} diff --git a/zig/src/families/qwen27/cuda/seq_pool.zig b/zig/src/families/qwen27/cuda/seq_pool.zig new file mode 100644 index 0000000000..2a20f88ef9 --- /dev/null +++ b/zig/src/families/qwen27/cuda/seq_pool.zig @@ -0,0 +1,120 @@ +//! Fixed ownership storage for the private exact-capacity short Seq experiment; no allocator or device work. +const std = @import("std"); +pub const max_slots = 16; +pub const max_capacity = 512; +pub const max_bytes = 3 * 1024 * 1024 * 1024; + +pub const Stats = struct { + enabled: bool = false, + failed: bool = false, + create_attempts: u64 = 0, + driver_creations: u64 = 0, + reuse_hits: u64 = 0, + evictions: u64 = 0, + discarded: u64 = 0, + retained_bytes: usize = 0, + retained_slots: usize = 0, +}; + +pub fn parse(raw: ?[]const u8) !bool { + const value = raw orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (std.mem.eql(u8, value, "1")) return true; + return error.InvalidSeqReuse; +} + +pub fn configured(cache_enabled: bool) !bool { + const raw = std.c.getenv("TF_PR608_EXACT_CACHE_SEQ_REUSE"); + return compatible(if (raw) |value| std.mem.span(value) else null, cache_enabled); +} + +pub fn compatible(raw: ?[]const u8, cache_enabled: bool) !bool { + const on = try parse(raw); + if (on and !cache_enabled) return error.SeqReuseNeedsExactCache; + return on; +} + +/// Normal completion requires proven full-N restoration, never a cached count alone. +pub fn completed(s: anytype, settled: bool, restored: bool, cache_ok: bool, cache_failed: bool) bool { + return cache_ok and !cache_failed and restored and settled and s.finished and (s.reason == .stop or s.reason == .length) and + !s.isCancelled() and s.cached == s.prompt_len and !s.reuse_failed and s.reuse.saved == null and + s.reuse.at == 0 and s.reuse.marks.len == 0 and s.reuse.hook == null; +} + +/// A taken item is caller-owned immediately; a retained item is pool-owned until take/pop. +pub fn Pool(comptime T: type) type { + return struct { + const Self = @This(); + const Entry = struct { ptr: *T, capacity: usize, bytes: usize }; + entries: [max_slots]Entry = undefined, + limit: usize = 0, + stats: Stats = .{}, + + pub fn init(enabled: bool, slots: usize) Self { + return .{ .limit = @min(slots, max_slots), .stats = .{ .enabled = enabled } }; + } + + pub fn active(p: *const Self) bool { + return p.stats.enabled and !p.stats.failed; + } + + fn remove(p: *Self, index: usize) *T { + const entry = p.entries[index]; + p.stats.retained_slots -= 1; + p.stats.retained_bytes -= entry.bytes; + for (index..p.stats.retained_slots) |i| p.entries[i] = p.entries[i + 1]; + return entry.ptr; + } + + pub fn take(p: *Self, capacity: usize) ?*T { + if (!p.active() or capacity > max_capacity) return null; + for (p.entries[0..p.stats.retained_slots], 0..) |entry, i| { + if (entry.capacity != capacity) continue; + p.stats.reuse_hits += 1; + return p.remove(i); + } + return null; + } + + /// Eviction detaches ownership; the caller frees exactly this item. + pub fn evict(p: *Self) ?*T { + if (p.stats.retained_slots == 0) return null; + p.stats.evictions += 1; + return p.remove(0); + } + + /// An exact-capacity miss drains all retained bytes, even when a count slot remains available. + pub fn beforeCreate(p: *Self, context: anytype, comptime free: anytype) void { + while (p.evict()) |ptr| free(context, ptr); + } + + pub fn hasRoom(p: *const Self, active_count: usize) bool { + return active_count < p.limit and p.stats.retained_slots < p.limit - active_count; + } + + pub fn canRetain(p: *const Self, active_count: usize, capacity: usize, bytes: usize) bool { + return p.active() and capacity <= max_capacity and bytes <= max_bytes and + p.stats.retained_bytes <= max_bytes - bytes and p.hasRoom(active_count); + } + + pub fn retain(p: *Self, ptr: *T, active_count: usize, capacity: usize, bytes: usize) bool { + if (!p.active()) return false; + for (p.entries[0..p.stats.retained_slots]) |entry| + if (entry.ptr == ptr) @panic("duplicate Seq pool ownership"); + if (!p.canRetain(active_count, capacity, bytes)) return false; + p.entries[p.stats.retained_slots] = .{ .ptr = ptr, .capacity = capacity, .bytes = bytes }; + p.stats.retained_slots += 1; + p.stats.retained_bytes += bytes; + return true; + } + + /// Failure disables future take/retain; the caller must evict and free outstanding ownership. + pub fn fail(p: *Self) void { + if (p.stats.enabled) p.stats.failed = true; + } + }; +} + +test { + _ = @import("seq_pool_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/seq_pool_test.zig b/zig/src/families/qwen27/cuda/seq_pool_test.zig new file mode 100644 index 0000000000..9ffe378818 --- /dev/null +++ b/zig/src/families/qwen27/cuda/seq_pool_test.zig @@ -0,0 +1,172 @@ +//! Exact-capacity ownership, footprint and failure checks independent of CUDA or model allocation. +const std = @import("std"); +const policy = @import("seq_pool.zig"); +const Object = struct { alive: bool = true }; +const Pool = policy.Pool(Object); + +fn free(object: *Object) !void { + try std.testing.expect(object.alive); + object.alive = false; +} + +test "default-off and invalid settings cannot silently enable retention" { + try std.testing.expect(!try policy.parse(null)); + try std.testing.expect(!try policy.parse("0")); + try std.testing.expect(try policy.parse("1")); + for ([_][]const u8{ "", "2", "true", "-1" }) |value| + try std.testing.expectError(error.InvalidSeqReuse, policy.parse(value)); + var p = Pool.init(false, 16); + var object: Object = .{}; + try std.testing.expect(!p.retain(&object, 0, 158, 100)); + try std.testing.expect(p.take(158) == null); + try std.testing.expect(object.alive); +} + +test "take transfers exclusive ownership and does not round capacity or change the object" { + var p = Pool.init(true, 2); + var first: Object = .{}; + var second: Object = .{}; + try std.testing.expect(p.retain(&first, 0, 158, 100)); + try std.testing.expect(p.retain(&second, 0, 272, 120)); + try std.testing.expect(p.take(157) == null); + try std.testing.expect(p.take(159) == null); + const taken = p.take(158).?; + try std.testing.expect(taken == &first and taken.alive); + try std.testing.expect(p.take(158) == null); + try std.testing.expectEqual(@as(usize, 1), p.stats.retained_slots); + try std.testing.expectEqual(@as(usize, 120), p.stats.retained_bytes); + try free(taken); + try free(p.evict().?); + try std.testing.expect(p.evict() == null); + try std.testing.expect(!first.alive and !second.alive); +} + +test "capacity miss must evict unused ownership before a fresh active slot fits" { + var p = Pool.init(true, 2); + var retained: Object = .{}; + var active: Object = .{}; + var replacement: Object = .{}; + try std.testing.expect(p.retain(&retained, 1, 158, 100)); + try std.testing.expect(!p.hasRoom(1)); + try std.testing.expect(p.take(272) == null); + try free(p.evict().?); + try std.testing.expect(p.hasRoom(1)); + // The caller can now own active and replacement, never active+retained+replacement. + try std.testing.expect(!p.retain(&active, 2, 158, 100)); + try std.testing.expectEqual(@as(u64, 1), p.stats.evictions); + try free(&active); + try free(&replacement); + try std.testing.expect(!retained.alive); +} + +test "capacity miss drains every retained byte even with room for a fresh large allocation" { + const Ledger = struct { + freed: usize = 0, + fn release(x: *@This(), object: *Object) void { + std.debug.assert(object.alive); + object.alive = false; + x.freed += 1; + } + }; + var p = Pool.init(true, 16); + var objects: [2]Object = @splat(.{}); + var ledger: Ledger = .{}; + try std.testing.expect(p.retain(&objects[0], 0, 158, 100)); + try std.testing.expect(p.retain(&objects[1], 0, 272, 120)); + try std.testing.expect(p.hasRoom(1)); + try std.testing.expect(p.take(16384) == null); + p.beforeCreate(&ledger, Ledger.release); + try std.testing.expectEqual(@as(usize, 2), ledger.freed); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_bytes); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_slots); + for (objects) |object| try std.testing.expect(!object.alive); +} + +test "total ownership byte cap short-capacity cap and configured stream bound are enforced" { + var p = Pool.init(true, 20); + var objects: [16]Object = @splat(.{}); + try std.testing.expectEqual(@as(usize, 16), p.limit); + try std.testing.expect(!p.retain(&objects[0], 0, 513, 1)); + try std.testing.expect(!p.retain(&objects[0], 0, 512, policy.max_bytes + 1)); + try std.testing.expect(p.retain(&objects[0], 15, 512, policy.max_bytes)); + try std.testing.expect(!p.retain(&objects[1], 14, 158, 1)); + try free(p.evict().?); + p = Pool.init(true, 2); + try std.testing.expect(!p.retain(&objects[1], 2, 158, 1)); + try std.testing.expect(p.retain(&objects[1], 1, 158, 1)); + try std.testing.expect(!p.retain(&objects[2], 1, 158, 1)); + try free(p.evict().?); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_bytes); +} + +test "failure disables reuse and shutdown drains each detached object once" { + var p = Pool.init(true, 3); + var objects: [3]Object = @splat(.{}); + for (&objects, 0..) |*object, i| try std.testing.expect(p.retain(object, 0, 158, i + 1)); + p.fail(); + try std.testing.expect(p.stats.failed); + try std.testing.expect(p.take(158) == null); + try std.testing.expect(!p.retain(&objects[0], 0, 158, 1)); + while (p.evict()) |object| try free(object); + for (objects) |object| try std.testing.expect(!object.alive); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_slots); + try std.testing.expectEqual(@as(usize, 0), p.stats.retained_bytes); + try std.testing.expect(p.evict() == null); +} + +const Stream = struct { + finished: bool = true, + reason: enum { none, stop, length, cancelled } = .length, + cancelled: bool = false, + cached: u32 = 14, + prompt_len: u32 = 14, + reuse_failed: bool = false, + reuse: struct { + saved: ?*anyopaque = null, + at: u32 = 0, + marks: []const u32 = &.{}, + hook: ?u8 = null, + } = .{}, + pub fn isCancelled(s: *Stream) bool { + return s.cancelled; + } +}; + +test "only normally completed settled uncancelled proven full-N restored streams may transfer to the pool" { + var s: Stream = .{}; + try std.testing.expect(policy.completed(&s, true, true, true, false)); + try std.testing.expect(!policy.completed(&s, false, true, true, false)); + try std.testing.expect(!policy.completed(&s, true, false, true, false)); + try std.testing.expect(!policy.completed(&s, true, true, false, false)); + try std.testing.expect(!policy.completed(&s, true, true, true, true)); + for (0..9) |case| { + s = .{}; + switch (case) { + 0 => s.finished = false, + 1 => s.reason = .cancelled, + 2 => s.reason = .none, + 3 => s.cancelled = true, + 4 => s.cached = 1, + 5 => s.reuse_failed = true, + 6 => s.reuse.at = 1, + 7 => s.reuse.marks = &.{1}, + 8 => s.reuse.hook = 1, + else => unreachable, + } + try std.testing.expect(!policy.completed(&s, true, true, true, false)); + } + s = .{}; + var sentinel: u8 = 0; + s.reuse.saved = &sentinel; + try std.testing.expect(!policy.completed(&s, true, true, true, false)); + s = .{ .reason = .stop }; + try std.testing.expect(policy.completed(&s, true, true, true, false)); +} + +test "independent pool flag requires the exact cache and remains strict" { + try std.testing.expect(!try policy.compatible(null, false)); + try std.testing.expect(!try policy.compatible("0", true)); + try std.testing.expect(try policy.compatible("1", true)); + try std.testing.expectError(error.SeqReuseNeedsExactCache, policy.compatible("1", false)); + try std.testing.expectError(error.InvalidSeqReuse, policy.compatible("yes", true)); +} diff --git a/zig/src/families/qwen27/cuda/state.zig b/zig/src/families/qwen27/cuda/state.zig index 8204da8fcd..55c606b093 100644 --- a/zig/src/families/qwen27/cuda/state.zig +++ b/zig/src/families/qwen27/cuda/state.zig @@ -69,6 +69,15 @@ pub const Seq = struct { /// A fresh stream: zero recurrences and conv windows, caches for `capacity` positions. pub fn init(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize, kv8: bool) !Seq { + return initState(gpa, ops, g, capacity, kv8, true); + } + + /// Private BF16 restore destination; actual restore or reset must precede any target work. + pub fn initRestoreDestination(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize) !Seq { + return initState(gpa, ops, g, capacity, false, false); + } + + fn initState(gpa: std.mem.Allocator, ops: kern.Ops, g: c.Geometry, capacity: usize, kv8: bool, initialize: bool) !Seq { var s: Seq = .{ .gpa = gpa, .arena = try Arena.init(ops.k.d, bytes(g, capacity, kv8)), .rec = &.{}, .conv = &.{}, .keys = &.{}, .values = &.{}, .kscale = &.{}, .vscale = &.{}, .kv8 = kv8, .head_in = 0, .head_xs = 0, .capacity = capacity }; errdefer s.arena.deinit(); s.rec = try gpa.alloc(u64, g.layers); @@ -106,7 +115,7 @@ pub const Seq = struct { } } } - try s.reset(ops, g); + if (initialize) try s.reset(ops, g); return s; } @@ -367,3 +376,7 @@ pub const Scratch = struct { s.* = undefined; } }; + +test { + _ = @import("reset_skip_factory_test.zig"); +} diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 16c2542444..568689e37e 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -235,7 +235,7 @@ const Host = struct { /// The family's backend, each call made with the context current on the calling thread. fn backend(h: *Host) lanes.backend.Backend { h.vtable = lane_backend.vtable(Host, h.inner.vtable.*); - return .{ .ptr = h, .vtable = &h.vtable }; + return .{ .ptr = h, .vtable = &h.vtable, .first_before_draft = h.inner.first_before_draft }; } }; From a356e981af5f53785711424d2ab51d3ef7472f4d Mon Sep 17 00:00:00 2001 From: Jon Regan-Beasley Date: Wed, 14 Oct 2026 09:10:50 +1100 Subject: [PATCH 16/19] test(qwen27): reuse quantized KV host rows during drafter startup --- zig/src/families/qwen27/cuda/draft_load.zig | 37 +++++- .../qwen27/cuda/draft_quantized_tail_test.zig | 117 ++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 zig/src/families/qwen27/cuda/draft_quantized_tail_test.zig diff --git a/zig/src/families/qwen27/cuda/draft_load.zig b/zig/src/families/qwen27/cuda/draft_load.zig index 8f9b0a7404..56b43c4354 100644 --- a/zig/src/families/qwen27/cuda/draft_load.zig +++ b/zig/src/families/qwen27/cuda/draft_load.zig @@ -52,6 +52,7 @@ pub const Weights = struct { pred: []f32 = &.{}, // candidate_selector.predecessor_codebook (vocab, 256) succ: []f32 = &.{}, buffers: std.ArrayList(cuda.DeviceBuffer) = .empty, + reused_kv_layers: u32 = 0, pub fn deinit(w: *Weights) void { for (w.buffers.items) |*b| b.free(); @@ -106,6 +107,29 @@ pub fn quantize4(src: []const u8, n: usize, k: usize, words: []u32, scales: []u1 }; } +/// A host-only view of independent already-quantized rows; no device projection aliases another. +pub const QuantizedRows = struct { words: []const u32, scales: []const u16, biases: []const u16 }; + +/// This startup-only private switch is strict and disabled when unset. +pub fn reuseKvFlag(raw: ?[]const u8) !bool { + const value = raw orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (std.mem.eql(u8, value, "1")) return true; + return error.InvalidQuantizedKvReuse; +} + +/// Views complete row-major quantized groups, validating all three arrays before any tiling. +pub fn quantizedTail(words: []const u32, scales: []const u16, biases: []const u16, first: usize, count: usize, k: usize) !QuantizedRows { + if (k == 0 or k % 64 != 0) return error.InvalidQuantizedTail; + const k8 = k / 8; + const kg = k / 64; + if (words.len % k8 != 0) return error.InvalidQuantizedTail; + const n = words.len / k8; + if (scales.len != n * kg or biases.len != n * kg or first > n or count > n - first) + return error.InvalidQuantizedTail; + return .{ .words = words[first * k8 ..][0 .. count * k8], .scales = scales[first * kg ..][0 .. count * kg], .biases = biases[first * kg ..][0 .. count * kg] }; +} + const Loader = struct { gpa: std.mem.Allocator, ops: kern.Ops, @@ -207,6 +231,8 @@ pub fn load(gpa: std.mem.Allocator, io: std.Io, ops: kern.Ops, dir: []const u8, L.biases.deinit(gpa); if (L.staging) |*b| b.free(); } + const env = std.c.getenv("TF_Q27_REUSE_QUANTIZED_KV"); + const reuse_kv = try reuseKvFlag(if (env) |value| std.mem.span(value) else null); var ck = try core.Checkpoint.openModel(gpa, io, dir); defer ck.close(); for (&w.layers, 0..) |*l, i| { @@ -214,7 +240,12 @@ pub fn load(gpa: std.mem.Allocator, io: std.Io, ops: kern.Ops, dir: []const u8, const k = try Loader.get(&ck, "layers.{d}.self_attn.k_proj.weight", .{i}); const v = try Loader.get(&ck, "layers.{d}.self_attn.v_proj.weight", .{i}); l.qkv = try L.quantized(&.{ q, k, v }, hidden); - l.kv = try L.quantized(&.{ k, v }, hidden); + l.kv = if (reuse_kv) blk: { + const rows = try quantizedTail(L.words.items, L.scales.items, L.biases.items, q.dim(0), k.dim(0) + v.dim(0), hidden); + const projection = try L.tile(rows.words, rows.scales, rows.biases, k.dim(0) + v.dim(0), hidden); + w.reused_kv_layers += 1; + break :blk projection; + } else try L.quantized(&.{ k, v }, hidden); l.o = try L.quantized(&.{try Loader.get(&ck, "layers.{d}.self_attn.o_proj.weight", .{i})}, heads * head_dim); l.q_norm = try L.raw(try Loader.get(&ck, "layers.{d}.self_attn.q_norm.weight", .{i})); l.k_norm = try L.raw(try Loader.get(&ck, "layers.{d}.self_attn.k_norm.weight", .{i})); @@ -272,3 +303,7 @@ test "quantize4 rounds as torch does" { try std.testing.expectEqual(roundEven(2.5), 2.0); try std.testing.expectEqual(roundEven(3.5), 4.0); } + +test { + _ = @import("draft_quantized_tail_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/draft_quantized_tail_test.zig b/zig/src/families/qwen27/cuda/draft_quantized_tail_test.zig new file mode 100644 index 0000000000..8e7cccb450 --- /dev/null +++ b/zig/src/families/qwen27/cuda/draft_quantized_tail_test.zig @@ -0,0 +1,117 @@ +//! Quantized host row views preserve independent K/V bits and decline invalid layout or ranges. +const std = @import("std"); +const dl = @import("draft_load.zig"); +const a = std.testing.allocator; + +fn bf16(value: f32) u16 { + const bits: u32 = @bitCast(value); + return @intCast((bits + 0x7fff + ((bits >> 16) & 1)) >> 16); +} + +fn fixture(nq: usize, nk: usize, nv: usize, width: usize) !void { + const total = nq + nk + nv; + const input = try a.alloc(u8, total * width * 2); + defer a.free(input); + for (0..total) |row| for (0..width) |col| { + const group = col / 64; + const scale: f32 = @floatFromInt(1 + (row + group) % 3); + const lo: f32 = @floatFromInt(@as(i32, @intCast((row * 7 + group * 3) % 17)) - 8); + const j = col % 64; + const fraction: f32 = switch (j) { + 0 => 0, + 1 => 15, + 2 => 2.5, + 3 => 3.5, + else => @floatFromInt((j * 7 + row * 3 + group) % 16), + }; + const value = if (group % 4 == 3) fraction * 1e-8 else lo + fraction * scale; + std.mem.writeInt(u16, input[(row * width + col) * 2 ..][0..2], bf16(value), .little); + }; + const words = try a.alloc(u32, total * width / 8); + defer a.free(words); + const scales = try a.alloc(u16, total * width / 64); + defer a.free(scales); + const biases = try a.alloc(u16, total * width / 64); + defer a.free(biases); + var first: usize = 0; + for ([_]usize{ nq, nk, nv }) |rows| { + dl.quantize4(input[first * width * 2 ..][0 .. rows * width * 2], rows, width, words[first * width / 8 ..][0 .. rows * width / 8], scales[first * width / 64 ..][0 .. rows * width / 64], biases[first * width / 64 ..][0 .. rows * width / 64]); + first += rows; + } + const original = try a.dupe(u32, words); + defer a.free(original); + const expected_words = try a.alloc(u32, (nk + nv) * width / 8); + defer a.free(expected_words); + const expected_scales = try a.alloc(u16, (nk + nv) * width / 64); + defer a.free(expected_scales); + const expected_biases = try a.alloc(u16, (nk + nv) * width / 64); + defer a.free(expected_biases); + first = 0; + for ([_]usize{ nk, nv }) |rows| { + dl.quantize4(input[(nq + first) * width * 2 ..][0 .. rows * width * 2], rows, width, expected_words[first * width / 8 ..][0 .. rows * width / 8], expected_scales[first * width / 64 ..][0 .. rows * width / 64], expected_biases[first * width / 64 ..][0 .. rows * width / 64]); + first += rows; + } + const tail = try dl.quantizedTail(words, scales, biases, nq, nk + nv, width); + try std.testing.expectEqualSlices(u32, expected_words, tail.words); + try std.testing.expectEqualSlices(u16, expected_scales, tail.scales); + try std.testing.expectEqualSlices(u16, expected_biases, tail.biases); + try std.testing.expectEqualSlices(u32, original, words); + try std.testing.expect(tail.words.ptr == words[nq * width / 8 ..].ptr); + try std.testing.expect(tail.scales.ptr == scales[nq * width / 64 ..].ptr); + try std.testing.expect(tail.biases.ptr == biases[nq * width / 64 ..].ptr); +} + +test "QKV tail equals separately quantized K/V across nonunit scales ties and unpadded rows" { + try fixture(7, 5, 3, 64); + try fixture(32, 8, 8, 256); + try fixture(1, 1, 1, 5120); +} + +test "RNE halfway values and signed-zero rows remain literal bits in the tail" { + var input: [128 * 2]u8 = @splat(0); + for (0..64) |j| { + std.mem.writeInt(u16, input[j * 2 ..][0..2], if (j % 2 == 0) 0x8000 else 0, .little); + const value: f32 = switch (j) { + 0 => 0, + 1 => 15, + 2 => 2.5, + 3 => 3.5, + else => 0, + }; + std.mem.writeInt(u16, input[(64 + j) * 2 ..][0..2], bf16(value), .little); + } + var words: [16]u32 = undefined; + var scales: [2]u16 = undefined; + var biases: [2]u16 = undefined; + dl.quantize4(&input, 2, 64, &words, &scales, &biases); + const tied = try dl.quantizedTail(&words, &scales, &biases, 1, 1, 64); + try std.testing.expectEqual(@as(u32, 2), (tied.words[0] >> 8) & 15); + try std.testing.expectEqual(@as(u32, 4), (tied.words[0] >> 12) & 15); + try std.testing.expectEqual(bf16(1), tied.scales[0]); + const zeros = try dl.quantizedTail(&words, &scales, &biases, 0, 1, 64); + try std.testing.expectEqual(@as(u32, 0), zeros.words[0]); +} + +test "tail validates all host arrays and bounds before exposing a borrowed view" { + const words: [32]u32 = @splat(0); + const scales: [4]u16 = @splat(0); + const biases: [4]u16 = @splat(0); + for ([_]usize{ 0, 1, 65 }) |width| + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(&words, &scales, &biases, 0, 1, width)); + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(words[0..31], &scales, &biases, 0, 1, 64)); + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(&words, scales[0..3], &biases, 0, 1, 64)); + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(&words, &scales, biases[0..3], 0, 1, 64)); + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(&words, &scales, &biases, 5, 0, 64)); + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(&words, &scales, &biases, 3, 2, 64)); + try std.testing.expectError(error.InvalidQuantizedTail, dl.quantizedTail(&words, &scales, &biases, 1, std.math.maxInt(usize), 64)); + const empty = try dl.quantizedTail(&words, &scales, &biases, 4, 0, 64); + try std.testing.expectEqual(@as(usize, 0), empty.words.len + empty.scales.len + empty.biases.len); +} + +test "startup KV reuse flag is strict and remains disabled by default" { + try std.testing.expect(!try dl.reuseKvFlag(null)); + try std.testing.expect(!try dl.reuseKvFlag("0")); + try std.testing.expect(try dl.reuseKvFlag("1")); + for ([_][]const u8{ "", "2", "true", " 1", "1 " }) |raw| + try std.testing.expectError(error.InvalidQuantizedKvReuse, dl.reuseKvFlag(raw)); +} From 5b6a80c7586e8407f408e7053b08ceaa254050d9 Mon Sep 17 00:00:00 2001 From: Jon Regan-Beasley Date: Wed, 14 Oct 2026 09:30:59 +1100 Subject: [PATCH 17/19] fix(cuda): remove duplicate integrated fatbin exports --- zig/src/cuda/kernels.zig | 2 -- 1 file changed, 2 deletions(-) diff --git a/zig/src/cuda/kernels.zig b/zig/src/cuda/kernels.zig index 81f4d5ed5d..e40301fa3b 100644 --- a/zig/src/cuda/kernels.zig +++ b/zig/src/cuda/kernels.zig @@ -58,8 +58,6 @@ pub const nemotron_mamba: []const u8 = if (available) &Blob("fatbin_nemotron_mam pub const nemotron_attention: []const u8 = if (available) &Blob("fatbin_nemotron_attention").bytes else &.{}; pub const nemotron_keyed: []const u8 = if (available) &Blob("fatbin_nemotron_keyed").bytes else &.{}; pub const torch_norm_rotary: []const u8 = if (available) &Blob("fatbin_torch_norm_rotary").bytes else &.{}; -pub const norms: []const u8 = if (available) &Blob("fatbin_norms").bytes else &.{}; -pub const embed_rows: []const u8 = if (available) &Blob("fatbin_embed_rows").bytes else &.{}; /// Symbols in the gdn image as cuobjdump lists them for the built fatbin (named namespace tf_gdn). pub const gdn_symbols = struct { From e22ab6dabdfb8be1a248db12d5b84e4dad60296d Mon Sep 17 00:00:00 2001 From: Jon Regan-Beasley Date: Wed, 14 Oct 2026 09:57:19 +1100 Subject: [PATCH 18/19] test(qwen27): port qualified post-first capture to current native --- zig/src/core/lane_host_first_test.zig | 23 ++- zig/src/core/lanes/backend.zig | 2 + zig/src/core/lanes/open_first.zig | 4 + zig/src/core/lanes/open_first_test.zig | 54 ++++- .../qwen27/cuda/backend_vtable_test.zig | 61 ++++++ .../families/qwen27/cuda/exact_cache_test.zig | 30 +-- zig/src/families/qwen27/cuda/exact_init.zig | 1 + .../families/qwen27/cuda/exact_post_first.zig | 95 +++++++++ .../qwen27/cuda/exact_post_first_test.zig | 190 ++++++++++++++++++ .../qwen27/cuda/exact_prompt_cache.zig | 34 ++++ zig/src/families/qwen27/cuda/lanes.zig | 18 +- zig/src/native/cuda_bound.zig | 39 ++++ 12 files changed, 527 insertions(+), 24 deletions(-) create mode 100644 zig/src/families/qwen27/cuda/backend_vtable_test.zig create mode 100644 zig/src/families/qwen27/cuda/exact_post_first.zig create mode 100644 zig/src/families/qwen27/cuda/exact_post_first_test.zig diff --git a/zig/src/core/lane_host_first_test.zig b/zig/src/core/lane_host_first_test.zig index 04b71df7b5..3b51adbb64 100644 --- a/zig/src/core/lane_host_first_test.zig +++ b/zig/src/core/lane_host_first_test.zig @@ -45,11 +45,13 @@ const Target = struct { drafts: usize = 0, releases: usize = 0, lane_memory: bool, + post_failure: bool = false, observed: *Box, delivered_before_draft: bool = false, fn backend(t: *Target) lanes.backend.Backend { t.table = t.fake.backend().vtable.*; t.table.draft = draft; + t.table.after_first = if (t.post_failure) after else null; t.table.release = release; return .{ .ptr = &t.fake, .vtable = &t.table, .first_before_draft = true }; } @@ -57,6 +59,13 @@ const Target = struct { const fake: *lanes.fake.Fake = @ptrCast(@alignCast(ptr)); return @fieldParentPtr("fake", fake); } + fn after(ptr: *anyopaque, _: *lanes.Stream) !void { + const t = self(ptr); + t.observed.mutex.lockUncancelable(std.testing.io); + t.delivered_before_draft = t.observed.tokens == 1 and t.observed.logprobs == 1; + t.observed.mutex.unlock(std.testing.io); + return error.LaneMemory; + } fn draft(ptr: *anyopaque, _: []const lanes.backend.DraftRequest) !void { const t = self(ptr); t.drafts += 1; @@ -72,10 +81,10 @@ const Target = struct { } }; -fn check(lane_memory: bool) !void { +fn check(lane_memory: bool, post_failure: bool) !void { var one: Box = .{}; var other: Box = .{}; - var t: Target = .{ .lane_memory = lane_memory, .observed = &other }; + var t: Target = .{ .lane_memory = lane_memory, .post_failure = post_failure, .observed = &other }; defer t.fake.deinit(); var cfg = try lanes.Config.init(a, .{ .exact_width = 8, .hidden_rows = true, .mtp = true, .speculate = true, .drafts = 7 }, 8, 7); defer cfg.deinit(a); @@ -105,15 +114,19 @@ fn check(lane_memory: bool) !void { try std.testing.expectEqual(@as(usize, 1), other.prefilled); try std.testing.expectEqual(@as(usize, 1), one.finishes); try std.testing.expectEqual(@as(usize, 1), other.finishes); - try std.testing.expectEqual(@as(usize, 1), t.drafts); + try std.testing.expectEqual(@as(usize, if (post_failure) 0 else 1), t.drafts); try std.testing.expectEqual(@as(usize, 2), t.releases); try std.testing.expect(t.delivered_before_draft); } test "first token and logprob sent once; terminal job survives another job's draft error" { - try check(false); + try check(false, false); } test "LaneMemory after the committed handoff fails instead of resetting and requeueing the reply" { - try check(true); + try check(true, false); +} + +test "post-first LaneMemory never requeues an emitted reply and terminal first still succeeds" { + try check(true, true); } diff --git a/zig/src/core/lanes/backend.zig b/zig/src/core/lanes/backend.zig index 8bfef2869b..5df7cee894 100644 --- a/zig/src/core/lanes/backend.zig +++ b/zig/src/core/lanes/backend.zig @@ -75,6 +75,8 @@ pub const Backend = struct { prefill_many: ?*const fn (ptr: *anyopaque, streams: []const *Stream, errs: []?anyerror) anyerror!void = null, /// Draw the first token from the prompt's last row at `position`; a handle to it. first: *const fn (ptr: *anyopaque, s: *Stream, position: u64) anyerror!u64, + /// Optional work after first commit and handoff, before any draft or target continuation. + after_first: ?*const fn (ptr: *anyopaque, s: *Stream) anyerror!void = null, /// Feed one token and queue the draw of the next at `position` (a one-token round); a handle to it. queue: *const fn (ptr: *anyopaque, s: *Stream, feed: Feed, position: u64) anyerror!u64, /// The value of a drawn token (waits for the GPU). diff --git a/zig/src/core/lanes/open_first.zig b/zig/src/core/lanes/open_first.zig index 8638695786..2d1c1c431e 100644 --- a/zig/src/core/lanes/open_first.zig +++ b/zig/src/core/lanes/open_first.zig @@ -46,6 +46,10 @@ pub fn opened(e: *Engine, s: *Stream, hook: ?be.Committed) !void { e.release(s); return error.Cancelled; } + if (e.backend.vtable.after_first) |after| after(e.backend.ptr, s) catch |err| { + if (err == error.Cancelled) e.release(s); + return err; + }; if (depth) |d| { e.backend.draft(&.{.{ .stream = s, .follow = &.{}, .first = .{ .value = value }, .rows = null, .start = s.prompt_len, .position = position + 1, .depth = d }}) catch |err| { if (err == error.Cancelled) e.release(s); diff --git a/zig/src/core/lanes/open_first_test.zig b/zig/src/core/lanes/open_first_test.zig index 274994dd10..c73fa5a1d6 100644 --- a/zig/src/core/lanes/open_first_test.zig +++ b/zig/src/core/lanes/open_first_test.zig @@ -19,10 +19,15 @@ const Probe = struct { fail_draft: bool = false, cancel: bool = false, selected: u32 = 0, + post: bool = false, + afters: usize = 0, + fail_after: bool = false, + saw_committed_handoff: bool = false, fn backend(p: *Probe, early: bool, stepped: bool) be.Backend { p.table = (if (stepped) p.target.stepped() else p.target.backend()).vtable.*; p.table.draft = draft; + p.table.after_first = if (p.post) after else null; p.table.release = release; return .{ .ptr = &p.target, .vtable = &p.table, .first_before_draft = early }; } @@ -30,6 +35,12 @@ const Probe = struct { const t: *fake.Fake = @ptrCast(@alignCast(ptr)); return @fieldParentPtr("target", t); } + fn after(ptr: *anyopaque, s: *sm.Stream) !void { + const p = self(ptr); + p.afters += 1; + p.saw_committed_handoff = p.hooks == 1 and s.emitted().len == 1 and s.pending != null; + if (p.fail_after) return error.LaneMemory; + } fn draft(ptr: *anyopaque, req: []const be.DraftRequest) !void { const p = self(ptr); p.drafts += 1; @@ -87,7 +98,7 @@ test "opt-in commits and hands off before draft; default retains old ordering" { test "max-one and EOS terminate successfully without preparing any drafts" { for ([_]bool{ false, true }) |eos| { - var p: Probe = .{ .fail_draft = true }; + var p: Probe = .{ .fail_draft = true, .post = true, .fail_after = true }; defer p.target.deinit(); var cfg = try config(); defer cfg.deinit(a); @@ -100,6 +111,7 @@ test "max-one and EOS terminate successfully without preparing any drafts" { try e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook }); try std.testing.expect(s.finished); try std.testing.expectEqual(@as(usize, 0), p.drafts); + try std.testing.expectEqual(@as(usize, 0), p.afters); try std.testing.expectEqual(@as(usize, 1), p.hooks); try std.testing.expectEqual(@as(usize, 1), p.releases); try std.testing.expectEqual(@as(usize, 0), e.activeCount()); @@ -128,7 +140,7 @@ test "forced first and its logprob are committed exactly once before a later dra } test "cancellation at handoff releases once and skips initial drafts" { - var p: Probe = .{ .cancel = true }; + var p: Probe = .{ .cancel = true, .post = true, .fail_after = true }; defer p.target.deinit(); var cfg = try config(); defer cfg.deinit(a); @@ -141,6 +153,7 @@ test "cancellation at handoff releases once and skips initial drafts" { try std.testing.expectEqual(@as(usize, 1), p.hooks); try std.testing.expectEqual(@as(usize, 1), p.releases); try std.testing.expectEqual(@as(usize, 0), p.drafts); + try std.testing.expectEqual(@as(usize, 0), p.afters); } test "stepped prefill uses the same committed handoff only after its final chunk" { @@ -161,3 +174,40 @@ test "stepped prefill uses the same committed handoff only after its final chunk try std.testing.expect(p.committed_in_draft and p.hook_in_draft); e.discard(&s); } + +test "optional post-first runs after committed handoff and before draft for forced sampled first" { + var p: Probe = .{ .post = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "after", .prompt = &ids, .max_new = 8, .logprobs = 2, .sampling = .{ .temperature = 1.2, .top_k = 0, .top_p = 1, .seed = 7 } }); + defer s.deinit(a); + try s.force.append(a, 42); + try e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook }); + try std.testing.expect(p.saw_committed_handoff and p.hook_in_draft); + try std.testing.expectEqual(@as(usize, 1), p.afters); + try std.testing.expectEqual(@as(u32, 42), p.selected); + try std.testing.expectEqual(@as(usize, 1), s.rows.items.len); + e.discard(&s); +} + +test "post-first error preserves one token and bypasses draft until the owner discards" { + var p: Probe = .{ .post = true, .fail_after = true }; + defer p.target.deinit(); + var cfg = try config(); + defer cfg.deinit(a); + var clock: fake.FixedClock = .{}; + var e = Engine.init(a, &cfg, p.backend(true, false), clock.clock()); + defer e.deinit(); + var s = try sm.Stream.init(a, .{ .id = "after-error", .prompt = &ids, .max_new = 8 }); + defer s.deinit(a); + try std.testing.expectError(error.LaneMemory, e.addStreamCommitted(&s, .{ .ptr = &p, .call = Probe.hook })); + try std.testing.expect(p.saw_committed_handoff); + try std.testing.expectEqual(@as(usize, 1), s.emitted().len); + try std.testing.expectEqual(@as(usize, 0), p.drafts); + e.discard(&s); + try std.testing.expectEqual(@as(usize, 1), p.releases); +} diff --git a/zig/src/families/qwen27/cuda/backend_vtable_test.zig b/zig/src/families/qwen27/cuda/backend_vtable_test.zig new file mode 100644 index 0000000000..35005811f9 --- /dev/null +++ b/zig/src/families/qwen27/cuda/backend_vtable_test.zig @@ -0,0 +1,61 @@ +//! Actual backend tables retain immutable lifetime across owners and post-first configuration changes. +const std = @import("std"); +const be = @import("lanes").backend; +const Cuda = @import("lanes.zig").Cuda; +const Cache = @import("exact_prompt_cache.zig").Cache; +const shape = @import("shape.zig"); +const a = std.testing.allocator; + +fn fixture(enabled: bool) !*Cuda { + const x = try a.create(Cuda); + x.exact_cache = Cache.init(a, true); + x.exact_cache.post_first.enabled = enabled; + return x; +} + +noinline fn obtain(x: *Cuda) be.Backend { + return x.backend(); +} + +test "captured off and on backend tables survive later calls and flag toggles" { + const x = try fixture(false); + defer a.destroy(x); + const off = obtain(x); + x.exact_cache.post_first.enabled = true; + const on = obtain(x); + x.exact_cache.post_first.enabled = false; + const off_again = obtain(x); + try std.testing.expect(off.vtable != on.vtable); + try std.testing.expect(off.vtable == off_again.vtable); + try std.testing.expect(off.vtable.after_first == null); + try std.testing.expect(on.vtable.after_first != null); + try std.testing.expect(off.first_before_draft and on.first_before_draft); + try std.testing.expectEqual(@as(u32, 123), try off.read(123)); + try std.testing.expectEqual(@as(u32, 321), try on.read(321)); + try std.testing.expectError(error.InvalidQwenToken, off.read(shape.vocab)); +} + +test "two owners share immutable mode tables while retaining distinct backend pointers" { + const first = try fixture(false); + defer a.destroy(first); + const second = try fixture(true); + defer a.destroy(second); + const off = obtain(first); + const on = obtain(second); + first.exact_cache.post_first.enabled = true; + const first_on = obtain(first); + second.exact_cache.post_first.enabled = false; + const second_off = obtain(second); + try std.testing.expect(off.ptr != on.ptr); + try std.testing.expect(off.ptr == first_on.ptr); + try std.testing.expect(on.ptr == second_off.ptr); + try std.testing.expect(off.vtable == second_off.vtable); + try std.testing.expect(on.vtable == first_on.vtable); + try std.testing.expect(off.vtable != on.vtable); + try std.testing.expect(off.vtable.after_first == null and on.vtable.after_first != null); + try std.testing.expect(off.vtable.prefill == on.vtable.prefill); + try std.testing.expect(off.vtable.first == on.vtable.first); + try std.testing.expect(off.vtable.draft == on.vtable.draft); + try std.testing.expectEqual(@as(u32, 456), try first_on.read(456)); + try std.testing.expectEqual(@as(u32, 654), try second_off.read(654)); +} diff --git a/zig/src/families/qwen27/cuda/exact_cache_test.zig b/zig/src/families/qwen27/cuda/exact_cache_test.zig index db3166b367..464b07a7b0 100644 --- a/zig/src/families/qwen27/cuda/exact_cache_test.zig +++ b/zig/src/families/qwen27/cuda/exact_cache_test.zig @@ -10,17 +10,20 @@ const dr = @import("draft.zig"); const g = shape.the_27b; const a = std.testing.allocator; -const Mock = struct { - var live: usize = 0; - var freed: usize = 0; +pub const Mock = struct { + pub var live: usize = 0; + pub var freed: usize = 0; const Copy = struct { dst: u64, src: u64, bytes: usize }; - var copy_rows: [512]Copy = undefined; - var copies: usize = 0; - var fences: usize = 0; - var fail_copy = false; - var fail_fence = false; + pub var copy_rows: [512]Copy = undefined; + pub var copies: usize = 0; + pub var fences: usize = 0; + pub var fail_copy = false; + pub var fail_fence = false; var next: u64 = 0x100000000; + pub var fail_alloc = false; + pub var copied: ?*const fn () void = null; fn alloc(ptr: *cuda.abi.DevicePtr, _: usize) callconv(.c) cuda.abi.Result { + if (fail_alloc) return cuda.abi.error_not_ready; ptr.* = next; next += 0x100000000; live += 1; @@ -36,6 +39,7 @@ const Mock = struct { std.debug.assert(copies < copy_rows.len); copy_rows[copies] = .{ .dst = dst, .src = src, .bytes = bytes }; copies += 1; + if (copied) |call| call(); return cuda.abi.success; } fn fence(_: cuda.abi.Stream) callconv(.c) cuda.abi.Result { @@ -43,24 +47,26 @@ const Mock = struct { fences += 1; return cuda.abi.success; } - fn reset() void { + pub fn reset() void { live = 0; freed = 0; copies = 0; fences = 0; fail_copy = false; fail_fence = false; + fail_alloc = false; + copied = null; } }; -const Fixture = struct { +pub const Fixture = struct { driver: cuda.Driver = undefined, kernels: kern.Kernels = undefined, rec: [64]u64 = @splat(0x1000), conv: [64]u64 = @splat(0x2000), keys: [64]u64 = @splat(0x3000), values: [64]u64 = @splat(0x4000), - fn ops(f: *Fixture) kern.Ops { + pub fn ops(f: *Fixture) kern.Ops { f.driver.api.cuMemAlloc_v2 = Mock.alloc; f.driver.api.cuMemFree_v2 = Mock.free; f.driver.api.cuMemcpyDtoDAsync_v2 = Mock.copy; @@ -68,7 +74,7 @@ const Fixture = struct { f.kernels.d = &f.driver; return .{ .k = &f.kernels, .s = .{ .d = &f.driver, .handle = null } }; } - fn seq(f: *Fixture, n: usize) st.Seq { + pub fn seq(f: *Fixture, n: usize) st.Seq { return .{ .gpa = a, .arena = .{ .buf = null }, .rec = &f.rec, .conv = &f.conv, .keys = &f.keys, .values = &f.values, .kscale = &.{}, .vscale = &.{}, .kv8 = false, .head_in = 0x5000, .head_xs = 0x6000, .pos = n, .capacity = n + 256 }; } }; diff --git a/zig/src/families/qwen27/cuda/exact_init.zig b/zig/src/families/qwen27/cuda/exact_init.zig index b4657aec4c..2043bf800d 100644 --- a/zig/src/families/qwen27/cuda/exact_init.zig +++ b/zig/src/families/qwen27/cuda/exact_init.zig @@ -14,6 +14,7 @@ pub fn init(gpa: std.mem.Allocator, e: *Engine, streams: u32) !Cuda { const on = try exact.policy.enabled(if (env) |v| std.mem.span(v) else null); var cache = exact.Cache.init(gpa, on); cache.logits_enabled = try exact.logits_policy.configured(on); + cache.post_first.enabled = try @import("exact_post_first.zig").configured(on); return .{ .gpa = gpa, .e = e, diff --git a/zig/src/families/qwen27/cuda/exact_post_first.zig b/zig/src/families/qwen27/cuda/exact_post_first.zig new file mode 100644 index 0000000000..95d08e2732 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_post_first.zig @@ -0,0 +1,95 @@ +//! A bounded host-only reservation moves optional snapshot work past the committed-token handoff. +const std = @import("std"); +const lanes = @import("lanes"); +const st = @import("state.zig"); +const dr = @import("draft.zig"); + +pub fn enabled(raw: ?[]const u8, cache_on: bool) !bool { + const value = raw orelse return false; + if (std.mem.eql(u8, value, "0")) return false; + if (!std.mem.eql(u8, value, "1")) return error.InvalidPostFirstCaptureFlag; + if (!cache_on) return error.PostFirstCaptureNeedsExactCache; + return true; +} + +pub fn configured(cache_on: bool) !bool { + const raw = std.c.getenv("TF_PR608_EXACT_CACHE_POST_FIRST"); + return enabled(if (raw) |value| std.mem.span(value) else null, cache_on); +} + +pub const Reservation = struct { + stream: *lanes.Stream, + seq: *st.Seq, + ctx: ?*dr.Context, + tokens: usize, + bytes: usize, + cur: u1, +}; + +pub const Control = struct { + enabled: bool = false, + reserved: ?Reservation = null, + reservations: u64 = 0, + saves: u64 = 0, + skips: u64 = 0, + failures: u64 = 0, + + pub fn take(control: *Control, s: *lanes.Stream) ?Reservation { + const value = control.reserved orelse return null; + if (value.stream != s) return null; + control.reserved = null; + return value; + } + + pub fn forget(control: *Control, s: *lanes.Stream) void { + if (control.take(s) != null) control.skips += 1; + } +}; + +/// No device work or active-entry eviction occurs while deciding the one pending owner. +pub fn reserve(cache: anytype, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, bytes: usize) void { + if (seq.kv8 or seq.media != null or seq.rope_delta != 0 or !cache.eligible(s) or + s.drafts != (ctx != null) or (cache.active != null and cache.active.?.matches(s))) return; + if (cache.post_first.reserved != null) { + cache.stats.refusals += 1; + return; + } + const policy = @import("exact_cache_policy.zig"); + switch (policy.admission(cache.stats.active_bytes, cache.stats.pending_bytes, bytes)) { + .refuse, .busy => { + cache.stats.refusals += 1; + return; + }, + .evict, .retain => {}, + } + cache.post_first.reserved = .{ .stream = s, .seq = seq, .ctx = ctx, .tokens = s.prompt().len, .bytes = bytes, .cur = if (ctx) |dc| dc.cur else 0 }; + cache.post_first.reservations += 1; +} + +/// This executes on the engine thread before initial proposals can consume shared scratch. +pub fn after(p: *anyopaque, s: *lanes.Stream) !void { + const Cuda = @import("lanes.zig").Cuda; + const x: *Cuda = @ptrCast(@alignCast(p)); + if (x.exact_cache.post_first.reserved == null or x.exact_cache.post_first.reserved.?.stream != s) return; + finish(x, s) catch |err| { + if (err != error.Cancelled) { + @import("exact_seq_reuse.zig").fail(x); + x.exact_cache.failed = true; + } + x.exact_cache.abandon(s, x.e.ops()) catch |purge_error| { + std.log.err("fatal unresolved post-first capture fence before device frees: {s}", .{@errorName(purge_error)}); + std.process.exit(1); + }; + return err; + }; +} + +fn finish(x: anytype, s: *lanes.Stream) !void { + const seq = x.seqs.get(s) orelse return error.UnknownQwenStream; + const ctx = if (x.drafting.get(s)) |ds| &ds.ctx else null; + try x.exact_cache.finishAfterFirst(s, seq, ctx, x.e.ops(), x.e.w.g, x.e.scratch.logits); +} + +test { + _ = @import("exact_post_first_test.zig"); +} diff --git a/zig/src/families/qwen27/cuda/exact_post_first_test.zig b/zig/src/families/qwen27/cuda/exact_post_first_test.zig new file mode 100644 index 0000000000..5db75b71f4 --- /dev/null +++ b/zig/src/families/qwen27/cuda/exact_post_first_test.zig @@ -0,0 +1,190 @@ +//! Actual snapshot driver mocks discriminate reservation-only handoff, valid save and fenced failure ownership. +const std = @import("std"); +const lanes = @import("lanes"); +const cache_mod = @import("exact_prompt_cache.zig"); +const control = @import("exact_post_first.zig"); +const mocks = @import("exact_cache_test.zig"); +const Mock = mocks.Mock; +const Fixture = mocks.Fixture; +const g = @import("shape.zig").the_27b; +const a = std.testing.allocator; + +fn stream() !lanes.Stream { + return lanes.Stream.init(a, .{ .id = "post-first", .prompt = &.{ 1, 2, 3 }, .max_new = 256, .drafts = false }); +} + +fn committed(s: *lanes.Stream) !void { + _ = try s.commit(a, &.{7}, &.{}); + s.pending = 7; +} + +fn cache() cache_mod.Cache { + var out = cache_mod.Cache.init(a, true); + out.logits_enabled = true; + out.post_first.enabled = true; + return out; +} + +test "post-first flag is strict default-off and requires exact cache" { + try std.testing.expect(!try control.enabled(null, false)); + try std.testing.expect(!try control.enabled("0", true)); + try std.testing.expect(try control.enabled("1", true)); + try std.testing.expectError(error.PostFirstCaptureNeedsExactCache, control.enabled("1", false)); + for ([_][]const u8{ "", "2", "true", " 1" }) |raw| + try std.testing.expectError(error.InvalidPostFirstCaptureFlag, control.enabled(raw, true)); +} + +test "one reservation performs no CUDA work and saves original state and raw logits only after commit" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var c = cache(); + defer c.deinit(ops) catch unreachable; + var s = try stream(); + defer s.deinit(a); + var other = try stream(); + defer other.deinit(a); + var seq = f.seq(3); + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try c.captureBeforeFirst(&other, &seq, null, ops, g); + try std.testing.expectEqual(@as(usize, 0), Mock.live + Mock.copies + Mock.fences); + try std.testing.expectEqual(@as(u64, 1), c.post_first.reservations); + try std.testing.expectEqual(@as(u64, 1), c.stats.refusals); + try c.finishAfterFirst(&other, &seq, null, ops, g, 0x7770); + try std.testing.expect(c.post_first.reserved.?.stream == &s); + try committed(&s); + try c.finishAfterFirst(&s, &seq, null, ops, g, 0x7770); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + try std.testing.expectEqual(@as(usize, 131), Mock.copies); + try std.testing.expectEqual(@as(usize, 0), Mock.fences); + try std.testing.expectEqual(@as(u64, 0x7770), Mock.copy_rows[130].src); + try std.testing.expectEqual(@as(usize, 496640), Mock.copy_rows[130].bytes); + try std.testing.expect(c.pending.?.logits_valid and c.post_first.reserved == null); + try std.testing.expectEqual(@as(u64, 1), c.post_first.saves); + s.finished = true; + s.reason = .length; + try c.release(&s, true, ops); + try std.testing.expectEqual(@as(u64, 1), c.stats.promotions); +} + +test "disabled path keeps capture before first and terminal reservation creates no snapshot" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var c = cache(); + defer c.deinit(ops) catch unreachable; + var s = try stream(); + defer s.deinit(a); + var seq = f.seq(3); + c.post_first.enabled = false; + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + try std.testing.expectEqual(@as(usize, 130), Mock.copies); + try c.abandon(&s, ops); + Mock.reset(); + c.post_first.enabled = true; + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try committed(&s); + s.finished = true; + s.reason = .length; + try c.release(&s, true, ops); + try std.testing.expectEqual(@as(usize, 0), Mock.live + Mock.copies + Mock.fences); + try std.testing.expect(c.post_first.reserved == null); +} + +test "allocation refusal after handoff remains uncached generation with no device copies" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var c = cache(); + defer c.deinit(ops) catch unreachable; + var s = try stream(); + defer s.deinit(a); + var seq = f.seq(3); + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try committed(&s); + Mock.fail_alloc = true; + try c.finishAfterFirst(&s, &seq, null, ops, g, 0x7770); + Mock.fail_alloc = false; + try std.testing.expectEqual(@as(usize, 0), Mock.live + Mock.copies + Mock.fences); + try std.testing.expect(!c.failed and c.pending == null); + try std.testing.expectEqual(@as(u64, 1), c.stats.refusals); + try std.testing.expectEqualSlices(u32, &.{7}, s.emitted()); +} + +test "post-send copy failure preserves pending ownership until a successful original fence" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var c = cache(); + defer c.deinit(ops) catch unreachable; + var s = try stream(); + defer s.deinit(a); + var seq = f.seq(3); + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try committed(&s); + Mock.fail_copy = true; + try std.testing.expectError(error.NotReady, c.finishAfterFirst(&s, &seq, null, ops, g, 0x7770)); + Mock.fail_copy = false; + try std.testing.expect(c.failed and c.pending != null and c.post_first.reserved == null); + try std.testing.expectEqual(@as(u64, 1), c.post_first.failures); + Mock.fail_fence = true; + try std.testing.expectError(error.NotReady, c.abandon(&s, ops)); + try std.testing.expectEqual(@as(usize, 1), Mock.live); + Mock.fail_fence = false; + try c.abandon(&s, ops); + try std.testing.expectEqual(@as(usize, 0), Mock.live); + try std.testing.expectEqualSlices(u32, &.{7}, s.emitted()); +} + +const Cancel = struct { + var value = false; + fn check(_: *anyopaque) bool { + return value; + } + fn copied() void { + value = true; + } +}; + +test "cancel before or during save clears reservation and fenced abandon purges copied state" { + for ([_]bool{ false, true }) |during| { + Mock.reset(); + Cancel.value = false; + var marker: u8 = 0; + var f: Fixture = .{}; + const ops = f.ops(); + var c = cache(); + defer c.deinit(ops) catch unreachable; + var s = try stream(); + defer s.deinit(a); + s.cancel_check = .{ .ptr = &marker, .check = Cancel.check }; + var seq = f.seq(3); + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try committed(&s); + if (during) Mock.copied = Cancel.copied else Cancel.value = true; + try std.testing.expectError(error.Cancelled, c.finishAfterFirst(&s, &seq, null, ops, g, 0x7770)); + Mock.copied = null; + try std.testing.expect(c.post_first.reserved == null); + try c.abandon(&s, ops); + try std.testing.expectEqual(@as(usize, 0), Mock.live); + try std.testing.expectEqual(@as(u64, 0), c.stats.promotions); + Cancel.value = false; + } +} + +test "state boundary changed after reservation rejects before allocation or copy" { + Mock.reset(); + var f: Fixture = .{}; + const ops = f.ops(); + var c = cache(); + defer c.deinit(ops) catch unreachable; + var s = try stream(); + defer s.deinit(a); + var seq = f.seq(3); + try c.captureBeforeFirst(&s, &seq, null, ops, g); + try committed(&s); + seq.pos += 1; + try std.testing.expectError(error.InvalidPostFirstCapture, c.finishAfterFirst(&s, &seq, null, ops, g, 0x7770)); + try std.testing.expectEqual(@as(usize, 0), Mock.live + Mock.copies + Mock.fences); +} diff --git a/zig/src/families/qwen27/cuda/exact_prompt_cache.zig b/zig/src/families/qwen27/cuda/exact_prompt_cache.zig index 41f5662879..9810a6db07 100644 --- a/zig/src/families/qwen27/cuda/exact_prompt_cache.zig +++ b/zig/src/families/qwen27/cuda/exact_prompt_cache.zig @@ -175,6 +175,7 @@ pub const Cache = struct { pending: ?*Snapshot = null, owner: ?*lanes.Stream = null, stats: Stats = .{}, + post_first: @import("exact_post_first.zig").Control = .{}, pub fn init(gpa: std.mem.Allocator, on: bool) Cache { return .{ .gpa = gpa, .enabled = on }; @@ -234,6 +235,35 @@ pub const Cache = struct { return false; } + /// Serving may reserve the exact same snapshot until the first token has been handed off. + pub fn captureBeforeFirst(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry) !void { + if (!cache.post_first.enabled) return cache.capture(s, seq, ctx, ops, g); + @import("exact_post_first.zig").reserve(cache, s, seq, ctx, Snapshot.bytesWithLogits(g, s.prompt().len, s.drafts, cache.logits_enabled)); + } + + /// A successful save remains pending until the original clean-completion promotion fence. + pub fn finishAfterFirst(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry, raw: u64) !void { + const reserved = cache.post_first.take(s) orelse return; + errdefer cache.post_first.failures += 1; + if (s.isCancelled()) return error.Cancelled; + if (s.finished or cache.failed) { + cache.post_first.skips += 1; + return; + } + if (reserved.seq != seq or reserved.ctx != ctx or reserved.tokens != seq.pos or + reserved.tokens != s.prompt().len or s.emitted().len != 1 or s.pending == null) + return error.InvalidPostFirstCapture; + if (ctx) |dc| if (dc.cur != reserved.cur or dc.len != reserved.tokens or dc.end != reserved.tokens) { + return error.InvalidPostFirstContext; + }; + try cache.capture(s, seq, ctx, ops, g); + if (cache.owner == s and cache.pending != null) { + _ = try cache.captureFirstLogits(s, seq, ops, raw); + cache.post_first.saves += 1; + } else cache.post_first.skips += 1; + if (s.isCancelled()) return error.Cancelled; + } + pub fn capture(cache: *Cache, s: *lanes.Stream, seq: *st.Seq, ctx: ?*dr.Context, ops: kern.Ops, g: c.Geometry) !void { if (seq.kv8 or seq.media != null or seq.rope_delta != 0 or !cache.eligible(s) or s.drafts != (ctx != null) or (cache.active != null and cache.active.?.matches(s))) return; const size = Snapshot.bytesWithLogits(g, s.prompt().len, s.drafts, cache.logits_enabled); @@ -292,6 +322,7 @@ pub const Cache = struct { } pub fn release(cache: *Cache, s: *lanes.Stream, settled: bool, ops: kern.Ops) !void { + cache.post_first.forget(s); if (!settled) { cache.failed = true; if (cache.pending != null) { @@ -319,6 +350,7 @@ pub const Cache = struct { } pub fn abandon(cache: *Cache, s: *lanes.Stream, ops: kern.Ops) !void { + cache.post_first.forget(s); if (cache.owner == s) { try cache.freePending(ops); cache.stats.purges += 1; @@ -326,6 +358,8 @@ pub const Cache = struct { } pub fn deinit(cache: *Cache, ops: kern.Ops) !void { + cache.post_first.reserved = null; + if (cache.post_first.enabled) std.debug.print("PR608_POST_FIRST_CAPTURE {f}\n", .{std.json.fmt(.{ .reservations = cache.post_first.reservations, .saves = cache.post_first.saves, .skips = cache.post_first.skips, .failures = cache.post_first.failures }, .{})}); if (cache.enabled) std.debug.print("PR608_EXACT_CACHE {{\"hits\":{d},\"misses\":{d},\"captures\":{d},\"promotions\":{d},\"purges\":{d},\"evictions\":{d},\"refusals\":{d},\"active_bytes\":{d},\"pending_bytes\":{d},\"failed\":{}}}\n", .{ cache.stats.hits, cache.stats.misses, cache.stats.captures, cache.stats.promotions, cache.stats.purges, cache.stats.evictions, cache.stats.refusals, cache.stats.active_bytes, cache.stats.pending_bytes, cache.failed }); if (cache.logits_enabled) std.debug.print("PR608_EXACT_CACHE_LOGITS {f}\n", .{std.json.fmt(.{ .captures = cache.stats.first_logits_captures, diff --git a/zig/src/families/qwen27/cuda/lanes.zig b/zig/src/families/qwen27/cuda/lanes.zig index 2008211bb6..a199cd6739 100644 --- a/zig/src/families/qwen27/cuda/lanes.zig +++ b/zig/src/families/qwen27/cuda/lanes.zig @@ -143,8 +143,15 @@ pub const Cuda = struct { x.cost_count = n; } + const ordinary_vtable: be.Backend.VTable = .{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany }; + const post_first_vtable: be.Backend.VTable = blk: { + var table = ordinary_vtable; + table.after_first = @import("exact_post_first.zig").after; + break :blk table; + }; + pub fn backend(x: *Cuda) be.Backend { - return .{ .ptr = x, .first_before_draft = true, .vtable = &.{ .prefill = prefill, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .probabilities = probabilities, .tree = tree, .release = release, .prefill_many = prefillMany } }; + return .{ .ptr = x, .first_before_draft = true, .vtable = if (x.exact_cache.post_first.enabled) &post_first_vtable else &ordinary_vtable }; } /// The stream's drafting (its DFlash2 context and copy index), made on first use. @@ -242,11 +249,11 @@ pub const Cuda = struct { if (x.drafter) |d| if (s.drafts) { const ds = try x.draftingOf(s); try lone.prefillFrom(x.e, seq, &ds.ctx, d, s.prompt(), from, media, marks); - try x.exact_cache.capture(s, seq, &ds.ctx, x.e.ops(), x.e.w.g); + try x.exact_cache.captureBeforeFirst(s, seq, &ds.ctx, x.e.ops(), x.e.w.g); return draft_reuse.initialized(x, s); }; try x.e.prefillFrom(seq, s.prompt(), from, media, marks); - try x.exact_cache.capture(s, seq, null, x.e.ops(), x.e.w.g); + try x.exact_cache.captureBeforeFirst(s, seq, null, x.e.ops(), x.e.w.g); } /// The stream's kept state restored on its sequence: where its prompt pass starts (0: none, off the grid, failed). @@ -331,7 +338,7 @@ pub const Cuda = struct { const d = x.drafter orelse { try f.chunks(pieces); if (x.exact_cache.enabled and !x.e.kv8) for (pieces, which) |pc, i| { - try x.exact_cache.capture(streams[i], pc.seq, null, x.e.ops(), x.e.w.g); + try x.exact_cache.captureBeforeFirst(streams[i], pc.seq, null, x.e.ops(), x.e.w.g); }; return; }; @@ -370,7 +377,7 @@ pub const Cuda = struct { for (pieces, which) |pc, i| { const s = streams[i]; const ctx = if (s.drafts) &(try x.draftingOf(s)).ctx else null; - if (!x.e.kv8) try x.exact_cache.capture(s, pc.seq, ctx, x.e.ops(), x.e.w.g); + if (!x.e.kv8) try x.exact_cache.captureBeforeFirst(s, pc.seq, ctx, x.e.ops(), x.e.w.g); if (ctx != null) try draft_reuse.initialized(x, s); } } @@ -597,4 +604,5 @@ pub fn draw(gpa: std.mem.Allocator, logits: []const u16, settings: ?lanes.Sampli test { _ = @import("exact_cache_test.zig"); _ = @import("exact_eligible.zig"); + _ = @import("backend_vtable_test.zig"); } diff --git a/zig/src/native/cuda_bound.zig b/zig/src/native/cuda_bound.zig index aba0a9636d..6012414197 100644 --- a/zig/src/native/cuda_bound.zig +++ b/zig/src/native/cuda_bound.zig @@ -30,6 +30,12 @@ pub fn vtable(comptime H: type, v: be.Backend.VTable) be.Backend.VTable { return x.inner.vtable.first(x.inner.ptr, s, position); } }.f, + .after_first = if (v.after_first != null) struct { + fn f(p: *anyopaque, s: *lanes.Stream) anyerror!void { + const x = H.bind(p); + return x.inner.vtable.after_first.?(x.inner.ptr, s); + } + }.f else null, .queue = struct { fn f(p: *anyopaque, s: *lanes.Stream, feed: be.Feed, position: u64) anyerror!u64 { const x = H.bind(p); @@ -143,3 +149,36 @@ test "every entry the family's backend gives is forwarded, and none it leaves ou } try std.testing.expect(vtable(H, fake.stepped().vtable.*).prefill_step != null); } + +test "post-first forwarding binds the host and preserves optional absence and errors" { + const Calls = struct { + count: usize = 0, + seen: ?*lanes.Stream = null, + fn after(p: *anyopaque, s: *lanes.Stream) !void { + const counter: *@This() = @ptrCast(@alignCast(p)); + counter.count += 1; + counter.seen = s; + if (counter.count == 2) return error.InjectedPostFirst; + } + }; + const H = struct { + inner: be.Backend, + pub fn bind(p: *anyopaque) *@This() { + return @ptrCast(@alignCast(p)); + } + }; + var fake: lanes.fake.Fake = .{ .gpa = std.testing.allocator }; + defer fake.deinit(); + var family_table = fake.backend().vtable.*; + try std.testing.expect(vtable(H, family_table).after_first == null); + family_table.after_first = Calls.after; + var calls: Calls = .{}; + var host: H = .{ .inner = .{ .ptr = &calls, .vtable = &family_table } }; + var stream = try lanes.Stream.init(std.testing.allocator, .{ .id = "forwarded", .prompt = &.{1}, .max_new = 1 }); + defer stream.deinit(std.testing.allocator); + const forwarded = vtable(H, family_table).after_first orelse return error.MissingForwardedCallback; + try forwarded(&host, &stream); + try std.testing.expect(calls.count == 1 and calls.seen == &stream); + try std.testing.expectError(error.InjectedPostFirst, forwarded(&host, &stream)); + try std.testing.expectEqual(@as(usize, 2), calls.count); +} From 5cbd313e7c8e0014a88f1037c2ecfd3fcd6c0fa2 Mon Sep 17 00:00:00 2001 From: Jon Regan-Beasley Date: Wed, 14 Oct 2026 14:23:13 +1100 Subject: [PATCH 19/19] Sync shared CUDA operators to upstream bf3f16a2 --- zig/kernels/cuda/act_rows.cu | 1 + zig/kernels/cuda/gdn_prep.cu | 17 ++-- zig/kernels/cuda/norms.cu | 161 ++++++++++++++++++----------------- zig/kernels/cuda/qk_rows.cu | 4 +- zig/src/core/act_ref.zig | 10 +-- zig/src/cuda/norms.zig | 20 +++-- 6 files changed, 119 insertions(+), 94 deletions(-) diff --git a/zig/kernels/cuda/act_rows.cu b/zig/kernels/cuda/act_rows.cu index 9fc24a05b6..a10b0ed2c9 100644 --- a/zig/kernels/cuda/act_rows.cu +++ b/zig/kernels/cuda/act_rows.cu @@ -27,6 +27,7 @@ __device__ __forceinline__ uint4 pack8(const float* f) { return u; } +// Exactly rounded division on purpose: the same bits on every GPU, where div.full and ex2.approx are not. __device__ __forceinline__ float sigmoid(float x) { return __fdiv_rn(1.f, __fadd_rn(1.f, expf(-x))); } // A thread's 8 stored values in order, then the group's 8 lanes in a fixed tree (every lane of the warp calls). diff --git a/zig/kernels/cuda/gdn_prep.cu b/zig/kernels/cuda/gdn_prep.cu index 04f330f4e7..7915bea2f7 100644 --- a/zig/kernels/cuda/gdn_prep.cu +++ b/zig/kernels/cuda/gdn_prep.cu @@ -30,6 +30,7 @@ struct Io { float* beta; // [rows, vh] }; +// Exactly rounded division on purpose: the same bits on every GPU, where div.full and ex2.approx are not. __device__ __forceinline__ float sigmoid(float x) { return __fdiv_rn(1.f, __fadd_rn(1.f, expf(-x))); } template @@ -44,16 +45,22 @@ __device__ __forceinline__ void prep(const Io& io, int c, int kh, int vh, int ke int src[MAXT]; #pragma unroll for (int t = 0; t < MAXT; ++t) src[t] = t <= keep ? io.win[r * (keep + 1) + t] : 0; + __nv_bfloat16 xv[MAXT][J], kv[MAXT][J]; +#pragma unroll + for (int t = 0; t < MAXT; ++t) { // every tap's loads issued before any is used + if (t > keep) break; + const __nv_bfloat16* x = src[t] < keep ? io.state + (srow + src[t]) * c : io.qkv + (src[t] - keep) * io.qkv_ld; +#pragma unroll + for (int i = 0; i < J; ++i) xv[t][i] = x[ch + i], kv[t][i] = io.conv[(ch + i) * (keep + 1) + t]; + } float acc[J]; #pragma unroll for (int i = 0; i < J; ++i) acc[i] = 0.f; #pragma unroll for (int t = 0; t < MAXT; ++t) { if (t > keep) break; - const __nv_bfloat16* x = src[t] < keep ? io.state + (srow + src[t]) * c : io.qkv + (src[t] - keep) * io.qkv_ld; #pragma unroll - for (int i = 0; i < J; ++i) - acc[i] = __fmaf_rn(__bfloat162float(x[ch + i]), __bfloat162float(io.conv[(ch + i) * (keep + 1) + t]), acc[i]); + for (int i = 0; i < J; ++i) acc[i] = __fmaf_rn(__bfloat162float(xv[t][i]), __bfloat162float(kv[t][i]), acc[i]); } float cv[J]; #pragma unroll @@ -63,9 +70,9 @@ __device__ __forceinline__ void prep(const Io& io, int c, int kh, int vh, int ke #pragma unroll for (int i = 0; i < J; ++i) part = __fmaf_rn(cv[i], cv[i], part); for (int m = 16; m > 0; m >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xFFFFFFFFu, part, m)); - const float inv = rsqrtf(__fadd_rn(__fdiv_rn(part, static_cast(DK)), eps)); + const float inv = __fdiv_rn(1.f, __fsqrt_rn(__fadd_rn(__fdiv_rn(part, static_cast(DK)), eps))); const bool is_q = hd < kh; - const float scale = is_q ? 1.f / DK : rsqrtf(static_cast(DK)); + const float scale = is_q ? 1.f / DK : __fdiv_rn(1.f, __fsqrt_rn(static_cast(DK))); __nv_bfloat16* out = (is_q ? io.q : io.k) + (r * kh + (is_q ? hd : hd - kh)) * DK + J * lane; #pragma unroll for (int i = 0; i < J; ++i) out[i] = __float2bfloat16_rn(__fmul_rn(__fmul_rn(cv[i], inv), scale)); diff --git a/zig/kernels/cuda/norms.cu b/zig/kernels/cuda/norms.cu index f194bc2e3e..9bf453ccc9 100644 --- a/zig/kernels/cuda/norms.cu +++ b/zig/kernels/cuda/norms.cu @@ -36,7 +36,7 @@ __device__ __forceinline__ void store(void* p, int kind, int64_t i, float v) { else static_cast<__nv_bfloat16*>(p)[i] = __float2bfloat16_rn(v); } -// rsqrt(sum / width + eps) of a block's per-thread partial sums, the same order for every row. +// 1 / sqrt(sum / width + eps), IEEE division and square root, of a block's partial sums in one order every row. __device__ __forceinline__ float inv_rms(float part, int width, float eps) { __shared__ float warps[NT / 32]; __shared__ float inv; @@ -46,7 +46,7 @@ __device__ __forceinline__ float inv_rms(float part, int width, float eps) { if (threadIdx.x < 32) { float s = warps[threadIdx.x]; for (int m = 16; m > 0; m >>= 1) s = __fadd_rn(s, __shfl_xor_sync(0xFFFFFFFFu, s, m)); - if (threadIdx.x == 0) inv = rsqrtf(__fadd_rn(__fdiv_rn(s, static_cast(width)), eps)); + if (threadIdx.x == 0) inv = __fdiv_rn(1.f, __fsqrt_rn(__fadd_rn(__fdiv_rn(s, static_cast(width)), eps))); } __syncthreads(); const float r = inv; @@ -68,7 +68,7 @@ __device__ __forceinline__ void rms_row(const void* __restrict__ x, const float* int width, float eps) { const int64_t at = static_cast(blockIdx.x) * width; float part = 0.f; - if (WIDE) { + if constexpr (WIDE) { for (int i = threadIdx.x; i < width; i += NT) { const float a = ld(x, at + i); part = __fmaf_rn(a, a, part); @@ -79,22 +79,22 @@ __device__ __forceinline__ void rms_row(const void* __restrict__ x, const float* if (w != nullptr) a = __fmul_rn(a, w[i]); st(y, at + i, a); } - return; - } - float v[CACHED], wv[CACHED]; + } else { + float v[CACHED], wv[CACHED]; #pragma unroll - for (int j = 0; j < CACHED; ++j) { // every load issued before any is used - const int i = threadIdx.x + j * NT; - v[j] = i < width ? ld(x, at + i) : 0.f; - wv[j] = (w != nullptr && i < width) ? w[i] : 1.f; - } + for (int j = 0; j < CACHED; ++j) { // every load issued before any is used + const int i = threadIdx.x + j * NT; + v[j] = i < width ? ld(x, at + i) : 0.f; + wv[j] = (w != nullptr && i < width) ? w[i] : 1.f; + } #pragma unroll - for (int j = 0; j < CACHED; ++j) part = __fmaf_rn(v[j], v[j], part); // zeros past the row add nothing - const float inv = inv_rms(part, width, eps); + for (int j = 0; j < CACHED; ++j) part = __fmaf_rn(v[j], v[j], part); // zeros past the row add nothing + const float inv = inv_rms(part, width, eps); #pragma unroll - for (int j = 0; j < CACHED; ++j) { - const int i = threadIdx.x + j * NT; - if (i < width) st(y, at + i, w != nullptr ? __fmul_rn(__fmul_rn(v[j], inv), wv[j]) : __fmul_rn(v[j], inv)); + for (int j = 0; j < CACHED; ++j) { + const int i = threadIdx.x + j * NT; + if (i < width) st(y, at + i, w != nullptr ? __fmul_rn(__fmul_rn(v[j], inv), wv[j]) : __fmul_rn(v[j], inv)); + } } } @@ -102,7 +102,7 @@ template __device__ __forceinline__ void add_rms_row(const void* __restrict__ x, float* __restrict__ res, const float* __restrict__ w1, const float* __restrict__ w2, void* __restrict__ y, int width, float eps) { const int64_t at = static_cast(blockIdx.x) * width; - if (WIDE) { + if constexpr (WIDE) { float inv1 = 1.f; if (w1 != nullptr) { float part = 0.f; @@ -122,39 +122,39 @@ __device__ __forceinline__ void add_rms_row(const void* __restrict__ x, float* _ } const float inv2 = inv_rms(part, width, eps); for (int i = threadIdx.x; i < width; i += NT) st(y, at + i, __fmul_rn(__fmul_rn(res[at + i], inv2), w2[i])); - return; - } - float v[CACHED], rv[CACHED], a1[CACHED], a2[CACHED]; -#pragma unroll - for (int j = 0; j < CACHED; ++j) { // every load issued before any is used - const int i = threadIdx.x + j * NT; - const bool in = i < width; - v[j] = in ? ld(x, at + i) : 0.f; - rv[j] = in ? res[at + i] : 0.f; - a1[j] = (w1 != nullptr && in) ? w1[i] : 0.f; - a2[j] = in ? w2[i] : 0.f; - } - float inv1 = 1.f; - if (w1 != nullptr) { - float part = 0.f; + } else { + float v[CACHED], rv[CACHED], a1[CACHED], a2[CACHED]; +#pragma unroll + for (int j = 0; j < CACHED; ++j) { // every load issued before any is used + const int i = threadIdx.x + j * NT; + const bool in = i < width; + v[j] = in ? ld(x, at + i) : 0.f; + rv[j] = in ? res[at + i] : 0.f; + a1[j] = (w1 != nullptr && in) ? w1[i] : 0.f; + a2[j] = in ? w2[i] : 0.f; + } + float inv1 = 1.f; + if (w1 != nullptr) { + float part = 0.f; #pragma unroll - for (int j = 0; j < CACHED; ++j) part = __fmaf_rn(v[j], v[j], part); - inv1 = inv_rms(part, width, eps); - } - float part = 0.f; + for (int j = 0; j < CACHED; ++j) part = __fmaf_rn(v[j], v[j], part); + inv1 = inv_rms(part, width, eps); + } + float part = 0.f; #pragma unroll - for (int j = 0; j < CACHED; ++j) { - const float a = w1 != nullptr ? __fmul_rn(__fmul_rn(v[j], inv1), a1[j]) : v[j]; - rv[j] = __fadd_rn(rv[j], a); - part = __fmaf_rn(rv[j], rv[j], part); - } - const float inv2 = inv_rms(part, width, eps); + for (int j = 0; j < CACHED; ++j) { + const float a = w1 != nullptr ? __fmul_rn(__fmul_rn(v[j], inv1), a1[j]) : v[j]; + rv[j] = __fadd_rn(rv[j], a); + part = __fmaf_rn(rv[j], rv[j], part); + } + const float inv2 = inv_rms(part, width, eps); #pragma unroll - for (int j = 0; j < CACHED; ++j) { - const int i = threadIdx.x + j * NT; - if (i < width) { - res[at + i] = rv[j]; - st(y, at + i, __fmul_rn(__fmul_rn(rv[j], inv2), a2[j])); + for (int j = 0; j < CACHED; ++j) { + const int i = threadIdx.x + j * NT; + if (i < width) { + res[at + i] = rv[j]; + st(y, at + i, __fmul_rn(__fmul_rn(rv[j], inv2), a2[j])); + } } } } @@ -224,20 +224,18 @@ __device__ __forceinline__ void store8f(float* p, const float* f) { } // rsqrt(sum / width + eps) over a VNT-thread block: warp trees, then warp 0's tree over the 8 warp sums. +template __device__ __forceinline__ float inv_rms_v(float part, int width, float eps) { __shared__ float warps[VNT / 32]; - __shared__ float inv; for (int m = 16; m > 0; m >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xFFFFFFFFu, part, m)); if ((threadIdx.x & 31) == 0) warps[threadIdx.x >> 5] = part; __syncthreads(); - if (threadIdx.x < 32) { - float s = threadIdx.x < VNT / 32 ? warps[threadIdx.x] : 0.f; - for (int m = 4; m > 0; m >>= 1) s = __fadd_rn(s, __shfl_xor_sync(0xFFFFFFFFu, s, m)); - if (threadIdx.x == 0) inv = rsqrtf(__fadd_rn(__fdiv_rn(s, static_cast(width)), eps)); - } - __syncthreads(); - const float r = inv; - __syncthreads(); + // every warp folds the warps' sums in the same tree, so each holds the same total: no broadcast + float s = (threadIdx.x & 31) < VNT / 32 ? warps[threadIdx.x & 31] : 0.f; + for (int m = 4; m > 0; m >>= 1) s = __fadd_rn(s, __shfl_xor_sync(0xFFFFFFFFu, s, m)); + s = __shfl_sync(0xFFFFFFFFu, s, 0); + const float r = __fdiv_rn(1.f, __fsqrt_rn(__fadd_rn(__fdiv_rn(s, static_cast(width)), eps))); + if constexpr (!ONCE) __syncthreads(); // `warps` is reused by a second norm in the same block return r; } @@ -358,15 +356,24 @@ __device__ __forceinline__ void add_rms_h(const __nv_bfloat16* __restrict__ x, c const int64_t at = static_cast(blockIdx.x) * width; const int chunks = width / 8; float v[VEC_MOST][8], wv[VEC_MOST][8]; + uint4 xp[VEC_MOST], rp[VEC_MOST]; #pragma unroll - for (int j = 0; j < VEC_MOST; ++j) { + for (int j = 0; j < VEC_MOST; ++j) { // every load issued before any is used const int c = threadIdx.x + j * VNT; if (c < chunks) { - unpack8(reinterpret_cast(x + at)[c], v[j]); + xp[j] = reinterpret_cast(x + at)[c]; + if (r != nullptr) rp[j] = reinterpret_cast(r + at)[c]; load8w(w + 8 * c, wv[j]); + } + } +#pragma unroll + for (int j = 0; j < VEC_MOST; ++j) { + const int c = threadIdx.x + j * VNT; + if (c < chunks) { + unpack8(xp[j], v[j]); if (r != nullptr) { float rv[8]; - unpack8(reinterpret_cast(r + at)[c], rv); + unpack8(rp[j], rv); #pragma unroll for (int i = 0; i < 8; ++i) v[j][i] = __fadd_rn(v[j][i], rv[i]); const uint4 hp = pack8(v[j]); @@ -381,7 +388,7 @@ __device__ __forceinline__ void add_rms_h(const __nv_bfloat16* __restrict__ x, c if (threadIdx.x + j * VNT < chunks) #pragma unroll for (int i = 0; i < 8; ++i) part = __fmaf_rn(v[j][i], v[j][i], part); - const float inv = inv_rms_v(part, width, eps); + const float inv = inv_rms_v(part, width, eps); #pragma unroll for (int j = 0; j < VEC_MOST; ++j) { const int c = threadIdx.x + j * VNT; @@ -431,26 +438,26 @@ __device__ __forceinline__ void gated_rms(const __nv_bfloat16* __restrict__ y, l unpack8(*reinterpret_cast(w + 8 * sub), wv); } else { #pragma unroll - for (int i = 0; i < 8; ++i) yv[i] = zv[i] = wv[i] = 0.f; - } - float part = 0.f; + for (int i = 0; i < 8; ++i) yv[i] = zv[i] = wv[i] = 0.f; + } + float part = 0.f; #pragma unroll - for (int i = 0; i < 8; ++i) part = __fmaf_rn(yv[i], yv[i], part); + for (int i = 0; i < 8; ++i) part = __fmaf_rn(yv[i], yv[i], part); #pragma unroll - for (int m = L / 2; m > 0; m >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xFFFFFFFFu, part, m)); - const float inv = rsqrtf(__fadd_rn(__fdiv_rn(part, static_cast(DV)), eps)); - float o[8]; + for (int m = L / 2; m > 0; m >>= 1) part = __fadd_rn(part, __shfl_xor_sync(0xFFFFFFFFu, part, m)); + const float inv = __fdiv_rn(1.f, __fsqrt_rn(__fadd_rn(__fdiv_rn(part, static_cast(DV)), eps))); + float o[8]; #pragma unroll - for (int i = 0; i < 8; ++i) { - const float sg = __fdiv_rn(1.f, __fadd_rn(1.f, expf(-zv[i]))); - o[i] = __fmul_rn(__fmul_rn(zv[i], sg), __fmul_rn(__fmul_rn(yv[i], inv), wv[i])); - } - const uint4 op = pack8(o); - const int64_t at = (r * heads + hd) * DV + 8 * sub; - if (live) *reinterpret_cast(out + at) = op; - if (xs != nullptr) { - const float s = group_sum8(op, live); - if (live && (sub & 7) == 0) xs[at / 64] = s; + for (int i = 0; i < 8; ++i) { + const float sg = __fdiv_rn(1.f, __fadd_rn(1.f, expf(-zv[i]))); + o[i] = __fmul_rn(__fmul_rn(zv[i], sg), __fmul_rn(__fmul_rn(yv[i], inv), wv[i])); + } + const uint4 op = pack8(o); + const int64_t at = (r * heads + hd) * DV + 8 * sub; + if (live) *reinterpret_cast(out + at) = op; + if (xs != nullptr) { + const float s = group_sum8(op, live); + if (live && (sub & 7) == 0) xs[at / 64] = s; } } diff --git a/zig/kernels/cuda/qk_rows.cu b/zig/kernels/cuda/qk_rows.cu index 146e14017c..395b1ca677 100644 --- a/zig/kernels/cuda/qk_rows.cu +++ b/zig/kernels/cuda/qk_rows.cu @@ -96,7 +96,7 @@ extern "C" __global__ void __launch_bounds__(MOST) tf_qk_rows(const QkSrc in, co __syncthreads(); float s = 0.f; for (int w = 0; w < dim / 32; ++w) s = __fadd_rn(s, warps[w]); - const float rstd = rsqrtf(__fadd_rn(__fdiv_rn(s, static_cast(dim)), eps)); + const float rstd = __fdiv_rn(1.f, __fsqrt_rn(__fadd_rn(__fdiv_rn(s, static_cast(dim)), eps))); y = __fmul_rn(__fmul_rn(x, rstd), wv(nw, hd >= h, d)); if (d < rot) yp = __fmul_rn(__fmul_rn(__bfloat162float(src[partner]), rstd), wv(nw, hd >= h, partner)); } else if (d < rot) { @@ -166,7 +166,7 @@ extern "C" __global__ void __launch_bounds__(128) tf_qk_rows_w(const QkSrc in, c float s = 0.f; #pragma unroll for (int j = 0; j < J; ++j) s = __fadd_rn(s, part[j]); - const float rstd = rsqrtf(__fadd_rn(__fdiv_rn(s, static_cast(D)), eps)); + const float rstd = __fdiv_rn(1.f, __fsqrt_rn(__fadd_rn(__fdiv_rn(s, static_cast(D)), eps))); #pragma unroll for (int j = 0; j < J; ++j) { y[j] = __fmul_rn(__fmul_rn(x[j], rstd), w[j]); diff --git a/zig/src/core/act_ref.zig b/zig/src/core/act_ref.zig index 606a3efd8e..f56f282175 100644 --- a/zig/src/core/act_ref.zig +++ b/zig/src/core/act_ref.zig @@ -10,16 +10,16 @@ pub fn swiglu(gate: f64, up: f64) f64 { return gate * sigmoid(gate) * up; } -/// A 4-bit affine table's value: q * scale + bias. -pub fn affine4(q: u4, scale: f64, bias: f64) f64 { - return @as(f64, @floatFromInt(q)) * scale + bias; -} - /// DFlash2's dynamic conv at one value: x k0 + prev k1 with the taps as rounded (k_t = bf16(base_t + dyn_t)). pub fn dconv(x: f64, prev: f64, k0: f64, k1: f64) f64 { return x * k0 + prev * k1; } +/// A 4-bit affine table's value: q * scale + bias. +pub fn affine4(q: u4, scale: f64, bias: f64) f64 { + return @as(f64, @floatFromInt(q)) * scale + bias; +} + /// A bf16 output of fp32 arithmetic through expf: half a bf16 ulp plus 2^-18 of |want|. pub fn tolerance(want: f64) f64 { return @abs(want) * (std.math.pow(f64, 2, -8) + std.math.pow(f64, 2, -18)) + 1e-30; diff --git a/zig/src/cuda/norms.zig b/zig/src/cuda/norms.zig index b9f827e0cd..fd343b4635 100644 --- a/zig/src/cuda/norms.zig +++ b/zig/src/cuda/norms.zig @@ -18,13 +18,21 @@ const threads = 1024; const vec_threads = 256; const vec_most = 4; -/// The 16-byte forms take the row: width a multiple of 8 within VEC_MOST * 8 * VNT, every address 16-byte aligned. -fn vectorOk(width: usize, ptrs: []const u64) bool { - if (width % 8 != 0 or width > vec_most * 8 * vec_threads) return false; +/// The 16-byte forms take a bf16 row by its width alone: a multiple of 8 within VEC_MOST * 8 * VNT. +fn vectorWidth(width: usize) bool { + return width % 8 == 0 and width <= vec_most * 8 * vec_threads; +} + +/// Every address 16-byte aligned, as the 16-byte forms load. +fn aligned(ptrs: []const u64) bool { for (ptrs) |p| if (p % 16 != 0) return false; return true; } +fn vectorOk(width: usize, ptrs: []const u64) bool { + return vectorWidth(width) and aligned(ptrs); +} + pub const Norms = struct { mod: Module, rms_fn: Function, @@ -58,7 +66,8 @@ pub const Norms = struct { /// `rows` rows of `width`: y = x * rsqrt(mean(x^2) + eps) * w (w null: no weight), fp32 weights. pub fn rms(n: *const Norms, s: Stream, x: u64, x_kind: Kind, w: ?u64, y: u64, y_kind: Kind, rows: usize, width: usize, eps: f32) !void { if (rows == 0) return; - if (x_kind == .bf16 and y_kind == .bf16 and vectorOk(width, &.{ x, y, w orelse 0 })) { + if (x_kind == .bf16 and y_kind == .bf16 and vectorWidth(width)) { + if (!aligned(&.{ x, y, w orelse 0 })) return error.Misaligned; // the other tree would round otherwise var a: launch_.Args = .{}; a.add(x); a.add(w orelse 0); @@ -81,7 +90,8 @@ pub const Norms = struct { /// The residual step: res (fp32, in place) += rms(x) * w1 (w1 null: res += x), then y = rms(res) * w2. pub fn addRms(n: *const Norms, s: Stream, x: u64, x_kind: Kind, res: u64, w1: ?u64, w2: u64, y: u64, y_kind: Kind, rows: usize, width: usize, eps: f32) !void { if (rows == 0) return; - if (x_kind == .bf16 and y_kind == .bf16 and vectorOk(width, &.{ x, y, res, w1 orelse 0, w2 })) { + if (x_kind == .bf16 and y_kind == .bf16 and vectorWidth(width)) { + if (!aligned(&.{ x, y, res, w1 orelse 0, w2 })) return error.Misaligned; var a: launch_.Args = .{}; a.add(x); a.add(res);