From ac958eff360efb157b15ddce560a13549af7192d Mon Sep 17 00:00:00 2001 From: David Charlot Date: Sat, 31 Oct 2026 00:12:50 +0000 Subject: [PATCH] Thor port: build and run on NVIDIA Jetson (non-AMD hosts) - npu: build as a no-op stub when AMD XRT headers are absent (npu_stub cfg); unchanged where XRT exists - kitten: portable dot4/sdpa_rows fallbacks for non-x86_64 (NEON via autovec) - vkrt: enable maintenance4 + vulkanMemoryModel(DeviceScope) device features (spec-required by our shaders; RADV tolerated the omission, NVIDIA does not) - jenergy: Rust energy recorder for Jetson INA3221/INA238 hwmon rails, same CLI/JSON as tools/energy/record.py - bench/devinfo: print vkrt DeviceInfo incl. supported coopmat configs - PORT_NOTES_THOR.md: lab notes - first Thor numbers (Nemotron 4B + 30B-A3B decode J/token vs llama.cpp CUDA) and known gaps Decode works and is correct on Thor; batched prefill (PF=1) still device-lost, narrowed by PF_PROBE bisect to coopMatLoad-fed MulAdd. See PORT_NOTES_THOR.md. Co-Authored-By: Claude Fable 5 --- Cargo.toml | 3 +- PORT_NOTES_THOR.md | 77 ++++++++++++++++++ crates/bench/src/bin/devinfo.rs | 9 +++ crates/jenergy/Cargo.toml | 8 ++ crates/jenergy/src/main.rs | 135 ++++++++++++++++++++++++++++++++ crates/kitten/src/dec.rs | 22 ++++++ crates/npu/build.rs | 29 ++++--- crates/npu/src/lib.rs | 23 ++++++ crates/vkrt/src/lib.rs | 4 +- 9 files changed, 296 insertions(+), 14 deletions(-) create mode 100644 PORT_NOTES_THOR.md create mode 100644 crates/bench/src/bin/devinfo.rs create mode 100644 crates/jenergy/Cargo.toml create mode 100644 crates/jenergy/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 3cfac3c..ffaef71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" -members = ["crates/gguf", "crates/vkrt", "crates/rtcore", "crates/qwen2", "crates/lfm2", "crates/qwen38", "crates/kitten", "crates/bench", "crates/npu", "crates/ternary", "crates/prefill", "crates/kev", "crates/laya", "crates/parakeet", "crates/qimage", "crates/yue2", "crates/k6", "crates/mcv", "crates/serve", "crates/gptoss", "crates/gemma4", "crates/glimmer", "crates/nemotron", "crates/cosmos3", "crates/nemotron35"] +members = [ + "crates/jenergy","crates/gguf", "crates/vkrt", "crates/rtcore", "crates/qwen2", "crates/lfm2", "crates/qwen38", "crates/kitten", "crates/bench", "crates/npu", "crates/ternary", "crates/prefill", "crates/kev", "crates/laya", "crates/parakeet", "crates/qimage", "crates/yue2", "crates/k6", "crates/mcv", "crates/serve", "crates/gptoss", "crates/gemma4", "crates/glimmer", "crates/nemotron", "crates/cosmos3", "crates/nemotron35"] [workspace.package] edition = "2021" version = "0.1.0" diff --git a/PORT_NOTES_THOR.md b/PORT_NOTES_THOR.md new file mode 100644 index 0000000..390c9b8 --- /dev/null +++ b/PORT_NOTES_THOR.md @@ -0,0 +1,77 @@ +# Thor port notes (lab notebook, 2026-10-30) + +Porting strix-rt to an NVIDIA Jetson AGX Thor devkit (Tegra T264: Blackwell iGPU, 20 SMs, CC 11.0, warp 32; +14-core Neoverse-class CPU; 122 GiB unified LPDDR5X at ~273 GB/s; JetPack 7.2 / L4T r39.2.1, Ubuntu 24.04, +NVIDIA driver 595.78, Vulkan 1.4.329). Everything below is from the logs on the Thor itself. + +## What it took to build (three changes) + +1. **`npu` crate hard-links AMD XRT.** `build.rs` now probes `/usr/include/xrt/xrt_bo.h`; when absent it skips the + shim compile/link, emits `cargo:rustc-cfg=npu_stub`, and `lib.rs` substitutes no-op stubs for the 13 extern fns + (open returns null -> every NPU path errors cleanly at runtime). Zero change on hosts that have XRT. +2. **Ubuntu 24.04's packaged glslc predates `GL_EXT_integer_dot_product`** (gemvq.comp fails). Fixed with + shaderc 2026.4 from conda-forge (`pixi global install -c conda-forge shaderc`); `~/.pixi/bin` first in PATH. +3. **`kitten`'s `dot4` / `sdpa_rows` were `#[cfg(target_arch = "x86_64")]` AVX2-only** with ungated call sites. + Added portable fallbacks (multi-accumulator scalar; LLVM auto-vectorizes to NEON). A tuned NEON/SVE2 pass is + an open slot. + +With those, the whole workspace builds (cargo build --release, all 60+ bins) and runs. + +## Correctness smoke (not a full parity run) + +Nemotron 3 Nano 4B Q4_K_M via `nh`: "The capital of France is" -> top-1 token 6993 (" Paris") at logit 18.0, +4 logits clear of #2. Tokenizer read from the GGUF by `glm_tok` matches round-trips. Weights import: 2693 MiB in +0.42 s (3 arenas on the 4.7 GiB Qwen3 file earlier: 4788 MiB in 0.76 s). Teacher-forced agreement vs a reference +run (FORCE mode) still to do. + +## First numbers (decode, differential long-short: n=256 vs n=16, 240-token delta) + +Energy from the board's INA3221 rails (VDD_GPU, VDD_CPU_SOC_MSS, VIN_SYS_5V0) and the INA238 module-input rail, +sampled at 20 Hz by `crates/jenergy` (`jrecord`, a Rust sibling of tools/energy/record.py with the same CLI/JSON). +Board rails, not wall power. Idle floor ~30 W module input. + +| Nemotron 3 Nano 4B Q4_K_M, decode | strix-rt (as ported, untuned) | llama.cpp CUDA (conda-forge 0.8.0-dev 7e2fea0) | +|---|---|---| +| tok/s | 21.9 | 66.6 (tg128), 67.0 (tg256) | +| J/token, module input (INA238) | 2.247 | 1.163 | +| J/token, VDD_GPU | 0.915 | 0.536 | +| J/token, VDD_CPU_SOC_MSS | 0.463 | 0.254 | +| prompt | 6-token prompt only (NOT a pp512 number) | pp512 2251 tok/s | + +Read it the honest way: the port is 3.05x slower and ~1.9x more joules per token than the CUDA bar. The implied +GPU-rail draw during decode is ~20 W (strix-rt) vs ~36 W (llama.cpp): our kernels starve the GPU rather than +strain it, so closing the speed gap should close most of the energy gap with it. + +## Leads (in rough order of expected payoff) + +- **NV_coopmat2**: this Vulkan driver exposes cooperative-matrix-2 (tensor cores) and `int dot: 1` + (llama.cpp's vulkan device line confirms both). strix-rt's integer-dot GEMV failed on RADV; Blackwell's DP4A + path may redeem it, and coopmat2 is a bigger lever than any subgroup tuning. +- Occupancy/tile sweep for 20 SMs x warp 32 (all shader tile sizes were swept for gfx1151's 40 CUs). +- Prefill batching through the same pp512 harness used for the Strix numbers, before comparing prefill at all. +- NEON/SVE2 versions of kitten's CPU kernels. +- Teacher-forced parity runs (FORCE) per model as they come up on this box. +- Same-model cross-silicon table once Nemotron 30B-A3B numbers land here (Strix Halo side is already published). + +## Environment pins + +rustc 1.99.1; shaderc 2026.4 (conda-forge); llama.cpp conda-forge 0.8.0-dev build 7e2fea0 (CUDA+Vulkan); +models under ~/models per MODELS.md; this clone: ~/work/strix-rt on the Thor (`cbio@thor.local`). + +## Prefill on NVIDIA: device-lost hunt (2026-10-30, evening) + +PF=1 batched prefill dies with VK_ERROR_DEVICE_LOST. Validation layers found real spec violations first: +vkrt did not enable `maintenance4` (modern glslc emits LocalSizeId) or `vulkanMemoryModel`(+DeviceScope) +(GL_KHR_memory_scope_semantics) — both now enabled in vkrt (correct on AMD too; RADV just tolerated it). +Remaining violations on this driver: int8 coopmat shaders use 16x16x16 but NVIDIA only lists K=32 for s8/u8 +(ternary/Bonsai path, needs a KSize-32 variant), and one shader wants 62592 B of shared vs 49152 max. + +Supported coopmat (NVIDIA 595.78, subgroup 32, max shared 49152): f16xf16->f16/f32 at 16x16x16, 16x8x16, 16x8x8; +s8/u8 at 16x16x32, 16x8x32; bf16 16x16x16->f32. `devinfo` (new bench bin) prints this table. + +The crash itself is narrower than any of that: PF_PROBE bisect on pf_gemm (Q4_K, f16 16x16x16 — a listed config): +PROBE=3 (skeleton) OK, PROBE=2 (dequant+staging, no MMA) OK, PROBE=1 (MMA on, loads off) device lost. +pf_peak (register-only coopmats) runs fine. So the kill is coopMatLoad-fed MulAdd — A loaded straight from the +activation buffer (ADIR) and/or B from shared with LS4 stride. Next: standalone repro shader, then sweep +layout/stride/BK variants. pf_peak rates for reference: f16 29.5 TFLOPS, s8 "1778 TOPS" (out-of-spec shape on NV, +treat as invalid until re-measured at K=32). diff --git a/crates/bench/src/bin/devinfo.rs b/crates/bench/src/bin/devinfo.rs new file mode 100644 index 0000000..870d1d0 --- /dev/null +++ b/crates/bench/src/bin/devinfo.rs @@ -0,0 +1,9 @@ +//! Print the Vulkan device info vkrt sees: limits, subgroים, every cooperative-matrix config. +fn main() -> anyhow::Result<()> { + let ctx = vkrt::Ctx::new()?; + let i = &ctx.info; + println!("{} | driver {} {} | api {}", i.name, i.driver_name, i.driver_info, i.api_version); + println!("subgroup {} (min {} max {}) | max shared {} B | max wg inv {}", i.subgroup_size, i.min_subgroup, i.max_subgroup, i.max_shared_mem, i.max_wg_invocations); + for c in &i.coop_matrix { println!("coopmat {}", c); } + Ok(()) +} diff --git a/crates/jenergy/Cargo.toml b/crates/jenergy/Cargo.toml new file mode 100644 index 0000000..95f3fe1 --- /dev/null +++ b/crates/jenergy/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "jenergy" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "jrecord" +path = "src/main.rs" diff --git a/crates/jenergy/src/main.rs b/crates/jenergy/src/main.rs new file mode 100644 index 0000000..63d84bd --- /dev/null +++ b/crates/jenergy/src/main.rs @@ -0,0 +1,135 @@ +//! Stand-alone energy recorder for NVIDIA Jetson (Thor and friends), using only standard sysfs hwmon files. +//! A Rust sibling of tools/energy/record.py with the same CLI and JSON shape: +//! +//! jrecord record [--hz N] [--idle S] [--json F] [--csv F] -- CMD [ARGS...] +//! Run CMD, sample power at N Hz (default 20) while it runs, integrate to joules (trapezoid), write JSON/CSV. +//! jrecord snapshot +//! Print one JSON line with the current readings. +//! +//! Channels (each reported only if its sysfs files exist and read): +//! VDD_GPU, VDD_CPU_SOC_MSS, VIN_SYS_5V0 INA3221 rails (inN_input mV x currN_input mA) +//! ina238_power INA238 power1_input (uW), the module input rail +//! Like the SMU numbers on the AMD box, these are board rails, NOT wall power: fans, SSD and PSU losses are not in them. +//! Method used throughout the repo: run the same workload with a short and a long token count and take +//! (J_long - J_short) / (N_long - N_short); load, prefill and warm-up are in both runs and cancel. + +use std::io::Write as _; +use std::time::{Duration, Instant}; + +struct Chan { name: String, source: Source } +enum Source { Vi { v: String, i: String }, Pw { p: String } } + +fn read_f(p: &str) -> Option { std::fs::read_to_string(p).ok()?.trim().parse().ok() } + +impl Chan { + fn watts(&self) -> Option { + match &self.source { + Source::Vi { v, i } => Some(read_f(v)? / 1000.0 * (read_f(i)? / 1000.0)), // mV * mA -> W + Source::Pw { p } => Some(read_f(p)? / 1e6), // uW -> W + } + } + fn path(&self) -> &str { match &self.source { Source::Vi { v, .. } => v, Source::Pw { p } => p } } +} + +fn discover() -> Vec { + let mut out = Vec::new(); + let Ok(rd) = std::fs::read_dir("/sys/class/hwmon") else { return out }; + for e in rd.flatten() { + let d = e.path(); let ds = d.to_string_lossy().to_string(); + let name = std::fs::read_to_string(d.join("name")).unwrap_or_default(); + match name.trim() { + "ina3221" => for ch in 1..=3u32 { + let (lbl, v, i) = (format!("{ds}/in{ch}_label"), format!("{ds}/in{ch}_input"), format!("{ds}/curr{ch}_input")); + if let Ok(l) = std::fs::read_to_string(&lbl) { + if read_f(&v).is_some() && read_f(&i).is_some() { + out.push(Chan { name: l.trim().to_string(), source: Source::Vi { v, i } }); + } + } + }, + "ina238" => { + let p = format!("{ds}/power1_input"); + if read_f(&p).is_some() { out.push(Chan { name: "ina238_power".into(), source: Source::Pw { p } }); } + } + _ => {} + } + } + out +} + +fn snapshot(chans: &[Chan]) -> String { + let fields: Vec = chans.iter().filter_map(|c| c.watts().map(|w| format!("\"{}\": {:.3}", c.name, w))).collect(); + format!("{{{}}}", fields.join(", ")) +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let chans = discover(); + if chans.is_empty() { eprintln!("jrecord: no ina3221/ina238 hwmon rails found"); std::process::exit(1) } + match args.first().map(|s| s.as_str()) { + Some("snapshot") => println!("{}", snapshot(&chans)), + Some("record") => record(&chans, &args[1..]), + _ => { eprintln!("usage: jrecord record [--hz N] [--idle S] [--json F] [--csv F] -- CMD [ARGS...] | jrecord snapshot"); std::process::exit(2) } + } +} + +fn record(chans: &[Chan], args: &[String]) { + let (mut hz, mut idle, mut json_f, mut csv_f) = (20.0f64, 0.0f64, None::, None::); + let mut cmd = Vec::new(); let mut it = args.iter(); + while let Some(a) = it.next() { + match a.as_str() { + "--hz" => hz = it.next().and_then(|v| v.parse().ok()).unwrap_or(20.0), + "--idle" => idle = it.next().and_then(|v| v.parse().ok()).unwrap_or(0.0), + "--json" => json_f = it.next().cloned(), + "--csv" => csv_f = it.next().cloned(), + "--" => { cmd = it.cloned().collect(); break } + other => { eprintln!("jrecord: unknown arg {other}"); std::process::exit(2) } + } + } + if cmd.is_empty() { eprintln!("jrecord: no command after --"); std::process::exit(2) } + + // samples[i] = (t_s, per-channel W at that instant) + let period = Duration::from_secs_f64(1.0 / hz); + let stop = std::sync::atomic::AtomicBool::new(false); + let samples = std::sync::Mutex::new(Vec::<(f64, Vec)>::new()); + let t0 = Instant::now(); + let status = std::thread::scope(|s| { + s.spawn(|| { + let mut next = Instant::now(); + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + let t = t0.elapsed().as_secs_f64(); + let w: Vec = chans.iter().map(|c| c.watts().unwrap_or(f64::NAN)).collect(); + samples.lock().unwrap().push((t, w)); + next += period; let now = Instant::now(); + if next > now { std::thread::sleep(next - now) } else { next = now } + } + }); + if idle > 0.0 { std::thread::sleep(Duration::from_secs_f64(idle)) } + let st = std::process::Command::new(&cmd[0]).args(&cmd[1..]).status(); + stop.store(true, std::sync::atomic::Ordering::Relaxed); + st + }); + let samples = samples.into_inner().unwrap(); + let dur = t0.elapsed().as_secs_f64(); + match &status { Ok(s) if s.success() => {}, Ok(s) => eprintln!("jrecord: CMD exited {s}"), Err(e) => eprintln!("jrecord: CMD failed: {e}") } + + // trapezoid per channel, skipping NaN gaps + let nc = chans.len(); + let mut joules = vec![0.0f64; nc]; + for k in 1..samples.len() { + let (ta, wa) = &samples[k - 1]; let (tb, wb) = &samples[k]; + for c in 0..nc { if wa[c].is_finite() && wb[c].is_finite() { joules[c] += (wa[c] + wb[c]) * 0.5 * (tb - ta) } } + } + let avg: Vec = (0..nc).map(|c| if dur > 0.0 { joules[c] / dur } else { 0.0 }).collect(); + + if let Some(f) = &csv_f { + let mut s = String::from("t_s"); for c in chans { s += &format!(",{}_w", c.name.to_lowercase()) } s += "\n"; + for (t, w) in &samples { s += &format!("{t:.4}"); for x in w { s += &format!(",{x:.3}") } s += "\n"; } + std::fs::write(f, s).unwrap_or_else(|e| eprintln!("jrecord: csv write: {e}")); + } + let brk: Vec = (0..nc).map(|c| format!( + "{{\"fabric\": \"{}\", \"joules\": {:.3}, \"avg_w\": {:.3}, \"source\": \"{}\"}}", chans[c].name, joules[c], avg[c], chans[c].path())).collect(); + let out = format!("{{\"duration_s\": {dur:.3}, \"hz\": {hz}, \"samples\": {}, \"breakdown\": {{\"channels\": [{}]}}}}\n", samples.len(), brk.join(", ")); + if let Some(f) = &json_f { std::fs::write(f, &out).unwrap_or_else(|e| eprintln!("jrecord: json write: {e}")) } + print!("{out}"); std::io::stdout().flush().ok(); + std::process::exit(status.ok().and_then(|s| s.code()).unwrap_or(1)); +} diff --git a/crates/kitten/src/dec.rs b/crates/kitten/src/dec.rs index 6a154aa..662f937 100644 --- a/crates/kitten/src/dec.rs +++ b/crates/kitten/src/dec.rs @@ -98,6 +98,17 @@ unsafe fn dot4(a: [&[f32]; 4], b: &[f32]) -> [f32; 4] { for r in 0..4 { let mut t = [0f32; 8]; _mm256_storeu_ps(t.as_mut_ptr(), acc[r]); let mut s = ((t[0] + t[4]) + (t[1] + t[5])) + ((t[2] + t[6]) + (t[3] + t[7])); for j in i..k { s += a[r][j] * b[j] } out[r] = s; } out } +#[cfg(not(target_arch = "x86_64"))] +unsafe fn dot4(a: [&[f32]; 4], b: &[f32]) -> [f32; 4] { + // portable 4-lane dot; 4-wide accumulators so LLVM can keep NEON lanes busy + let k = b.len(); let mut out = [0f32; 4]; + for r in 0..4 { + let ar = a[r]; let (mut s0, mut s1, mut s2, mut s3) = (0f32, 0f32, 0f32, 0f32); let mut i = 0; + while i + 4 <= k { s0 += ar[i] * b[i]; s1 += ar[i + 1] * b[i + 1]; s2 += ar[i + 2] * b[i + 2]; s3 += ar[i + 3] * b[i + 3]; i += 4; } + let mut s = (s0 + s1) + (s2 + s3); for j in i..k { s += ar[j] * b[j] } out[r] = s; + } + out +} pub fn gemm_nt(a: &[f32], b: &[f32], m: usize, n: usize, k: usize, bias: Option<&[f32]>) -> Vec { let mut c = vec![0f32; m * n]; let work = |rows: std::ops::Range, c: &mut [f32]| { @@ -475,6 +486,17 @@ unsafe fn sdpa_rows(q: &[f32], k: &[f32], v: &[f32], e: usize, s: usize, ev: usi for j in 0..s { let pj = p[j]; if pj == 0.0 { continue } let vj = &v[j * ev..j * ev + ev]; for t in 0..ev { o[t] += pj * vj[t] } } } } +#[cfg(not(target_arch = "x86_64"))] +unsafe fn sdpa_rows(q: &[f32], k: &[f32], v: &[f32], e: usize, s: usize, ev: usize, sc: f32, mrow: &dyn Fn(usize, usize) -> f32, mbool: bool, rows: std::ops::Range, out: &mut [f32]) { + let mut p = vec![0f32; s]; + for (oi, i) in rows.enumerate() { + let qi = &q[i * e..i * e + e]; + for j in 0..s { let kj = &k[j * e..j * e + e]; let mut d = 0f32; for t in 0..e { d += qi[t] * kj[t] } let mut x = d * sc; let mv = mrow(i, j); if mbool { if mv == 0.0 { x = f32::NEG_INFINITY } } else { x += mv } p[j] = x; } + softmax(&mut p); + let o = &mut out[oi * ev..oi * ev + ev]; o.fill(0.0); + for j in 0..s { let pj = p[j]; if pj == 0.0 { continue } let vj = &v[j * ev..j * ev + ev]; for t in 0..ev { o[t] += pj * vj[t] } } + } +} fn sdpa(q: &T, k: &T, v: &T, mask: Option<&T>, sc: f32) -> T { // q [B,H,L,E] k [B,H,S,E] v [B,H,S,Ev]; mask broadcast to [B,H,L,S] (bool: true = keep, float: additive), read through strides let (b, h, l, e) = (q.sh[0], q.sh[1], q.sh[2], q.sh[3]); let s = k.sh[2]; let ev = v.sh[3]; diff --git a/crates/npu/build.rs b/crates/npu/build.rs index 7bd9e6a..241f9db 100644 --- a/crates/npu/build.rs +++ b/crates/npu/build.rs @@ -3,17 +3,24 @@ fn main() { let out = std::env::var("OUT_DIR").unwrap(); println!("cargo:rerun-if-changed=csrc/xrt_shim.cpp"); - let obj = format!("{out}/xrt_shim.o"); - let s = std::process::Command::new("g++") - .args(["-O2", "-std=c++17", "-fPIC", "-c", "csrc/xrt_shim.cpp", "-o", &obj, "-I/usr/include/xrt"]) - .status().expect("g++"); - assert!(s.success(), "g++ failed on xrt_shim.cpp"); - let s = std::process::Command::new("ar").args(["crs", &format!("{out}/libxrt_shim.a"), &obj]).status().expect("ar"); - assert!(s.success()); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=xrt_shim"); - println!("cargo:rustc-link-lib=dylib=xrt_coreutil"); - println!("cargo:rustc-link-lib=dylib=stdc++"); + println!("cargo:rustc-check-cfg=cfg(npu_stub)"); + let have_xrt = std::path::Path::new("/usr/include/xrt/xrt_bo.h").exists(); + if have_xrt { + let obj = format!("{out}/xrt_shim.o"); + let s = std::process::Command::new("g++") + .args(["-O2", "-std=c++17", "-fPIC", "-c", "csrc/xrt_shim.cpp", "-o", &obj, "-I/usr/include/xrt"]) + .status().expect("g++"); + assert!(s.success(), "g++ failed on xrt_shim.cpp"); + let s = std::process::Command::new("ar").args(["crs", &format!("{out}/libxrt_shim.a"), &obj]).status().expect("ar"); + assert!(s.success()); + println!("cargo:rustc-link-search=native={out}"); + println!("cargo:rustc-link-lib=static=xrt_shim"); + println!("cargo:rustc-link-lib=dylib=xrt_coreutil"); + println!("cargo:rustc-link-lib=dylib=stdc++"); + } else { + println!("cargo:warning=XRT headers not found; building npu as a no-op stub (no NPU offload)"); + println!("cargo:rustc-cfg=npu_stub"); + } println!("cargo:rerun-if-changed=shaders"); for e in std::fs::read_dir("shaders").unwrap() { let p = e.unwrap().path(); diff --git a/crates/npu/src/lib.rs b/crates/npu/src/lib.rs index 3fa9f56..0d57347 100644 --- a/crates/npu/src/lib.rs +++ b/crates/npu/src/lib.rs @@ -3,6 +3,7 @@ use anyhow::{bail, Context, Result}; use std::ffi::{c_char, c_int, c_void, CStr, CString}; +#[cfg(not(npu_stub))] extern "C" { fn npu_last_error() -> *const c_char; fn npu_open(dev: u32, xclbin: *const c_char, kname: *const c_char, instr: *const u32, n: usize) -> *mut c_void; @@ -18,6 +19,28 @@ extern "C" { fn npu_bo_free(b: *mut c_void); fn npu_run(c: *mut c_void, a: *mut c_void, b: *mut c_void, o: *mut c_void) -> c_int; } + +#[cfg(npu_stub)] +mod npu_stub_impl { + #![allow(unused_variables, clippy::missing_safety_doc)] + use std::ffi::{c_char, c_int, c_void}; + static ERR: &[u8] = b"npu: built without XRT (stub backend, no NPU on this host)\0"; + pub unsafe fn npu_last_error() -> *const c_char { ERR.as_ptr() as *const c_char } + pub unsafe fn npu_open(_d: u32, _x: *const c_char, _k: *const c_char, _i: *const u32, _n: usize) -> *mut c_void { std::ptr::null_mut() } + pub unsafe fn npu_close(_c: *mut c_void) {} + pub unsafe fn npu_bo_alloc(_c: *mut c_void, _s: usize, _a: c_int) -> *mut c_void { std::ptr::null_mut() } + pub unsafe fn npu_bo_userptr(_c: *mut c_void, _p: *mut c_void, _s: usize, _a: c_int) -> *mut c_void { std::ptr::null_mut() } + pub unsafe fn npu_bo_import_fd(_c: *mut c_void, _f: c_int) -> *mut c_void { std::ptr::null_mut() } + pub unsafe fn npu_bo_export_fd(_b: *mut c_void) -> c_int { -1 } + pub unsafe fn npu_bo_map(_b: *mut c_void) -> *mut c_void { std::ptr::null_mut() } + pub unsafe fn npu_bo_size(_b: *mut c_void) -> usize { 0 } + pub unsafe fn npu_bo_address(_b: *mut c_void) -> u64 { 0 } + pub unsafe fn npu_bo_sync(_b: *mut c_void, _t: c_int) -> c_int { -1 } + pub unsafe fn npu_bo_free(_b: *mut c_void) {} + pub unsafe fn npu_run(_c: *mut c_void, _a: *mut c_void, _b: *mut c_void, _o: *mut c_void) -> c_int { -1 } +} +#[cfg(npu_stub)] +use npu_stub_impl::*; fn last_err() -> String { unsafe { CStr::from_ptr(npu_last_error()).to_string_lossy().into_owned() } } /// A loaded xclbin + instruction stream on a hardware context (one kernel, IRON GEMM argument convention: diff --git a/crates/vkrt/src/lib.rs b/crates/vkrt/src/lib.rs index 9b230bf..032639c 100644 --- a/crates/vkrt/src/lib.rs +++ b/crates/vkrt/src/lib.rs @@ -134,8 +134,8 @@ impl Ctx { let qp = [1.0f32]; let qci = [vk::DeviceQueueCreateInfo::default().queue_family_index(qfam).queue_priorities(&qp)]; let mut f11 = vk::PhysicalDeviceVulkan11Features::default().storage_buffer16_bit_access(true); - let mut f12 = vk::PhysicalDeviceVulkan12Features::default().shader_float16(true).shader_int8(true).storage_buffer8_bit_access(true).buffer_device_address(true).shader_buffer_int64_atomics(true); - let mut f13 = vk::PhysicalDeviceVulkan13Features::default().subgroup_size_control(true).compute_full_subgroups(true); + let mut f12 = vk::PhysicalDeviceVulkan12Features::default().shader_float16(true).shader_int8(true).storage_buffer8_bit_access(true).buffer_device_address(true).shader_buffer_int64_atomics(true).vulkan_memory_model(true).vulkan_memory_model_device_scope(true); + let mut f13 = vk::PhysicalDeviceVulkan13Features::default().subgroup_size_control(true).compute_full_subgroups(true).maintenance4(true); let mut fcm = vk::PhysicalDeviceCooperativeMatrixFeaturesKHR::default().cooperative_matrix(true); let mut dev_exts = vec![]; if has_coop_matrix { dev_exts.push(ash::khr::cooperative_matrix::NAME.as_ptr()); }