YOLO end-to-end example, CircularTensor hardening, CPU backend, ThreadFusion, multi-GPU routing

.github/workflows/wheels.yml

@@ -115,9 +115,8 @@ jobs:
           fklroot = os.environ["FKL_ROOT"]
           inc = os.environ["FKL_INCLUDE"]
           arch = os.environ.get("FKL_ARCH", "sm_90")
-          failed = []
-          for name, ops, dt, shape, n in CASES:
-              cu = generate_cu(ops, dt, shape, n)
+
+          def compile_cu(name, cu):
               with tempfile.NamedTemporaryFile("w", suffix=".cu", delete=False) as f:
                   f.write(cu); path = f.name
               r = subprocess.run(
@@ -129,9 +128,30 @@ jobs:
               print(f"{name}: {status}")
               if r.returncode != 0:
                   print(r.stderr[-3000:])
+              return r.returncode == 0
+
+          failed = []
+          for name, ops, dt, shape, n in CASES:
+              if not compile_cu(name, generate_cu(ops, dt, shape, n)):
                   failed.append(name)
+
+          # CircularTensor: stateful, separate codegen path (PyCT + divergent
+          # launch). Compile-check both layouts/orders + a preproc chain.
+          CT_CASES = [
+              ("ct_packed_f32", fkl.CircularTensor(32, 16, batch=4, dtype="float32"),
+               []),
+              ("ct_packed_oldest", fkl.CircularTensor(32, 16, batch=3,
+               dtype="float32", order="oldest_first"), [fkl.Mul(2.0)]),
+              ("ct_planar_u8c3", fkl.CircularTensor(16, 8, batch=2, dtype="uint8",
+               channels=3, layout="planar", out_dtype="float32"),
+               [fkl.Cast("float32"), fkl.Div(255.0)]),
+          ]
+          for name, ct, ops in CT_CASES:
+              if not compile_cu(name, ct.generate_source(ops)):
+                  failed.append(name)
+
           assert not failed, f"codegen compile failures: {failed}"
-          print("all codegen cases compiled")
+          print("all codegen cases compiled (chains + CircularTensor)")
           EOF
 
   release:

.gitignore

@@ -9,3 +9,5 @@ build/
 .venv/
 .cache/
 err.log
+examples/assets/
+.venv-yolo/

README.md

@@ -83,10 +83,54 @@ backends (clang with auto-shims, nvcc):
 | test_warping_splitwrite          | 8     | PASS  | PASS |
 | test_niche_ops                   | 9     | PASS  | PASS |
 | test_dlpack                      | 9     | PASS  | PASS |
-| test_circular_tensor             | 10    | PASS  | PASS |
+| test_circular_tensor             | 21    | PASS  | PASS |
+| test_thread_fusion               | 5     | PASS  | PASS |
 | test_e2e (timing/cache)          | 1     | PASS  | PASS |
 
-112 checks per backend. Steady-state hot launch ~70 µs/call; cache-hit
+Plus, separately: test_torch_integration (9 checks, real torch 2.12+cu130:
+zero-copy both ways, external streams, CircularTensor->torch) and
+test_cpu_backend (5 checks, ParArch::CPU with numpy in/out, CPU==GPU
+cross-validated).
+
+## CPU backend (no CUDA required)
+
+```python
+pipe = fkl.compose(fkl.TensorRead(), fkl.Mul(2.0), fkl.TensorWrite(),
+                   target="cpu")
+out = pipe(numpy_array)        # numpy in -> numpy out, ParArch::CPU
+```
+Same fused chains, FKL's CPU executor, plain C++ .so (clang++; g++ rejects
+Stream_<ParArch::CPU>'s ctor spelling). Results cross-validated against
+the GPU backend.
+
+## ThreadFusion
+
+```python
+pipe = fkl.compose(..., thread_fusion=True)   # TF::ENABLED, GPU only
+```
+Vectorized multi-element threads. Auto-falls back to scalar for shapes
+whose row bytes aren't 16-aligned (external tight-pitch pointers would
+fault on float4 loads).
+
+## Multi-GPU
+
+`DeviceBuffer(..., device=N)`, `CircularTensor(..., device=N)`; `compose`
+pipelines follow the input tensor's device (torch `cuda:N`, cupy device id,
+DeviceBuffer.device). DLPack exports carry the right device id. NOTE: this
+box has 1 GPU — device routing is implemented and exercised on device 0;
+true multi-GPU runs still need a 2+ GPU machine.
+
+## Known structural limits
+
+- `fk::Equal` and other Tuple-input ops need a multi-source read (a read
+  producing `Tuple<A,B>` from two pointers). FKL has no such Read op yet —
+  upstream feature candidate, not wrappable from here.
+- Divergent HF goes through a direct kernel launch (not the Executor)
+  because of upstream issue
+  [#250](https://github.com/Libraries-Openly-Fused/FusedKernelLibrary/issues/250)
+  (grid.z = sum of sequence z-extents).
+
+128 checks per backend (+ 9 torch + 5 cpu separately). Steady-state hot launch ~70 µs/call; cache-hit
 compose ~0 ms (lazy); cold compile ~1-3 s once per chain signature.
 
 ## Temporal video: CircularTensor

examples/11_yolo_inference.py

@@ -0,0 +1,180 @@
+"""Example 11 — REAL end-to-end pipeline: YOLOv8 object detection with
+fkl-python GPU preprocessing.
+
+    JPEG -> [GPU: ONE fused kernel] -> YOLOv8n (onnxruntime) -> boxes
+
+The preprocessing that ultralytics does in several numpy/torch steps
+(letterbox resize + RGB normalize + HWC->CHW) is ONE fused FKL kernel:
+
+    TensorRead          uchar3 frame (any size)
+    Resize(640, 640,    bilinear letterbox, gray(114) padding --
+      preserve AR)        the exact YOLO convention
+    Div(255)            normalize to [0,1]
+    TensorSplit         packed HWC -> planar CHW float, NCHW-ready
+
+Requirements (see examples/assets/):
+    pip install numpy pillow onnxruntime
+    assets/bus.jpg       (ultralytics sample image)
+    assets/yolov8n.onnx  (COCO-pretrained YOLOv8n, 1x3x640x640 input)
+
+Run:  python3 11_yolo_inference.py [image.jpg]
+"""
+import ctypes
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+import onnxruntime as ort
+
+import fkl
+
+ASSETS = Path(__file__).parent / "assets"
+NET = 640
+CONF_THRES, IOU_THRES = 0.25, 0.45
+
+COCO = ("person bicycle car motorcycle airplane bus train truck boat traffic-light "
+        "fire-hydrant stop-sign parking-meter bench bird cat dog horse sheep cow "
+        "elephant bear zebra giraffe backpack umbrella handbag tie suitcase frisbee "
+        "skis snowboard sports-ball kite baseball-bat baseball-glove skateboard "
+        "surfboard tennis-racket bottle wine-glass cup fork knife spoon bowl banana "
+        "apple sandwich orange broccoli carrot hot-dog pizza donut cake chair couch "
+        "potted-plant bed dining-table toilet tv laptop mouse remote keyboard "
+        "cell-phone microwave oven toaster sink refrigerator book clock vase "
+        "scissors teddy-bear hair-drier toothbrush").split()
+
+
+# --------------------------------------------------------------------------
+# GPU preprocessing: ONE fused kernel, compiled once, reused every frame
+# --------------------------------------------------------------------------
+
+def build_preprocessor():
+    return fkl.compose(
+        fkl.TensorRead(),                                  # uchar3 HxW
+        fkl.Resize(NET, NET, interp="linear",
+                   aspect_ratio="preserve",                # letterbox
+                   background=(114.0, 114.0, 114.0)),      # YOLO gray pad
+        fkl.Div((255.0, 255.0, 255.0)),                    # [0,1]
+        fkl.TensorSplit(),                                 # HWC -> CHW float
+    )
+
+
+def upload_image(img_np):
+    """HxWx3 uint8 numpy -> fkl.DeviceBuffer (zero further copies)."""
+    h, w, _ = img_np.shape
+    buf = fkl.DeviceBuffer(w, h, "uint8", channels=3)
+    buf.copy_from_host(img_np.tobytes())
+    return buf
+
+
+def gpu_preprocess(pipe, frame_dev):
+    """Run the fused kernel; return a (1,3,640,640) float32 numpy view."""
+    out = pipe(frame_dev)                                  # ONE kernel launch
+    chw = np.frombuffer(out.copy_to_host(), dtype=np.float32)
+    return chw.reshape(1, 3, NET, NET)
+
+
+# --------------------------------------------------------------------------
+# YOLO postprocessing (standard: decode + NMS), CPU numpy
+# --------------------------------------------------------------------------
+
+def letterbox_params(w, h):
+    """Replicate FKL's PRESERVE_AR placement: scale + centered offsets."""
+    scale = min(NET / w, NET / h)
+    nw, nh = round(w * scale), round(h * scale)
+    return scale, (NET - nw) // 2, (NET - nh) // 2
+
+
+def postprocess(pred, orig_w, orig_h):
+    """pred: (1, 84, 8400) -> [(x1,y1,x2,y2,conf,cls), ...] in image coords."""
+    p = pred[0].T                                          # (8400, 84)
+    boxes, scores = p[:, :4], p[:, 4:]
+    cls = scores.argmax(1)
+    conf = scores.max(1)
+    keep = conf >= CONF_THRES
+    boxes, conf, cls = boxes[keep], conf[keep], cls[keep]
+
+    # xywh (letterbox space) -> xyxy (original image space)
+    scale, ox, oy = letterbox_params(orig_w, orig_h)
+    x, y, w, h = boxes.T
+    x1 = (x - w / 2 - ox) / scale
+    y1 = (y - h / 2 - oy) / scale
+    x2 = (x + w / 2 - ox) / scale
+    y2 = (y + h / 2 - oy) / scale
+    boxes = np.stack([x1.clip(0, orig_w), y1.clip(0, orig_h),
+                      x2.clip(0, orig_w), y2.clip(0, orig_h)], 1)
+
+    # class-aware NMS
+    order = conf.argsort()[::-1]
+    result = []
+    while order.size:
+        i = order[0]
+        result.append(i)
+        if order.size == 1:
+            break
+        rest = order[1:]
+        same = cls[rest] == cls[i]
+        xx1 = np.maximum(boxes[i, 0], boxes[rest, 0])
+        yy1 = np.maximum(boxes[i, 1], boxes[rest, 1])
+        xx2 = np.minimum(boxes[i, 2], boxes[rest, 2])
+        yy2 = np.minimum(boxes[i, 3], boxes[rest, 3])
+        inter = np.maximum(0, xx2 - xx1) * np.maximum(0, yy2 - yy1)
+        area_i = (boxes[i, 2] - boxes[i, 0]) * (boxes[i, 3] - boxes[i, 1])
+        area_r = (boxes[rest, 2] - boxes[rest, 0]) * (boxes[rest, 3] - boxes[rest, 1])
+        iou = inter / (area_i + area_r - inter + 1e-9)
+        order = rest[~(same & (iou > IOU_THRES))]
+    return [(boxes[i], conf[i], cls[i]) for i in result]
+
+
+# --------------------------------------------------------------------------
+
+def main():
+    img_path = Path(sys.argv[1]) if len(sys.argv) > 1 else ASSETS / "bus.jpg"
+    model_path = ASSETS / "yolov8n.onnx"
+    for p in (img_path, model_path):
+        if not p.exists():
+            sys.exit(f"missing {p} — see the docstring for setup")
+
+    img = np.asarray(Image.open(img_path).convert("RGB"))
+    H, W = img.shape[:2]
+    print(f"image: {img_path.name} {W}x{H}")
+
+    session = ort.InferenceSession(str(model_path),
+                                   providers=["CPUExecutionProvider"])
+    pipe = build_preprocessor()
+    frame = upload_image(img)
+
+    # warmup (compiles the fused kernel once; cached on disk afterwards)
+    t0 = time.perf_counter()
+    blob = gpu_preprocess(pipe, frame)
+    t_first = (time.perf_counter() - t0) * 1e3
+
+    # steady-state preprocessing timing
+    t0 = time.perf_counter()
+    N = 50
+    for _ in range(N):
+        blob = gpu_preprocess(pipe, frame)
+    t_pre = (time.perf_counter() - t0) * 1e3 / N
+
+    t0 = time.perf_counter()
+    pred = session.run(None, {"images": blob})[0]
+    t_inf = (time.perf_counter() - t0) * 1e3
+
+    dets = postprocess(pred, W, H)
+    print(f"\npreprocess (fused GPU kernel + D2H): first {t_first:.1f} ms, "
+          f"steady {t_pre:.2f} ms")
+    print(f"inference (onnxruntime CPU): {t_inf:.1f} ms")
+    print(f"\n{len(dets)} detections:")
+    for box, conf, c in dets:
+        x1, y1, x2, y2 = (int(v) for v in box)
+        print(f"  {COCO[c]:<14} {conf:.2f}  [{x1},{y1} -> {x2},{y2}]")
+
+    persons = sum(1 for _, _, c in dets if COCO[c] == "person")
+    buses = sum(1 for _, _, c in dets if COCO[c] == "bus")
+    print(f"\nsanity: {persons} persons + {buses} bus "
+          f"({'OK - expected for bus.jpg' if persons >= 3 and buses >= 1 else 'check image'})")
+
+
+if __name__ == "__main__":
+    main()

examples/README.md

@@ -31,6 +31,9 @@ disk cache in `~/.cache/fkl/` and start instantly.
 | `06_warping_and_borders.py` | affine rotation / perspective warp (animatable, no recompile), border policies on OOB crops |
 | `07_torch_interop_dlpack.py` | torch/cupy inputs, DLPack zero-copy outputs, external streams, preallocated outputs |
 | `08_performance.py` | cold compile vs cache hit vs hot launch; fused chain vs 6 separate kernels |
+| `09_video_temporal_window.py` | CircularTensor: rolling N-frame window for temporal models, one fused update per frame |
+| `10_torch_realworld.py` | torch-specific recipe (needs torch+cuda installed) |
+| `11_yolo_inference.py` | **REAL end-to-end**: JPEG -> ONE fused GPU kernel (letterbox+normalize+CHW) -> YOLOv8n ONNX -> NMS -> boxes. Run `./fetch_yolo_assets.sh` first; needs `pip install numpy pillow onnxruntime` |
 
 ## The one-page mental model
 

examples/fetch_yolo_assets.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Fetch the assets for examples/11_yolo_inference.py:
+#   bus.jpg      ultralytics sample image (810x1080)
+#   yolov8n.onnx COCO-pretrained YOLOv8n exported to ONNX (1x3x640x640)
+set -euo pipefail
+cd "$(dirname "$0")/assets" 2>/dev/null || { mkdir -p "$(dirname "$0")/assets"; cd "$(dirname "$0")/assets"; }
+
+if [ ! -f bus.jpg ]; then
+  curl -sL -o bus.jpg \
+    https://raw.githubusercontent.com/ultralytics/ultralytics/main/ultralytics/assets/bus.jpg
+  echo "downloaded bus.jpg"
+fi
+
+if [ ! -f yolov8n.onnx ]; then
+  curl -sL -o yolov8n.onnx \
+    "https://huggingface.co/salim4n/yolov8n-detect-onnx/resolve/main/yolov8n-onnx-web/yolov8n.onnx"
+  echo "downloaded yolov8n.onnx"
+fi
+
+python3 - <<'EOF'
+import struct
+ok = open("yolov8n.onnx", "rb").read(4)[:2] == b"\x08\x08"
+print("yolov8n.onnx looks like ONNX protobuf:", ok)
+EOF
+echo "assets ready"

fkl/backend.py

@@ -61,6 +61,7 @@ def _fkl_version() -> str:
 class CompilerBackend:
     CLANG = "clang"
     NVCC = "nvcc"
+    CPU = "cpu"      # plain C++ (clang++/g++), ParArch::CPU, no CUDA at all
 
     def __init__(self, kind: str | None = None):
         self.kind = kind or self._auto()
@@ -108,8 +109,12 @@ def compile(self, cu_source: str, sig: str, force: bool = False) -> Path:
         src_path = so.with_suffix(".cu")
         src_path.write_text(cu_source)
 
-        cmd = self._clang_cmd(src_path, so) if self.kind == self.CLANG \
-            else self._nvcc_cmd(src_path, so)
+        if self.kind == self.CPU:
+            cmd = self._cpu_cmd(src_path, so)
+        elif self.kind == self.CLANG:
+            cmd = self._clang_cmd(src_path, so)
+        else:
+            cmd = self._nvcc_cmd(src_path, so)
         proc = subprocess.run(cmd, capture_output=True, text=True)
 
         # Automatic fallback: clang's CUDA support can lag the installed CUDA
@@ -139,6 +144,21 @@ def compile(self, cu_source: str, sig: str, force: bool = False) -> Path:
         self._remember()
         return so
 
+    def _cpu_cmd(self, src: Path, so: Path):
+        # FKL's CPU path compiles as plain C++. NOTE: g++ rejects the
+        # Stream_<ParArch::CPU>() constructor spelling in stream.h; clang++
+        # accepts it, so CPU mode requires clang++.
+        cxx = shutil.which("clang++")
+        if cxx is None:
+            raise RuntimeError("CPU backend requires clang++ (g++ chokes on "
+                               "Stream_<ParArch::CPU>'s ctor spelling)")
+        cpp = src.with_suffix(".cpp")
+        if not cpp.exists():
+            cpp.write_text(src.read_text())
+        return [cxx, f"-std={_STD}", "-O2", "-shared", "-fPIC",
+                "-I", _FKL_INCLUDE, "-I", _FKL_ROOT,
+                str(cpp), "-o", str(so)]
+
     def _nvcc_cmd(self, src: Path, so: Path):
         nvcc = str(Path(_CUDA_HOME) / "bin" / "nvcc")
         return [