From 7455ba102daed60591a3f654bf2b6be7c5cc9676 Mon Sep 17 00:00:00 2001 From: Gyeongseo Park Date: Fri, 23 Oct 2026 10:11:46 +0900 Subject: [PATCH 1/5] fix: KV-cache generation at batch size >= 2 (#164) (#187) The exported graph ties position_ids' batch dim to input_ids'. Prefill passed position_ids [1, S] and decode [1, 1], and decode fed the [B] sampled tokens as [1, B]. Prefill now builds [B, S] positions, decode [B, 1] positions and a [B, 1] token input. Adds tests/test_kv_cache_batch.py: greedy KV-cache generation matches HF at B=1 and B=2 (GPU only). Co-authored-by: Claude Opus 5.5 (cherry picked from commit d46b8ffcada3f0839340689afb890626d291bba7) --- opt_prime/opt_prime/inference.py | 4 +-- opt_prime/opt_prime/schedule_infer.py | 9 ++++--- opt_prime/tests/test_kv_cache_batch.py | 37 ++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 opt_prime/tests/test_kv_cache_batch.py diff --git a/opt_prime/opt_prime/inference.py b/opt_prime/opt_prime/inference.py index c09ae220..73f6cb94 100644 --- a/opt_prime/opt_prime/inference.py +++ b/opt_prime/opt_prime/inference.py @@ -1047,9 +1047,7 @@ def _generate_with_kv_cache( # Forward single token via decode_step if self.tpl.is_first_stage(): - token_input = next_token.unsqueeze(0) if next_token.dim() == 1 else next_token - if token_input.dim() == 1: - token_input = token_input.unsqueeze(0) # [1, 1] + token_input = next_token.reshape(-1, 1) # [B] -> [B, 1] logits = self._schedule_gen.decode_step(token_input, cur_pos) else: logits = self._schedule_gen.decode_step(None, cur_pos) diff --git a/opt_prime/opt_prime/schedule_infer.py b/opt_prime/opt_prime/schedule_infer.py index 10bf1576..e68f79ed 100644 --- a/opt_prime/opt_prime/schedule_infer.py +++ b/opt_prime/opt_prime/schedule_infer.py @@ -370,7 +370,7 @@ def prefill( Args: input_ids: Input token IDs tensor of shape [batch_size, seq_len] - position_ids: Position IDs tensor of shape [1, seq_len] for RoPE. + position_ids: Position IDs tensor of shape [batch_size, seq_len] for RoPE. If None, auto-generated as arange(seq_len). Returns: @@ -383,9 +383,10 @@ def prefill( # Set position_ids for RoPE (critical for LLaMA) if position_ids is None and input_ids is not None: seq_len = input_ids.size(1) if input_ids.dim() > 1 else input_ids.size(0) + # The export ties position_ids' batch dim to input_ids' position_ids = torch.arange( seq_len, device=input_ids.device - ).unsqueeze(0) + ).expand(input_ids.size(0) if input_ids.dim() > 1 else 1, -1) if position_ids is not None: self.optimus.run_info.env[0]["position_ids"] = \ position_ids.to(self.optimus.run_info.device) @@ -427,8 +428,8 @@ def decode_step( if self.optimus.tpl.is_first_stage(): self.get_input(input_token) # Set position_ids for RoPE — must be the absolute position - position_ids = torch.tensor( - [[position]], device=input_token.device + position_ids = torch.full( + (input_token.size(0), 1), position, device=input_token.device ) self.optimus.run_info.env[0]["position_ids"] = \ position_ids.to(self.optimus.run_info.device) diff --git a/opt_prime/tests/test_kv_cache_batch.py b/opt_prime/tests/test_kv_cache_batch.py new file mode 100644 index 00000000..d1a094c3 --- /dev/null +++ b/opt_prime/tests/test_kv_cache_batch.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2026-present, ETRI, All rights reserved. +# +# opt_prime.inference: greedy generation with the KV cache (torch.export path) matches HF at batch size 1 and 2. +import copy +import os +import sys + +import pytest +import torch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="Optimus_Inference needs a GPU") + + +@pytest.fixture(scope="module") +def tiny(): + from transformers import AutoConfig, AutoModelForCausalLM + torch.manual_seed(0) + cfg = AutoConfig.for_model("llama", hidden_size=64, intermediate_size=128, num_hidden_layers=2, + num_attention_heads=4, num_key_value_heads=2, vocab_size=128, + attn_implementation="sdpa") + return AutoModelForCausalLM.from_config(cfg).eval() + + +@pytest.mark.parametrize("batch", [1, 2]) +def test_kv_cache_generation_matches_hf(tiny, batch): + from opt_prime.inference import Optimus_Inference + ids = torch.randint(0, 128, (batch, 6), generator=torch.Generator().manual_seed(batch)) + with torch.no_grad(): + ref = copy.deepcopy(tiny).cuda().generate(ids.cuda(), max_new_tokens=5, do_sample=False, + pad_token_id=0).cpu() + eng = Optimus_Inference(copy.deepcopy(tiny), use_kv_cache=True, dynamo_capture=True, dtype=torch.float32) + eng.eval() + out = eng.generate(ids, max_new_tokens=5, do_sample=False, verbose=False).cpu() + assert torch.equal(out, ref) From 3d07128a9b0059e5609ac20883b0d1372cdf0a5d Mon Sep 17 00:00:00 2001 From: Gyeongseo Park Date: Fri, 23 Oct 2026 10:11:54 +0900 Subject: [PATCH 2/5] fix: per-rank checkpoint names for TP/DP (#165) (#188) save_ckpt named files by pipeline stage only, so with tp > 1 the TP ranks overwrote each other's shards, and all DP replicas wrote and pruned the same files. save_lora_ckpt already named TP shards but let every DP replica write. Following save_hf_ckpt, only DP rank 0 writes and save_ckpt/load_ckpt add a _tp{rank} suffix when tp > 1 (PP-only names unchanged). The checkpoint directory no longer becomes checkpoint_None_ without a postfix (the "or" fallback never triggered). Adds tests/test_ckpt_names.py. Co-authored-by: Claude Opus 5.5 (cherry picked from commit fb09a618fed5ec20e511afd9b945da3461108f1f) --- opt_prime/opt_prime/opti_pri.py | 17 +++++-- opt_prime/tests/test_ckpt_names.py | 76 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 opt_prime/tests/test_ckpt_names.py diff --git a/opt_prime/opt_prime/opti_pri.py b/opt_prime/opt_prime/opti_pri.py index 4dd36604..204b40ca 100644 --- a/opt_prime/opt_prime/opti_pri.py +++ b/opt_prime/opt_prime/opti_pri.py @@ -461,7 +461,7 @@ def __init__(self, module:nn.Module, num_mb, use_gpu=True, pp_size=1, dp_size=1, self.tpl = Topology(rank, local_rank, world_size, pp_size, dp_size, tp_size) self.checkpoint = checkpoint - self.ckpt_dir = f"checkpoint_{ckpt_dir_postfix}_{self.tpl.stage}" or f"checkpoint_{self.tpl.stage}" + self.ckpt_dir = f"checkpoint_{ckpt_dir_postfix}_{self.tpl.stage}" if ckpt_dir_postfix else f"checkpoint_{self.tpl.stage}" if self.checkpoint == True: os.makedirs(self.ckpt_dir, exist_ok=True) @@ -1165,6 +1165,10 @@ def save_lora_ckpt(self, step, epoch, lora_dir=None): print("[LoRA] No LoRA config set, skipping save.") return + # DP replicas are identical → only DP rank 0 saves + if (self.get_rank() // self.tpl.tp_size) % self.tpl.dp_size != 0: + return + if lora_dir is None: lora_dir = f"lora_checkpoint_stage_{self.tpl.stage}" else: @@ -1234,7 +1238,11 @@ def save_ckpt(self, step, epoch): if self.checkpoint != True: return - ckpt_path = os.path.join(self.ckpt_dir, f"ckpt_step_{step}_epoch_{epoch}.pt") + # DP replicas are identical → only DP rank 0 saves, one file per TP shard + if (self.get_rank() // self.tpl.tp_size) % self.tpl.dp_size != 0: + return + tp = f"_tp{self.get_rank() % self.tpl.tp_size}" if self.tpl.tp_size > 1 else "" + ckpt_path = os.path.join(self.ckpt_dir, f"ckpt_step_{step}_epoch_{epoch}{tp}.pt") torch.save({ 'epoch': epoch, @@ -1245,7 +1253,7 @@ def save_ckpt(self, step, epoch): print(f"Checkpoint saved: {ckpt_path}") ckpt_files = sorted( - glob.glob(os.path.join(self.ckpt_dir, "ckpt_step_*.pt")), + glob.glob(os.path.join(self.ckpt_dir, f"ckpt_step_*{tp}.pt")), key=os.path.getmtime, reverse=True ) @@ -1257,7 +1265,8 @@ def load_ckpt(self, step, epoch): if self.checkpoint != True: return - ckpt_path = os.path.join(self.ckpt_dir, f"ckpt_step_{step}_epoch_{epoch}.pt") + tp = f"_tp{self.get_rank() % self.tpl.tp_size}" if self.tpl.tp_size > 1 else "" + ckpt_path = os.path.join(self.ckpt_dir, f"ckpt_step_{step}_epoch_{epoch}{tp}.pt") ckpt = torch.load(ckpt_path) diff --git a/opt_prime/tests/test_ckpt_names.py b/opt_prime/tests/test_ckpt_names.py new file mode 100644 index 00000000..1b4d385a --- /dev/null +++ b/opt_prime/tests/test_ckpt_names.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2026-present, ETRI, All rights reserved. +# +# opt_prime.opti_pri checkpoint naming: TP shards get their own files, only DP rank 0 writes, and the directory +# name never contains "None". Ranks are laid out as the (pp, dp, tp) device mesh: rank = (pp*dp + dp_i)*tp + tp_i. +import os +import sys +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from opt_prime import opti_pri # noqa: E402 +from opt_prime.opti_pri import Optimus_p # noqa: E402 + + +def _rank(rank, ckpt_dir, tp_size=2, dp_size=2): + o = Optimus_p.__new__(Optimus_p) # no process group: just the state the save/load paths read + o.tpl = SimpleNamespace(rank=rank, stage=0, tp_size=tp_size, dp_size=dp_size) + o.checkpoint, o.ckpt_dir, o.lora_config, o.device = True, str(ckpt_dir), "lora", torch.device("cpu") + torch.manual_seed(rank) # every rank holds different weights + submod = nn.ModuleDict({"lora_A": nn.Linear(2, 2)}) + o.run_info = SimpleNamespace(submod=submod) + o.optimizer = torch.optim.SGD(submod.parameters(), lr=0.1) + return o + + +def _weight(o): + return o.run_info.submod["lora_A"].weight.detach().clone() + + +def test_save_ckpt_one_file_per_tp_shard_from_dp_rank_0(tmp_path): + ranks = [_rank(r, tmp_path) for r in range(4)] # tp_i = r % 2, dp_i = r // 2 + for o in ranks: + o.save_ckpt(1, 0) + assert sorted(os.listdir(tmp_path)) == ["ckpt_step_1_epoch_0_tp0.pt", "ckpt_step_1_epoch_0_tp1.pt"] + expected = _weight(ranks[1]) # TP rank 1 of DP rank 0 + reload = _rank(3, tmp_path) # TP rank 1 of DP rank 1 reloads the same shard + reload.load_ckpt(1, 0) + assert torch.equal(_weight(reload), expected) + + +def test_save_ckpt_pp_only_name_unchanged(tmp_path): + _rank(0, tmp_path, tp_size=1, dp_size=1).save_ckpt(1, 0) # existing PP-only checkpoints keep loading + assert os.listdir(tmp_path) == ["ckpt_step_1_epoch_0.pt"] + + +def test_save_lora_ckpt_only_dp_rank_0_writes(tmp_path): + ranks = [_rank(r, tmp_path) for r in range(4)] + for o in ranks: + o.save_lora_ckpt(1, 0, lora_dir=str(tmp_path / "lora")) + files = sorted(os.listdir(str(tmp_path / "lora") + "_stage_0")) + assert files == ["lora_step_1_epoch_0_tp0.pt", "lora_step_1_epoch_0_tp1.pt"] + expected = _weight(ranks[1]) + reload = _rank(3, tmp_path) + reload.load_lora_ckpt(1, 0, lora_dir=str(tmp_path / "lora")) + assert torch.equal(_weight(reload), expected) + + +class _Stop(Exception): + pass + + +@pytest.mark.parametrize("postfix, name", [(None, "checkpoint_0"), ("run1", "checkpoint_run1_0")]) +def test_ckpt_dir_name(tmp_path, monkeypatch, postfix, name): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(opti_pri, "Comm", lambda **kw: SimpleNamespace(rank=0, world_size=1, local_rank=0)) + + def stop(*a): # __init__ names the directory just before picking the GPU + raise _Stop + monkeypatch.setattr(torch.cuda, "set_device", stop) + with pytest.raises(_Stop): + Optimus_p(nn.Linear(2, 2), 1, checkpoint=True, ckpt_dir_postfix=postfix) + assert os.listdir(tmp_path) == [name] From 0e441d2fad1f0bd311f3439d85173911f3ca8d25 Mon Sep 17 00:00:00 2001 From: Gyeongseo Park Date: Fri, 23 Oct 2026 10:12:01 +0900 Subject: [PATCH 3/5] fix: scope MPS daemon detection and cleanup to our pipe dir (#166) (#189) _mps_daemon_running fell back to a host-wide pgrep, so another job's MPS daemon (other pipe directory) looked like ours; the health probe on our pipe dir then failed and the stale cleanup ran pkill -u $UID on every MPS server and control process, killing the other job's daemon. Liveness is now our control socket only, and the stale cleanup sends quit to our pipe dir and removes our socket, without pkill. Adds tests/test_mps_manager.py with a fake pipe dir. Co-authored-by: Claude Opus 5.5 (cherry picked from commit 0e69cc9d54d19e336d33920298c39507d1415811) --- opt_prime/opt_prime/mps_manager.py | 44 +++++------------------------ opt_prime/tests/test_mps_manager.py | 36 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 opt_prime/tests/test_mps_manager.py diff --git a/opt_prime/opt_prime/mps_manager.py b/opt_prime/opt_prime/mps_manager.py index 71b5cfb7..63ba6fb8 100644 --- a/opt_prime/opt_prime/mps_manager.py +++ b/opt_prime/opt_prime/mps_manager.py @@ -502,34 +502,12 @@ def cleanup_barrier_files(pipe_dir: str) -> None: # --------------------------------------------------------------------------- def _mps_daemon_running(pipe_dir: str) -> bool: - """Best-effort check whether an MPS daemon is already running. + """Whether an MPS daemon serves this pipe directory (its control socket exists). - Looks for the pipe directory's control socket and probes pgrep. - Returns True if either signal is present. + A host-wide process check would mistake another job's daemon (other pipe + directory) for ours, and the stale cleanup would then hit it. """ - # 1) control socket presence (most reliable when MPS daemon started cleanly) - if os.path.exists(os.path.join(pipe_dir, "control")): - return True - # 2) Process check - try: - out = subprocess.run( - ["pgrep", "-x", "nvidia-cuda-mps-server"], - capture_output=True, text=True, timeout=2, - ) - if out.returncode == 0 and out.stdout.strip(): - return True - except (FileNotFoundError, subprocess.TimeoutExpired, OSError): - pass - try: - out = subprocess.run( - ["pgrep", "-x", "nvidia-cuda-mps-control"], - capture_output=True, text=True, timeout=2, - ) - if out.returncode == 0 and out.stdout.strip(): - return True - except (FileNotFoundError, subprocess.TimeoutExpired, OSError): - pass - return False + return os.path.exists(os.path.join(pipe_dir, "control")) # --------------------------------------------------------------------------- @@ -786,7 +764,7 @@ def _daemon_responsive(self, timeout_s: float = 2.0) -> bool: return False def _force_cleanup_stale_daemon(self) -> None: - """Best-effort cleanup of a stale daemon: try quit, then pkill (own user).""" + """Best-effort cleanup of a stale daemon on our pipe dir: quit, drop socket.""" try: subprocess.run( ["nvidia-cuda-mps-control"], @@ -798,16 +776,8 @@ def _force_cleanup_stale_daemon(self) -> None: ) except (subprocess.TimeoutExpired, OSError): pass - # Force kill stragglers (only this user's processes) - for proc_name in ("nvidia-cuda-mps-server", "nvidia-cuda-mps-control"): - try: - subprocess.run( - ["pkill", "-u", str(os.geteuid()), "-x", proc_name], - capture_output=True, timeout=5, - ) - except (FileNotFoundError, subprocess.TimeoutExpired, OSError): - pass - # Remove control socket if present + # No pkill: other jobs' daemons (other pipe directories) share the + # process names. Remove our control socket so a new daemon can bind. ctrl = os.path.join(self.pipe_dir, "control") try: os.unlink(ctrl) diff --git a/opt_prime/tests/test_mps_manager.py b/opt_prime/tests/test_mps_manager.py new file mode 100644 index 00000000..d72d7237 --- /dev/null +++ b/opt_prime/tests/test_mps_manager.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2026-present, ETRI, All rights reserved. +# +# opt_prime.mps_manager: daemon detection and stale cleanup stay scoped to this job's pipe directory. +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from opt_prime import mps_manager # noqa: E402 + + +def _fake_run(calls): + # Every command "succeeds" as if another job's MPS daemon were running on this host. + def run(cmd, *a, **kw): + calls.append(cmd) + return subprocess.CompletedProcess(cmd, 0, stdout="4242\n", stderr="") + return run + + +def test_daemon_running_only_from_own_control_socket(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(subprocess, "run", _fake_run(calls)) + assert not mps_manager._mps_daemon_running(str(tmp_path)) # another job's daemon is not ours + (tmp_path / "control").touch() + assert mps_manager._mps_daemon_running(str(tmp_path)) + + +def test_stale_cleanup_does_not_kill_other_daemons(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr(subprocess, "run", _fake_run(calls)) + monkeypatch.setattr(mps_manager.time, "sleep", lambda s: None) + (tmp_path / "control").touch() + mps_manager.MPSManager(pipe_dir=str(tmp_path), log_dir=str(tmp_path / "log"))._force_cleanup_stale_daemon() + assert not any(c[0] in ("pkill", "killall") for c in calls), calls + assert not (tmp_path / "control").exists() From 6b5d8f39a10d37fb0b439236fc091ee4664eb9f3 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 23 Oct 2026 17:39:05 +0900 Subject: [PATCH 4/5] fix: initialize hybrid ATOM cache tables and request masks (#207) --- opt_prime/docs/validation/batching/ATOM.md | 4 + .../runtime207/.research/events.jsonl | 1 + .../validation/batching/runtime207/STATUS.md | 3 + .../2026-10-23-atom-hybrid-runtime.md | 36 +++++++++ .../runtime207/validate_runtime207.py | 11 +++ .../accel/platforms/atom/batching.py | 20 ++++- opt_prime/tests/test_atom_hybrid.py | 79 +++++++++++++++++++ 7 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 opt_prime/docs/validation/batching/runtime207/.research/events.jsonl create mode 100644 opt_prime/docs/validation/batching/runtime207/STATUS.md create mode 100644 opt_prime/docs/validation/batching/runtime207/experiments/2026-10-23-atom-hybrid-runtime.md create mode 100644 opt_prime/docs/validation/batching/runtime207/validate_runtime207.py create mode 100644 opt_prime/tests/test_atom_hybrid.py diff --git a/opt_prime/docs/validation/batching/ATOM.md b/opt_prime/docs/validation/batching/ATOM.md index 3bda29e8..5cfebc15 100644 --- a/opt_prime/docs/validation/batching/ATOM.md +++ b/opt_prime/docs/validation/batching/ATOM.md @@ -100,3 +100,7 @@ shared (load average 75–135 from other users' jobs), so absolute values differ With one cache block the host tier quadruples the reused prefix (2048 → 11264 tokens). Under that run's host load the reloads did not turn into throughput: host staging is CPU-side work. PD mode (KV prefilled elsewhere, ATOM decodes only) is measured in `PD.md`. + +## Hybrid attention (#207) + +Gemma3 standalone external-cache serving uses global block ownership and lane-local sliding cache ownership. Fresh prefill resets the owning lane’s decode attention mask. Validation: [runtime integration](runtime207/experiments/2026-10-23-atom-hybrid-runtime.md). Prefix caching, host caching and remote prefill are rejected for hybrid models until sliding-state restoration is implemented. The verified hardware case is Gemma3-1B B1/context9216 with the upstream causal-prefill correction; this does not establish other hybrid model or multi-lane hardware support. diff --git a/opt_prime/docs/validation/batching/runtime207/.research/events.jsonl b/opt_prime/docs/validation/batching/runtime207/.research/events.jsonl new file mode 100644 index 00000000..dcce6224 --- /dev/null +++ b/opt_prime/docs/validation/batching/runtime207/.research/events.jsonl @@ -0,0 +1 @@ +{"id": "1b738ab428584b0f822c8c9392b3d203", "ts": "2026-10-23T17:38:12+09:00", "task": "Gemma hybrid KV tables (aicomp #207)", "result": "Actual AtomBatcher integration:16/16 Gemma outputs exact vs verified bridge through8K;20 regression tests pass,2 optional PD suites skipped. Hybrid unsupported cache modes now fail before SDK calls.", "next": "Review engine PR targeting optimus-rm; implement sliding-state restoration for hybrid prefix/PD; continue HTTP and cross-backend numerical validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"model": "Gemma3-1B", "backend": "ATOM", "batch": "1"}, "artifacts": ["results/atom-gemma3-1b-runtime207.jsonl"], "experiments": ["experiments/2026-10-23-atom-hybrid-runtime.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "RNGD+ATOM host", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/docs/validation/batching/runtime207/STATUS.md b/opt_prime/docs/validation/batching/runtime207/STATUS.md new file mode 100644 index 00000000..f5865d2b --- /dev/null +++ b/opt_prime/docs/validation/batching/runtime207/STATUS.md @@ -0,0 +1,3 @@ +# ATOM hybrid runtime integration + +State: review. Issue #207. Real batcher16/16 Gemma outputs match the experimental bridge through8K;20 local tests pass,2 optional suites skip. Prefix/host cache and PD restoration, B>1 hardware and cross-backend numerical parity remain open. diff --git a/opt_prime/docs/validation/batching/runtime207/experiments/2026-10-23-atom-hybrid-runtime.md b/opt_prime/docs/validation/batching/runtime207/experiments/2026-10-23-atom-hybrid-runtime.md new file mode 100644 index 00000000..39f61759 --- /dev/null +++ b/opt_prime/docs/validation/batching/runtime207/experiments/2026-10-23-atom-hybrid-runtime.md @@ -0,0 +1,36 @@ +# Integrate hybrid cache ownership into AtomBatcher + +- **Date**: 2026-10-23 +- **Task**: aicomp #207; broader matrix #201 +- **Host / device**: RNGD+ATOM host, one allocated ATOM; rebel-compiler0.11.0, optimum-rbln0.11.0.post1, torch2.11.0+cpu, Transformers5.8.1 +- **Commit**: baseline optimus-rm0e69cc9d54d19e336d33920298c39507d1415811; candidate this PR's batching.py patch (uncommitted during measurement) + +## Question + +Does the verified experimental table/mask fix also work inside the actual engine batcher, without replacing its methods in a validation wrapper? + +## Setup + +Gemma3-1B revision5b11413a10db4e486ef16a20101fd028f8f2499c, B1/context9216/three KV blocks, no prefix/host cache or remote prefill. Reuse the artifact rebuilt with the causal-prefill correction from optimum-rbln PR827. Config SHA256204d77f67d2566125248a0cc3c74b6a40da7bf718632a14eb70726f4ad47b170. Four input lengths128/512/2048/8192, outputs32/128, two repeats. Fixed-output tests ignore EOS. + +Only the candidate batching module is loaded in the isolated validation process. validate_runtime207.py imports atom_batching207.py as the engine module and invokes the PR209 Gemma matrix harness. It does not alter the shared installed package. The harness has no experimental-hybrid-tables flag in this run; normal AtomBatcher methods execute. + +## Command + +Within the matrix allocation/bootstrap environment, with PR203/209 matrix helpers on the working directory/search path: + +```bash +cp "$ENGINE/opt_prime/opt_prime/accel/platforms/atom/batching.py" atom_batching207.py +python container_bootstrap.py validate_runtime207.py --model unsloth/gemma-3-1b-it --cache-dir atom-artifacts --output atom-gemma3-1b-runtime207.jsonl +PYTHONPATH=opt_prime python -m pytest -q -rs opt_prime/tests/test_atom_hybrid.py opt_prime/tests/test_atom_batching.py opt_prime/tests/test_pd_atom_reuse.py opt_prime/tests/test_pd_native_atom.py +``` + +## Results + +Before implementation, all4 initial regression cases fail: missing local table and absent early rejections for prefix/host/PD. After implementation and an additional staggered cancellation/refill case,20 tests pass. Two optional suites skip because hetero_p2p and rngd_pd packages are unavailable locally; they are not counted as passing hardware or PD coverage. + +Hardware:16/16 runs complete;8/8 repeat pairs identical;16/16 outputs equal the experimentally validated bridge from PR209. That bridge independently matched complete32-step logits against SDK-owned tables at all four lengths. This integration run compares output tokens, not every logit. Raw results: results/atom-gemma3-1b-runtime207.jsonl. + +## Conclusion + +The real engine can execute this standalone Gemma configuration without the experimental method replacement. Static-cache Qwen/Llama behavior is covered by the existing fake-runtime regression suite. Prefix/host caching and PD for hybrid models are rejected before SDK work because local sliding-state restoration remains unimplemented. B>1 is covered by ownership/cancellation unit tests, not Gemma hardware validation. GPU/ATOM numerical differences from PR209 remain; this is not a complete model-quality or serving-performance certification. diff --git a/opt_prime/docs/validation/batching/runtime207/validate_runtime207.py b/opt_prime/docs/validation/batching/runtime207/validate_runtime207.py new file mode 100644 index 00000000..b233b3c0 --- /dev/null +++ b/opt_prime/docs/validation/batching/runtime207/validate_runtime207.py @@ -0,0 +1,11 @@ +"""Load the candidate engine module only in this validation process.""" +import importlib.util +import runpy +import sys + +name='opt_prime.accel.platforms.atom.batching' +spec=importlib.util.spec_from_file_location(name,'atom_batching207.py') +module=importlib.util.module_from_spec(spec) +sys.modules[name]=module +spec.loader.exec_module(module) +runpy.run_module('atom_gemma_sweep',run_name='__main__') diff --git a/opt_prime/opt_prime/accel/platforms/atom/batching.py b/opt_prime/opt_prime/accel/platforms/atom/batching.py index d29064a8..2be40067 100644 --- a/opt_prime/opt_prime/accel/platforms/atom/batching.py +++ b/opt_prime/opt_prime/accel/platforms/atom/batching.py @@ -262,6 +262,9 @@ def __init__(self, model, *, pad_token_id=None, prefix_cache=False, host_kv_gib= slow prefill device is therefore used only for overflow, and a fast one takes most prompts. local_prefill=False keeps every prefill remote (decode never stalls on a prompt; TPOT first).""" rc = model.rbln_config + self.hybrid_cache = getattr(rc, "cache_impl", None) == "hybrid" + if self.hybrid_cache and (prefix_cache or host_kv_gib or prefill_worker is not None): + raise ValueError("hybrid sliding cache restoration is not implemented for prefix/host cache or PD") self.lanes = rc.batch_size decoders = getattr(model, "decoders", None) or {} if self.lanes not in decoders: @@ -313,9 +316,18 @@ def kv_usage(self): def _prefill(self, lane, prompt, start=0): kw = {} if self.blocks is None else dict(block_tables=torch.tensor([self.blocks.lane_block[lane]], dtype=torch.int16)) - return self.model.prefill_decoder(input_ids=torch.tensor([list(prompt[start:])], dtype=torch.int64), - cache_position=torch.arange(start, len(prompt), dtype=torch.int32)[None], - batch_idx=lane, **kw) + if self.hybrid_cache and self.blocks is not None: + if start: + raise ValueError("hybrid sliding cache requires a fresh prefill") + kw["local_block_tables"] = torch.tensor([lane], dtype=torch.int16) + out = self.model.prefill_decoder(input_ids=torch.tensor([list(prompt[start:])], dtype=torch.int64), + cache_position=torch.arange(start, len(prompt), dtype=torch.int32)[None], + batch_idx=lane, **kw) + if self.hybrid_cache and self.blocks is not None: + # External tables bypass the SDK's prompt mask reset; decode only sets new bits. + self.decoder.dec_attn_mask[lane].zero_() + self.decoder.dec_attn_mask[lane, :len(prompt)] = 1 + return out @torch.inference_mode() def step(self): @@ -367,6 +379,8 @@ def _decode_step(self): tokens[r.slot], positions[r.slot] = r.tokens[-1], self.kv.length[r.slot] kw = {} if self.blocks is None else dict(block_tables=self.blocks.table([r.slot for r in rows])) cache_position = torch.tensor(positions, dtype=torch.int32)[:, None].contiguous() + if self.hybrid_cache and self.blocks is not None: + kw["local_block_tables"] = torch.arange(self.lanes, dtype=torch.int16)[:, None] if getattr(self.model.rbln_config, "use_position_ids", False): # e.g. Gemma 3 (sliding attention) kw["position_ids"] = cache_position.clone() out = self.decoder(input_ids=torch.tensor(tokens, dtype=torch.int64)[:, None].contiguous(), diff --git a/opt_prime/tests/test_atom_hybrid.py b/opt_prime/tests/test_atom_hybrid.py new file mode 100644 index 00000000..0d3b32dc --- /dev/null +++ b/opt_prime/tests/test_atom_hybrid.py @@ -0,0 +1,79 @@ +"""Hybrid SDK contract: local cache belongs to lanes, global cache to blocks.""" + +import pytest +import torch +from opt_prime.accel.pd import PDRequest +from opt_prime.accel.platforms.atom.batching import AtomBatcher +from test_atom_batching import FakeBlocks, _greedy, _llama, _prompts + + +class HybridFake(FakeBlocks): + def __init__(self): + super().__init__(_llama(), batch=2, blocks=4) + self.rbln_config.cache_impl = "hybrid" + self.rbln_config.use_position_ids = True + self.dec_attn_mask = torch.ones(2, 128) + self.decoders = {2: self} + self.local_calls = [] + + def prefill_decoder(self, *, local_block_tables=None, **kw): + assert local_block_tables is not None, "missing local cache ownership" + assert local_block_tables.tolist() == [kw["batch_idx"]] + self.local_calls.append((kw["batch_idx"], kw["block_tables"].tolist())) + return super().prefill_decoder(**kw) + + def __call__(self, *, local_block_tables=None, position_ids=None, **kw): + assert local_block_tables.tolist() == [[0], [1]] + assert torch.equal(position_ids, kw["cache_position"]) + for lane in range(2): + if int(kw["block_tables"][lane, 0]) == 3: + continue + pos = int(kw["cache_position"][lane, 0]) + self.dec_attn_mask[lane, pos] = 1 + expected = torch.arange(128) <= pos + assert torch.equal(self.dec_attn_mask[lane].bool(), expected), ( + "stale or missing prompt mask" + ) + return self._decode(**kw) + + +def test_hybrid_refill_reinitializes_only_request_lane(): + model = HybridFake() + b = AtomBatcher(model) + prompts = _prompts(4, 20, 32) + for i, p in enumerate(prompts): + f = b.submit(PDRequest(str(i), p, 3)) + b.run_until_idle() + assert f.result() == _greedy(model.m, p, 3) + assert any(lane != blocks[0] for lane, blocks in model.local_calls) + + +@pytest.mark.parametrize( + "options", + [{"prefix_cache": True}, {"host_kv_gib": 1}, {"prefill_worker": object()}], +) +def test_hybrid_unrestored_cache_modes_fail_before_sdk(options): + model = HybridFake() + with pytest.raises(ValueError, match="sliding"): + AtomBatcher(model, **options) + assert not model.local_calls + + +def test_hybrid_staggered_cancel_and_refill_preserves_other_lane(): + from opt_prime.accel.pd.runtime import RequestCancelled + + model = HybridFake() + b = AtomBatcher(model) + p, q, r = _prompts(3, 20, 35) + first = b.submit(PDRequest("a", p, 12)) + b.step() + second = b.submit(PDRequest("b", q, 8)) + b.step() + assert b.cancel("a") + third = b.submit(PDRequest("c", r, 4)) + b.run_until_idle() + with pytest.raises(RequestCancelled): + first.result() + assert second.result() == _greedy(model.m, q, 8) + assert third.result() == _greedy(model.m, r, 4) + assert sorted(b.free) == [0, 1] From 21f2c87bcdcebeeec73699b9f6f93b6fe4558077 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 23 Oct 2026 17:39:23 +0900 Subject: [PATCH 5/5] test: attach measured hybrid runtime outputs --- .../results/atom-gemma3-1b-runtime207.jsonl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 opt_prime/docs/validation/batching/runtime207/results/atom-gemma3-1b-runtime207.jsonl diff --git a/opt_prime/docs/validation/batching/runtime207/results/atom-gemma3-1b-runtime207.jsonl b/opt_prime/docs/validation/batching/runtime207/results/atom-gemma3-1b-runtime207.jsonl new file mode 100644 index 00000000..b7793176 --- /dev/null +++ b/opt_prime/docs/validation/batching/runtime207/results/atom-gemma3-1b-runtime207.jsonl @@ -0,0 +1,16 @@ +{"input_tokens": 128, "output_tokens": 32, "repeat": 0, "batch": 1, "input_sha256": "557b0eedab01df046189400093ccf00995f3e74642d1159283992762e18d1e70", "ttft_s": 0.03371518000494689, "e2e_s": 0.40370554593391716, "tpot_s": 0.011932162417759818, "tokens": [9079, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669], "status": "passed"} +{"input_tokens": 128, "output_tokens": 32, "repeat": 1, "batch": 1, "input_sha256": "557b0eedab01df046189400093ccf00995f3e74642d1159283992762e18d1e70", "ttft_s": 0.02560331905260682, "e2e_s": 0.42961790901608765, "tpot_s": 0.013029861902337401, "tokens": [9079, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669], "status": "passed"} +{"input_tokens": 128, "output_tokens": 128, "repeat": 0, "batch": 1, "input_sha256": "557b0eedab01df046189400093ccf00995f3e74642d1159283992762e18d1e70", "ttft_s": 0.028461545007303357, "e2e_s": 1.9107618549605832, "tpot_s": 0.014820677000024597, "tokens": [9079, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107], "status": "passed"} +{"input_tokens": 128, "output_tokens": 128, "repeat": 1, "batch": 1, "input_sha256": "557b0eedab01df046189400093ccf00995f3e74642d1159283992762e18d1e70", "ttft_s": 0.02486087998840958, "e2e_s": 1.8448207559995353, "tpot_s": 0.01432968326844275, "tokens": [9079, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107], "status": "passed"} +{"input_tokens": 512, "output_tokens": 32, "repeat": 0, "batch": 1, "input_sha256": "70e4b24e2cf34281354882fbe1b899615c206153ac7f3be6cf01d6f2e11dee96", "ttft_s": 0.04974888893775642, "e2e_s": 0.5968867749907076, "tpot_s": 0.017646442742026862, "tokens": [9079, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669], "status": "passed"} +{"input_tokens": 512, "output_tokens": 32, "repeat": 1, "batch": 1, "input_sha256": "70e4b24e2cf34281354882fbe1b899615c206153ac7f3be6cf01d6f2e11dee96", "ttft_s": 0.048813628032803535, "e2e_s": 0.4256832260871306, "tpot_s": 0.012154390870202933, "tokens": [9079, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669], "status": "passed"} +{"input_tokens": 512, "output_tokens": 128, "repeat": 0, "batch": 1, "input_sha256": "70e4b24e2cf34281354882fbe1b899615c206153ac7f3be6cf01d6f2e11dee96", "ttft_s": 0.056926797959022224, "e2e_s": 1.986548307002522, "tpot_s": 0.015193108716744959, "tokens": [9079, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107], "status": "passed"} +{"input_tokens": 512, "output_tokens": 128, "repeat": 1, "batch": 1, "input_sha256": "70e4b24e2cf34281354882fbe1b899615c206153ac7f3be6cf01d6f2e11dee96", "ttft_s": 0.04907623992767185, "e2e_s": 1.5254827969474718, "tpot_s": 0.011624555629641405, "tokens": [9079, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107], "status": "passed"} +{"input_tokens": 2048, "output_tokens": 32, "repeat": 0, "batch": 1, "input_sha256": "2d38f6b9337dc2f56fff99b2995ef6cda76f7da1821ddd9bb7f9f4efc4e324e5", "ttft_s": 0.29576452693436295, "e2e_s": 0.7815623859642074, "tpot_s": 0.015668201549429325, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761], "status": "passed"} +{"input_tokens": 2048, "output_tokens": 32, "repeat": 1, "batch": 1, "input_sha256": "2d38f6b9337dc2f56fff99b2995ef6cda76f7da1821ddd9bb7f9f4efc4e324e5", "ttft_s": 0.19869860005564988, "e2e_s": 0.703318142099306, "tpot_s": 0.016275610031199553, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761], "status": "passed"} +{"input_tokens": 2048, "output_tokens": 128, "repeat": 0, "batch": 1, "input_sha256": "2d38f6b9337dc2f56fff99b2995ef6cda76f7da1821ddd9bb7f9f4efc4e324e5", "ttft_s": 0.20976138697005808, "e2e_s": 2.4802555789938197, "tpot_s": 0.01787703976357781, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761], "status": "passed"} +{"input_tokens": 2048, "output_tokens": 128, "repeat": 1, "batch": 1, "input_sha256": "2d38f6b9337dc2f56fff99b2995ef6cda76f7da1821ddd9bb7f9f4efc4e324e5", "ttft_s": 0.2570356719661504, "e2e_s": 2.3116221609525383, "tpot_s": 0.016177268724477903, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761], "status": "passed"} +{"input_tokens": 8192, "output_tokens": 32, "repeat": 0, "batch": 1, "input_sha256": "0a550211766d20a9650d69b1ffabfa3b26aca460e3cf07d640acbd722c18704a", "ttft_s": 0.9133721810067073, "e2e_s": 1.4257266820641235, "tpot_s": 0.01652411677529134, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761], "status": "passed"} +{"input_tokens": 8192, "output_tokens": 32, "repeat": 1, "batch": 1, "input_sha256": "0a550211766d20a9650d69b1ffabfa3b26aca460e3cf07d640acbd722c18704a", "ttft_s": 0.8985766950063407, "e2e_s": 1.2889886510092765, "tpot_s": 0.012590600804786288, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761], "status": "passed"} +{"input_tokens": 8192, "output_tokens": 128, "repeat": 0, "batch": 1, "input_sha256": "0a550211766d20a9650d69b1ffabfa3b26aca460e3cf07d640acbd722c18704a", "ttft_s": 0.8298595099477097, "e2e_s": 2.5776417369488627, "tpot_s": 0.013760936968981868, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761], "status": "passed"} +{"input_tokens": 8192, "output_tokens": 128, "repeat": 1, "batch": 1, "input_sha256": "0a550211766d20a9650d69b1ffabfa3b26aca460e3cf07d640acbd722c18704a", "ttft_s": 0.8298346350202337, "e2e_s": 2.563930473057553, "tpot_s": 0.013652600275579928, "tokens": [9079, 236761, 108, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761, 107, 2094, 563, 1695, 4403, 236761, 669, 2604, 3673, 6097, 36157, 236761, 669, 3730, 3673, 6097, 6249, 236761], "status": "passed"}